xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'fix/rocksdbState' into enh/rocksRevert

This commit is contained in:
yihaoDeng 2023-04-23 02:37:26 +00:00
parent 0750e0d43a 9e22bd07a4
commit 917b07348c
96 changed files with 4029 additions and 1272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 0681d8b GIT_TAG ffc2e6f
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

2
examples/JDBC/taosdemo/pom.xml
View File

@ -10,7 +10,7 @@
<description>Demo project for TDengine</description> <description>Demo project for TDengine</description>
<properties> <properties>
<spring.version>5.3.26</spring.version> <spring.version>5.3.27</spring.version>
</properties> </properties>
<dependencies> <dependencies>

3
include/client/taos.h
View File

@ -225,6 +225,9 @@ DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *tabl
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
// set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner
DLL_EXPORT void taos_set_hb_quit(int8_t quitByKill);
/* --------------------------schemaless INTERFACE------------------------------- */ /* --------------------------schemaless INTERFACE------------------------------- */
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);

4
include/common/tcommon.h
View File

@ -341,6 +341,8 @@ typedef struct {
float f; float f;
}; };
size_t length; size_t length;
bool keyEscaped;
bool valueEscaped;
} SSmlKv; } SSmlKv;
#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_ASC_FORWARD_STEP 1
@ -379,6 +381,8 @@ typedef struct STUidTagInfo {
#define UD_GROUPID_COLUMN_INDEX 1 #define UD_GROUPID_COLUMN_INDEX 1
#define UD_TAG_COLUMN_INDEX 2 #define UD_TAG_COLUMN_INDEX 2
int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

6
include/common/tmsg.h
View File

@ -2071,7 +2071,6 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq
typedef struct { typedef struct {
char key[TSDB_SUBSCRIBE_KEY_LEN]; char key[TSDB_SUBSCRIBE_KEY_LEN];
SArray* lostConsumers; // SArray<int64_t>
SArray* removedConsumers; // SArray<int64_t> SArray* removedConsumers; // SArray<int64_t>
SArray* newConsumers; // SArray<int64_t> SArray* newConsumers; // SArray<int64_t>
} SMqRebInfo; } SMqRebInfo;
@ -2082,10 +2081,6 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) {
return NULL; return NULL;
} }
tstrncpy(pRebInfo->key, key, TSDB_SUBSCRIBE_KEY_LEN); tstrncpy(pRebInfo->key, key, TSDB_SUBSCRIBE_KEY_LEN);
pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t));
if (pRebInfo->lostConsumers == NULL) {
goto _err;
}
pRebInfo->removedConsumers = taosArrayInit(0, sizeof(int64_t)); pRebInfo->removedConsumers = taosArrayInit(0, sizeof(int64_t));
if (pRebInfo->removedConsumers == NULL) { if (pRebInfo->removedConsumers == NULL) {
goto _err; goto _err;
@ -2096,7 +2091,6 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) {
} }
return pRebInfo; return pRebInfo;
_err: _err:
taosArrayDestroy(pRebInfo->lostConsumers);
taosArrayDestroy(pRebInfo->removedConsumers); taosArrayDestroy(pRebInfo->removedConsumers);
taosArrayDestroy(pRebInfo->newConsumers); taosArrayDestroy(pRebInfo->newConsumers);
taosMemoryFreeClear(pRebInfo); taosMemoryFreeClear(pRebInfo);

659
include/common/ttokendef.h
View File

@ -16,335 +16,336 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #define TK_OR 1
#define TK_AND 2 #define TK_AND 2
#define TK_UNION 3 #define TK_UNION 3
#define TK_ALL 4 #define TK_ALL 4
#define TK_MINUS 5 #define TK_MINUS 5
#define TK_EXCEPT 6 #define TK_EXCEPT 6
#define TK_INTERSECT 7 #define TK_INTERSECT 7
#define TK_NK_BITAND 8 #define TK_NK_BITAND 8
#define TK_NK_BITOR 9 #define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10 #define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11 #define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12 #define TK_NK_PLUS 12
#define TK_NK_MINUS 13 #define TK_NK_MINUS 13
#define TK_NK_STAR 14 #define TK_NK_STAR 14
#define TK_NK_SLASH 15 #define TK_NK_SLASH 15
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_ACCOUNT 19 #define TK_ACCOUNT 19
#define TK_NK_ID 20 #define TK_NK_ID 20
#define TK_PASS 21 #define TK_PASS 21
#define TK_NK_STRING 22 #define TK_NK_STRING 22
#define TK_ALTER 23 #define TK_ALTER 23
#define TK_PPS 24 #define TK_PPS 24
#define TK_TSERIES 25 #define TK_TSERIES 25
#define TK_STORAGE 26 #define TK_STORAGE 26
#define TK_STREAMS 27 #define TK_STREAMS 27
#define TK_QTIME 28 #define TK_QTIME 28
#define TK_DBS 29 #define TK_DBS 29
#define TK_USERS 30 #define TK_USERS 30
#define TK_CONNS 31 #define TK_CONNS 31
#define TK_STATE 32 #define TK_STATE 32
#define TK_USER 33 #define TK_USER 33
#define TK_ENABLE 34 #define TK_ENABLE 34
#define TK_NK_INTEGER 35 #define TK_NK_INTEGER 35
#define TK_SYSINFO 36 #define TK_SYSINFO 36
#define TK_DROP 37 #define TK_DROP 37
#define TK_GRANT 38 #define TK_GRANT 38
#define TK_ON 39 #define TK_ON 39
#define TK_TO 40 #define TK_TO 40
#define TK_REVOKE 41 #define TK_REVOKE 41
#define TK_FROM 42 #define TK_FROM 42
#define TK_SUBSCRIBE 43 #define TK_SUBSCRIBE 43
#define TK_NK_COMMA 44 #define TK_NK_COMMA 44
#define TK_READ 45 #define TK_READ 45
#define TK_WRITE 46 #define TK_WRITE 46
#define TK_NK_DOT 47 #define TK_NK_DOT 47
#define TK_WITH 48 #define TK_WITH 48
#define TK_DNODE 49 #define TK_DNODE 49
#define TK_PORT 50 #define TK_PORT 50
#define TK_DNODES 51 #define TK_DNODES 51
#define TK_NK_IPTOKEN 52 #define TK_NK_IPTOKEN 52
#define TK_FORCE 53 #define TK_FORCE 53
#define TK_LOCAL 54 #define TK_LOCAL 54
#define TK_QNODE 55 #define TK_QNODE 55
#define TK_BNODE 56 #define TK_BNODE 56
#define TK_SNODE 57 #define TK_SNODE 57
#define TK_MNODE 58 #define TK_MNODE 58
#define TK_DATABASE 59 #define TK_DATABASE 59
#define TK_USE 60 #define TK_USE 60
#define TK_FLUSH 61 #define TK_FLUSH 61
#define TK_TRIM 62 #define TK_TRIM 62
#define TK_COMPACT 63 #define TK_COMPACT 63
#define TK_IF 64 #define TK_IF 64
#define TK_NOT 65 #define TK_NOT 65
#define TK_EXISTS 66 #define TK_EXISTS 66
#define TK_BUFFER 67 #define TK_BUFFER 67
#define TK_CACHEMODEL 68 #define TK_CACHEMODEL 68
#define TK_CACHESIZE 69 #define TK_CACHESIZE 69
#define TK_COMP 70 #define TK_COMP 70
#define TK_DURATION 71 #define TK_DURATION 71
#define TK_NK_VARIABLE 72 #define TK_NK_VARIABLE 72
#define TK_MAXROWS 73 #define TK_MAXROWS 73
#define TK_MINROWS 74 #define TK_MINROWS 74
#define TK_KEEP 75 #define TK_KEEP 75
#define TK_PAGES 76 #define TK_PAGES 76
#define TK_PAGESIZE 77 #define TK_PAGESIZE 77
#define TK_TSDB_PAGESIZE 78 #define TK_TSDB_PAGESIZE 78
#define TK_PRECISION 79 #define TK_PRECISION 79
#define TK_REPLICA 80 #define TK_REPLICA 80
#define TK_VGROUPS 81 #define TK_VGROUPS 81
#define TK_SINGLE_STABLE 82 #define TK_SINGLE_STABLE 82
#define TK_RETENTIONS 83 #define TK_RETENTIONS 83
#define TK_SCHEMALESS 84 #define TK_SCHEMALESS 84
#define TK_WAL_LEVEL 85 #define TK_WAL_LEVEL 85
#define TK_WAL_FSYNC_PERIOD 86 #define TK_WAL_FSYNC_PERIOD 86
#define TK_WAL_RETENTION_PERIOD 87 #define TK_WAL_RETENTION_PERIOD 87
#define TK_WAL_RETENTION_SIZE 88 #define TK_WAL_RETENTION_SIZE 88
#define TK_WAL_ROLL_PERIOD 89 #define TK_WAL_ROLL_PERIOD 89
#define TK_WAL_SEGMENT_SIZE 90 #define TK_WAL_SEGMENT_SIZE 90
#define TK_STT_TRIGGER 91 #define TK_STT_TRIGGER 91
#define TK_TABLE_PREFIX 92 #define TK_TABLE_PREFIX 92
#define TK_TABLE_SUFFIX 93 #define TK_TABLE_SUFFIX 93
#define TK_NK_COLON 94 #define TK_NK_COLON 94
#define TK_MAX_SPEED 95 #define TK_MAX_SPEED 95
#define TK_START 96 #define TK_START 96
#define TK_TIMESTAMP 97 #define TK_TIMESTAMP 97
#define TK_END 98 #define TK_END 98
#define TK_TABLE 99 #define TK_TABLE 99
#define TK_NK_LP 100 #define TK_NK_LP 100
#define TK_NK_RP 101 #define TK_NK_RP 101
#define TK_STABLE 102 #define TK_STABLE 102
#define TK_ADD 103 #define TK_ADD 103
#define TK_COLUMN 104 #define TK_COLUMN 104
#define TK_MODIFY 105 #define TK_MODIFY 105
#define TK_RENAME 106 #define TK_RENAME 106
#define TK_TAG 107 #define TK_TAG 107
#define TK_SET 108 #define TK_SET 108
#define TK_NK_EQ 109 #define TK_NK_EQ 109
#define TK_USING 110 #define TK_USING 110
#define TK_TAGS 111 #define TK_TAGS 111
#define TK_BOOL 112 #define TK_BOOL 112
#define TK_TINYINT 113 #define TK_TINYINT 113
#define TK_SMALLINT 114 #define TK_SMALLINT 114
#define TK_INT 115 #define TK_INT 115
#define TK_INTEGER 116 #define TK_INTEGER 116
#define TK_BIGINT 117 #define TK_BIGINT 117
#define TK_FLOAT 118 #define TK_FLOAT 118
#define TK_DOUBLE 119 #define TK_DOUBLE 119
#define TK_BINARY 120 #define TK_BINARY 120
#define TK_NCHAR 121 #define TK_NCHAR 121
#define TK_UNSIGNED 122 #define TK_UNSIGNED 122
#define TK_JSON 123 #define TK_JSON 123
#define TK_VARCHAR 124 #define TK_VARCHAR 124
#define TK_MEDIUMBLOB 125 #define TK_MEDIUMBLOB 125
#define TK_BLOB 126 #define TK_BLOB 126
#define TK_VARBINARY 127 #define TK_VARBINARY 127
#define TK_DECIMAL 128 #define TK_DECIMAL 128
#define TK_COMMENT 129 #define TK_COMMENT 129
#define TK_MAX_DELAY 130 #define TK_MAX_DELAY 130
#define TK_WATERMARK 131 #define TK_WATERMARK 131
#define TK_ROLLUP 132 #define TK_ROLLUP 132
#define TK_TTL 133 #define TK_TTL 133
#define TK_SMA 134 #define TK_SMA 134
#define TK_DELETE_MARK 135 #define TK_DELETE_MARK 135
#define TK_FIRST 136 #define TK_FIRST 136
#define TK_LAST 137 #define TK_LAST 137
#define TK_SHOW 138 #define TK_SHOW 138
#define TK_PRIVILEGES 139 #define TK_PRIVILEGES 139
#define TK_DATABASES 140 #define TK_DATABASES 140
#define TK_TABLES 141 #define TK_TABLES 141
#define TK_STABLES 142 #define TK_STABLES 142
#define TK_MNODES 143 #define TK_MNODES 143
#define TK_QNODES 144 #define TK_QNODES 144
#define TK_FUNCTIONS 145 #define TK_FUNCTIONS 145
#define TK_INDEXES 146 #define TK_INDEXES 146
#define TK_ACCOUNTS 147 #define TK_ACCOUNTS 147
#define TK_APPS 148 #define TK_APPS 148
#define TK_CONNECTIONS 149 #define TK_CONNECTIONS 149
#define TK_LICENCES 150 #define TK_LICENCES 150
#define TK_GRANTS 151 #define TK_GRANTS 151
#define TK_QUERIES 152 #define TK_QUERIES 152
#define TK_SCORES 153 #define TK_SCORES 153
#define TK_TOPICS 154 #define TK_TOPICS 154
#define TK_VARIABLES 155 #define TK_VARIABLES 155
#define TK_CLUSTER 156 #define TK_CLUSTER 156
#define TK_BNODES 157 #define TK_BNODES 157
#define TK_SNODES 158 #define TK_SNODES 158
#define TK_TRANSACTIONS 159 #define TK_TRANSACTIONS 159
#define TK_DISTRIBUTED 160 #define TK_DISTRIBUTED 160
#define TK_CONSUMERS 161 #define TK_CONSUMERS 161
#define TK_SUBSCRIPTIONS 162 #define TK_SUBSCRIPTIONS 162
#define TK_VNODES 163 #define TK_VNODES 163
#define TK_ALIVE 164 #define TK_ALIVE 164
#define TK_LIKE 165 #define TK_LIKE 165
#define TK_TBNAME 166 #define TK_TBNAME 166
#define TK_QTAGS 167 #define TK_QTAGS 167
#define TK_AS 168 #define TK_AS 168
#define TK_INDEX 169 #define TK_INDEX 169
#define TK_FUNCTION 170 #define TK_FUNCTION 170
#define TK_INTERVAL 171 #define TK_INTERVAL 171
#define TK_COUNT 172 #define TK_COUNT 172
#define TK_LAST_ROW 173 #define TK_LAST_ROW 173
#define TK_TOPIC 174 #define TK_TOPIC 174
#define TK_META 175 #define TK_META 175
#define TK_CONSUMER 176 #define TK_CONSUMER 176
#define TK_GROUP 177 #define TK_GROUP 177
#define TK_DESC 178 #define TK_DESC 178
#define TK_DESCRIBE 179 #define TK_DESCRIBE 179
#define TK_RESET 180 #define TK_RESET 180
#define TK_QUERY 181 #define TK_QUERY 181
#define TK_CACHE 182 #define TK_CACHE 182
#define TK_EXPLAIN 183 #define TK_EXPLAIN 183
#define TK_ANALYZE 184 #define TK_ANALYZE 184
#define TK_VERBOSE 185 #define TK_VERBOSE 185
#define TK_NK_BOOL 186 #define TK_NK_BOOL 186
#define TK_RATIO 187 #define TK_RATIO 187
#define TK_NK_FLOAT 188 #define TK_NK_FLOAT 188
#define TK_OUTPUTTYPE 189 #define TK_OUTPUTTYPE 189
#define TK_AGGREGATE 190 #define TK_AGGREGATE 190
#define TK_BUFSIZE 191 #define TK_BUFSIZE 191
#define TK_LANGUAGE 192 #define TK_LANGUAGE 192
#define TK_REPLACE 193 #define TK_REPLACE 193
#define TK_STREAM 194 #define TK_STREAM 194
#define TK_INTO 195 #define TK_INTO 195
#define TK_TRIGGER 196 #define TK_TRIGGER 196
#define TK_AT_ONCE 197 #define TK_AT_ONCE 197
#define TK_WINDOW_CLOSE 198 #define TK_WINDOW_CLOSE 198
#define TK_IGNORE 199 #define TK_IGNORE 199
#define TK_EXPIRED 200 #define TK_EXPIRED 200
#define TK_FILL_HISTORY 201 #define TK_FILL_HISTORY 201
#define TK_UPDATE 202 #define TK_UPDATE 202
#define TK_SUBTABLE 203 #define TK_SUBTABLE 203
#define TK_KILL 204 #define TK_KILL 204
#define TK_CONNECTION 205 #define TK_CONNECTION 205
#define TK_TRANSACTION 206 #define TK_TRANSACTION 206
#define TK_BALANCE 207 #define TK_BALANCE 207
#define TK_VGROUP 208 #define TK_VGROUP 208
#define TK_LEADER 209 #define TK_LEADER 209
#define TK_MERGE 210 #define TK_MERGE 210
#define TK_REDISTRIBUTE 211 #define TK_REDISTRIBUTE 211
#define TK_SPLIT 212 #define TK_SPLIT 212
#define TK_DELETE 213 #define TK_DELETE 213
#define TK_INSERT 214 #define TK_INSERT 214
#define TK_NULL 215 #define TK_NULL 215
#define TK_NK_QUESTION 216 #define TK_NK_QUESTION 216
#define TK_NK_ARROW 217 #define TK_NK_ARROW 217
#define TK_ROWTS 218 #define TK_ROWTS 218
#define TK_QSTART 219 #define TK_QSTART 219
#define TK_QEND 220 #define TK_QEND 220
#define TK_QDURATION 221 #define TK_QDURATION 221
#define TK_WSTART 222 #define TK_WSTART 222
#define TK_WEND 223 #define TK_WEND 223
#define TK_WDURATION 224 #define TK_WDURATION 224
#define TK_IROWTS 225 #define TK_IROWTS 225
#define TK_ISFILLED 226 #define TK_ISFILLED 226
#define TK_CAST 227 #define TK_CAST 227
#define TK_NOW 228 #define TK_NOW 228
#define TK_TODAY 229 #define TK_TODAY 229
#define TK_TIMEZONE 230 #define TK_TIMEZONE 230
#define TK_CLIENT_VERSION 231 #define TK_CLIENT_VERSION 231
#define TK_SERVER_VERSION 232 #define TK_SERVER_VERSION 232
#define TK_SERVER_STATUS 233 #define TK_SERVER_STATUS 233
#define TK_CURRENT_USER 234 #define TK_CURRENT_USER 234
#define TK_CASE 235 #define TK_CASE 235
#define TK_WHEN 236 #define TK_WHEN 236
#define TK_THEN 237 #define TK_THEN 237
#define TK_ELSE 238 #define TK_ELSE 238
#define TK_BETWEEN 239 #define TK_BETWEEN 239
#define TK_IS 240 #define TK_IS 240
#define TK_NK_LT 241 #define TK_NK_LT 241
#define TK_NK_GT 242 #define TK_NK_GT 242
#define TK_NK_LE 243 #define TK_NK_LE 243
#define TK_NK_GE 244 #define TK_NK_GE 244
#define TK_NK_NE 245 #define TK_NK_NE 245
#define TK_MATCH 246 #define TK_MATCH 246
#define TK_NMATCH 247 #define TK_NMATCH 247
#define TK_CONTAINS 248 #define TK_CONTAINS 248
#define TK_IN 249 #define TK_IN 249
#define TK_JOIN 250 #define TK_JOIN 250
#define TK_INNER 251 #define TK_INNER 251
#define TK_SELECT 252 #define TK_SELECT 252
#define TK_DISTINCT 253 #define TK_DISTINCT 253
#define TK_WHERE 254 #define TK_WHERE 254
#define TK_PARTITION 255 #define TK_PARTITION 255
#define TK_BY 256 #define TK_BY 256
#define TK_SESSION 257 #define TK_SESSION 257
#define TK_STATE_WINDOW 258 #define TK_STATE_WINDOW 258
#define TK_EVENT_WINDOW 259 #define TK_EVENT_WINDOW 259
#define TK_SLIDING 260 #define TK_SLIDING 260
#define TK_FILL 261 #define TK_FILL 261
#define TK_VALUE 262 #define TK_VALUE 262
#define TK_VALUE_F 263 #define TK_VALUE_F 263
#define TK_NONE 264 #define TK_NONE 264
#define TK_PREV 265 #define TK_PREV 265
#define TK_NULL_F 266 #define TK_NULL_F 266
#define TK_LINEAR 267 #define TK_LINEAR 267
#define TK_NEXT 268 #define TK_NEXT 268
#define TK_HAVING 269 #define TK_HAVING 269
#define TK_RANGE 270 #define TK_RANGE 270
#define TK_EVERY 271 #define TK_EVERY 271
#define TK_ORDER 272 #define TK_ORDER 272
#define TK_SLIMIT 273 #define TK_SLIMIT 273
#define TK_SOFFSET 274 #define TK_SOFFSET 274
#define TK_LIMIT 275 #define TK_LIMIT 275
#define TK_OFFSET 276 #define TK_OFFSET 276
#define TK_ASC 277 #define TK_ASC 277
#define TK_NULLS 278 #define TK_NULLS 278
#define TK_ABORT 279 #define TK_ABORT 279
#define TK_AFTER 280 #define TK_AFTER 280
#define TK_ATTACH 281 #define TK_ATTACH 281
#define TK_BEFORE 282 #define TK_BEFORE 282
#define TK_BEGIN 283 #define TK_BEGIN 283
#define TK_BITAND 284 #define TK_BITAND 284
#define TK_BITNOT 285 #define TK_BITNOT 285
#define TK_BITOR 286 #define TK_BITOR 286
#define TK_BLOCKS 287 #define TK_BLOCKS 287
#define TK_CHANGE 288 #define TK_CHANGE 288
#define TK_COMMA 289 #define TK_COMMA 289
#define TK_CONCAT 290 #define TK_CONCAT 290
#define TK_CONFLICT 291 #define TK_CONFLICT 291
#define TK_COPY 292 #define TK_COPY 292
#define TK_DEFERRED 293 #define TK_DEFERRED 293
#define TK_DELIMITERS 294 #define TK_DELIMITERS 294
#define TK_DETACH 295 #define TK_DETACH 295
#define TK_DIVIDE 296 #define TK_DIVIDE 296
#define TK_DOT 297 #define TK_DOT 297
#define TK_EACH 298 #define TK_EACH 298
#define TK_FAIL 299 #define TK_FAIL 299
#define TK_FILE 300 #define TK_FILE 300
#define TK_FOR 301 #define TK_FOR 301
#define TK_GLOB 302 #define TK_GLOB 302
#define TK_ID 303 #define TK_ID 303
#define TK_IMMEDIATE 304 #define TK_IMMEDIATE 304
#define TK_IMPORT 305 #define TK_IMPORT 305
#define TK_INITIALLY 306 #define TK_INITIALLY 306
#define TK_INSTEAD 307 #define TK_INSTEAD 307
#define TK_ISNULL 308 #define TK_ISNULL 308
#define TK_KEY 309 #define TK_KEY 309
#define TK_MODULES 310 #define TK_MODULES 310
#define TK_NK_BITNOT 311 #define TK_NK_BITNOT 311
#define TK_NK_SEMI 312 #define TK_NK_SEMI 312
#define TK_NOTNULL 313 #define TK_NOTNULL 313
#define TK_OF 314 #define TK_OF 314
#define TK_PLUS 315 #define TK_PLUS 315
#define TK_PRIVILEGE 316 #define TK_PRIVILEGE 316
#define TK_RAISE 317 #define TK_RAISE 317
#define TK_RESTRICT 318 #define TK_RESTRICT 318
#define TK_ROW 319 #define TK_ROW 319
#define TK_SEMI 320 #define TK_SEMI 320
#define TK_STAR 321 #define TK_STAR 321
#define TK_STATEMENT 322 #define TK_STATEMENT 322
#define TK_STRICT 323 #define TK_STRICT 323
#define TK_STRING 324 #define TK_STRING 324
#define TK_TIMES 325 #define TK_TIMES 325
#define TK_VALUES 326 #define TK_VALUES 326
#define TK_VARIABLE 327 #define TK_VARIABLE 327
#define TK_VIEW 328 #define TK_VIEW 328
#define TK_WAL 329 #define TK_WAL 329
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

1
include/libs/executor/dataSinkMgt.h
View File

@ -29,7 +29,6 @@ extern "C" {
#define DS_BUF_FULL 2 #define DS_BUF_FULL 2
#define DS_BUF_EMPTY 3 #define DS_BUF_EMPTY 3
struct SDataSink;
struct SSDataBlock; struct SSDataBlock;
typedef struct SDeleterRes { typedef struct SDeleterRes {

56
include/libs/stream/tstream.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h"
#include "executor.h" #include "executor.h"
#include "os.h"
#include "query.h" #include "query.h"
#include "streamState.h" #include "streamState.h"
#include "tdatablock.h" #include "tdatablock.h"
@ -206,14 +206,12 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) {
atomic_store_8(&queue->status, STREAM_QUEUE__FAILED); atomic_store_8(&queue->status, STREAM_QUEUE__FAILED);
} }
static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; }
return queue->qItem;
}
void* streamQueueNextItem(SStreamQueue* queue); void* streamQueueNextItem(SStreamQueue* queue);
SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type); SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type);
void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit); void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit);
SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit); SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit);
@ -272,7 +270,8 @@ typedef struct SStreamId {
typedef struct SCheckpointInfo { typedef struct SCheckpointInfo {
int64_t id; int64_t id;
int64_t version; // offset in WAL int64_t version; // offset in WAL
int64_t currentVer; // current offset in WAL, not serialize it
} SCheckpointInfo; } SCheckpointInfo;
typedef struct SStreamStatus { typedef struct SStreamStatus {
@ -335,18 +334,17 @@ struct SStreamTask {
// meta // meta
typedef struct SStreamMeta { typedef struct SStreamMeta {
char* path; char* path;
TDB* db; TDB* db;
TTB* pTaskDb; TTB* pTaskDb;
TTB* pCheckpointDb; TTB* pCheckpointDb;
SHashObj* pTasks; SHashObj* pTasks;
void* ahandle; void* ahandle;
TXN* txn; TXN* txn;
FTaskExpand* expandFunc; FTaskExpand* expandFunc;
int32_t vgId; int32_t vgId;
SRWLatch lock; SRWLatch lock;
int32_t walScan; int32_t walScan;
bool quit;
} SStreamMeta; } SStreamMeta;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@ -359,10 +357,6 @@ void tFreeStreamTask(SStreamTask* pTask);
int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem); int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem);
bool tInputQueueIsFull(const SStreamTask* pTask); bool tInputQueueIsFull(const SStreamTask* pTask);
static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) {
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
}
typedef struct { typedef struct {
SMsgHead head; SMsgHead head;
int64_t streamId; int64_t streamId;
@ -538,9 +532,11 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); // int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
void streamTaskInputFail(SStreamTask* pTask);
int32_t streamTryExec(SStreamTask* pTask); int32_t streamTryExec(SStreamTask* pTask);
int32_t streamSchedExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask);
int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock); int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock);
bool streamTaskShouldStop(const SStreamStatus* pStatus);
int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz); int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz);
@ -568,19 +564,19 @@ int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId);
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId);
void streamMetaClose(SStreamMeta* streamMeta); void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask);
int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen);
int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta);
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaBegin(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta);
int32_t streamMetaRollBack(SStreamMeta* pMeta); int32_t streamMetaRollBack(SStreamMeta* pMeta);
int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver);
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);

8
include/util/tdef.h
View File

@ -371,11 +371,11 @@ typedef enum ELogicConditionType {
#define TSDB_MIN_STT_TRIGGER 1 #define TSDB_MIN_STT_TRIGGER 1
#define TSDB_MAX_STT_TRIGGER 16 #define TSDB_MAX_STT_TRIGGER 16
#define TSDB_DEFAULT_SST_TRIGGER 1 #define TSDB_DEFAULT_SST_TRIGGER 1
#define TSDB_MIN_HASH_PREFIX 0 #define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN)
#define TSDB_MAX_HASH_PREFIX 128 #define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_PREFIX 0 #define TSDB_DEFAULT_HASH_PREFIX 0
#define TSDB_MIN_HASH_SUFFIX 0 #define TSDB_MIN_HASH_SUFFIX (2 - TSDB_TABLE_NAME_LEN)
#define TSDB_MAX_HASH_SUFFIX 128 #define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DEFAULT_HASH_SUFFIX 0
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1

1
include/util/tlog.h
View File

@ -102,7 +102,6 @@ bool taosAssertRelease(bool condition);
void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo); void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo);
void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr *pFd); void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr *pFd);
void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile);
int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime);
// clang-format off // clang-format off
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}

18
include/util/tutil.h
View File

@ -81,14 +81,22 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix, static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
int32_t suffix) { int32_t suffix) {
if (prefix == 0 && suffix == 0) { if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) {
return MurmurHash3_32(tbname, tblen); return MurmurHash3_32(tbname, tblen);
} else if (prefix > 0 || suffix > 0) {
return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
} else { } else {
if (tblen <= (prefix + suffix)) { char tbName[TSDB_TABLE_FNAME_LEN];
return MurmurHash3_32(tbname, tblen); int32_t offset = 0;
} else { if (prefix < 0) {
return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); offset = -1 * prefix;
strncpy(tbName, tbname, offset);
} }
if (suffix < 0) {
strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix);
offset += -1 *suffix;
}
return MurmurHash3_32(tbName, offset);
} }
} }

5
packaging/docker/bin/entrypoint.sh
View File

@ -42,8 +42,9 @@ if [ "$DISABLE_ADAPTER" = "0" ]; then
done done
fi fi
# if has mnode ep set or the host is first ep or not for cluster, just start. # if dnode has been created or has mnode ep set or the host is first ep or not for cluster, just start.
if [ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] || if [ -f "$DATA_DIR/dnode/dnode.json" ] ||
[ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] ||
[ "$TAOS_FQDN" = "$FIRST_EP_HOST" ]; then [ "$TAOS_FQDN" = "$FIRST_EP_HOST" ]; then
$@ $@
# others will first wait the first ep ready. # others will first wait the first ep ready.

3
packaging/tools/makeclient.sh
View File

@ -197,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then
mkdir -p ${install_dir}/connector mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
[ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l)
[ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
fi fi
git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go
rm -rf ${install_dir}/connector/go/.git ||: rm -rf ${install_dir}/connector/go/.git ||:

15
packaging/tools/makepkg.sh
View File

@ -338,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
connector_dir="${code_dir}/connector" connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
[ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : tmp_pwd=`pwd`
cd ${install_dir}/connector
if [ ! -d taos-connector-jdbc ];then
git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||:
fi
cd taos-connector-jdbc
mvn clean package -Dmaven.test.skip=true
echo ${build_dir}/lib/
cp target/*.jar ${build_dir}/lib/
cd ${install_dir}/connector
rm -rf taos-connector-jdbc
cd ${tmp_pwd}
jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l)
[ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go
rm -rf ${install_dir}/connector/go/.git ||: rm -rf ${install_dir}/connector/go/.git ||:

1
source/client/inc/clientInt.h
View File

@ -80,6 +80,7 @@ typedef struct {
int64_t appId; int64_t appId;
// ctl // ctl
int8_t threadStop; int8_t threadStop;
int8_t quitByKill;
TdThread thread; TdThread thread;
TdThreadMutex lock; // used when app init and cleanup TdThreadMutex lock; // used when app init and cleanup
SHashObj* appSummary; SHashObj* appSummary;

17
source/client/inc/clientSml.h
View File

@ -70,7 +70,7 @@ extern "C" {
#define VALUE_LEN 6 #define VALUE_LEN 6
#define OTD_JSON_FIELDS_NUM 4 #define OTD_JSON_FIELDS_NUM 4
#define MAX_RETRY_TIMES 100 #define MAX_RETRY_TIMES 10
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
typedef enum { typedef enum {
@ -107,6 +107,7 @@ typedef struct {
int32_t colsLen; int32_t colsLen;
int32_t timestampLen; int32_t timestampLen;
bool measureEscaped;
SArray *colArray; SArray *colArray;
} SSmlLineInfo; } SSmlLineInfo;
@ -206,6 +207,19 @@ typedef struct {
#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) #define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \
if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \
MOVE_FORWARD_ONE(key + i, keyLen - i); \
keyLen--; \
} \
}
extern int64_t smlFactorNS[3]; extern int64_t smlFactorNS[3];
extern int64_t smlFactorS[3]; extern int64_t smlFactorS[3];
@ -237,6 +251,7 @@ uint8_t smlGetTimestampLen(int64_t num);
void clearColValArray(SArray* pCols); void clearColValArray(SArray* pCols);
void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag);
void freeSSmlKv(void* data);
int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseJSON(SSmlHandle *info, char *payload); int32_t smlParseJSON(SSmlHandle *info, char *payload);

12
source/client/src/clientHb.c
View File

@ -845,7 +845,12 @@ static void hbStopThread() {
return; return;
} }
taosThreadJoin(clientHbMgr.thread, NULL); // thread quit mode kill or inner exit from self-thread
if (clientHbMgr.quitByKill) {
taosThreadKill(clientHbMgr.thread, 0);
} else {
taosThreadJoin(clientHbMgr.thread, NULL);
}
tscDebug("hb thread stopped"); tscDebug("hb thread stopped");
} }
@ -1037,3 +1042,8 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1);
} }
// set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner
void taos_set_hb_quit(int8_t quitByKill) {
clientHbMgr.quitByKill = quitByKill;
}

46
source/client/src/clientSml.c
View File

@ -534,8 +534,9 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL;
if (index) { if (index) {
if (colField[*index].type != kv->type) { if (colField[*index].type != kv->type) {
uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key); uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id,
return TSDB_CODE_TSC_INVALID_VALUE; colField[*index].type, kv->type, kv->key);
return TSDB_CODE_SML_INVALID_DATA;
} }
if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR &&
@ -765,8 +766,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
size_t superTableLen = 0; size_t superTableLen = 0;
void *superTable = taosHashGetKey(tmp, &superTableLen); void *superTable = taosHashGetKey(tmp, &superTableLen);
char *measure = taosMemoryMalloc(superTableLen);
memcpy(measure, superTable, superTableLen);
PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen);
memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
memcpy(pName.tname, superTable, superTableLen); memcpy(pName.tname, measure, superTableLen);
taosMemoryFree(measure);
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
@ -1049,7 +1054,7 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
// } // }
// taosMemoryFree(tag->key); // taosMemoryFree(tag->key);
taosArrayDestroy(tag->cols); taosArrayDestroy(tag->cols);
taosArrayDestroy(tag->tags); taosArrayDestroyEx(tag->tags, freeSSmlKv);
taosMemoryFree(tag); taosMemoryFree(tag);
} }
@ -1063,6 +1068,12 @@ void clearColValArray(SArray *pCols) {
} }
} }
void freeSSmlKv(void *data) {
SSmlKv *kv = (SSmlKv *)data;
if (kv->keyEscaped) taosMemoryFree((void *)(kv->key));
if (kv->valueEscaped) taosMemoryFree((void *)(kv->value));
}
void smlDestroyInfo(SSmlHandle *info) { void smlDestroyInfo(SSmlHandle *info) {
if (!info) return; if (!info) return;
qDestroyQuery(info->pQuery); qDestroyQuery(info->pQuery);
@ -1098,11 +1109,11 @@ void smlDestroyInfo(SSmlHandle *info) {
} }
taosArrayDestroy(info->valueJsonArray); taosArrayDestroy(info->valueJsonArray);
taosArrayDestroy(info->preLineTagKV); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv);
if (!info->dataFormat) { if (!info->dataFormat) {
for (int i = 0; i < info->lineNum; i++) { for (int i = 0; i < info->lineNum; i++) {
taosArrayDestroy(info->lines[i].colArray); taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
if (info->parseJsonByLib) { if (info->parseJsonByLib) {
taosMemoryFree(info->lines[i].tags); taosMemoryFree(info->lines[i].tags);
} }
@ -1165,6 +1176,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
} }
for (size_t i = 0; i < taosArrayGetSize(cols); i++) { for (size_t i = 0; i < taosArrayGetSize(cols); i++) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
terrno = 0;
taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
if (terrno == TSDB_CODE_DUP_KEY) { if (terrno == TSDB_CODE_DUP_KEY) {
taosHashCleanup(kvHash); taosHashCleanup(kvHash);
@ -1242,6 +1254,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
info->lineNum); info->lineNum);
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
terrno = 0;
smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
if (terrno == TSDB_CODE_DUP_KEY) { if (terrno == TSDB_CODE_DUP_KEY) {
return terrno; return terrno;
@ -1305,9 +1318,15 @@ static int32_t smlInsertData(SSmlHandle *info) {
uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname,
tableData->uid, info->dataFormat); tableData->uid, info->dataFormat);
int measureLen = tableData->sTableNameLen;
char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen);
memcpy(measure, tableData->sTableName, tableData->sTableNameLen);
PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf,
info->ttl, info->msgBuf.buf, info->msgBuf.len); info->msgBuf.len);
taosMemoryFree(measure);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBindData failed", info->id); uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
return code; return code;
@ -1421,14 +1440,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
char cTmp = 0; // for print tmp if is raw char cTmp = 0; // for print tmp if is raw
if (info->isRawLine) { if (info->isRawLine) {
cTmp = tmp[len - 1]; cTmp = tmp[len];
tmp[len - 1] = '\0'; tmp[len] = '\0';
} }
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
info->isRawLine, numLines, info->protocol, len, tmp); info->isRawLine, numLines, info->protocol, len, tmp);
if (info->isRawLine) { if (info->isRawLine) {
tmp[len - 1] = cTmp; tmp[len] = cTmp;
} }
if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
@ -1450,6 +1469,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tmp[len] = '\0';
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
return code; return code;
} }
@ -1495,8 +1515,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
do { do {
code = smlModifyDBSchemas(info); code = smlModifyDBSchemas(info);
if (code == 0) break; if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA) break;
taosMsleep(500); taosMsleep(100);
uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum);
} while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES);

174
source/client/src/clientSmlLine.c
View File

@ -21,32 +21,33 @@
#include "clientSml.h" #include "clientSml.h"
// comma , // comma ,
// #define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH)
#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
// space // space
// #define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH)
#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
// equal = // equal =
// #define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH)
#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
// quote " // quote "
// #define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) //#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
// SLASH // SLASH
// #define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH)
#define IS_SLASH_LETTER(sql) \ #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || \ (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
*(sql) == SLASH)) // (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) ||
// IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql))
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) #define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL))
#define PROCESS_SLASH(key, keyLen) \ #define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \ for (int i = 1; i < keyLen; ++i) { \
if (IS_SLASH_LETTER(key + i)) { \ if (IS_SLASH_LETTER_IN_FIELD_VALUE(key + i)) { \
MOVE_FORWARD_ONE(key + i, keyLen - i); \
keyLen--; \
} \
}
#define PROCESS_SLASH_IN_TAG_FIELD_KEY(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(key + i)) { \
MOVE_FORWARD_ONE(key + i, keyLen - i); \ MOVE_FORWARD_ONE(key + i, keyLen - i); \
i--; \
keyLen--; \ keyLen--; \
} \ } \
} }
@ -151,7 +152,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
SSmlSTableMeta *sMeta = NULL; SSmlSTableMeta *sMeta = NULL;
if (unlikely(tmp == NULL)) { if (unlikely(tmp == NULL)) {
STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); char* measure = currElement->measure;
int measureLen = currElement->measureLen;
if(currElement->measureEscaped){
measure = (char*)taosMemoryMalloc(currElement->measureLen);
memcpy(measure, currElement->measure, currElement->measureLen);
PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
}
STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
if(currElement->measureEscaped){
taosMemoryFree(measure);
}
if (pTableMeta == NULL) { if (pTableMeta == NULL) {
info->dataFormat = false; info->dataFormat = false;
info->reRun = true; info->reRun = true;
@ -171,17 +182,18 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
info->maxTagKVs = (*tmp)->tags; info->maxTagKVs = (*tmp)->tags;
} }
} }
taosArrayClear(preLineKV); taosArrayClearEx(preLineKV, freeSSmlKv);
while (*sql < sqlEnd) { while (*sql < sqlEnd) {
if (unlikely(IS_SPACE(*sql))) { if (unlikely(IS_SPACE(*sql))) {
break; break;
} }
bool hasSlash = false;
// parse key // parse key
const char *key = *sql; const char *key = *sql;
size_t keyLen = 0; size_t keyLen = 0;
bool keyEscaped = false;
size_t keyLenEscaped = 0;
while (*sql < sqlEnd) { while (*sql < sqlEnd) {
if (unlikely(IS_COMMA(*sql))) { if (unlikely(IS_COMMA(*sql))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
@ -192,16 +204,14 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
(*sql)++; (*sql)++;
break; break;
} }
if (!hasSlash) { if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
hasSlash = (*(*sql) == SLASH); keyLenEscaped++;
keyEscaped = true;
} }
(*sql)++; (*sql)++;
} }
if (unlikely(hasSlash)) {
PROCESS_SLASH(key, keyLen)
}
if (unlikely(IS_INVALID_COL_LEN(keyLen))) { if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
} }
@ -209,7 +219,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
// parse value // parse value
const char *value = *sql; const char *value = *sql;
size_t valueLen = 0; size_t valueLen = 0;
hasSlash = false; bool valueEscaped = false;
size_t valueLenEscaped = 0;
while (*sql < sqlEnd) { while (*sql < sqlEnd) {
// parse value // parse value
if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
@ -219,8 +230,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
if (!hasSlash) { if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
hasSlash = (*(*sql) == SLASH); valueLenEscaped++;
valueEscaped = true;
} }
(*sql)++; (*sql)++;
@ -232,15 +244,24 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
if (unlikely(hasSlash)) { if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
PROCESS_SLASH(value, valueLen)
}
if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
} }
SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; if (keyEscaped){
char *tmp = (char*)taosMemoryMalloc(keyLen);
memcpy(tmp, key, keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen);
key = tmp;
}
if (valueEscaped){
char *tmp = (char*)taosMemoryMalloc(valueLen);
memcpy(tmp, value, valueLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen);
value = tmp;
}
SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped};
taosArrayPush(preLineKV, &kv);
if (info->dataFormat) { if (info->dataFormat) {
if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
info->dataFormat = false; info->dataFormat = false;
@ -266,7 +287,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
info->needModifySchema = true; info->needModifySchema = true;
} }
} }
taosArrayPush(preLineKV, &kv);
cnt++; cnt++;
if (IS_SPACE(*sql)) { if (IS_SPACE(*sql)) {
@ -285,6 +305,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
tinfo->tags = taosArrayDup(preLineKV, NULL); tinfo->tags = taosArrayDup(preLineKV, NULL);
for(size_t i = 0; i < taosArrayGetSize(preLineKV); i++){
SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
if(kv->keyEscaped)kv->key = NULL;
if(kv->valueEscaped)kv->value = NULL;
}
smlSetCTableName(tinfo); smlSetCTableName(tinfo);
tinfo->uid = info->uid++; tinfo->uid = info->uid++;
@ -321,7 +346,17 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
SSmlSTableMeta **tmp = SSmlSTableMeta **tmp =
(SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
if (unlikely(tmp == NULL)) { if (unlikely(tmp == NULL)) {
STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); char* measure = currElement->measure;
int measureLen = currElement->measureLen;
if(currElement->measureEscaped){
measure = (char*)taosMemoryMalloc(currElement->measureLen);
memcpy(measure, currElement->measure, currElement->measureLen);
PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
}
STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
if(currElement->measureEscaped){
taosMemoryFree(measure);
}
if (pTableMeta == NULL) { if (pTableMeta == NULL) {
info->dataFormat = false; info->dataFormat = false;
info->reRun = true; info->reRun = true;
@ -352,10 +387,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
break; break;
} }
bool hasSlash = false;
// parse key // parse key
const char *key = *sql; const char *key = *sql;
size_t keyLen = 0; size_t keyLen = 0;
bool keyEscaped = false;
size_t keyLenEscaped = 0;
while (*sql < sqlEnd) { while (*sql < sqlEnd) {
if (unlikely(IS_COMMA(*sql))) { if (unlikely(IS_COMMA(*sql))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
@ -366,16 +402,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
(*sql)++; (*sql)++;
break; break;
} }
if (!hasSlash) { if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
hasSlash = (*(*sql) == SLASH); keyLenEscaped++;
keyEscaped = true;
} }
(*sql)++; (*sql)++;
} }
if (unlikely(hasSlash)) {
PROCESS_SLASH(key, keyLen)
}
if (unlikely(IS_INVALID_COL_LEN(keyLen))) { if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
} }
@ -383,11 +417,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
// parse value // parse value
const char *value = *sql; const char *value = *sql;
size_t valueLen = 0; size_t valueLen = 0;
hasSlash = false; bool valueEscaped = false;
bool isInQuote = false; size_t valueLenEscaped = 0;
bool isInQuote = false;
const char *escapeChar = NULL;
while (*sql < sqlEnd) { while (*sql < sqlEnd) {
// parse value // parse value
if (unlikely(IS_QUOTE(*sql))) { if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) {
isInQuote = !isInQuote; isInQuote = !isInQuote;
(*sql)++; (*sql)++;
continue; continue;
@ -395,13 +431,12 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
if (!isInQuote) { if (!isInQuote) {
if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
break; break;
} else if (unlikely(IS_EQUAL(*sql))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
return TSDB_CODE_SML_INVALID_DATA;
} }
} }
if (!hasSlash) { if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) {
hasSlash = (*(*sql) == SLASH); escapeChar = *sql;
valueEscaped = true;
valueLenEscaped++;
} }
(*sql)++; (*sql)++;
@ -416,9 +451,6 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
if (unlikely(hasSlash)) {
PROCESS_SLASH(value, valueLen)
}
SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen};
int32_t ret = smlParseValue(&kv, &info->msgBuf); int32_t ret = smlParseValue(&kv, &info->msgBuf);
@ -427,11 +459,28 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return ret; return ret;
} }
if (keyEscaped){
char *tmp = (char*)taosMemoryMalloc(kv.keyLen);
memcpy(tmp, key, kv.keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen);
kv.key = tmp;
kv.keyEscaped = keyEscaped;
}
if (valueEscaped){
char *tmp = (char*)taosMemoryMalloc(kv.length);
memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
kv.value = tmp;
kv.valueEscaped = valueEscaped;
}
if (info->dataFormat) { if (info->dataFormat) {
// cnt begin 0, add ts so + 2 // cnt begin 0, add ts so + 2
if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
info->dataFormat = false; info->dataFormat = false;
info->reRun = true; info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// bind data // bind data
@ -440,22 +489,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
uDebug("smlBuildCol error, retry"); uDebug("smlBuildCol error, retry");
info->dataFormat = false; info->dataFormat = false;
info->reRun = true; info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (cnt >= taosArrayGetSize(info->masColKVs)) { if (cnt >= taosArrayGetSize(info->masColKVs)) {
info->dataFormat = false; info->dataFormat = false;
info->reRun = true; info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt); SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt);
if (kv.type != maxKV->type) { if (kv.type != maxKV->type) {
info->dataFormat = false; info->dataFormat = false;
info->reRun = true; info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (unlikely(!IS_SAME_KEY)) { if (unlikely(!IS_SAME_KEY)) {
info->dataFormat = false; info->dataFormat = false;
info->reRun = true; info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -463,6 +516,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
maxKV->length = kv.length; maxKV->length = kv.length;
info->needModifySchema = true; info->needModifySchema = true;
} }
freeSSmlKv(&kv);
} else { } else {
if (currElement->colArray == NULL) { if (currElement->colArray == NULL) {
currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@ -487,10 +541,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
elements->measure = sql; elements->measure = sql;
// parse measure // parse measure
size_t measureLenEscaped = 0;
while (sql < sqlEnd) { while (sql < sqlEnd) {
if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) {
MOVE_FORWARD_ONE(sql, sqlEnd - sql); elements->measureEscaped = true;
sqlEnd--; measureLenEscaped++;
sql++;
continue; continue;
} }
if (unlikely(IS_COMMA(sql))) { if (unlikely(IS_COMMA(sql))) {
@ -503,7 +559,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
sql++; sql++;
} }
elements->measureLen = sql - elements->measure; elements->measureLen = sql - elements->measure;
if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
} }
@ -581,7 +637,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
.keyLen = TS_LEN, .keyLen = TS_LEN,
.type = TSDB_DATA_TYPE_TIMESTAMP, .type = TSDB_DATA_TYPE_TIMESTAMP,
.i = ts, .i = ts,
.length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
.keyEscaped = false,
.valueEscaped = false};
if (info->dataFormat) { if (info->dataFormat) {
uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts); uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);

9
source/client/src/clientTmq.c
View File

@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
SArray* container = &list->container; SArray* container = &list->container;
if (src == NULL || src[0] == 0) return -1; if (src == NULL || src[0] == 0) return -1;
char* topic = taosStrdup(src); char* topic = taosStrdup(src);
if (topic[0] != '`') {
strtolower(topic, src);
}
if (taosArrayPush(container, &topic) == NULL) return -1; if (taosArrayPush(container, &topic) == NULL) return -1;
return 0; return 0;
} }
@ -1243,9 +1240,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pParam); taosMemoryFree(pParam);
if (code != 0) { if (code != 0) {
tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
vgId, epoch, tstrerror(code), requestId);
if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pData) taosMemoryFree(pMsg->pData);
if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet);
@ -1267,6 +1261,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosWriteQitem(tmq->mqueue, pRspWrapper); taosWriteQitem(tmq->mqueue, pRspWrapper);
} else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert
taosMsleep(500); taosMsleep(500);
} else{
tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
vgId, epoch, tstrerror(code), requestId);
} }
goto CREATE_MSG_FAIL; goto CREATE_MSG_FAIL;

36
source/client/test/smlTest.cpp
View File

@ -50,8 +50,9 @@ TEST(testCase, smlParseInfluxString_Test) {
int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
ASSERT_EQ(elements.measureLen, strlen(",st")); ASSERT_EQ(elements.measureLen, strlen("\\,st"));
ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.measureEscaped, true);
ASSERT_EQ(elements.measureTagsLen, strlen("\\,st,t1=3,t2=4,t3=t3"));
ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); ASSERT_EQ(elements.tags, sql + elements.measureLen + 1);
ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3"));
@ -204,7 +205,26 @@ TEST(testCase, smlParseCols_Error_Test) {
"st,t=1 c=-3.402823466e+39u64 1626006833639000000", "st,t=1 c=-3.402823466e+39u64 1626006833639000000",
"st,t=1 c=-339u64 1626006833639000000", "st,t=1 c=-339u64 1626006833639000000",
"st,t=1 c=18446744073709551616u64 1626006833639000000", "st,t=1 c=18446744073709551616u64 1626006833639000000",
"st,t=1 c=1=2 1626006833639000000"}; "st,t=1 c=1=2 1626006833639000000,",
// escape error test
// measure comma,space
"s,t,t=1 c=1 1626006833639000000,",
"s t,t=1 c=1 1626006833639000000,",
//tag key comma,equal,space
"st,t,t=1 c=2 1626006833639000000,",
"st,t=t=1 c=2 1626006833639000000,",
"st,t t=1 c=2 1626006833639000000,",
//tag value comma,equal,space
"st,tt=a,a c=2 1626006833639000000,",
"st,t=t=a a c=2 1626006833639000000,",
"st,t t=a=a c=2 1626006833639000000,",
//field key comma,equal,space
"st,tt=aa c,1=2 1626006833639000000,",
"st,tt=aa c=1=2 1626006833639000000,",
"st,tt=aa c 1=2 1626006833639000000,",
//field value double quote,slash
"st,tt=aa c=\"a\"a\" 1626006833639000000,",
};
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = smlBuildSmlInfo(NULL);
info->protocol = TSDB_SML_LINE_PROTOCOL; info->protocol = TSDB_SML_LINE_PROTOCOL;
@ -256,16 +276,18 @@ TEST(testCase, smlParseCols_Test) {
ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
ASSERT_EQ(kv->length, 17); ASSERT_EQ(kv->length, 18);
ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); ASSERT_EQ(kv->keyEscaped, true);
ASSERT_EQ(kv->valueEscaped, false);
ASSERT_EQ(strncasecmp(kv->value, "pass\\,it ", 9), 0);
// nchar // nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); kv = (SSmlKv *)taosArrayGet(elements.colArray, 2);
ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->length, 9);
ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); ASSERT_EQ(strncasecmp(kv->value, "ii\\=sd", 5), 0);
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); kv = (SSmlKv *)taosArrayGet(elements.colArray, 3);

8
source/common/src/tdataformat.c
View File

@ -2503,9 +2503,11 @@ _exit:
int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) {
int32_t code = 0; int32_t code = 0;
ASSERT(pColData->type == pBind->buffer_type); if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) {
ASSERT(pColData->type == pBind->buffer_type);
if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type }
if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type
for (int32_t i = 0; i < pBind->num; ++i) { for (int32_t i = 0; i < pBind->num; ++i) {
if (pBind->is_null && pBind->is_null[i]) { if (pBind->is_null && pBind->is_null[i]) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);

2
source/common/src/tglobal.c
View File

@ -198,7 +198,7 @@ int32_t tsTransPullupInterval = 2;
int32_t tsMqRebalanceInterval = 2; int32_t tsMqRebalanceInterval = 2;
int32_t tsStreamCheckpointTickInterval = 1; int32_t tsStreamCheckpointTickInterval = 1;
int32_t tsTtlUnit = 86400; int32_t tsTtlUnit = 86400;
int32_t tsTtlPushInterval = 86400; int32_t tsTtlPushInterval = 3600;
int32_t tsGrantHBInterval = 60; int32_t tsGrantHBInterval = 60;
int32_t tsUptimeInterval = 300; // seconds int32_t tsUptimeInterval = 300; // seconds
char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits

61
source/common/src/tmisce.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tmisce.h" #include "tmisce.h"
#include "tjson.h"
#include "tglobal.h" #include "tglobal.h"
#include "tlog.h" #include "tlog.h"
#include "tname.h" #include "tname.h"
@ -87,3 +88,63 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet) {
return ep; return ep;
} }
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) {
SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return -1;
char tmp[4096] = {0};
tjsonAddDoubleToObject(pJson, "reportVersion", 1);
tjsonAddIntegerToObject(pJson, "clusterId", clusterId);
tjsonAddIntegerToObject(pJson, "startTime", startTime);
// Do NOT invoke the taosGetFqdn here.
// this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked
// environment. The lock operation by taosGetFqdn may cause this program deadlock.
tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn);
tjsonAddIntegerToObject(pJson, "pid", taosGetPId());
taosGetAppName(tmp, NULL);
tjsonAddStringToObject(pJson, "appName", tmp);
if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) {
tjsonAddStringToObject(pJson, "os", tmp);
}
float numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) {
tjsonAddStringToObject(pJson, "cpuModel", tmp);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
} else {
tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores);
}
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
tjsonAddStringToObject(pJson, "version", version);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo);
tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
tjsonAddIntegerToObject(pJson, "crashSig", signum);
tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs());
#ifdef _TD_DARWIN_64
taosLogTraceToBuf(tmp, sizeof(tmp), 4);
#elif !defined(WINDOWS)
taosLogTraceToBuf(tmp, sizeof(tmp), 3);
#else
taosLogTraceToBuf(tmp, sizeof(tmp), 8);
#endif
tjsonAddStringToObject(pJson, "stackInfo", tmp);
char* pCont = tjsonToString(pJson);
tjsonDelete(pJson);
*pMsg = pCont;
return TSDB_CODE_SUCCESS;
}

2
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -119,6 +119,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
pVnode->pFetchQ->threadId); pVnode->pFetchQ->threadId);
while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
tqNotifyClose(pVnode->pImpl->pTq);
dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ); dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ);
while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10);
@ -141,7 +142,6 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
dInfo("vgId:%d, vnode is closed", pVnode->vgId); dInfo("vgId:%d, vnode is closed", pVnode->vgId);
if (commitAndRemoveWal) { if (commitAndRemoveWal) {
char path[TSDB_FILENAME_LEN] = {0};
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP); snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP);
dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path); dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path);
tfsRmdir(pMgmt->pTfs, path); tfsRmdir(pMgmt->pTfs, path);

6
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -137,12 +137,12 @@ typedef enum {
} EDndReason; } EDndReason;
typedef enum { typedef enum {
CONSUMER_UPDATE__TOUCH = 1, CONSUMER_UPDATE__TOUCH = 1, // rebalance req do not need change consume topic
CONSUMER_UPDATE__ADD, CONSUMER_UPDATE__ADD,
CONSUMER_UPDATE__REMOVE, CONSUMER_UPDATE__REMOVE,
CONSUMER_UPDATE__LOST, CONSUMER_UPDATE__LOST,
CONSUMER_UPDATE__RECOVER, CONSUMER_UPDATE__RECOVER,
CONSUMER_UPDATE__MODIFY, CONSUMER_UPDATE__MODIFY, // subscribe req need change consume topic
} ECsmUpdateType; } ECsmUpdateType;
typedef struct { typedef struct {
@ -624,7 +624,7 @@ typedef struct {
SArray* rebVgs; // SArray<SMqRebOutputVg> SArray* rebVgs; // SArray<SMqRebOutputVg>
SArray* newConsumers; // SArray<int64_t> SArray* newConsumers; // SArray<int64_t>
SArray* removedConsumers; // SArray<int64_t> SArray* removedConsumers; // SArray<int64_t>
SArray* touchedConsumers; // SArray<int64_t> SArray* modifyConsumers; // SArray<int64_t>
SMqSubscribeObj* pSub; SMqSubscribeObj* pSub;
SMqSubActionLogEntry* pLogEntry; SMqSubActionLogEntry* pLogEntry;
} SMqRebOutputObj; } SMqRebOutputObj;

51
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -247,7 +247,6 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
static void freeRebalanceItem(void *param) { static void freeRebalanceItem(void *param) {
SMqRebInfo *pInfo = param; SMqRebInfo *pInfo = param;
taosArrayDestroy(pInfo->lostConsumers);
taosArrayDestroy(pInfo->newConsumers); taosArrayDestroy(pInfo->newConsumers);
taosArrayDestroy(pInfo->removedConsumers); taosArrayDestroy(pInfo->removedConsumers);
} }
@ -335,7 +334,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
} }
taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pConsumer->lock);
} else if (status == MQ_CONSUMER_STATUS__MODIFY) { } else {
taosRLockLatch(&pConsumer->lock); taosRLockLatch(&pConsumer->lock);
int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
@ -356,8 +355,6 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
} }
taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pConsumer->lock);
} else {
// do nothing
} }
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
@ -917,34 +914,22 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
taosWLockLatch(&pOldConsumer->lock); taosWLockLatch(&pOldConsumer->lock);
if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) {
/*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ SArray *tmp = pOldConsumer->rebNewTopics;
/*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics;
pNewConsumer->rebNewTopics = tmp;
// this new consumer has identical topics with one existed consumers. tmp = pOldConsumer->rebRemovedTopics;
if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics;
pOldConsumer->status = MQ_CONSUMER_STATUS__READY; pNewConsumer->rebRemovedTopics = tmp;
} else {
SArray *tmp = pOldConsumer->rebNewTopics;
pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics;
pNewConsumer->rebNewTopics = tmp;
tmp = pOldConsumer->rebRemovedTopics; tmp = pOldConsumer->assignedTopics;
pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; pOldConsumer->assignedTopics = pNewConsumer->assignedTopics;
pNewConsumer->rebRemovedTopics = tmp; pNewConsumer->assignedTopics = tmp;
tmp = pOldConsumer->assignedTopics; pOldConsumer->subscribeTime = pNewConsumer->upTime;
pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
pNewConsumer->assignedTopics = tmp;
pOldConsumer->subscribeTime = pNewConsumer->upTime;
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
}
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
/*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/
/*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/
int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
/*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i)); char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i));
taosArrayPush(pOldConsumer->rebRemovedTopics, &topic); taosArrayPush(pOldConsumer->rebRemovedTopics, &topic);
@ -958,9 +943,6 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status), pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status),
pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) {
/*A(taosArrayGetSize(pOldConsumer->currentTopics) == 0);*/
/*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/
int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i));
@ -976,7 +958,6 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->rebalanceTime = pNewConsumer->upTime; pOldConsumer->rebalanceTime = pNewConsumer->upTime;
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
// not exist in current topic // not exist in current topic
@ -1015,15 +996,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
(int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
/*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/
/*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/
char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
#if 0
for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
A(strcmp(topic, removedTopic) != 0);
}
#endif
// remove from removed topic // remove from removed topic
removeFromRemoveTopicList(pOldConsumer, removedTopic); removeFromRemoveTopicList(pOldConsumer, removedTopic);

16
source/dnode/mnode/impl/src/mndDb.c
View File

@ -373,6 +373,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1; if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
if ((pCfg->hashSuffix * pCfg->hashPrefix) < 0) return -1;
if ((pCfg->hashPrefix + pCfg->hashSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) return -1;
if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1; if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
@ -409,8 +411,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD; if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD;
if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
} }
@ -553,6 +553,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
int32_t dbLen = strlen(dbObj.name) + 1; int32_t dbLen = strlen(dbObj.name) + 1;
mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen); mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen);
dbObj.cfg.hashPrefix += dbLen; dbObj.cfg.hashPrefix += dbLen;
} else if (dbObj.cfg.hashPrefix < 0) {
int32_t dbLen = strlen(dbObj.name) + 1;
mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix - dbLen);
dbObj.cfg.hashPrefix -= dbLen;
} }
SVgObj *pVgroups = NULL; SVgObj *pVgroups = NULL;
@ -1437,7 +1441,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
return 0; return 0;
} }
static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, SRpcMsg *pReq) { static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -1459,7 +1463,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, SRpcMsg *pReq) {
pHead->vgId = htonl(pVgroup->vgId); pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), contLen, &trimReq); tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), contLen, &trimReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen, .info = pReq->info}; SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg); int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) { if (code != 0) {
@ -1495,7 +1499,7 @@ static int32_t mndProcessTrimDbReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
code = mndTrimDb(pMnode, pDb, pReq); code = mndTrimDb(pMnode, pDb);
_OVER: _OVER:
if (code != 0) { if (code != 0) {
@ -1788,6 +1792,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
int16_t hashPrefix = pDb->cfg.hashPrefix; int16_t hashPrefix = pDb->cfg.hashPrefix;
if (hashPrefix > 0) { if (hashPrefix > 0) {
hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1; hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1;
} else if (hashPrefix < 0) {
hashPrefix = pDb->cfg.hashPrefix + strlen(pDb->name) + 1;
} }
colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false); colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false);

138
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -269,7 +269,7 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) {
}; };
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", pSubKey, pVgEp->vgId); mInfo("sub:%s mq re-balance addUnassignedVgroups vgId:%d from unassigned", pSubKey, pVgEp->vgId);
} }
} }
@ -289,9 +289,9 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
// all old consumers still existing are touched // all old consumers still existing need to be modified
// TODO optimize: touch only consumer whose vgs changed // TODO optimize: modify only consumer whose vgs changed
taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId); taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId);
if (consumerVgNum > minVgCnt) { if (consumerVgNum > minVgCnt) {
if (imbCnt < imbConsumerNum) { if (imbCnt < imbConsumerNum) {
if (consumerVgNum == minVgCnt + 1) { if (consumerVgNum == minVgCnt + 1) {
@ -339,13 +339,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
mInfo("sub:%s mq re-balance %d vgroups, existed consumers:%d, added:%d, removed:%d", pSubKey, totalVgNum, mInfo("sub:%s mq re-balance %d vgroups, existed consumers:%d, added:%d, removed:%d", pSubKey, totalVgNum,
pInput->oldConsumerNum, numOfAdded, numOfRemoved); pInput->oldConsumerNum, numOfAdded, numOfRemoved);
// 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg // 1. build temporary hash(vgId -> SMqRebOutputVg) to store vg that need to be assigned
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
// 2. check and get actual removed consumers, put their vg into hash // 2. check and get actual removed consumers, put their vg into pHash
doRemoveExistedConsumers(pOutput, pHash, pInput); doRemoveExistedConsumers(pOutput, pHash, pInput);
// 3. if previously no consumer, there are vgs not assigned // 3. if previously no consumer, there are vgs not assigned, put these vg into pHash
addUnassignedVgroups(pOutput, pHash); addUnassignedVgroups(pOutput, pHash);
// 4. calc vg number of each consumer // 4. calc vg number of each consumer
@ -364,19 +364,17 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
mInfo("sub:%s no consumer subscribe this topic", pSubKey); mInfo("sub:%s no consumer subscribe this topic", pSubKey);
} }
// 5. first scan: remove vgroups from te consumers, who have more vgroups than the threashold value that is // 5. remove vgroups from consumers who have more vgroups than the threshold value(minVgCnt or minVgCnt + 1), and then another vg into pHash
// minVgCnt, and then put them into the recycled hash list
transferVgroupsForConsumers(pOutput, pHash, minVgCnt, imbConsumerNum); transferVgroupsForConsumers(pOutput, pHash, minVgCnt, imbConsumerNum);
// 6. add new consumer into sub // 6. add new consumer into sub
doAddNewConsumers(pOutput, pInput); doAddNewConsumers(pOutput, pInput);
// 7. second scan: find consumer do not have enough vgroups, extract from temporary hash and assign to them
// All related vg should be put into rebVgs
SMqRebOutputVg *pRebVg = NULL; SMqRebOutputVg *pRebVg = NULL;
void *pRemovedIter = NULL; void *pRemovedIter = NULL;
void *pIter = NULL; void *pIter = NULL;
// 7. extract bgroups from pHash and assign to consumers that do not have enough vgroups
while (1) { while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) { if (pIter == NULL) {
@ -390,68 +388,52 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
// iter hash and find one vg // iter hash and find one vg
pRemovedIter = taosHashIterate(pHash, pRemovedIter); pRemovedIter = taosHashIterate(pHash, pRemovedIter);
if (pRemovedIter == NULL) { if (pRemovedIter == NULL) {
mError("sub:%s removed iter is null", pSubKey); mError("sub:%s removed iter is null, never can reach hear", pSubKey);
break; break;
} }
pRebVg = (SMqRebOutputVg *)pRemovedIter; pRebVg = (SMqRebOutputVg *)pRemovedIter;
// push
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pConsumerEp->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId, mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
pConsumerEp->consumerId);
} }
} }
// 7. handle unassigned vg while (1) {
if (taosHashGetSize(pOutput->pSub->consumerHash) != 0) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
// if has consumer, assign all left vg if (pIter == NULL) {
while (1) { break;
SMqConsumerEp *pConsumerEp = NULL; }
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
if (taosArrayGetSize(pConsumerEp->vgs) == minVgCnt) {
pRemovedIter = taosHashIterate(pHash, pRemovedIter); pRemovedIter = taosHashIterate(pHash, pRemovedIter);
if (pRemovedIter == NULL) { if (pRemovedIter == NULL) {
if (pIter != NULL) { mInfo("sub:%s removed iter is null", pSubKey);
taosHashCancelIterate(pOutput->pSub->consumerHash, pIter);
pIter = NULL;
}
break; break;
} }
while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
pConsumerEp = (SMqConsumerEp *)pIter;
if (taosArrayGetSize(pConsumerEp->vgs) == minVgCnt) {
break;
}
}
pRebVg = (SMqRebOutputVg *)pRemovedIter; pRebVg = (SMqRebOutputVg *)pRemovedIter;
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pConsumerEp->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
if (pRebVg->newConsumerId == pRebVg->oldConsumerId) { taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
mInfo("mq rebalance: skip vg %d for same consumer:0x%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId, mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
pConsumerEp->consumerId);
continue;
}
taosArrayPush(pOutput->rebVgs, pRebVg);
mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId);
} }
} else { }
// if all consumer is removed, put all vg into unassigned
pIter = NULL;
SMqRebOutputVg *pRebOutput = NULL;
while (1) {
pIter = taosHashIterate(pHash, pIter);
if (pIter == NULL) {
break;
}
pRebOutput = (SMqRebOutputVg *)pIter; // All assigned vg should be put into pOutput->rebVgs
if(pRemovedIter != NULL){
mError("sub:%s error pRemovedIter should be NULL", pSubKey);
}
while (1) {
pRemovedIter = taosHashIterate(pHash, pRemovedIter);
if (pRemovedIter == NULL) {
break;
}
SMqRebOutputVg* pRebOutput = (SMqRebOutputVg *)pRemovedIter;
taosArrayPush(pOutput->rebVgs, pRebOutput);
if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed, put all vg into unassigned
taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
taosArrayPush(pOutput->rebVgs, pRebOutput);
mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", pSubKey, pRebOutput->pVgEp->vgId);
} }
} }
@ -462,19 +444,18 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, pSubKey, mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, pSubKey,
pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId);
} }
{
pIter = NULL; pIter = NULL;
while (1) { while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
int32_t sz = taosArrayGetSize(pConsumerEp->vgs); int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", pSubKey, pConsumerEp->consumerId, sz); mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", pSubKey, pConsumerEp->consumerId, sz);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, pSubKey, pVgEp->vgId, mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, pSubKey, pVgEp->vgId,
pConsumerEp->consumerId); pConsumerEp->consumerId);
}
} }
} }
@ -515,9 +496,9 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
// 3. commit log: consumer to update status and epoch // 3. commit log: consumer to update status and epoch
// 3.1 set touched consumer // 3.1 set touched consumer
int32_t consumerNum = taosArrayGetSize(pOutput->touchedConsumers); int32_t consumerNum = taosArrayGetSize(pOutput->modifyConsumers);
for (int32_t i = 0; i < consumerNum; i++) { for (int32_t i = 0; i < consumerNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->touchedConsumers, i); int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->modifyConsumers, i);
SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH; pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH;
@ -597,15 +578,15 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
SMqDoRebalanceMsg *pReq = pMsg->pCont; SMqDoRebalanceMsg *pReq = pMsg->pCont;
void *pIter = NULL; void *pIter = NULL;
bool rebalanceOnce = false; // to ensure only once. // bool rebalanceOnce = false; // to ensure only once.
mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash));
// here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction. // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction.
while (1) { while (1) {
if (rebalanceOnce) { // if (rebalanceOnce) {
break; // break;
} // }
pIter = taosHashIterate(pReq->rebSubHash, pIter); pIter = taosHashIterate(pReq->rebSubHash, pIter);
if (pIter == NULL) { if (pIter == NULL) {
@ -617,7 +598,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
SMqRebOutputObj rebOutput = {0}; SMqRebOutputObj rebOutput = {0};
rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t)); rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t));
rebOutput.removedConsumers = taosArrayInit(0, sizeof(int64_t)); rebOutput.removedConsumers = taosArrayInit(0, sizeof(int64_t));
rebOutput.touchedConsumers = taosArrayInit(0, sizeof(int64_t)); rebOutput.modifyConsumers = taosArrayInit(0, sizeof(int64_t));
rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg)); rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg));
SMqRebInfo *pRebInfo = (SMqRebInfo *)pIter; SMqRebInfo *pRebInfo = (SMqRebInfo *)pIter;
@ -653,13 +634,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
rebInput.oldConsumerNum = 0; rebInput.oldConsumerNum = 0;
mInfo("topic:%s has no consumers sub yet", topic); mInfo("sub topic:%s has no consumers sub yet", pRebInfo->key);
} else { } else {
taosRLockLatch(&pSub->lock); taosRLockLatch(&pSub->lock);
rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash); rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash);
rebOutput.pSub = tCloneSubscribeObj(pSub); rebOutput.pSub = tCloneSubscribeObj(pSub);
taosRUnLockLatch(&pSub->lock); taosRUnLockLatch(&pSub->lock);
mInfo("topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum); mInfo("sub topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum);
mndReleaseSubscribe(pMnode, pSub); mndReleaseSubscribe(pMnode, pSub);
} }
@ -675,13 +656,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
} }
taosArrayDestroy(rebOutput.newConsumers); taosArrayDestroy(rebOutput.newConsumers);
taosArrayDestroy(rebOutput.touchedConsumers); taosArrayDestroy(rebOutput.modifyConsumers);
taosArrayDestroy(rebOutput.removedConsumers); taosArrayDestroy(rebOutput.removedConsumers);
taosArrayDestroy(rebOutput.rebVgs); taosArrayDestroy(rebOutput.rebVgs);
tDeleteSubscribeObj(rebOutput.pSub); tDeleteSubscribeObj(rebOutput.pSub);
taosMemoryFree(rebOutput.pSub); taosMemoryFree(rebOutput.pSub);
rebalanceOnce = true; // taosSsleep(100);
// rebalanceOnce = true;
} }
// reset flag // reset flag

2
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -196,13 +196,13 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SSdbRow *pOldRow = *ppOldRow; SSdbRow *pOldRow = *ppOldRow;
pOldRow->status = pRaw->status; pOldRow->status = pRaw->status;
sdbPrintOper(pSdb, pOldRow, "update"); sdbPrintOper(pSdb, pOldRow, "update");
sdbUnLock(pSdb, type);
int32_t code = 0; int32_t code = 0;
SdbUpdateFp updateFp = pSdb->updateFps[type]; SdbUpdateFp updateFp = pSdb->updateFps[type];
if (updateFp != NULL) { if (updateFp != NULL) {
code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj);
} }
sdbUnLock(pSdb, type);
// sdbUnLock(pSdb, type); // sdbUnLock(pSdb, type);
sdbFreeRow(pSdb, pNewRow, false); sdbFreeRow(pSdb, pNewRow, false);

8
source/dnode/vnode/src/inc/tq.h
View File

@ -179,16 +179,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqStreamTasksScanWal(STQ* pTq); int32_t tqStreamTasksScanWal(STQ* pTq);
// tq util // tq util
void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId); char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver);
int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset);
int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver);
void saveOffsetForAllTasks(STQ* pTq, int64_t ver);
void initOffsetForAllRestoreTasks(STQ* pTq);
int32_t transferToWalReadTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -190,6 +190,7 @@ int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
int tqInit(); int tqInit();
void tqCleanUp(); void tqCleanUp();
STQ* tqOpen(const char* path, SVnode* pVnode); STQ* tqOpen(const char* path, SVnode* pVnode);
void tqNotifyClose(STQ*);
void tqClose(STQ*); void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp,

137
source/dnode/vnode/src/tq/tq.c
View File

@ -18,7 +18,9 @@
// 0: not init // 0: not init
// 1: already inited // 1: already inited
// 2: wait to be inited or cleaup // 2: wait to be inited or cleaup
#define WAL_READ_TASKS_ID (-1) #define WAL_READ_TASKS_ID (-1)
static int32_t tqInitialize(STQ* pTq);
int32_t tqInit() { int32_t tqInit() {
int8_t old; int8_t old;
@ -109,25 +111,32 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo);
tqInitialize(pTq);
return pTq;
}
int32_t tqInitialize(STQ* pTq) {
if (tqMetaOpen(pTq) < 0) { if (tqMetaOpen(pTq) < 0) {
return NULL; return -1;
} }
pTq->pOffsetStore = tqOffsetOpen(pTq); pTq->pOffsetStore = tqOffsetOpen(pTq);
if (pTq->pOffsetStore == NULL) { if (pTq->pOffsetStore == NULL) {
return NULL; return -1;
} }
pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId);
if (pTq->pStreamMeta == NULL) { if (pTq->pStreamMeta == NULL) {
return NULL; return -1;
} }
if (streamLoadTasks(pTq->pStreamMeta, walGetCommittedVer(pVnode->pWal)) < 0) { // the version is kept in task's meta data
return NULL; // todo check if this version is required or not
if (streamLoadTasks(pTq->pStreamMeta, walGetCommittedVer(pTq->pVnode->pWal)) < 0) {
return -1;
} }
return pTq; return 0;
} }
void tqClose(STQ* pTq) { void tqClose(STQ* pTq) {
@ -145,6 +154,31 @@ void tqClose(STQ* pTq) {
taosMemoryFree(pTq); taosMemoryFree(pTq);
} }
void tqNotifyClose(STQ* pTq) {
if (pTq != NULL) {
taosWLockLatch(&pTq->pStreamMeta->lock);
void* pIter = NULL;
while (1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) {
break;
}
SStreamTask* pTask = *(SStreamTask**)pIter;
tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr);
pTask->status.taskStatus = TASK_STATUS__STOP;
int64_t st = taosGetTimestampMs();
qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS);
int64_t el = taosGetTimestampMs() - st;
tqDebug("vgId:%d s-task:%s is closed in %" PRId64 "ms", pTq->pStreamMeta->vgId, pTask->id.idStr, el);
}
taosWUnLockLatch(&pTq->pStreamMeta->lock);
}
}
static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch,
int64_t consumerId, int32_t type) { int64_t consumerId, int32_t type) {
int32_t len = 0; int32_t len = 0;
@ -548,12 +582,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
} }
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
// todo extract method
char buf[128] = {0};
sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId);
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
pTask->id.idStr = taosStrdup(buf); pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId);
pTask->refCnt = 1; pTask->refCnt = 1;
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
pTask->inputQueue = streamQueueOpen(512 << 10); pTask->inputQueue = streamQueueOpen(512 << 10);
@ -568,6 +598,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask->pMsgCb = &pTq->pVnode->msgCb; pTask->pMsgCb = &pTq->pVnode->msgCb;
pTask->pMeta = pTq->pStreamMeta; pTask->pMeta = pTq->pStreamMeta;
pTask->chkInfo.version = ver; pTask->chkInfo.version = ver;
pTask->chkInfo.currentVer = ver;
// expand executor // expand executor
if (pTask->fillHistory) { if (pTask->fillHistory) {
@ -596,8 +627,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
return -1; return -1;
} }
int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo);
SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; SReadHandle mgHandle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState};
pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId);
if (pTask->exec.pExecutor == NULL) { if (pTask->exec.pExecutor == NULL) {
@ -623,7 +654,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
SSchemaWrapper* pschemaWrapper = pTask->tbSink.pSchemaWrapper; SSchemaWrapper* pschemaWrapper = pTask->tbSink.pSchemaWrapper;
pTask->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1); pTask->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
if(pTask->tbSink.pTSchema == NULL) { if (pTask->tbSink.pTSchema == NULL) {
return -1; return -1;
} }
} }
@ -633,8 +664,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
} }
streamSetupTrigger(pTask); streamSetupTrigger(pTask);
tqInfo("vgId:%d expand stream task, s-task:%s, ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr, tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId,
pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); pTask->id.idStr, pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel);
// next valid version will add one
pTask->chkInfo.version += 1;
return 0; return 0;
} }
@ -692,7 +726,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
tEncodeSStreamTaskCheckRsp(&encoder, &rsp); tEncodeSStreamTaskCheckRsp(&encoder, &rsp);
tEncoderClear(&encoder); tEncoderClear(&encoder);
SRpcMsg rspMsg = { .code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info }; SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info};
tmsgSendRsp(&rspMsg); tmsgSendRsp(&rspMsg);
return 0; return 0;
} }
@ -750,7 +784,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
tDecoderClear(&decoder); tDecoderClear(&decoder);
// 2.save task // 2.save task, use the newest commit version as the initial start version of stream task.
code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask); code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask);
if (code < 0) { if (code < 0) {
tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr, tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr,
@ -958,14 +992,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
int32_t* pRef = taosMemoryMalloc(sizeof(int32_t)); int32_t* pRef = taosMemoryMalloc(sizeof(int32_t));
*pRef = 1; *pRef = 1;
taosWLockLatch(&pTq->pStreamMeta->lock);
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) break; if (pIter == NULL) {
SStreamTask* pTask = *(SStreamTask**)pIter; break;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; }
qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver); SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
continue;
}
qDebug("s-task:%s delete req enqueue, ver: %" PRId64, pTask->id.idStr, ver);
if (!failed) { if (!failed) {
SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);
@ -975,15 +1016,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
atomic_add_fetch_32(pRefBlock->dataRef, 1); atomic_add_fetch_32(pRefBlock->dataRef, 1);
if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
qError("stream task input del failed, task id %d", pTask->id.taskId);
atomic_sub_fetch_32(pRef, 1); atomic_sub_fetch_32(pRef, 1);
taosFreeQitem(pRefBlock); taosFreeQitem(pRefBlock);
continue; continue;
} }
if (streamSchedExec(pTask) < 0) { if (streamSchedExec(pTask) < 0) {
qError("stream task launch failed, task id %d", pTask->id.taskId); qError("s-task:%s stream task launch failed", pTask->id.idStr);
continue; continue;
} }
@ -992,8 +1031,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
} }
} }
taosWUnLockLatch(&pTq->pStreamMeta->lock);
int32_t ref = atomic_sub_fetch_32(pRef, 1); int32_t ref = atomic_sub_fetch_32(pRef, 1);
/*A(ref >= 0);*/
if (ref == 0) { if (ref == 0) {
blockDataDestroy(pDelBlock); blockDataDestroy(pDelBlock);
taosMemoryFree(pRef); taosMemoryFree(pRef);
@ -1024,23 +1064,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
} }
blockDataDestroy(pDelBlock); blockDataDestroy(pDelBlock);
#endif #endif
return 0; return 0;
} }
static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit,
const char* key, int64_t ver) {
doSaveTaskOffset(pOffsetStore, key, ver);
int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver);
// remove the offset, if all functions are completed successfully.
if (code == TSDB_CODE_SUCCESS) {
tqOffsetDelete(pOffsetStore, key);
}
return code;
}
int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
#if 0 #if 0
void* pIter = NULL; void* pIter = NULL;
@ -1122,8 +1148,8 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr); tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
streamProcessRunReq(pTask); streamProcessRunReq(pTask);
} else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) { } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) {
tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, pTask->id.idStr,
pTask->id.idStr, pTask->chkInfo.version); pTask->chkInfo.version);
streamProcessRunReq(pTask); streamProcessRunReq(pTask);
} else { } else {
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr);
@ -1149,11 +1175,12 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) {
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId);
if (pTask) { if (pTask) {
SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
streamProcessDispatchReq(pTask, &req, &rsp, exec); streamProcessDispatchReq(pTask, &req, &rsp, exec);
streamMetaReleaseTask(pTq->pStreamMeta, pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask);
return 0; return 0;
} else { } else {
tDeleteStreamDispatchReq(&req);
return -1; return -1;
} }
} }
@ -1190,12 +1217,13 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) {
int32_t taskId = req.dstTaskId; int32_t taskId = req.dstTaskId;
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
if (pTask) { if (pTask) {
SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
streamProcessRetrieveReq(pTask, &req, &rsp); streamProcessRetrieveReq(pTask, &req, &rsp);
streamMetaReleaseTask(pTq->pStreamMeta, pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask);
tDeleteStreamRetrieveReq(&req); tDeleteStreamRetrieveReq(&req);
return 0; return 0;
} else { } else {
tDeleteStreamRetrieveReq(&req);
return -1; return -1;
} }
} }
@ -1226,7 +1254,7 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId);
if (pTask) { if (pTask) {
SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
streamProcessDispatchReq(pTask, &req, &rsp, false); streamProcessDispatchReq(pTask, &req, &rsp, false);
streamMetaReleaseTask(pTq->pStreamMeta, pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
@ -1243,7 +1271,7 @@ FAIL:
SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp));
if (pRspHead == NULL) { if (pRspHead == NULL) {
SRpcMsg rsp = { .code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info }; SRpcMsg rsp = {.code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info};
tqDebug("send dispatch error rsp, code: %x", code); tqDebug("send dispatch error rsp, code: %x", code);
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
@ -1276,6 +1304,13 @@ int32_t tqStartStreamTasks(STQ* pTq) {
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
taosWLockLatch(&pMeta->lock); taosWLockLatch(&pMeta->lock);
int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks);
if (numOfTasks == 0) {
tqInfo("vgId:%d no stream tasks exists", vgId);
taosWUnLockLatch(&pTq->pStreamMeta->lock);
return 0;
}
pMeta->walScan += 1; pMeta->walScan += 1;
if (pMeta->walScan > 1) { if (pMeta->walScan > 1) {
@ -1287,16 +1322,12 @@ int32_t tqStartStreamTasks(STQ* pTq) {
SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
if (pRunReq == NULL) { if (pRunReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr(terrno)); tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr());
taosWUnLockLatch(&pTq->pStreamMeta->lock); taosWUnLockLatch(&pTq->pStreamMeta->lock);
return -1; return -1;
} }
int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks);
tqDebug("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks); tqDebug("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks);
initOffsetForAllRestoreTasks(pTq);
pRunReq->head.vgId = vgId; pRunReq->head.vgId = vgId;
pRunReq->streamId = 0; pRunReq->streamId = 0;
pRunReq->taskId = WAL_READ_TASKS_ID; pRunReq->taskId = WAL_READ_TASKS_ID;

3
source/dnode/vnode/src/tq/tqRead.c
View File

@ -1023,6 +1023,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
} }
// update the table list handle for each stream scanner/wal reader // update the table list handle for each stream scanner/wal reader
taosWLockLatch(&pTq->pStreamMeta->lock);
while (1) { while (1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) { if (pIter == NULL) {
@ -1039,5 +1040,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
} }
} }
taosWUnLockLatch(&pTq->pStreamMeta->lock);
return 0; return 0;
} }

123
source/dnode/vnode/src/tq/tqRestore.c
View File

@ -15,25 +15,23 @@
#include "tq.h" #include "tq.h"
static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle); static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle);
static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
// this function should be executed by stream threads. // this function should be executed by stream threads.
// there is a case that the WAL increases more fast than the restore procedure, and this restore procedure // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure
// will not stop eventually. // will not stop eventually.
int tqStreamTasksScanWal(STQ* pTq) { int32_t tqStreamTasksScanWal(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
while (1) { while (1) {
int32_t scan = pMeta->walScan; int32_t scan = pMeta->walScan;
tqDebug("vgId:%d continue check if data in wal are available, scan:%d", vgId, scan); tqDebug("vgId:%d continue check if data in wal are available, scan:%d", vgId, scan);
ASSERT(scan >= 1);
// check all restore tasks // check all restore tasks
bool shouldIdle = true; bool shouldIdle = true;
streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &shouldIdle); createStreamRunReq(pTq->pStreamMeta, &shouldIdle);
int32_t times = 0; int32_t times = 0;
@ -51,50 +49,19 @@ int tqStreamTasksScanWal(STQ* pTq) {
taosWUnLockLatch(&pMeta->lock); taosWUnLockLatch(&pMeta->lock);
tqDebug("vgId:%d scan wal for stream tasks for %d times", vgId, times); tqDebug("vgId:%d scan wal for stream tasks for %d times", vgId, times);
} else {
tqDebug("vgId:%d no idle, scan wal for stream tasks for %d times", vgId, pMeta->walScan);
ASSERT(pMeta->walScan >= 1);
} }
} }
int64_t el = (taosGetTimestampMs() - st); int64_t el = (taosGetTimestampMs() - st);
tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%"PRId64" ms", vgId, el); tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%" PRId64 " ms", vgId, el);
// restore wal scan flag
// atomic_store_8(&pTq->pStreamMeta->walScan, 0);
return 0; return 0;
} }
//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) { static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) {
// int32_t numOfTask = taosArrayGetSize(pTaskList); SArray* pTaskIdList = taosArrayInit(numOfTasks, sizeof(int32_t));
// if (numOfTask <= 0) {
// return TSDB_CODE_SUCCESS;
// }
//
// // todo: add lock
// for (int32_t i = 0; i < numOfTask; ++i) {
// SStreamTask* pTask = taosArrayGetP(pTaskList, i);
// tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
// pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
// taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
//
// // NOTE: do not change the following order
// atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
// taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
// }
//
// return TSDB_CODE_SUCCESS;
//}
int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) {
void* pIter = NULL; void* pIter = NULL;
int32_t vgId = pStreamMeta->vgId;
*pScanIdle = true;
bool allWalChecked = true;
tqDebug("vgId:%d start to check wal to extract new submit block", vgId);
taosWLockLatch(&pStreamMeta->lock);
while (1) { while (1) {
pIter = taosHashIterate(pStreamMeta->pTasks, pIter); pIter = taosHashIterate(pStreamMeta->pTasks, pIter);
if (pIter == NULL) { if (pIter == NULL) {
@ -102,44 +69,71 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto
} }
SStreamTask* pTask = *(SStreamTask**)pIter; SStreamTask* pTask = *(SStreamTask**)pIter;
taosArrayPush(pTaskIdList, &pTask->id.taskId);
}
taosWUnLockLatch(&pStreamMeta->lock);
return pTaskIdList;
}
int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
*pScanIdle = true;
bool noNewDataInWal = true;
int32_t vgId = pStreamMeta->vgId;
int32_t numOfTasks = taosHashGetSize(pStreamMeta->pTasks);
if (numOfTasks == 0) {
return TSDB_CODE_SUCCESS;
}
tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks);
SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks);
// update the new task number
numOfTasks = taosArrayGetSize(pTaskIdList);
for (int32_t i = 0; i < numOfTasks; ++i) {
int32_t* pTaskId = taosArrayGet(pTaskIdList, i);
SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId);
if (pTask == NULL) {
continue;
}
int32_t status = pTask->status.taskStatus;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) { if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
tqDebug("s-task:%s not source task, no need to start", pTask->id.idStr);
streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE ||
pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { status == TASK_STATUS__WAIT_DOWNSTREAM) {
tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, status);
pTask->status.taskStatus); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
// check if offset value exists
char key[128] = {0};
createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
if (tInputQueueIsFull(pTask)) { if (tInputQueueIsFull(pTask)) {
tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr); tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr);
streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
*pScanIdle = false; *pScanIdle = false;
// check if offset value exists
STqOffset* pOffset = tqOffsetRead(pOffsetStore, key);
ASSERT(pOffset != NULL);
// seek the stored version and extract data from WAL // seek the stored version and extract data from WAL
int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version); int32_t code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer);
if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit
streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
// append the data for the stream // append the data for the stream
tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr); tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer);
SPackedData packData = {0}; SPackedData packData = {0};
code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData);
if (code != TSDB_CODE_SUCCESS) { // failed, continue if (code != TSDB_CODE_SUCCESS) { // failed, continue
streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
@ -147,28 +141,31 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto
if (p == NULL) { if (p == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr); tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr);
streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
allWalChecked = false; noNewDataInWal = false;
tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr);
code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader); pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader);
tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
pOffset->val.version); pTask->chkInfo.currentVer);
} else { } else {
// do nothing tqError("s-task:%s append input queue failed, ver:%" PRId64, pTask->id.idStr, pTask->chkInfo.currentVer);
} }
streamDataSubmitDestroy(p); streamDataSubmitDestroy(p);
taosFreeQitem(p); taosFreeQitem(p);
streamMetaReleaseTask(pStreamMeta, pTask);
} }
if (allWalChecked) { // all wal are checked, and no new data available in wal.
if (noNewDataInWal) {
*pScanIdle = true; *pScanIdle = true;
} }
taosArrayDestroy(pTaskIdList);
return 0; return 0;
} }

103
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -19,19 +19,10 @@
static int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp); static int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp);
// stream_task:stream_id:task_id char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) {
void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) { char buf[128] = {0};
int32_t n = 12; sprintf(buf, "0x%" PRIx64 "-%d", streamId, taskId);
char* p = dst; return taosStrdup(buf);
memcpy(p, "stream_task:", n);
p += n;
int32_t inc = tintToHex(streamId, p);
p += inc;
*(p++) = ':';
tintToHex(taskId, p);
} }
int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) { int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) {
@ -49,75 +40,6 @@ int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueI
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void initOffsetForAllRestoreTasks(STQ* pTq) {
void* pIter = NULL;
while(1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) {
break;
}
SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
continue;
}
if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus);
continue;
}
char key[128] = {0};
createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
if (pOffset == NULL) {
doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version);
}
}
}
void saveOffsetForAllTasks(STQ* pTq, int64_t ver) {
void* pIter = NULL;
while(1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) {
break;
}
SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
continue;
}
if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
pTask->status.taskStatus);
continue;
}
char key[128] = {0};
createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
if (pOffset == NULL) {
doSaveTaskOffset(pTq->pOffsetStore, key, ver);
}
}
}
void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) {
STqOffset offset = {0};
tqOffsetResetToLog(&offset.val, ver);
tstrncpy(offset.subKey, pKey, tListLen(offset.subKey));
// keep the offset info in the offset store
tqOffsetWrite(pOffsetStore, &offset);
}
static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) {
pRsp->reqOffset = pReq->reqOffset; pRsp->reqOffset = pReq->reqOffset;
@ -144,6 +66,21 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
if (pRsp->blockData != NULL) {
pRsp->blockData = taosArrayDestroy(pRsp->blockData);
}
if (pRsp->blockDataLen != NULL) {
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);
}
if (pRsp->blockTbName != NULL) {
pRsp->blockTbName = taosArrayDestroy(pRsp->blockTbName);
}
if (pRsp->blockSchema != NULL) {
pRsp->blockSchema = taosArrayDestroy(pRsp->blockSchema);
}
return -1; return -1;
} }
@ -277,6 +214,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
if (offset->type != TMQ_OFFSET__LOG) { if (offset->type != TMQ_OFFSET__LOG) {
if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) {
tDeleteSTaosxRsp(&taosxRsp);
return -1; return -1;
} }
@ -352,6 +290,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
tDeleteSTaosxRsp(&taosxRsp); tDeleteSTaosxRsp(&taosxRsp);
return code; return code;
} }
code = 0; code = 0;
taosMemoryFreeClear(pCkHead); taosMemoryFreeClear(pCkHead);
tDeleteSTaosxRsp(&taosxRsp); tDeleteSTaosxRsp(&taosxRsp);

56
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -1881,8 +1881,8 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_
return pReader->pSchema; return pReader->pSchema;
} }
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema);
if (pReader->pSchema == NULL) { if (code != TSDB_CODE_SUCCESS || pReader->pSchema == NULL) {
tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr);
} }
@ -1890,9 +1890,15 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_
} }
static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) {
int32_t code = 0;
// always set the newest schema version in pReader->pSchema // always set the newest schema version in pReader->pSchema
if (pReader->pSchema == NULL) { if (pReader->pSchema == NULL) {
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
}
} }
if (pReader->pSchema && sversion == pReader->pSchema->version) { if (pReader->pSchema && sversion == pReader->pSchema->version) {
@ -1905,7 +1911,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader*
} }
STSchema* ptr = NULL; STSchema* ptr = NULL;
int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
return NULL; return NULL;
@ -1969,7 +1975,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
// DESC: mem -----> imem -----> last block -----> file block // DESC: mem -----> imem -----> last block -----> file block
if (pReader->order == TSDB_ORDER_ASC) { if (pReader->order == TSDB_ORDER_ASC) {
if (minKey == key) { if (minKey == key) {
init = true; init = true; // todo check if pReader->pSchema is null or not
int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
@ -2014,6 +2020,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
if (minKey == k.ts) { if (minKey == k.ts) {
init = true; init = true;
STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
if (pSchema == NULL) {
return terrno;
}
int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
@ -2222,6 +2232,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
if (pSchema == NULL) { if (pSchema == NULL) {
return code; return code;
} }
STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
if (piSchema == NULL) { if (piSchema == NULL) {
return code; return code;
@ -3843,11 +3854,8 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter,
return terrno; return terrno;
} }
if (pReader->pSchema == NULL) { STSchema* ps = (pReader->pSchema != NULL)? pReader->pSchema:pTSchema;
pReader->pSchema = pTSchema; code = tsdbRowMergerInit(&merge, ps, &current, pTSchema);
}
code = tsdbRowMergerInit(&merge, pReader->pSchema, &current, pTSchema);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -3891,7 +3899,14 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p
TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY k = TSDBROW_KEY(pRow);
TSDBKEY ik = TSDBROW_KEY(piRow); TSDBKEY ik = TSDBROW_KEY(piRow);
STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
if (pSchema == NULL) {
return terrno;
}
STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
if (piSchema == NULL) {
return terrno;
}
if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem
int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema);
@ -4000,10 +4015,11 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT
int64_t uid = pScanInfo->uid; int64_t uid = pScanInfo->uid;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid);
if (pSchema == NULL) {
return terrno;
}
SColVal colVal = {0}; SColVal colVal = {0};
int32_t i = 0, j = 0; int32_t i = 0, j = 0;
@ -4235,8 +4251,8 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
} }
static void freeSchemaFunc(void* param) { static void freeSchemaFunc(void* param) {
void* p = *(void**)param; void **p = (void **)param;
taosMemoryFree(p); taosMemoryFreeClear(*p);
} }
// ====================================== EXPOSED APIs ====================================== // ====================================== EXPOSED APIs ======================================
@ -5187,8 +5203,6 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) {
} }
int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) {
int32_t sversion = 1;
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pVnode->pMeta, 0); metaReaderInit(&mr, pVnode->pMeta, 0);
int32_t code = metaGetTableEntryByUidCache(&mr, uid); int32_t code = metaGetTableEntryByUidCache(&mr, uid);
@ -5200,6 +5214,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
*suid = 0; *suid = 0;
// only child table and ordinary table is allowed, super table is not allowed.
if (mr.me.type == TSDB_CHILD_TABLE) { if (mr.me.type == TSDB_CHILD_TABLE) {
tDecoderClear(&mr.coder); tDecoderClear(&mr.coder);
*suid = mr.me.ctbEntry.suid; *suid = mr.me.ctbEntry.suid;
@ -5209,9 +5224,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
metaReaderClear(&mr); metaReaderClear(&mr);
return terrno; return terrno;
} }
sversion = mr.me.stbEntry.schemaRow.version; } else if (mr.me.type == TSDB_NORMAL_TABLE) { // do nothing
} else if (mr.me.type == TSDB_NORMAL_TABLE) {
sversion = mr.me.ntbEntry.schemaRow.version;
} else { } else {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
metaReaderClear(&mr); metaReaderClear(&mr);
@ -5219,9 +5232,10 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
} }
metaReaderClear(&mr); metaReaderClear(&mr);
*pSchema = metaGetTbTSchema(pVnode->pMeta, uid, sversion, 1);
return TSDB_CODE_SUCCESS; // get the newest table schema version
code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema);
return code;
} }
int32_t tsdbTakeReadSnap(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { int32_t tsdbTakeReadSnap(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) {

2
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -547,7 +547,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
walApplyVer(pVnode->pWal, commitIdx); walApplyVer(pVnode->pWal, commitIdx);
pVnode->restored = true; pVnode->restored = true;
vInfo("vgId:%d, sync restore finished", pVnode->config.vgId); vInfo("vgId:%d, sync restore finished, start to restore stream tasks by replay wal", pVnode->config.vgId);
// start to restore all stream tasks // start to restore all stream tasks
tqStartStreamTasks(pVnode->pTq); tqStartStreamTasks(pVnode->pTq);

7
source/libs/command/src/command.c
View File

@ -278,7 +278,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
char* retentions = buildRetension(pCfg->pRetensions); char* retentions = buildRetension(pCfg->pRetensions);
int32_t dbFNameLen = strlen(dbFName); int32_t dbFNameLen = strlen(dbFName);
int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0; int32_t hashPrefix = 0;
if (pCfg->hashPrefix > 0) {
hashPrefix = pCfg->hashPrefix - dbFNameLen - 1;
} else if (pCfg->hashPrefix < 0) {
hashPrefix = pCfg->hashPrefix + dbFNameLen + 1;
}
len += sprintf( len += sprintf(
buf2 + VARSTR_HEADER_SIZE, buf2 + VARSTR_HEADER_SIZE,

1
source/libs/executor/inc/executil.h
View File

@ -109,6 +109,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList);
uint64_t tableListGetSuid(const STableListInfo* pTableList); uint64_t tableListGetSuid(const STableListInfo* pTableList);
STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index); STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index);
int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex); int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex);
void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psuid, uint64_t* uid, int32_t* type);
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
void initResultRowInfo(SResultRowInfo* pResultRowInfo); void initResultRowInfo(SResultRowInfo* pResultRowInfo);

1
source/libs/executor/inc/executorimpl.h
View File

@ -429,7 +429,6 @@ typedef struct STimeWindowAggSupp {
} STimeWindowAggSupp; } STimeWindowAggSupp;
typedef struct SStreamScanInfo { typedef struct SStreamScanInfo {
uint64_t tableUid; // queried super table uid
SExprInfo* pPseudoExpr; SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr; int32_t numOfPseudoExpr;
SExprSupp tbnameCalSup; SExprSupp tbnameCalSup;

1
source/libs/executor/src/dataDeleter.c
View File

@ -240,6 +240,7 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData
SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle));
if (NULL == deleter) { if (NULL == deleter) {
taosMemoryFree(pParam);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
} }

1
source/libs/executor/src/dataInserter.c
View File

@ -408,6 +408,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
void* pParam) { void* pParam) {
SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle));
if (NULL == inserter) { if (NULL == inserter) {
taosMemoryFree(pParam);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }

44
source/libs/executor/src/executil.c
View File

@ -27,15 +27,21 @@
#include "executorimpl.h" #include "executorimpl.h"
#include "tcompression.h" #include "tcompression.h"
typedef struct STableListIdInfo {
uint64_t suid;
uint64_t uid;
int32_t tableType;
} STableListIdInfo;
// If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly // If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly
// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups // The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups
struct STableListInfo { struct STableListInfo {
bool oneTableForEachGroup; bool oneTableForEachGroup;
int32_t numOfOuputGroups; // the data block will be generated one by one int32_t numOfOuputGroups; // the data block will be generated one by one
int32_t* groupOffset; // keep the offset value for each group in the tableList int32_t* groupOffset; // keep the offset value for each group in the tableList
SArray* pTableList; SArray* pTableList;
SHashObj* map; // speedup acquire the tableQueryInfo by table uid SHashObj* map; // speedup acquire the tableQueryInfo by table uid
uint64_t suid; STableListIdInfo idInfo; // this maybe the super table or ordinary table
}; };
typedef struct tagFilterAssist { typedef struct tagFilterAssist {
@ -470,7 +476,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
} }
// int64_t stt = taosGetTimestampUs(); // int64_t stt = taosGetTimestampUs();
code = metaGetTableTags(metaHandle, pTableListInfo->suid, pUidTagList); code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
@ -953,7 +959,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
FilterCondType condType = checkTagCond(pTagCond); FilterCondType condType = checkTagCond(pTagCond);
int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->suid, pUidTagList, pTagCond); int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond);
if (filter == 0) { // tbname in filter is activated, do nothing and return if (filter == 0) { // tbname in filter is activated, do nothing and return
taosArrayClear(pUidList); taosArrayClear(pUidList);
@ -966,12 +972,12 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
terrno = 0; terrno = 0;
} else { } else {
if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) {
code = metaGetTableTagsByUids(metaHandle, pListInfo->suid, pUidTagList); code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList);
} else { } else {
code = metaGetTableTags(metaHandle, pListInfo->suid, pUidTagList); code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList);
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->suid); qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid);
terrno = code; terrno = code;
goto end; goto end;
} }
@ -1025,15 +1031,17 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
size_t numOfTables = 0; size_t numOfTables = 0;
pListInfo->suid = pScanNode->suid; pListInfo->idInfo.suid = pScanNode->suid;
pListInfo->idInfo.tableType = pScanNode->tableType;
SArray* pUidList = taosArrayInit(8, sizeof(uint64_t)); SArray* pUidList = taosArrayInit(8, sizeof(uint64_t));
SIdxFltStatus status = SFLT_NOT_INDEX; SIdxFltStatus status = SFLT_NOT_INDEX;
if (pScanNode->tableType != TSDB_SUPER_TABLE) { if (pScanNode->tableType != TSDB_SUPER_TABLE) {
pListInfo->idInfo.uid = pScanNode->uid;
if (metaIsTableExist(metaHandle, pScanNode->uid)) { if (metaIsTableExist(metaHandle, pScanNode->uid)) {
taosArrayPush(pUidList, &pScanNode->uid); taosArrayPush(pUidList, &pScanNode->uid);
} }
code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
@ -1794,7 +1802,9 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) {
return taosArrayGetSize(pTableList->pTableList); return taosArrayGetSize(pTableList->pTableList);
} }
uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->suid; } uint64_t tableListGetSuid(const STableListInfo* pTableList) {
return pTableList->idInfo.suid;
}
STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) {
if (taosArrayGetSize(pTableList->pTableList) == 0) { if (taosArrayGetSize(pTableList->pTableList) == 0) {
@ -1819,6 +1829,12 @@ int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t st
return -1; return -1;
} }
void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psuid, uint64_t* uid, int32_t* type) {
*psuid = pTableList->idInfo.suid;
*uid = pTableList->idInfo.uid;
*type = pTableList->idInfo.tableType;
}
uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) {
int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid)); int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid));
ASSERT(pTableList->map != NULL && slot != NULL); ASSERT(pTableList->map != NULL && slot != NULL);

35
source/libs/executor/src/executor.c
View File

@ -14,6 +14,7 @@
*/ */
#include "executor.h" #include "executor.h"
#include <vnode.h>
#include "executorimpl.h" #include "executorimpl.h"
#include "planner.h" #include "planner.h"
#include "tdatablock.h" #include "tdatablock.h"
@ -327,6 +328,13 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
return qa; return qa;
} }
STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info;
uint64_t suid = 0;
uint64_t uid = 0;
int32_t type = 0;
tableListGetSourceTableInfo(pTableScanInfo->base.pTableListInfo, &suid, &uid, &type);
// let's discard the tables those are not created according to the queried super table. // let's discard the tables those are not created according to the queried super table.
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); metaReaderInit(&mr, pScanInfo->readHandle.meta, 0);
@ -341,9 +349,21 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
tDecoderClear(&mr.coder); tDecoderClear(&mr.coder);
// TODO handle ntb case if (mr.me.type == TSDB_SUPER_TABLE) {
if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) {
continue; continue;
} else {
if (type == TSDB_SUPER_TABLE) {
// this new created child table does not belong to the scanned super table.
if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != suid) {
continue;
}
} else { // ordinary table
// In case that the scanned target table is an ordinary table. When replay the WAL during restore the vnode, we
// should check all newly created ordinary table to make sure that this table isn't the destination table.
if (mr.me.uid != uid) {
continue;
}
}
} }
if (pScanInfo->pTagCond != NULL) { if (pScanInfo->pTagCond != NULL) {
@ -382,7 +402,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
SStreamScanInfo* pScanInfo = pInfo->info; SStreamScanInfo* pScanInfo = pInfo->info;
if (isAdd) { // add new table id if (isAdd) { // add new table id
SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id);
int32_t numOfQualifiedTables = taosArrayGetSize(qa); int32_t numOfQualifiedTables = taosArrayGetSize(qa);
qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id);
code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
@ -497,10 +517,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
goto _error; goto _error;
} }
// pSinkParam has been freed during create sinker.
code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pSinkParam);
}
} }
qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId);
@ -1090,6 +1108,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pScanBaseInfo->dataReader = NULL; pScanBaseInfo->dataReader = NULL;
// let's seek to the next version in wal file // let's seek to the next version in wal file
int64_t firstVer = walGetFirstVer(pInfo->tqReader->pWalReader->pWal);
if (pOffset->version + 1 < firstVer){
pOffset->version = firstVer - 1;
}
if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
return -1; return -1;

3
source/libs/executor/src/scanoperator.c
View File

@ -691,7 +691,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
} }
uint32_t status = 0; uint32_t status = 0;
int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
@ -2437,7 +2437,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
} }
pInfo->readHandle = *pHandle; pInfo->readHandle = *pHandle;
pInfo->tableUid = pScanPhyNode->uid;
pTaskInfo->streamInfo.snapshotVer = pHandle->version; pTaskInfo->streamInfo.snapshotVer = pHandle->version;
pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup);
blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); blockDataEnsureCapacity(pInfo->pCreateTbRes, 8);

6
source/libs/executor/src/sysscanoperator.c
View File

@ -1482,11 +1482,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
pInfo->pIdx->init = 1; pInfo->pIdx->init = 1;
SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator);
return blk; return blk;
} else if (flt == -2) { } else if ((flt == -1) || (flt == -2)) {
qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo));
return NULL;
} else if (flt == -1) {
// not idx
qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo));
} }
} else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) {

15
source/libs/executor/src/timewindowoperator.c
View File

@ -2529,6 +2529,20 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
} }
while (1) { while (1) {
if (isTaskKilled(pTaskInfo)) {
if (pInfo->pUpdated != NULL) {
pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated);
}
if (pInfo->pUpdatedMap != NULL) {
tSimpleHashCleanup(pInfo->pUpdatedMap);
pInfo->pUpdatedMap = NULL;
}
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) { if (pBlock == NULL) {
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
@ -2605,6 +2619,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {
taosArrayPush(pInfo->pUpdated, pIte); taosArrayPush(pInfo->pUpdated, pIte);
} }
tSimpleHashCleanup(pInfo->pUpdatedMap); tSimpleHashCleanup(pInfo->pUpdatedMap);
pInfo->pUpdatedMap = NULL; pInfo->pUpdatedMap = NULL;
taosArraySort(pInfo->pUpdated, winPosCmprImpl); taosArraySort(pInfo->pUpdated, winPosCmprImpl);

40
source/libs/function/CMakeLists.txt
View File

@ -79,6 +79,26 @@ ENDIF ()
target_link_libraries( target_link_libraries(
udf1 PUBLIC os ${LINK_JEMALLOC}) udf1 PUBLIC os ${LINK_JEMALLOC})
add_library(udf1_dup STATIC MODULE test/udf1_dup.c)
target_include_directories(
udf1_dup
PUBLIC
"${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/client"
"${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEPENDENCIES(udf1_dup jemalloc)
ENDIF ()
target_link_libraries(
udf1_dup PUBLIC os ${LINK_JEMALLOC})
add_library(udf2 STATIC MODULE test/udf2.c) add_library(udf2 STATIC MODULE test/udf2.c)
target_include_directories( target_include_directories(
udf2 udf2
@ -99,6 +119,26 @@ target_link_libraries(
udf2 PUBLIC os ${LINK_JEMALLOC} udf2 PUBLIC os ${LINK_JEMALLOC}
) )
add_library(udf2_dup STATIC MODULE test/udf2_dup.c)
target_include_directories(
udf2_dup
PUBLIC
"${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/client"
"${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEPENDENCIES(udf2_dup jemalloc)
ENDIF ()
target_link_libraries(
udf2_dup PUBLIC os ${LINK_JEMALLOC}
)
#SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin)
add_executable(udfd src/udfd.c) add_executable(udfd src/udfd.c)
target_include_directories( target_include_directories(

42
source/libs/function/test/udf1_dup.c Normal file
View File

@ -0,0 +1,42 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef LINUX
#include <unistd.h>
#endif
#ifdef WINDOWS
#include <windows.h>
#endif
#include "taosudf.h"
DLL_EXPORT int32_t udf1_dup_init() { return 0; }
DLL_EXPORT int32_t udf1_dup_destroy() { return 0; }
DLL_EXPORT int32_t udf1_dup(SUdfDataBlock *block, SUdfColumn *resultCol) {
SUdfColumnData *resultData = &resultCol->colData;
for (int32_t i = 0; i < block->numOfRows; ++i) {
int j = 0;
for (; j < block->numOfCols; ++j) {
if (udfColDataIsNull(block->udfCols[j], i)) {
udfColDataSetNull(resultCol, i);
break;
}
}
if (j == block->numOfCols) {
int32_t luckyNum = 2;
udfColDataSet(resultCol, i, (char *)&luckyNum, false);
}
}
// to simulate actual processing delay by udf
#ifdef LINUX
usleep(1 * 1000); // usleep takes sleep time in us (1 millionth of a second)
#endif
#ifdef WINDOWS
Sleep(1);
#endif
resultData->numOfRows = block->numOfRows;
return 0;
}

78
source/libs/function/test/udf2_dup.c Normal file
View File

@ -0,0 +1,78 @@
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "taosudf.h"
DLL_EXPORT int32_t udf2_dup_init() { return 0; }
DLL_EXPORT int32_t udf2_dup_destroy() { return 0; }
DLL_EXPORT int32_t udf2_dup_start(SUdfInterBuf* buf) {
*(int64_t*)(buf->buf) = 0;
buf->bufLen = sizeof(double);
buf->numOfResult = 1;
return 0;
}
DLL_EXPORT int32_t udf2_dup(SUdfDataBlock* block, SUdfInterBuf* interBuf, SUdfInterBuf* newInterBuf) {
double sumSquares = 0;
if (interBuf->numOfResult == 1) {
sumSquares = *(double*)interBuf->buf;
}
int8_t numNotNull = 0;
for (int32_t i = 0; i < block->numOfCols; ++i) {
SUdfColumn* col = block->udfCols[i];
if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) {
return TSDB_CODE_UDF_INVALID_INPUT;
}
}
for (int32_t i = 0; i < block->numOfCols; ++i) {
for (int32_t j = 0; j < block->numOfRows; ++j) {
SUdfColumn* col = block->udfCols[i];
if (udfColDataIsNull(col, j)) {
continue;
}
switch (col->colMeta.type) {
case TSDB_DATA_TYPE_INT: {
char* cell = udfColDataGetData(col, j);
int32_t num = *(int32_t*)cell;
sumSquares += (double)num * num;
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
char* cell = udfColDataGetData(col, j);
double num = *(double*)cell;
sumSquares += num * num;
break;
}
default:
break;
}
++numNotNull;
}
}
*(double*)(newInterBuf->buf) = sumSquares;
newInterBuf->bufLen = sizeof(double);
if (interBuf->numOfResult == 0 && numNotNull == 0) {
newInterBuf->numOfResult = 0;
} else {
newInterBuf->numOfResult = 1;
}
return 0;
}
DLL_EXPORT int32_t udf2_dup_finish(SUdfInterBuf* buf, SUdfInterBuf* resultData) {
if (buf->numOfResult == 0) {
resultData->numOfResult = 0;
return 0;
}
double sumSquares = *(double*)(buf->buf);
*(double*)(resultData->buf) = sqrt(sumSquares) + 100;
resultData->bufLen = sizeof(double);
resultData->numOfResult = 1;
return 0;
}

4
source/libs/parser/inc/sql.y Normal file → Executable file
View File

@ -221,8 +221,8 @@ db_options(A) ::= db_options(B) WAL_RETENTION_SIZE NK_MINUS(D) NK_INTEGER(C).
db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); } db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); }
db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); } db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); }
db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); } db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, &C); } db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, &C); } db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); }

24
source/libs/parser/src/parAstCreater.c
View File

@ -1024,12 +1024,28 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
case DB_OPTION_STT_TRIGGER: case DB_OPTION_STT_TRIGGER:
pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_TABLE_PREFIX: case DB_OPTION_TABLE_PREFIX: {
pDbOptions->tablePrefix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); SValueNode *pNode = (SValueNode *)pVal;
if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
pDbOptions->tablePrefix = taosStr2Int32(pNode->literal, NULL, 10);
} else {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_prefix data type");
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
}
nodesDestroyNode((SNode*)pNode);
break; break;
case DB_OPTION_TABLE_SUFFIX: }
pDbOptions->tableSuffix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); case DB_OPTION_TABLE_SUFFIX:{
SValueNode *pNode = (SValueNode *)pVal;
if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
pDbOptions->tableSuffix = taosStr2Int32(pNode->literal, NULL, 10);
} else {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_suffix data type");
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
}
nodesDestroyNode((SNode*)pNode);
break; break;
}
default: default:
break; break;
} }

2
source/libs/parser/src/parInsertStmt.c
View File

@ -251,7 +251,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in
goto _return; goto _return;
} }
if (bind[c].buffer_type != pColSchema->type) { if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type
code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
goto _return; goto _return;
} }

2
source/libs/parser/src/parTokenizer.c
View File

@ -692,7 +692,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC
len = tGetToken(&str[*i + t0.n + 1], &type); len = tGetToken(&str[*i + t0.n + 1], &type);
// only id and string are valid // only id and string are valid
if ((TK_NK_STRING != t0.type) && (TK_NK_ID != t0.type)) { if (((TK_NK_STRING != t0.type) && (TK_NK_ID != t0.type)) || ((TK_NK_STRING != type) && (TK_NK_ID != type))) {
t0.type = TK_NK_ILLEGAL; t0.type = TK_NK_ILLEGAL;
t0.n = 0; t0.n = 0;

33
source/libs/parser/src/parTranslater.c
View File

@ -4184,6 +4184,34 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbPrefix, int32_t tbSuffix) {
if (tbPrefix < TSDB_MIN_HASH_PREFIX || tbPrefix > TSDB_MAX_HASH_PREFIX) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix: %d valid range: [%d, %d]", tbPrefix,
TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
}
if (tbSuffix < TSDB_MIN_HASH_SUFFIX || tbSuffix > TSDB_MAX_HASH_SUFFIX) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_suffix: %d valid range: [%d, %d]", tbSuffix,
TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
}
if ((tbPrefix * tbSuffix) < 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix & table_suffix: mixed usage not allowed");
}
if ((tbPrefix + tbSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix & table_suffix: exceed max table name length");
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
int32_t daysPerFile = pOptions->daysPerFile; int32_t daysPerFile = pOptions->daysPerFile;
int64_t daysToKeep0 = pOptions->keep[0]; int64_t daysToKeep0 = pOptions->keep[0];
@ -4291,10 +4319,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER); code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX); code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "tableSuffix", pOptions->tableSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkOptionsDependency(pCxt, pDbName, pOptions); code = checkOptionsDependency(pCxt, pDbName, pOptions);

2289
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

2
source/libs/planner/src/planOptimizer.c
View File

@ -2529,7 +2529,7 @@ static bool tbCntScanOptIsEligibleAggFuncs(SNodeList* pAggFuncs) {
return false; return false;
} }
} }
return true; return LIST_LENGTH(pAggFuncs) > 0;
} }
static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) { static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) {

6
source/libs/stream/src/stream.c
View File

@ -16,7 +16,7 @@
#include "streamInc.h" #include "streamInc.h"
#include "ttimer.h" #include "ttimer.h"
#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 2000 #define STREAM_TASK_INPUT_QUEUEU_CAPACITY 3000
int32_t streamInit() { int32_t streamInit() {
int8_t old; int8_t old;
@ -356,4 +356,8 @@ void* streamQueueNextItem(SStreamQueue* queue) {
} }
return streamQueueCurItem(queue); return streamQueueCurItem(queue);
} }
}
void streamTaskInputFail(SStreamTask* pTask) {
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
} }

12
source/libs/stream/src/streamDispatch.c
View File

@ -230,23 +230,21 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq*
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen); tEncoderInit(&encoder, abuf, tlen);
if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) { if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) {
goto FAIL; rpcFreeCont(buf);
return code;
} }
tEncoderClear(&encoder); tEncoderClear(&encoder);
msg.contLen = tlen + sizeof(SMsgHead); msg.contLen = tlen + sizeof(SMsgHead);
msg.pCont = buf; msg.pCont = buf;
msg.msgType = TDMT_STREAM_TASK_CHECK; msg.msgType = TDMT_STREAM_TASK_CHECK;
qDebug("dispatch from s-task:%s to downstream s-task:%"PRIx64":%d node %d: check msg", pTask->id.idStr, qDebug("dispatch from s-task:%s to downstream s-task:%" PRIx64 ":%d node %d: check msg", pTask->id.idStr,
pReq->streamId, pReq->downstreamTaskId, nodeId); pReq->streamId, pReq->downstreamTaskId, nodeId);
tmsgSendReq(pEpSet, &msg); tmsgSendReq(pEpSet, &msg);
return 0; return 0;
FAIL:
if (buf) rpcFreeCont(buf);
return code;
} }
int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId,

22
source/libs/stream/src/streamExec.c
View File

@ -17,6 +17,11 @@
#define STREAM_EXEC_MAX_BATCH_NUM 1024 #define STREAM_EXEC_MAX_BATCH_NUM 1024
bool streamTaskShouldStop(const SStreamStatus* pStatus) {
int32_t status = atomic_load_8((int8_t*) &pStatus->taskStatus);
return (status == TASK_STATUS__STOP) || (status == TASK_STATUS__DROPPING);
}
static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
void* pExecutor = pTask->exec.pExecutor; void* pExecutor = pTask->exec.pExecutor;
@ -66,7 +71,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
// pExecutor // pExecutor
while (1) { while (1) {
if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { if (streamTaskShouldStop(&pTask->status)) {
return 0; return 0;
} }
@ -106,7 +111,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
continue; continue;
} }
qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId); qDebug("s-task:%s (child %d) executed and get block", pTask->id.idStr, pTask->selfChildId);
SSDataBlock block = {0}; SSDataBlock block = {0};
assignOneDataBlock(&block, output); assignOneDataBlock(&block, output);
@ -136,7 +141,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
int32_t batchCnt = 0; int32_t batchCnt = 0;
while (1) { while (1) {
if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { if (streamTaskShouldStop(&pTask->status)) {
taosArrayDestroy(pRes); taosArrayDestroy(pRes);
return 0; return 0;
} }
@ -195,7 +200,10 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
qDebug("task %d scan exec dispatch block num %d", pTask->id.taskId, batchCnt); qDebug("task %d scan exec dispatch block num %d", pTask->id.taskId, batchCnt);
streamDispatch(pTask); streamDispatch(pTask);
} }
if (finished) break;
if (finished) {
break;
}
} }
return 0; return 0;
} }
@ -274,7 +282,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
} }
} }
if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { if (streamTaskShouldStop(&pTask->status)) {
if (pInput) { if (pInput) {
streamFreeQitem(pInput); streamFreeQitem(pInput);
} }
@ -305,7 +313,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
", checkPoint id:%" PRId64 " -> %" PRId64, ", checkPoint id:%" PRId64 " -> %" PRId64,
pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId); pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId);
pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId}; pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer};
taosWLockLatch(&pTask->pMeta->lock); taosWLockLatch(&pTask->pMeta->lock);
streamMetaSaveTask(pTask->pMeta, pTask); streamMetaSaveTask(pTask->pMeta, pTask);
@ -374,7 +382,7 @@ int32_t streamTryExec(SStreamTask* pTask) {
atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
qDebug("s-task:%s exec completed", pTask->id.idStr); qDebug("s-task:%s exec completed", pTask->id.idStr);
if (!taosQueueEmpty(pTask->inputQueue->queue)) { if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status))) {
streamSchedExec(pTask); streamSchedExec(pTask);
} }
} }

30
source/libs/stream/src/streamMeta.c
View File

@ -85,11 +85,6 @@ void streamMetaClose(SStreamMeta* pMeta) {
tdbClose(pMeta->db); tdbClose(pMeta->db);
void* pIter = NULL; void* pIter = NULL;
// while(pMeta->walScan) {
// qDebug("wait stream daemon quit");
// taosMsleep(100);
// }
while (1) { while (1) {
pIter = taosHashIterate(pMeta->pTasks, pIter); pIter = taosHashIterate(pMeta->pTasks, pIter);
if (pIter == NULL) { if (pIter == NULL) {
@ -103,7 +98,6 @@ void streamMetaClose(SStreamMeta* pMeta) {
} }
tFreeStreamTask(pTask); tFreeStreamTask(pTask);
/*streamMetaReleaseTask(pMeta, pTask);*/
} }
taosHashCleanup(pMeta->pTasks); taosHashCleanup(pMeta->pTasks);
@ -177,10 +171,12 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
// add to the ready tasks hash map, not the restored tasks hash map // add to the ready tasks hash map, not the restored tasks hash map
int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) {
if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) {
tFreeStreamTask(pTask);
return -1; return -1;
} }
if (streamMetaSaveTask(pMeta, pTask) < 0) { if (streamMetaSaveTask(pMeta, pTask) < 0) {
tFreeStreamTask(pTask);
return -1; return -1;
} }
@ -196,10 +192,12 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) {
taosRLockLatch(&pMeta->lock); taosRLockLatch(&pMeta->lock);
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) { if (ppTask != NULL) {
atomic_add_fetch_32(&(*ppTask)->refCnt, 1); if (!streamTaskShouldStop(&(*ppTask)->status)) {
taosRUnLockLatch(&pMeta->lock); atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
return *ppTask; taosRUnLockLatch(&pMeta->lock);
return *ppTask;
}
} }
taosRUnLockLatch(&pMeta->lock); taosRUnLockLatch(&pMeta->lock);
@ -210,7 +208,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1); int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1);
ASSERT(left >= 0); ASSERT(left >= 0);
if (left == 0) { if (left == 0) {
ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING); ASSERT(streamTaskShouldStop(&pTask->status));
tFreeStreamTask(pTask); tFreeStreamTask(pTask);
} }
} }
@ -221,11 +219,8 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
SStreamTask* pTask = *ppTask; SStreamTask* pTask = *ppTask;
taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn); tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn);
/*if (pTask->timer) {
* taosTmrStop(pTask->timer);*/ atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__STOP);
/*pTask->timer = NULL;*/
/*}*/
atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
taosWLockLatch(&pMeta->lock); taosWLockLatch(&pMeta->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -297,8 +292,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
tDecodeStreamTask(&decoder, pTask); tDecodeStreamTask(&decoder, pTask);
tDecoderClear(&decoder); tDecoderClear(&decoder);
// todo set correct initial version. if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) {
if (pMeta->expandFunc(pMeta->ahandle, pTask, 0) < 0) {
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
tdbTbcClose(pCur); tdbTbcClose(pCur);

8
source/libs/stream/src/streamRecover.c
View File

@ -102,8 +102,10 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp
.downstreamNodeId = pRsp->downstreamNodeId, .downstreamNodeId = pRsp->downstreamNodeId,
.childId = pRsp->childId, .childId = pRsp->childId,
}; };
qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->id.taskId, pTask->nodeId,
qDebug("s-task:%s at node %d check downstream task %d at node %d (recheck)", pTask->id.idStr, pTask->nodeId,
req.downstreamTaskId, req.downstreamNodeId); req.downstreamTaskId, req.downstreamNodeId);
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet);
} else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
@ -116,6 +118,7 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp
} }
} }
} }
return 0; return 0;
} }
@ -158,9 +161,10 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp*
} else { } else {
ASSERT(0); ASSERT(0);
} }
} else { } else { // not ready, it should wait for at least 100ms and then retry
streamRecheckOneDownstream(pTask, pRsp); streamRecheckOneDownstream(pTask, pRsp);
} }
return 0; return 0;
} }

2
source/libs/sync/inc/syncCommit.h
View File

@ -48,8 +48,6 @@ extern "C" {
void syncOneReplicaAdvance(SSyncNode* pSyncNode); void syncOneReplicaAdvance(SSyncNode* pSyncNode);
void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode); void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode);
bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index);
bool syncAgree(SSyncNode* pSyncNode, SyncIndex index);
bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index); bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index);
int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex); int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex);

1
source/libs/sync/inc/syncReplication.h
View File

@ -55,7 +55,6 @@ int32_t syncNodeReplicateReset(SSyncNode* pSyncNode, SRaftId* pDestId);
int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode); int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode);
int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg); int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

13
source/libs/sync/src/syncAppendEntries.c
View File

@ -89,17 +89,6 @@
// /\ UNCHANGED <<candidateVars, leaderVars>> // /\ UNCHANGED <<candidateVars, leaderVars>>
// //
SSyncRaftEntry* syncBuildRaftEntryFromAppendEntries(const SyncAppendEntries* pMsg) {
SSyncRaftEntry* pEntry = taosMemoryMalloc(pMsg->dataLen);
if (pEntry == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
(void)memcpy(pEntry, pMsg->data, pMsg->dataLen);
ASSERT(pEntry->bytes == pMsg->dataLen);
return pEntry;
}
int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SyncAppendEntries* pMsg = pRpcMsg->pCont; SyncAppendEntries* pMsg = pRpcMsg->pCont;
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
@ -146,7 +135,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
goto _IGNORE; goto _IGNORE;
} }
pEntry = syncBuildRaftEntryFromAppendEntries(pMsg); pEntry = syncEntryBuildFromAppendEntries(pMsg);
if (pEntry == NULL) { if (pEntry == NULL) {
sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr()); sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr());
goto _IGNORE; goto _IGNORE;

29
source/libs/sync/src/syncCommit.c
View File

@ -44,22 +44,6 @@
// /\ UNCHANGED <<messages, serverVars, candidateVars, leaderVars, log>> // /\ UNCHANGED <<messages, serverVars, candidateVars, leaderVars, log>>
// //
bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index) {
// I am leader, I agree
if (syncUtilSameId(pRaftId, &(pSyncNode->myRaftId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
return true;
}
// follower agree
SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, pRaftId);
if (matchIndex >= index) {
return true;
}
// not agree
return false;
}
static inline int64_t syncNodeAbs64(int64_t a, int64_t b) { static inline int64_t syncNodeAbs64(int64_t a, int64_t b) {
ASSERT(a >= 0); ASSERT(a >= 0);
ASSERT(b >= 0); ASSERT(b >= 0);
@ -85,19 +69,6 @@ bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index) {
return count >= pNode->quorum; return count >= pNode->quorum;
} }
bool syncAgree(SSyncNode* pNode, SyncIndex index) {
int agreeCount = 0;
for (int i = 0; i < pNode->replicaNum; ++i) {
if (syncAgreeIndex(pNode, &(pNode->replicasId[i]), index)) {
++agreeCount;
}
if (agreeCount >= pNode->quorum) {
return true;
}
}
return false;
}
int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex) { int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex) {
SyncIndex lastVer = ths->pLogStore->syncLogLastIndex(ths->pLogStore); SyncIndex lastVer = ths->pLogStore->syncLogLastIndex(ths->pLogStore);
commitIndex = TMAX(commitIndex, ths->commitIndex); commitIndex = TMAX(commitIndex, ths->commitIndex);

9
source/libs/sync/src/syncRaftEntry.c
View File

@ -64,10 +64,13 @@ SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, Syn
} }
SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) { SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) {
SSyncRaftEntry* pEntry = syncEntryBuild((int32_t)(pMsg->dataLen)); SSyncRaftEntry* pEntry = taosMemoryMalloc(pMsg->dataLen);
if (pEntry == NULL) return NULL; if (pEntry == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memcpy(pEntry, pMsg->data, pMsg->dataLen); memcpy(pEntry, pMsg->data, pMsg->dataLen);
ASSERT(pEntry->bytes == pMsg->dataLen);
return pEntry; return pEntry;
} }

16
source/libs/sync/src/syncReplication.c
View File

@ -46,8 +46,6 @@
// mdest |-> j]) // mdest |-> j])
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>> // /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) { int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) {
SSyncLogBuffer* pBuf = pNode->pLogBuf; SSyncLogBuffer* pBuf = pNode->pLogBuf;
taosThreadMutexLock(&pBuf->mutex); taosThreadMutexLock(&pBuf->mutex);
@ -86,20 +84,6 @@ int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftI
return 0; return 0;
} }
int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {
int32_t ret = 0;
SyncAppendEntries* pMsg = pRpcMsg->pCont;
if (syncNodeNeedSendAppendEntries(pSyncNode, destRaftId, pMsg)) {
ret = syncNodeSendAppendEntries(pSyncNode, destRaftId, pRpcMsg);
} else {
sNTrace(pSyncNode, "do not repcate to dnode:%d for index:%" PRId64, DID(destRaftId), pMsg->prevLogIndex + 1);
rpcFreeCont(pRpcMsg->pCont);
}
return ret;
}
int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcMsg* pMsg) { int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcMsg* pMsg) {
return syncNodeSendMsgById(destId, pSyncNode, pMsg); return syncNodeSendMsgById(destId, pSyncNode, pMsg);
} }

4
source/os/src/osSocket.c
View File

@ -1008,7 +1008,7 @@ int32_t taosGetFqdn(char *fqdn) {
// hints.ai_family = AF_INET; // hints.ai_family = AF_INET;
strcpy(fqdn, hostname); strcpy(fqdn, hostname);
strcpy(fqdn + strlen(hostname), ".local"); strcpy(fqdn + strlen(hostname), ".local");
#else // __APPLE__ #else // linux
struct addrinfo hints = {0}; struct addrinfo hints = {0};
struct addrinfo *result = NULL; struct addrinfo *result = NULL;
hints.ai_flags = AI_CANONNAME; hints.ai_flags = AI_CANONNAME;
@ -1020,7 +1020,7 @@ int32_t taosGetFqdn(char *fqdn) {
} }
strcpy(fqdn, result->ai_canonname); strcpy(fqdn, result->ai_canonname);
freeaddrinfo(result); freeaddrinfo(result);
#endif // __APPLE__ #endif // linux
return 0; return 0;
} }

60
source/util/src/tlog.c
View File

@ -17,7 +17,6 @@
#include "tlog.h" #include "tlog.h"
#include "os.h" #include "os.h"
#include "tconfig.h" #include "tconfig.h"
#include "tutil.h"
#include "tjson.h" #include "tjson.h"
#include "tglobal.h" #include "tglobal.h"
@ -781,65 +780,6 @@ bool taosAssertDebug(bool condition, const char *file, int32_t line, const char
return true; return true;
} }
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) {
SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return -1;
char tmp[4096] = {0};
tjsonAddDoubleToObject(pJson, "reportVersion", 1);
tjsonAddIntegerToObject(pJson, "clusterId", clusterId);
tjsonAddIntegerToObject(pJson, "startTime", startTime);
taosGetFqdn(tmp);
tjsonAddStringToObject(pJson, "fqdn", tmp);
tjsonAddIntegerToObject(pJson, "pid", taosGetPId());
taosGetAppName(tmp, NULL);
tjsonAddStringToObject(pJson, "appName", tmp);
if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) {
tjsonAddStringToObject(pJson, "os", tmp);
}
float numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) {
tjsonAddStringToObject(pJson, "cpuModel", tmp);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
} else {
tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores);
}
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
tjsonAddStringToObject(pJson, "version", version);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo);
tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
tjsonAddIntegerToObject(pJson, "crashSig", signum);
tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs());
#ifdef _TD_DARWIN_64
taosLogTraceToBuf(tmp, sizeof(tmp), 4);
#elif !defined(WINDOWS)
taosLogTraceToBuf(tmp, sizeof(tmp), 3);
#else
taosLogTraceToBuf(tmp, sizeof(tmp), 8);
#endif
tjsonAddStringToObject(pJson, "stackInfo", tmp);
char* pCont = tjsonToString(pJson);
tjsonDelete(pJson);
*pMsg = pCont;
return TSDB_CODE_SUCCESS;
}
void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo) { void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo) {
const char *flags = "UTL FATAL "; const char *flags = "UTL FATAL ";
ELogLevel level = DEBUG_FATAL; ELogLevel level = DEBUG_FATAL;

4
source/util/src/tworker.c
View File

@ -217,8 +217,8 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem
int32_t queueNum = taosGetQueueNumber(pool->qset); int32_t queueNum = taosGetQueueNumber(pool->qset);
int32_t curWorkerNum = taosArrayGetSize(pool->workers); int32_t curWorkerNum = taosArrayGetSize(pool->workers);
int32_t dstWorkerNum = ceil(queueNum * pool->ratio); int32_t dstWorkerNum = ceilf(queueNum * pool->ratio);
if (dstWorkerNum < 2) dstWorkerNum = 2; if (dstWorkerNum < 1) dstWorkerNum = 1;
// spawn a thread to process queue // spawn a thread to process queue
while (curWorkerNum < dstWorkerNum) { while (curWorkerNum < dstWorkerNum) {

3
tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json
View File

@ -11,6 +11,7 @@
"confirm_parameter_prompt": "no", "confirm_parameter_prompt": "no",
"prepared_rand": 100, "prepared_rand": 100,
"chinese": "no", "chinese": "no",
"escape_character": "yes",
"insert_interval": 0, "insert_interval": 0,
"num_of_records_per_req": 10, "num_of_records_per_req": 10,
"databases": [{ "databases": [{
@ -29,7 +30,6 @@
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 8, "childtable_count": 8,
"childtable_prefix": "stb_", "childtable_prefix": "stb_",
"escape_character": "yes",
"auto_create_table": "yes", "auto_create_table": "yes",
"batch_create_tbl_num": 10, "batch_create_tbl_num": 10,
"data_source": "rand", "data_source": "rand",
@ -54,7 +54,6 @@
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 8, "childtable_count": 8,
"childtable_prefix": "stb3-2_", "childtable_prefix": "stb3-2_",
"escape_character": "yes",
"auto_create_table": "yes", "auto_create_table": "yes",
"batch_create_tbl_num": 10, "batch_create_tbl_num": 10,
"data_source": "rand", "data_source": "rand",

3
tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json
View File

@ -11,6 +11,7 @@
"confirm_parameter_prompt": "no", "confirm_parameter_prompt": "no",
"prepared_rand": 100, "prepared_rand": 100,
"chinese": "no", "chinese": "no",
"escape_character": "yes",
"insert_interval": 0, "insert_interval": 0,
"num_of_records_per_req": 10, "num_of_records_per_req": 10,
"databases": [{ "databases": [{
@ -29,7 +30,6 @@
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 8, "childtable_count": 8,
"childtable_prefix": "stb_", "childtable_prefix": "stb_",
"escape_character": "yes",
"auto_create_table": "yes", "auto_create_table": "yes",
"batch_create_tbl_num": 10, "batch_create_tbl_num": 10,
"data_source": "rand", "data_source": "rand",
@ -54,7 +54,6 @@
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 8, "childtable_count": 8,
"childtable_prefix": "stb4-2_", "childtable_prefix": "stb4-2_",
"escape_character": "yes",
"auto_create_table": "yes", "auto_create_table": "yes",
"batch_create_tbl_num": 10, "batch_create_tbl_num": 10,
"data_source": "rand", "data_source": "rand",

3
tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json
View File

@ -11,6 +11,7 @@
"confirm_parameter_prompt": "no", "confirm_parameter_prompt": "no",
"prepared_rand": 100, "prepared_rand": 100,
"chinese": "no", "chinese": "no",
"escape_character": "yes",
"insert_interval": 0, "insert_interval": 0,
"num_of_records_per_req": 10, "num_of_records_per_req": 10,
"databases": [{ "databases": [{
@ -29,7 +30,6 @@
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 8, "childtable_count": 8,
"childtable_prefix": "stb_", "childtable_prefix": "stb_",
"escape_character": "yes",
"auto_create_table": "yes", "auto_create_table": "yes",
"batch_create_tbl_num": 10, "batch_create_tbl_num": 10,
"data_source": "rand", "data_source": "rand",
@ -54,7 +54,6 @@
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 8, "childtable_count": 8,
"childtable_prefix": "stb2-2_", "childtable_prefix": "stb2-2_",
"escape_character": "yes",
"auto_create_table": "yes", "auto_create_table": "yes",
"batch_create_tbl_num": 10, "batch_create_tbl_num": 10,
"data_source": "rand", "data_source": "rand",

3
tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json
View File

@ -11,6 +11,7 @@
"confirm_parameter_prompt": "no", "confirm_parameter_prompt": "no",
"prepared_rand": 100, "prepared_rand": 100,
"chinese": "no", "chinese": "no",
"escape_character": "yes",
"insert_interval": 0, "insert_interval": 0,
"num_of_records_per_req": 10, "num_of_records_per_req": 10,
"databases": [{ "databases": [{
@ -29,7 +30,6 @@
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 8, "childtable_count": 8,
"childtable_prefix": "stb_", "childtable_prefix": "stb_",
"escape_character": "yes",
"auto_create_table": "yes", "auto_create_table": "yes",
"batch_create_tbl_num": 10, "batch_create_tbl_num": 10,
"data_source": "rand", "data_source": "rand",
@ -55,7 +55,6 @@
"child_table_exists":"no", "child_table_exists":"no",
"childtable_count": 8, "childtable_count": 8,
"childtable_prefix": "stb1-2_", "childtable_prefix": "stb1-2_",
"escape_character": "yes",
"auto_create_table": "yes", "auto_create_table": "yes",
"batch_create_tbl_num": 10, "batch_create_tbl_num": 10,
"data_source": "rand", "data_source": "rand",

1
tests/parallel_test/cases.task
View File

@ -1162,6 +1162,7 @@
,,y,script,./test.sh -f tsim/db/show_create_table.sim ,,y,script,./test.sh -f tsim/db/show_create_table.sim
,,y,script,./test.sh -f tsim/db/tables.sim ,,y,script,./test.sh -f tsim/db/tables.sim
,,y,script,./test.sh -f tsim/db/taosdlog.sim ,,y,script,./test.sh -f tsim/db/taosdlog.sim
,,y,script,./test.sh -f tsim/db/table_prefix_suffix.sim
,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim ,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim
,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim ,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim
,,y,script,./test.sh -f tsim/dnode/balance1.sim ,,y,script,./test.sh -f tsim/dnode/balance1.sim

5
tests/parallel_test/container_build.sh
View File

@ -68,8 +68,13 @@ docker run \
-v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
-v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
<<<<<<< HEAD
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=false -DPORTABLE=true -DWITH_GFLAGS=false;make -j || exit 1" --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=false -DPORTABLE=true -DWITH_GFLAGS=false;make -j || exit 1"
# -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
=======
-v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=true;make -j || exit 1"
>>>>>>> fix/rocksdbState
if [[ -d ${WORKDIR}/debugNoSan ]] ;then if [[ -d ${WORKDIR}/debugNoSan ]] ;then
echo "delete ${WORKDIR}/debugNoSan" echo "delete ${WORKDIR}/debugNoSan"

182
tests/script/tsim/db/table_prefix_suffix.sim Normal file
View File

@ -0,0 +1,182 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql drop database if exists db1;
sql create database db1 vgroups 5 TABLE_PREFIX 1 TABLE_SUFFIX 2;
sql use db1;
sql create table atb1aa (ts timestamp, f1 int);
sql create table btb1bb (ts timestamp, f1 int);
sql create table ctb1cc (ts timestamp, f1 int);
sql create table dtb1dd (ts timestamp, f1 int);
sql create table atb2aa (ts timestamp, f1 int);
sql create table btb2bb (ts timestamp, f1 int);
sql create table ctb2cc (ts timestamp, f1 int);
sql create table dtb2dd (ts timestamp, f1 int);
sql create table etb2ee (ts timestamp, f1 int);
sql show create database db1;
sql select count(*) a from information_schema.ins_tables where db_name='db1' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db1;
sql drop database if exists db2;
sql create database db2 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX -2;
sql use db2;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table taaa22 (ts timestamp, f1 int);
sql create table tbbb22 (ts timestamp, f1 int);
sql create table tccc22 (ts timestamp, f1 int);
sql create table tddd22 (ts timestamp, f1 int);
sql create table teee22 (ts timestamp, f1 int);
sql show create database db2;
sql select count(*) a from information_schema.ins_tables where db_name='db2' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db2;
sql drop database if exists db3;
sql create database db3 vgroups 5 TABLE_PREFIX -1;
sql use db3;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table zaaa22 (ts timestamp, f1 int);
sql create table zbbb22 (ts timestamp, f1 int);
sql create table zccc22 (ts timestamp, f1 int);
sql create table zddd22 (ts timestamp, f1 int);
sql create table zeee22 (ts timestamp, f1 int);
sql show create database db3;
sql select count(*) a from information_schema.ins_tables where db_name='db3' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db3;
sql drop database if exists db4;
sql create database db4 vgroups 5 TABLE_SUFFIX -2;
sql use db4;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table zaaa22 (ts timestamp, f1 int);
sql create table zbbb22 (ts timestamp, f1 int);
sql create table zccc22 (ts timestamp, f1 int);
sql create table zddd22 (ts timestamp, f1 int);
sql create table zeee22 (ts timestamp, f1 int);
sql show create database db4;
sql select count(*) a from information_schema.ins_tables where db_name='db4' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db4;
sql drop database if exists db5;
sql create database db5 vgroups 5 TABLE_PREFIX 1;
sql use db5;
sql create table taaa11 (ts timestamp, f1 int);
sql create table baaa11 (ts timestamp, f1 int);
sql create table caaa11 (ts timestamp, f1 int);
sql create table daaa11 (ts timestamp, f1 int);
sql create table faaa11 (ts timestamp, f1 int);
sql create table gbbb11 (ts timestamp, f1 int);
sql create table hbbb11 (ts timestamp, f1 int);
sql create table ibbb11 (ts timestamp, f1 int);
sql create table jbbb11 (ts timestamp, f1 int);
sql show create database db5;
sql select count(*) a from information_schema.ins_tables where db_name='db5' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db5;
sql drop database if exists db6;
sql create database db6 vgroups 5 TABLE_SUFFIX 2;
sql use db6;
sql create table taaa11 (ts timestamp, f1 int);
sql create table taaa12 (ts timestamp, f1 int);
sql create table taaa13 (ts timestamp, f1 int);
sql create table taaa14 (ts timestamp, f1 int);
sql create table tbbb23 (ts timestamp, f1 int);
sql create table tbbb24 (ts timestamp, f1 int);
sql create table tbbb31 (ts timestamp, f1 int);
sql create table tbbb32 (ts timestamp, f1 int);
sql create table tbbb33 (ts timestamp, f1 int);
sql show create database db6;
sql select count(*) a from information_schema.ins_tables where db_name='db6' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db6;
sql drop database if exists db7;
sql create database db7 vgroups 5 TABLE_PREFIX -100 TABLE_SUFFIX -92;
sql use db7;
sql create table taaa11 (ts timestamp, f1 int);
sql create table taaa12 (ts timestamp, f1 int);
sql create table taaa13 (ts timestamp, f1 int);
sql create table tbbb21 (ts timestamp, f1 int);
sql create table tbbb22 (ts timestamp, f1 int);
sql create table tbbb23 (ts timestamp, f1 int);
sql create table tbbb24 (ts timestamp, f1 int);
sql create table tccc31 (ts timestamp, f1 int);
sql create table tccc32 (ts timestamp, f1 int);
sql create table tccc33 (ts timestamp, f1 int);
sql create table tddd24 (ts timestamp, f1 int);
sql create table tddd31 (ts timestamp, f1 int);
sql create table tddd32 (ts timestamp, f1 int);
sql create table tddd33 (ts timestamp, f1 int);
sql show create database db7;
sql select count(*) a from information_schema.ins_tables where db_name='db7' group by vgroup_id having(count(*) > 0) order by a;
sql drop database if exists db7;
sql_error create database db8 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX 2;
sql_error create database db8 vgroups 5 TABLE_PREFIX 191 TABLE_SUFFIX 192;
sql_error create database db8 vgroups 5 TABLE_PREFIX -192 TABLE_SUFFIX -191;
sql_error create database db8 vgroups 5 TABLE_PREFIX 100 TABLE_SUFFIX 92;
system sh/exec.sh -n dnode1 -s stop -x SIGINT

21
tests/script/tsim/query/sys_tbname.sim
View File

@ -109,5 +109,26 @@ if $rows != 5000 then
return -1 return -1
endi endi
sql create database d1;
sql create stable d1.st1 (ts timestamp, f int) tags(t int);
sql create stable d1.st2 (ts timestamp, f int) tags(t int);
sql create table d1.ct1 using d1.st1 tags(1);
sql create table d1.ct2 using d1.st2 tags(2);
sql create database d2;
sql create stable d2.st1(ts timestamp, f int) tags(t int);
sql create stable d2.st2(ts timestamp, f int) tags(t int);
sql create table d2.ct1 using d2.st1 tags(1);
sql create table d2.ct2 using d2.st2 tags(2);
sql create database d3;
sql create stable d3.st1(ts timestamp, f int) tags(t int);
sql create stable d3.st2(ts timestamp, f int) tags(t int);
sql create table d3.ct1 using d3.st1 tags(1);
sql create table d3.ct2 using d3.st2 tags(2);
sql select count(*), stable_name, db_name from information_schema.ins_tables where db_name != 'd2' group by stable_name,db_name
print $rows
if $rows != 9 then
return -1
endi
#system sh/exec.sh -n dnode1 -s stop -x SIGINT #system sh/exec.sh -n dnode1 -s stop -x SIGINT

5
tests/script/tsim/query/tableCount.sim
View File

@ -104,4 +104,9 @@ if $data62 != 5 then
return -1 return -1
endi endi
sql select distinct db_name from information_schema.ins_tables;
print $rows
if $rows != 4 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT

2
tests/script/tsim/stream/basic1.sim
View File

@ -37,7 +37,7 @@ if $loop_count == 20 then
endi endi
if $rows != 4 then if $rows != 4 then
print =====rows=$rows, expect 4 print =====rows=$rows expect 4
goto loop0 goto loop0
endi endi

71
tests/system-test/0-others/TS-3131.tsql Normal file
View File

@ -0,0 +1,71 @@
drop database if exists d0;
create database d0 replica 1 keep 365 minRows 100 maxRows 4096 comp 2 vgroups 2 precision 'ms';
use d0;
create table if not exists almlog (starttime timestamp,endtime timestamp,durationtime int, alarmno int, alarmtext nchar(256),isactive nchar(64)) tags (mcid nchar(16));
create table if not exists mplog (starttime timestamp,mpid int, paravalue nchar(256),mptype nchar(32)) tags (mcid nchar(16));
create table if not exists mdlog (starttime timestamp,endtime timestamp,durationtime int, statuscode int, npcgmname nchar(256),attr int) tags (mcid nchar(16));
create table if not exists nrglog (updatetime timestamp,energyvalue double,enerygyincrease double) tags (mcid nchar(16),enerygytype nchar(16));
create table almlog_m201 using almlog tags("m201");
create table almlog_m0103 using almlog tags("m0103");
create table almlog_m0103_20031 using almlog tags("m0103");
create table almlog_m0103_20032 using almlog tags("m0103");
create table almlog_m0103_3003 using almlog tags("m0103");
create table almlog_m0103_20033 using almlog tags("m0103");
create table almlog_m0103_30031 using almlog tags("m0103");
create table almlog_m0201 using almlog tags("m0201");
create table almlog_m0102 using almlog tags("m0102");
create table almlog_m0101 using almlog tags("m0101");
create table almlog_m1002 using almlog tags("m1002");
create table mplog_m0204_4 using mplog tags("m0204");
create table mplog_m0204_5 using mplog tags("m0204");
create table mplog_m0204_6 using mplog tags("m0204");
create table mplog_m0204_12 using mplog tags("m0204");
create table mplog_m0204 using mplog tags("m0204");
create table mplog_m201 using mplog tags("m201");
create table mplog_m0102 using mplog tags("m0102");
create table mplog_m1101 using mplog tags("m1101");
create table mdlog_m0102 using mplog tags("m0102");
create table mdlog_m0504 using mplog tags("m0504");
create table mdlog_m0505 using mplog tags("m0505");
create table mdlog_m0507 using mplog tags("m0507");
create table mdlog_m1002 using mplog tags("m1002");
create table mdlog_m3201 using mplog tags("m3201");
create table mdlog_m0201 using mplog tags("m0201");
create table mdlog_m1102 using mplog tags("m1102");
create table mdlog_m201 using mplog tags("m201");
create table nrglog_m201_electricvalue1 using nrglog tags("m201","electricValue1");
create table nrglog_m201_oilvalue1 using nrglog tags("m201","oilValue1");
create table nrglog_m201_gasvalue1 using nrglog tags("m201","gasValue1");
create table nrglog_m201_watervalue1 using nrglog tags("m201","waterValue1");
create table nrglog_m0101_oilvalue1 using nrglog tags("m0101","oilValue1");
create table nrglog_m0101_watervalue1 using nrglog tags("m0101","waterValue1");
create table nrglog_m0102_gasvalue1 using nrglog tags("m0102","gasValue1");
create table nrglog_m1903 using nrglog tags("m1903",NULL);
create table nrglog_m2802 using nrglog tags("m2802",NULL);
create table nrglog_m2101 using nrglog tags("m2101",NULL);
create table nrglog_m0102 using nrglog tags("m0102",NULL);
create table nrglog_m0101_electricvalue1 using nrglog tags("m0101","electricValue1");
create table nrglog_m0101_gasvalue1 using nrglog tags("m0101","gasValue1");
create table nrglog_m0102_electricvalue1 using nrglog tags("m0102","electricValue1");
create table nrglog_m0102_oilvalue1 using nrglog tags("m0102","oilValue1");
create table nrglog_m0102_watervalue1 using nrglog tags("m0102","waterValue1");
insert into almlog_m0103 values(now,now+1s,10,0,'','dismissed');
insert into almlog_m0103_20031 values(now,now+1s,10,20031,'','dismissed');
insert into almlog_m0103_20032 values(now,now+1s,10,20032,'','dismissed');
insert into almlog_m0103_3003 values(now,now+1s,10,3003,'','dismissed');
insert into almlog_m0103_20033 values(now,now+1s,10,20033,'','dismissed');
insert into almlog_m0103_30031 values(now,now+1s,10,30031,'','dismissed');
flush database d0;
show table tags from almlog;
select *,tbname from d0.almlog where mcid='m0103';
select table_name from information_schema.ins_tables where db_name='d0';

14
tests/system-test/0-others/compa4096.json
View File

@ -53,18 +53,8 @@
"sample_format": "csv", "sample_format": "csv",
"sample_file": "./sample.csv", "sample_file": "./sample.csv",
"tags_file": "", "tags_file": "",
"columns": [ "columns": [{ "type": "INT","count": 4093}],
{ "tags": [{"type": "TINYINT", "count": 1},{"type": "NCHAR","count": 1}]
"type": "INT",
"count": 4094
}
],
"tags": [
{
"type": "TINYINT",
"count": 1
}
]
} }
] ]
} }

20
tests/system-test/0-others/compatibility.py
View File

@ -98,9 +98,12 @@ class TDTestCase:
def buildTaosd(self,bPath): def buildTaosd(self,bPath):
# os.system(f"mv {bPath}/build_bak {bPath}/build ") # os.system(f"mv {bPath}/build_bak {bPath}/build ")
os.system(f" cd {bPath} ") os.system(f" cd {bPath} ")
def is_list_same_as_ordered_list(self,unordered_list, ordered_list):
sorted_list = sorted(unordered_list)
return sorted_list == ordered_list
def run(self): def run(self):
scriptsPath = os.path.dirname(os.path.realpath(__file__)) scriptsPath = os.path.dirname(os.path.realpath(__file__))
distro_id = distro.id() distro_id = distro.id()
@ -146,6 +149,8 @@ class TDTestCase:
tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ")
os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y")
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '")
os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql")
cmd = f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taos -h localhost ;" cmd = f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taos -h localhost ;"
if os.system(cmd) == 0: if os.system(cmd) == 0:
raise Exception("failed to execute system command. cmd: %s" % cmd) raise Exception("failed to execute system command. cmd: %s" % cmd)
@ -220,6 +225,17 @@ class TDTestCase:
tdLog.exit("%s(%d) failed" % args) tdLog.exit("%s(%d) failed" % args)
tdsql.query("show streams;") tdsql.query("show streams;")
tdsql.checkRows(2) tdsql.checkRows(2)
tdsql.query("select *,tbname from d0.almlog where mcid='m0103';")
tdsql.checkRows(6)
expectList = [0,3003,20031,20032,20033,30031]
resultList = []
for i in range(6):
resultList.append(tdsql.queryResult[i][3])
print(resultList)
if self.is_list_same_as_ordered_list(resultList,expectList):
print("The unordered list is the same as the ordered list.")
else:
tdlog.error("The unordered list is not the same as the ordered list.")
tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);")
tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);")

23
tests/system-test/0-others/udf_create.py
View File

@ -47,17 +47,27 @@ class TDTestCase:
if platform.system().lower() == 'windows': if platform.system().lower() == 'windows':
self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf1_dup = subprocess.Popen('(for /r %s %%i in ("udf1_dup.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf2_dup = subprocess.Popen('(for /r %s %%i in ("udf2_dup.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if (not tdDnodes.dnodes[0].remoteIP == ""): if (not tdDnodes.dnodes[0].remoteIP == ""):
tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\")
tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1_dup.so',projPath+"\\debug\\build\\lib\\")
tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\")
tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2_dup.so',projPath+"\\debug\\build\\lib\\")
self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so')
self.libudf1_dup = self.libudf1_dup.replace('udf1_dup.dll','libudf1_dup.so')
self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so')
self.libudf2_dup = self.libudf2_dup.replace('udf2_dup.dll','libudf2_dup.so')
else: else:
self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf1_dup = subprocess.Popen('find %s -name "libudf1_dup.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf2_dup = subprocess.Popen('find %s -name "libudf2_dup.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
self.libudf1 = self.libudf1.replace('\r','').replace('\n','') self.libudf1 = self.libudf1.replace('\r','').replace('\n','')
self.libudf1_dup = self.libudf1_dup.replace('\r','').replace('\n','')
self.libudf2 = self.libudf2.replace('\r','').replace('\n','') self.libudf2 = self.libudf2.replace('\r','').replace('\n','')
self.libudf2_dup = self.libudf2_dup.replace('\r','').replace('\n','')
def prepare_data(self): def prepare_data(self):
@ -174,10 +184,12 @@ class TDTestCase:
# create scalar functions # create scalar functions
tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1)
tdSql.execute("create function udf1_dup as '%s' outputtype int;"%self.libudf1_dup)
# create aggregate functions # create aggregate functions
tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
tdSql.execute("create aggregate function udf2_dup as '%s' outputtype double bufSize 8;"%self.libudf2_dup)
functions = tdSql.getResult("show functions") functions = tdSql.getResult("show functions")
function_nums = len(functions) function_nums = len(functions)
@ -188,6 +200,13 @@ class TDTestCase:
# scalar functions # scalar functions
# udf1_dup
tdSql.query("select udf1(num1) ,udf1_dup(num1) from tb")
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,2)
tdSql.checkData(2,0,1)
tdSql.checkData(2,1,2)
tdSql.execute("use db ") tdSql.execute("use db ")
tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb")
tdSql.checkData(0,0,None) tdSql.checkData(0,0,None)
@ -238,6 +257,10 @@ class TDTestCase:
# aggregate functions # aggregate functions
tdSql.query("select udf2(num1) ,udf2_dup(num1) from tb")
val = tdSql.queryResult[0][0] + 100
tdSql.checkData(0,1,val)
tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb") tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb")
tdSql.checkData(0,0,15.362291496) tdSql.checkData(0,0,15.362291496)
tdSql.checkData(0,1,10000949.553189287) tdSql.checkData(0,1,10000949.553189287)

2
tests/system-test/1-insert/delete_childtable.py
View File

@ -27,7 +27,7 @@ class TDTestCase:
def init(self, conn, logSql, replicaVar=1): def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar) self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__) tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), True)
self.dbname = 'db_test' self.dbname = 'db_test'
self.setsql = TDSetSql() self.setsql = TDSetSql()
self.stbname = 'stb' self.stbname = 'stb'

37
tests/system-test/1-insert/delete_stable.py
View File

@ -29,6 +29,9 @@ class TDTestCase:
tdLog.debug("start to execute %s" % __file__) tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor()) tdSql.init(conn.cursor())
self.dbname = 'db_test' self.dbname = 'db_test'
self.ns_dbname = 'ns_test'
self.us_dbname = 'us_test'
self.ms_dbname = 'ms_test'
self.setsql = TDSetSql() self.setsql = TDSetSql()
self.stbname = 'stb' self.stbname = 'stb'
self.ntbname = 'ntb' self.ntbname = 'ntb'
@ -220,11 +223,45 @@ class TDTestCase:
tdSql.query(f'select {func}(*) from {self.stbname}') tdSql.query(f'select {func}(*) from {self.stbname}')
tdSql.execute(f'drop table {self.stbname}') tdSql.execute(f'drop table {self.stbname}')
tdSql.execute(f'drop database {self.dbname}') tdSql.execute(f'drop database {self.dbname}')
def precision_now_check(self):
for dbname in [self.ms_dbname, self.us_dbname, self.ns_dbname]:
self.ts = 1537146000000
if dbname == self.us_dbname:
self.ts = int(self.ts*1000)
precision = "us"
elif dbname == self.ns_dbname:
precision = "ns"
self.ts = int(self.ts*1000000)
else:
precision = "ms"
self.ts = int(self.ts)
tdSql.execute(f'drop database if exists {dbname}')
tdSql.execute(f'create database if not exists {dbname} precision "{precision}"')
tdSql.execute(f'use {dbname}')
self.base_data = {
'tinyint':self.tinyint_val
}
self.column_dict = {
'col1': 'tinyint'
}
for col_name,col_type in self.column_dict.items():
tdSql.execute(f'create table if not exists {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)')
for i in range(self.tbnum):
tdSql.execute(f'create table if not exists {self.stbname}_{i} using {self.stbname} tags(1)')
self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data)
tdSql.query(f'select * from {self.stbname}')
tdSql.checkEqual(tdSql.queryRows, self.tbnum*self.rowNum)
tdSql.execute(f'delete from {self.stbname} where ts < now()')
tdSql.query(f'select * from {self.stbname}')
tdSql.checkEqual(tdSql.queryRows, 0)
def run(self): def run(self):
self.delete_data_stb() self.delete_data_stb()
tdDnodes.stoptaosd(1) tdDnodes.stoptaosd(1)
tdDnodes.starttaosd(1) tdDnodes.starttaosd(1)
self.delete_data_stb() self.delete_data_stb()
self.precision_now_check()
def stop(self): def stop(self):
tdSql.close() tdSql.close()
tdLog.success("%s successfully executed" % __file__) tdLog.success("%s successfully executed" % __file__)

8
tests/system-test/7-tmq/stbTagFilter-1ctb.py
View File

@ -111,7 +111,7 @@ class TDTestCase:
topicFromStb1 = 'topic_UpperCase_stb1' topicFromStb1 = 'topic_UpperCase_stb1'
# queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicFromStb1, queryString) sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString)
tdLog.info("create topic sql: %s"%sqlString) tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString) tdSql.execute(sqlString)
@ -148,7 +148,7 @@ class TDTestCase:
tmqCom.checkFileContent(consumerId, queryString) tmqCom.checkFileContent(consumerId, queryString)
tdSql.query("drop topic %s"%topicFromStb1) tdSql.query("drop topic `%s`"%topicFromStb1)
tdLog.printNoPrefix("======== test case 1 end ...... ") tdLog.printNoPrefix("======== test case 1 end ...... ")
def tmqCase2(self): def tmqCase2(self):
@ -196,7 +196,7 @@ class TDTestCase:
topicFromStb1 = 'topic_UpperCase_stb1' topicFromStb1 = 'topic_UpperCase_stb1'
queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
# queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicFromStb1, queryString) sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString)
tdLog.info("create topic sql: %s"%sqlString) tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString) tdSql.execute(sqlString)
@ -242,7 +242,7 @@ class TDTestCase:
# tmqCom.checkFileContent(consumerId, queryString) # tmqCom.checkFileContent(consumerId, queryString)
tdSql.query("drop topic %s"%topicFromStb1) tdSql.query("drop topic `%s`"%topicFromStb1)
tdLog.printNoPrefix("======== test case 2 end ...... ") tdLog.printNoPrefix("======== test case 2 end ...... ")

8
tests/system-test/7-tmq/stbTagFilter-multiCtb.py
View File

@ -111,7 +111,7 @@ class TDTestCase:
topicFromStb1 = 'topic_UpperCase_stb1' topicFromStb1 = 'topic_UpperCase_stb1'
queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
# queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicFromStb1, queryString) sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString)
tdLog.info("create topic sql: %s"%sqlString) tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString) tdSql.execute(sqlString)
@ -148,7 +148,7 @@ class TDTestCase:
# tmqCom.checkFileContent(consumerId, queryString) # tmqCom.checkFileContent(consumerId, queryString)
tdSql.query("drop topic %s"%topicFromStb1) tdSql.query("drop topic `%s`"%topicFromStb1)
tdLog.printNoPrefix("======== test case 1 end ...... ") tdLog.printNoPrefix("======== test case 1 end ...... ")
def tmqCase2(self): def tmqCase2(self):
@ -196,7 +196,7 @@ class TDTestCase:
topicFromStb1 = 'topic_UpperCase_stb1' topicFromStb1 = 'topic_UpperCase_stb1'
# queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicFromStb1, queryString) sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString)
tdLog.info("create topic sql: %s"%sqlString) tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString) tdSql.execute(sqlString)
@ -244,7 +244,7 @@ class TDTestCase:
# tmqCom.checkFileContent(consumerId, queryString) # tmqCom.checkFileContent(consumerId, queryString)
tdSql.query("drop topic %s"%topicFromStb1) tdSql.query("drop topic `%s`"%topicFromStb1)
tdLog.printNoPrefix("======== test case 2 end ...... ") tdLog.printNoPrefix("======== test case 2 end ...... ")

13
tests/system-test/7-tmq/subscribeDb3.py
View File

@ -82,7 +82,7 @@ class TDTestCase:
tdSql.query("select * from %s.notifyinfo"%cdbName) tdSql.query("select * from %s.notifyinfo"%cdbName)
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
if tdSql.getRows() == 2 : if tdSql.getRows() == 2 :
print(tdSql.getData(0, 1), tdSql.getData(1, 1)) tdLog.info("row[0][1]: %d, row[1][1]: %d"%(tdSql.getData(0, 1), tdSql.getData(1, 1)))
if tdSql.getData(1, 1) == 1: if tdSql.getData(1, 1) == 1:
break break
time.sleep(0.1) time.sleep(0.1)
@ -122,6 +122,7 @@ class TDTestCase:
os.system(shellCmd) os.system(shellCmd)
def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl):
tdLog.info("start create tables......")
tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups))
tsql.execute("use %s" %dbName) tsql.execute("use %s" %dbName)
tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
@ -137,11 +138,11 @@ class TDTestCase:
tsql.execute(sql) tsql.execute(sql)
event.set() event.set()
tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) tdLog.info("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum))
return return
def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs):
tdLog.debug("start to insert data ............") tdLog.info("start to insert data ............")
tsql.execute("use %s" %dbName) tsql.execute("use %s" %dbName)
pre_insert = "insert into " pre_insert = "insert into "
sql = pre_insert sql = pre_insert
@ -163,7 +164,7 @@ class TDTestCase:
if sql != pre_insert: if sql != pre_insert:
#print("insert sql:%s"%sql) #print("insert sql:%s"%sql)
tsql.execute(sql) tsql.execute(sql)
tdLog.debug("insert data ............ [OK]") tdLog.info("insert data ............ [OK]")
return return
def prepareEnv(self, **parameterDict): def prepareEnv(self, **parameterDict):
@ -226,7 +227,7 @@ class TDTestCase:
event.wait() event.wait()
tdLog.info("start consume processor") tdLog.info("start consume processor")
pollDelay = 100 pollDelay = 20
showMsg = 1 showMsg = 1
showRow = 1 showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
@ -286,7 +287,7 @@ class TDTestCase:
prepareEnvThread.start() prepareEnvThread.start()
tdLog.info("create topics from db") tdLog.info("create topics from db")
topicName1 = 'topic_db1' topicName1 = 'topic_db11'
tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName']))
consumerId = 0 consumerId = 0

115
tests/system-test/7-tmq/tmqConsumerGroup.py
View File

@ -42,7 +42,7 @@ class TDTestCase:
'showRow': 1} 'showRow': 1}
topicNameList = ['topic1', 'topic2'] topicNameList = ['topic1', 'topic2']
expectRowsList = [] queryRowsList = []
tmqCom.initConsumerTable() tmqCom.initConsumerTable()
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName']))
@ -60,7 +60,7 @@ class TDTestCase:
tdLog.info("create topic sql: %s"%sqlString) tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString) tdSql.execute(sqlString)
tdSql.query(queryString) tdSql.query(queryString)
expectRowsList.append(tdSql.getRows()) queryRowsList.append(tdSql.getRows())
# create one stb2 # create one stb2
paraDict["stbName"] = 'stb2' paraDict["stbName"] = 'stb2'
@ -77,7 +77,7 @@ class TDTestCase:
tdLog.info("create topic sql: %s"%sqlString) tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString) tdSql.execute(sqlString)
# tdSql.query(queryString) # tdSql.query(queryString)
# expectRowsList.append(tdSql.getRows()) # queryRowsList.append(tdSql.getRows())
# init consume info, and start tmq_sim, then check consume result # init consume info, and start tmq_sim, then check consume result
tdLog.info("insert consume info to consume processor") tdLog.info("insert consume info to consume processor")
@ -99,7 +99,8 @@ class TDTestCase:
pThread = tmqCom.asyncInsertData(paraDict) pThread = tmqCom.asyncInsertData(paraDict)
tdLog.info("wait consumer commit notify") tdLog.info("wait consumer commit notify")
tmqCom.getStartCommitNotifyFromTmqsim(rows=4) # tmqCom.getStartCommitNotifyFromTmqsim(rows=4)
tmqCom.getStartConsumeNotifyFromTmqsim(rows=2)
tdLog.info("pkill one consume processor") tdLog.info("pkill one consume processor")
tmqCom.stopTmqSimProcess('tmq_sim_new') tmqCom.stopTmqSimProcess('tmq_sim_new')
@ -109,19 +110,21 @@ class TDTestCase:
tdLog.info("wait the consume result") tdLog.info("wait the consume result")
expectRows = 2 expectRows = 2
resultList = tmqCom.selectConsumeResult(expectRows) resultList = tmqCom.selectConsumeResult(expectRows)
actTotalRows = 0 actConsumTotalRows = 0
for i in range(len(resultList)): for i in range(len(resultList)):
actTotalRows += resultList[i] actConsumTotalRows += resultList[i]
tdLog.info("act consumer1 rows: %d, consumer2 rows: %d"%(resultList[0], resultList[1]))
tdSql.query(queryString) tdSql.query(queryString)
expectRowsList.append(tdSql.getRows()) queryRowsList.append(tdSql.getRows())
expectTotalRows = 0 queryTotalRows = 0
for i in range(len(expectRowsList)): for i in range(len(queryRowsList)):
expectTotalRows += expectRowsList[i] queryTotalRows += queryRowsList[i]
tdLog.info("act consume rows: %d, expect consume rows: %d"%(actTotalRows, expectTotalRows)) tdLog.info("act consume rows: %d, query consume rows: %d"%(actConsumTotalRows, queryTotalRows))
if expectTotalRows <= resultList[0]: if actConsumTotalRows < queryTotalRows:
tdLog.info("act consume rows: %d should >= expect consume rows: %d"%(actTotalRows, expectTotalRows)) tdLog.info("act consume rows: %d should >= query consume rows: %d"%(actConsumTotalRows, queryTotalRows))
tdLog.exit("0 tmq consume rows error!") tdLog.exit("0 tmq consume rows error!")
# time.sleep(10) # time.sleep(10)
@ -130,9 +133,95 @@ class TDTestCase:
tdLog.printNoPrefix("======== test case 1 end ...... ") tdLog.printNoPrefix("======== test case 1 end ...... ")
def tmqCase2(self):
tdLog.printNoPrefix("======== test case 2: ")
paraDict = {'dbName': 'db1',
'dropFlag': 1,
'event': '',
'vgroups': 4,
'stbName': 'stb',
'colPrefix': 'c',
'tagPrefix': 't',
'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
'ctbPrefix': 'ctb',
'ctbNum': 10,
'rowsPerTbl': 1000,
'batchNum': 10,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 20,
'showMsg': 1,
'showRow': 1}
topicNameList = ['topic3', 'topic4']
queryRowsList = []
tmqCom.initConsumerTable()
tdLog.info("create topics from stb with filter")
# queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString)
tdSql.query(queryString)
queryRowsList.append(tdSql.getRows())
# create one stb2
paraDict["stbName"] = 'stb2'
# queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName'])
queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
sqlString = "create topic %s as %s" %(topicNameList[1], queryString)
tdLog.info("create topic sql: %s"%sqlString)
tdSql.execute(sqlString)
tdSql.query(queryString)
queryRowsList.append(tdSql.getRows())
# init consume info, and start tmq_sim, then check consume result
tdLog.info("insert consume info to consume processor")
consumerId = 0
paraDict["rowsPerTbl"] = 5000
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
topicList = "%s,%s"%(topicNameList[0],topicNameList[1])
ifcheckdata = 1
ifManualCommit = 1
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:3000, auto.offset.reset:earliest'
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor 1")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
tdLog.info("start consume processor 2")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'],'cdb',0,1)
tdLog.info("wait the consume result")
expectRows = 2
resultList = tmqCom.selectConsumeResult(expectRows)
actConsumTotalRows = 0
for i in range(len(resultList)):
actConsumTotalRows += resultList[i]
tdLog.info("act consumer1 rows: %d, consumer2 rows: %d"%(resultList[0], resultList[1]))
queryTotalRows = 0
for i in range(len(queryRowsList)):
queryTotalRows += queryRowsList[i]
tdLog.info("act consume rows: %d, query consume rows: %d"%(actConsumTotalRows, queryTotalRows))
if actConsumTotalRows < queryTotalRows:
tdLog.info("act consume rows: %d should >= query consume rows: %d"%(actConsumTotalRows, queryTotalRows))
tdLog.exit("0 tmq consume rows error!")
# time.sleep(10)
# for i in range(len(topicNameList)):
# tdSql.query("drop topic %s"%topicNameList[i])
tdLog.printNoPrefix("======== test case 2 end ...... ")
def run(self): def run(self):
tdSql.prepare() tdSql.prepare()
self.tmqCase1() self.tmqCase1()
self.tmqCase2()
def stop(self): def stop(self):
tdSql.close() tdSql.close()

4
tests/system-test/7-tmq/tmqDropStb.py
View File

@ -82,7 +82,7 @@ class TDTestCase:
tdLog.info("create topics from db") tdLog.info("create topics from db")
topicName1 = 'UpperCasetopic_%s'%(self.paraDict['dbName']) topicName1 = 'UpperCasetopic_%s'%(self.paraDict['dbName'])
tdSql.execute("create topic %s as database %s" %(topicName1, self.paraDict['dbName'])) tdSql.execute("create topic `%s` as database %s" %(topicName1, self.paraDict['dbName']))
topicList = topicName1 + ',' +topicName1 topicList = topicName1 + ',' +topicName1
keyList = '%s,%s,%s,%s'%(self.groupId,self.autoCommit,self.autoCommitInterval,self.autoOffset) keyList = '%s,%s,%s,%s'%(self.groupId,self.autoCommit,self.autoCommitInterval,self.autoOffset)
@ -113,7 +113,7 @@ class TDTestCase:
tdLog.exit("tmq consume rows error!") tdLog.exit("tmq consume rows error!")
time.sleep(10) time.sleep(10)
tdSql.query("drop topic %s"%topicName1) tdSql.query("drop topic `%s`"%topicName1)
tdLog.printNoPrefix("======== test case 1 end ...... ") tdLog.printNoPrefix("======== test case 1 end ...... ")

2
tools/shell/src/shellEngine.c
View File

@ -58,7 +58,6 @@ int32_t shellRunSingleCommand(char *command) {
} }
if (shellRegexMatch(command, "^[ \t]*(quit|q|exit)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { if (shellRegexMatch(command, "^[ \t]*(quit|q|exit)[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
shellWriteHistory();
return -1; return -1;
} }
@ -887,7 +886,6 @@ void shellWriteHistory() {
} }
i = (i + 1) % SHELL_MAX_HISTORY_SIZE; i = (i + 1) % SHELL_MAX_HISTORY_SIZE;
} }
taosFsyncFile(pFile);
taosCloseFile(&pFile); taosCloseFile(&pFile);
} }

3
tools/shell/src/shellMain.c
View File

@ -83,6 +83,9 @@ int main(int argc, char *argv[]) {
#endif #endif
taos_init(); taos_init();
// kill heart-beat thread when quit
taos_set_hb_quit(1);
if (shell.args.is_dump_config) { if (shell.args.is_dump_config) {
shellDumpConfig(); shellDumpConfig();
taos_cleanup(); taos_cleanup();

110
utils/test/c/sml_test.c
View File

@ -895,6 +895,63 @@ int smlProcess_18784_Test() {
return code; return code;
} }
int sml_escape_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape");
taos_free_result(pRes);
pRes = taos_query(taos, "use db_escape");
taos_free_result(pRes);
const char *sql[] = {
"d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_used=176059i,total=1076048383523889174i 1661943960000000000",
"d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_f\\\\ree=\"\\\"id,= ei\\\\\\f\" 1661943960000000000",
};
pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0);
printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes));
int code = taos_errno(pRes);
ASSERT(!code);
ASSERT(taos_affected_rows(pRes) == 1);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from `d,i= s\\k\"`"); //check stable name
ASSERT(pRes);
int fieldNum = taos_field_count(pRes);
ASSERT(fieldNum == 5);
printf("fieldNum:%d\n", fieldNum);
int numFields = taos_num_fields(pRes);
TAOS_FIELD *fields = taos_fetch_fields(pRes);
ASSERT(numFields == 5);
ASSERT(strncmp(fields[1].name, "inode\"i,= s_used", sizeof("inode\"i,= s_used") - 1) == 0);
ASSERT(strncmp(fields[2].name, "total", sizeof("total") - 1) == 0);
ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\\\ree", sizeof("inode\"i,= s_f\\\\ree") - 1) == 0);
ASSERT(strncmp(fields[4].name, "dev\"i,= ce", sizeof("dev\"i,= ce") - 1) == 0);
TAOS_ROW row = NULL;
int32_t rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
int64_t ts = *(int64_t *)row[0];
int64_t used = *(int64_t *)row[1];
int64_t total = *(int64_t *)row[2];
if (rowIndex == 0) {
ASSERT(ts == 1661943960000);
ASSERT(used == 176059);
ASSERT(total == 1076048383523889174);
ASSERT(strncmp(row[3], "\"id,= ei\\\\f", sizeof("\"id,= ei\\\\f") - 1) == 0);
ASSERT(strncmp(row[4], "s\"i,= dc", sizeof("s\"i,= dc") - 1) == 0);
}
rowIndex++;
}
taos_free_result(pRes);
taos_close(taos);
return code;
}
int sml_19221_Test() { int sml_19221_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
@ -961,6 +1018,55 @@ int sml_ts2164_Test() {
return code; return code;
} }
int sml_ts3116_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS_RES *pRes =
taos_query(taos, "DROP DATABASE IF EXISTS ts3116");
taos_free_result(pRes);
pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts3116 BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'");
taos_free_result(pRes);
char *sql = {
"meters,location=la,groupid=ca current=11.8,voltage=221",
};
pRes = taos_query(taos, "use ts3116");
taos_free_result(pRes);
int32_t totalRows = 0;
char *tmp = (char *)taosMemoryCalloc(1024, 1);
memcpy(tmp, sql, strlen(sql));
totalRows = 0;
pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL,
TSDB_SML_TIMESTAMP_MILLI_SECONDS);
taosMemoryFree(tmp);
printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
int code = taos_errno(pRes);
taos_free_result(pRes);
char *sql1 = {
"meters,location=la,groupid=ca\\=3 current=11.8,voltage=221\nmeters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27",
};
pRes = taos_query(taos, "use ts3116");
taos_free_result(pRes);
tmp = (char *)taosMemoryCalloc(1024, 1);
memcpy(tmp, sql1, strlen(sql1));
totalRows = 0;
pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL,
TSDB_SML_TIMESTAMP_MILLI_SECONDS);
taosMemoryFree(tmp);
printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
code = taos_errno(pRes);
taos_free_result(pRes);
taos_close(taos);
return code;
}
int sml_td22898_Test() { int sml_td22898_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
@ -1195,6 +1301,10 @@ int main(int argc, char *argv[]) {
} }
int ret = 0; int ret = 0;
ret = sml_escape_Test();
ASSERT(!ret);
ret = sml_ts3116_Test();
ASSERT(!ret);
ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file
ASSERT(!ret); ASSERT(!ret);
// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){