diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d8bf3a09b4..4838e97dd7 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 0681d8b + GIT_TAG ffc2e6f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/examples/JDBC/taosdemo/pom.xml b/examples/JDBC/taosdemo/pom.xml index 4731d8e237..0d47663bba 100644 --- a/examples/JDBC/taosdemo/pom.xml +++ b/examples/JDBC/taosdemo/pom.xml @@ -10,7 +10,7 @@ Demo project for TDengine - 5.3.26 + 5.3.27 diff --git a/include/client/taos.h b/include/client/taos.h index cf410a42da..a59e203644 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -225,6 +225,9 @@ DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *tabl DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); +// set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner +DLL_EXPORT void taos_set_hb_quit(int8_t quitByKill); + /* --------------------------schemaless INTERFACE------------------------------- */ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a97c68be49..8b4d5f16df 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -341,6 +341,8 @@ typedef struct { float f; }; size_t length; + bool keyEscaped; + bool valueEscaped; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 @@ -379,6 +381,8 @@ typedef struct STUidTagInfo { #define UD_GROUPID_COLUMN_INDEX 1 #define UD_TAG_COLUMN_INDEX 2 +int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime); + #ifdef __cplusplus } #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6d1d3ebce6..0c86c2c4d8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2071,7 +2071,6 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; - SArray* lostConsumers; // SArray SArray* removedConsumers; // SArray SArray* newConsumers; // SArray } SMqRebInfo; @@ -2082,10 +2081,6 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { return NULL; } tstrncpy(pRebInfo->key, key, TSDB_SUBSCRIBE_KEY_LEN); - pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); - if (pRebInfo->lostConsumers == NULL) { - goto _err; - } pRebInfo->removedConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebInfo->removedConsumers == NULL) { goto _err; @@ -2096,7 +2091,6 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { } return pRebInfo; _err: - taosArrayDestroy(pRebInfo->lostConsumers); taosArrayDestroy(pRebInfo->removedConsumers); taosArrayDestroy(pRebInfo->newConsumers); taosMemoryFreeClear(pRebInfo); diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 641cbbb588..10b5328e6d 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -16,335 +16,336 @@ #ifndef _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_ -#define TK_OR 1 -#define TK_AND 2 -#define TK_UNION 3 -#define TK_ALL 4 -#define TK_MINUS 5 -#define TK_EXCEPT 6 -#define TK_INTERSECT 7 -#define TK_NK_BITAND 8 -#define TK_NK_BITOR 9 -#define TK_NK_LSHIFT 10 -#define TK_NK_RSHIFT 11 -#define TK_NK_PLUS 12 -#define TK_NK_MINUS 13 -#define TK_NK_STAR 14 -#define TK_NK_SLASH 15 -#define TK_NK_REM 16 -#define TK_NK_CONCAT 17 -#define TK_CREATE 18 -#define TK_ACCOUNT 19 -#define TK_NK_ID 20 -#define TK_PASS 21 -#define TK_NK_STRING 22 -#define TK_ALTER 23 -#define TK_PPS 24 -#define TK_TSERIES 25 -#define TK_STORAGE 26 -#define TK_STREAMS 27 -#define TK_QTIME 28 -#define TK_DBS 29 -#define TK_USERS 30 -#define TK_CONNS 31 -#define TK_STATE 32 -#define TK_USER 33 -#define TK_ENABLE 34 -#define TK_NK_INTEGER 35 -#define TK_SYSINFO 36 -#define TK_DROP 37 -#define TK_GRANT 38 -#define TK_ON 39 -#define TK_TO 40 -#define TK_REVOKE 41 -#define TK_FROM 42 -#define TK_SUBSCRIBE 43 -#define TK_NK_COMMA 44 -#define TK_READ 45 -#define TK_WRITE 46 -#define TK_NK_DOT 47 -#define TK_WITH 48 -#define TK_DNODE 49 -#define TK_PORT 50 -#define TK_DNODES 51 -#define TK_NK_IPTOKEN 52 -#define TK_FORCE 53 -#define TK_LOCAL 54 -#define TK_QNODE 55 -#define TK_BNODE 56 -#define TK_SNODE 57 -#define TK_MNODE 58 -#define TK_DATABASE 59 -#define TK_USE 60 -#define TK_FLUSH 61 -#define TK_TRIM 62 -#define TK_COMPACT 63 -#define TK_IF 64 -#define TK_NOT 65 -#define TK_EXISTS 66 -#define TK_BUFFER 67 -#define TK_CACHEMODEL 68 -#define TK_CACHESIZE 69 -#define TK_COMP 70 -#define TK_DURATION 71 -#define TK_NK_VARIABLE 72 -#define TK_MAXROWS 73 -#define TK_MINROWS 74 -#define TK_KEEP 75 -#define TK_PAGES 76 -#define TK_PAGESIZE 77 -#define TK_TSDB_PAGESIZE 78 -#define TK_PRECISION 79 -#define TK_REPLICA 80 -#define TK_VGROUPS 81 -#define TK_SINGLE_STABLE 82 -#define TK_RETENTIONS 83 -#define TK_SCHEMALESS 84 -#define TK_WAL_LEVEL 85 -#define TK_WAL_FSYNC_PERIOD 86 -#define TK_WAL_RETENTION_PERIOD 87 -#define TK_WAL_RETENTION_SIZE 88 -#define TK_WAL_ROLL_PERIOD 89 -#define TK_WAL_SEGMENT_SIZE 90 -#define TK_STT_TRIGGER 91 -#define TK_TABLE_PREFIX 92 -#define TK_TABLE_SUFFIX 93 -#define TK_NK_COLON 94 -#define TK_MAX_SPEED 95 -#define TK_START 96 -#define TK_TIMESTAMP 97 -#define TK_END 98 -#define TK_TABLE 99 -#define TK_NK_LP 100 -#define TK_NK_RP 101 -#define TK_STABLE 102 -#define TK_ADD 103 -#define TK_COLUMN 104 -#define TK_MODIFY 105 -#define TK_RENAME 106 -#define TK_TAG 107 -#define TK_SET 108 -#define TK_NK_EQ 109 -#define TK_USING 110 -#define TK_TAGS 111 -#define TK_BOOL 112 -#define TK_TINYINT 113 -#define TK_SMALLINT 114 -#define TK_INT 115 -#define TK_INTEGER 116 -#define TK_BIGINT 117 -#define TK_FLOAT 118 -#define TK_DOUBLE 119 -#define TK_BINARY 120 -#define TK_NCHAR 121 -#define TK_UNSIGNED 122 -#define TK_JSON 123 -#define TK_VARCHAR 124 -#define TK_MEDIUMBLOB 125 -#define TK_BLOB 126 -#define TK_VARBINARY 127 -#define TK_DECIMAL 128 -#define TK_COMMENT 129 -#define TK_MAX_DELAY 130 -#define TK_WATERMARK 131 -#define TK_ROLLUP 132 -#define TK_TTL 133 -#define TK_SMA 134 -#define TK_DELETE_MARK 135 -#define TK_FIRST 136 -#define TK_LAST 137 -#define TK_SHOW 138 -#define TK_PRIVILEGES 139 -#define TK_DATABASES 140 -#define TK_TABLES 141 -#define TK_STABLES 142 -#define TK_MNODES 143 -#define TK_QNODES 144 -#define TK_FUNCTIONS 145 -#define TK_INDEXES 146 -#define TK_ACCOUNTS 147 -#define TK_APPS 148 -#define TK_CONNECTIONS 149 -#define TK_LICENCES 150 -#define TK_GRANTS 151 -#define TK_QUERIES 152 -#define TK_SCORES 153 -#define TK_TOPICS 154 -#define TK_VARIABLES 155 -#define TK_CLUSTER 156 -#define TK_BNODES 157 -#define TK_SNODES 158 -#define TK_TRANSACTIONS 159 -#define TK_DISTRIBUTED 160 -#define TK_CONSUMERS 161 -#define TK_SUBSCRIPTIONS 162 -#define TK_VNODES 163 -#define TK_ALIVE 164 -#define TK_LIKE 165 -#define TK_TBNAME 166 -#define TK_QTAGS 167 -#define TK_AS 168 -#define TK_INDEX 169 -#define TK_FUNCTION 170 -#define TK_INTERVAL 171 -#define TK_COUNT 172 -#define TK_LAST_ROW 173 -#define TK_TOPIC 174 -#define TK_META 175 -#define TK_CONSUMER 176 -#define TK_GROUP 177 -#define TK_DESC 178 -#define TK_DESCRIBE 179 -#define TK_RESET 180 -#define TK_QUERY 181 -#define TK_CACHE 182 -#define TK_EXPLAIN 183 -#define TK_ANALYZE 184 -#define TK_VERBOSE 185 -#define TK_NK_BOOL 186 -#define TK_RATIO 187 -#define TK_NK_FLOAT 188 -#define TK_OUTPUTTYPE 189 -#define TK_AGGREGATE 190 -#define TK_BUFSIZE 191 -#define TK_LANGUAGE 192 -#define TK_REPLACE 193 -#define TK_STREAM 194 -#define TK_INTO 195 -#define TK_TRIGGER 196 -#define TK_AT_ONCE 197 -#define TK_WINDOW_CLOSE 198 -#define TK_IGNORE 199 -#define TK_EXPIRED 200 -#define TK_FILL_HISTORY 201 -#define TK_UPDATE 202 -#define TK_SUBTABLE 203 -#define TK_KILL 204 -#define TK_CONNECTION 205 -#define TK_TRANSACTION 206 -#define TK_BALANCE 207 -#define TK_VGROUP 208 -#define TK_LEADER 209 -#define TK_MERGE 210 -#define TK_REDISTRIBUTE 211 -#define TK_SPLIT 212 -#define TK_DELETE 213 -#define TK_INSERT 214 -#define TK_NULL 215 -#define TK_NK_QUESTION 216 -#define TK_NK_ARROW 217 -#define TK_ROWTS 218 -#define TK_QSTART 219 -#define TK_QEND 220 -#define TK_QDURATION 221 -#define TK_WSTART 222 -#define TK_WEND 223 -#define TK_WDURATION 224 -#define TK_IROWTS 225 -#define TK_ISFILLED 226 -#define TK_CAST 227 -#define TK_NOW 228 -#define TK_TODAY 229 -#define TK_TIMEZONE 230 -#define TK_CLIENT_VERSION 231 -#define TK_SERVER_VERSION 232 -#define TK_SERVER_STATUS 233 -#define TK_CURRENT_USER 234 -#define TK_CASE 235 -#define TK_WHEN 236 -#define TK_THEN 237 -#define TK_ELSE 238 -#define TK_BETWEEN 239 -#define TK_IS 240 -#define TK_NK_LT 241 -#define TK_NK_GT 242 -#define TK_NK_LE 243 -#define TK_NK_GE 244 -#define TK_NK_NE 245 -#define TK_MATCH 246 -#define TK_NMATCH 247 -#define TK_CONTAINS 248 -#define TK_IN 249 -#define TK_JOIN 250 -#define TK_INNER 251 -#define TK_SELECT 252 -#define TK_DISTINCT 253 -#define TK_WHERE 254 -#define TK_PARTITION 255 -#define TK_BY 256 -#define TK_SESSION 257 -#define TK_STATE_WINDOW 258 -#define TK_EVENT_WINDOW 259 -#define TK_SLIDING 260 -#define TK_FILL 261 -#define TK_VALUE 262 -#define TK_VALUE_F 263 -#define TK_NONE 264 -#define TK_PREV 265 -#define TK_NULL_F 266 -#define TK_LINEAR 267 -#define TK_NEXT 268 -#define TK_HAVING 269 -#define TK_RANGE 270 -#define TK_EVERY 271 -#define TK_ORDER 272 -#define TK_SLIMIT 273 -#define TK_SOFFSET 274 -#define TK_LIMIT 275 -#define TK_OFFSET 276 -#define TK_ASC 277 -#define TK_NULLS 278 -#define TK_ABORT 279 -#define TK_AFTER 280 -#define TK_ATTACH 281 -#define TK_BEFORE 282 -#define TK_BEGIN 283 -#define TK_BITAND 284 -#define TK_BITNOT 285 -#define TK_BITOR 286 -#define TK_BLOCKS 287 -#define TK_CHANGE 288 -#define TK_COMMA 289 -#define TK_CONCAT 290 -#define TK_CONFLICT 291 -#define TK_COPY 292 -#define TK_DEFERRED 293 -#define TK_DELIMITERS 294 -#define TK_DETACH 295 -#define TK_DIVIDE 296 -#define TK_DOT 297 -#define TK_EACH 298 -#define TK_FAIL 299 -#define TK_FILE 300 -#define TK_FOR 301 -#define TK_GLOB 302 -#define TK_ID 303 -#define TK_IMMEDIATE 304 -#define TK_IMPORT 305 -#define TK_INITIALLY 306 -#define TK_INSTEAD 307 -#define TK_ISNULL 308 -#define TK_KEY 309 -#define TK_MODULES 310 -#define TK_NK_BITNOT 311 -#define TK_NK_SEMI 312 -#define TK_NOTNULL 313 -#define TK_OF 314 -#define TK_PLUS 315 -#define TK_PRIVILEGE 316 -#define TK_RAISE 317 -#define TK_RESTRICT 318 -#define TK_ROW 319 -#define TK_SEMI 320 -#define TK_STAR 321 -#define TK_STATEMENT 322 -#define TK_STRICT 323 -#define TK_STRING 324 -#define TK_TIMES 325 -#define TK_VALUES 326 -#define TK_VARIABLE 327 -#define TK_VIEW 328 -#define TK_WAL 329 +#define TK_OR 1 +#define TK_AND 2 +#define TK_UNION 3 +#define TK_ALL 4 +#define TK_MINUS 5 +#define TK_EXCEPT 6 +#define TK_INTERSECT 7 +#define TK_NK_BITAND 8 +#define TK_NK_BITOR 9 +#define TK_NK_LSHIFT 10 +#define TK_NK_RSHIFT 11 +#define TK_NK_PLUS 12 +#define TK_NK_MINUS 13 +#define TK_NK_STAR 14 +#define TK_NK_SLASH 15 +#define TK_NK_REM 16 +#define TK_NK_CONCAT 17 +#define TK_CREATE 18 +#define TK_ACCOUNT 19 +#define TK_NK_ID 20 +#define TK_PASS 21 +#define TK_NK_STRING 22 +#define TK_ALTER 23 +#define TK_PPS 24 +#define TK_TSERIES 25 +#define TK_STORAGE 26 +#define TK_STREAMS 27 +#define TK_QTIME 28 +#define TK_DBS 29 +#define TK_USERS 30 +#define TK_CONNS 31 +#define TK_STATE 32 +#define TK_USER 33 +#define TK_ENABLE 34 +#define TK_NK_INTEGER 35 +#define TK_SYSINFO 36 +#define TK_DROP 37 +#define TK_GRANT 38 +#define TK_ON 39 +#define TK_TO 40 +#define TK_REVOKE 41 +#define TK_FROM 42 +#define TK_SUBSCRIBE 43 +#define TK_NK_COMMA 44 +#define TK_READ 45 +#define TK_WRITE 46 +#define TK_NK_DOT 47 +#define TK_WITH 48 +#define TK_DNODE 49 +#define TK_PORT 50 +#define TK_DNODES 51 +#define TK_NK_IPTOKEN 52 +#define TK_FORCE 53 +#define TK_LOCAL 54 +#define TK_QNODE 55 +#define TK_BNODE 56 +#define TK_SNODE 57 +#define TK_MNODE 58 +#define TK_DATABASE 59 +#define TK_USE 60 +#define TK_FLUSH 61 +#define TK_TRIM 62 +#define TK_COMPACT 63 +#define TK_IF 64 +#define TK_NOT 65 +#define TK_EXISTS 66 +#define TK_BUFFER 67 +#define TK_CACHEMODEL 68 +#define TK_CACHESIZE 69 +#define TK_COMP 70 +#define TK_DURATION 71 +#define TK_NK_VARIABLE 72 +#define TK_MAXROWS 73 +#define TK_MINROWS 74 +#define TK_KEEP 75 +#define TK_PAGES 76 +#define TK_PAGESIZE 77 +#define TK_TSDB_PAGESIZE 78 +#define TK_PRECISION 79 +#define TK_REPLICA 80 +#define TK_VGROUPS 81 +#define TK_SINGLE_STABLE 82 +#define TK_RETENTIONS 83 +#define TK_SCHEMALESS 84 +#define TK_WAL_LEVEL 85 +#define TK_WAL_FSYNC_PERIOD 86 +#define TK_WAL_RETENTION_PERIOD 87 +#define TK_WAL_RETENTION_SIZE 88 +#define TK_WAL_ROLL_PERIOD 89 +#define TK_WAL_SEGMENT_SIZE 90 +#define TK_STT_TRIGGER 91 +#define TK_TABLE_PREFIX 92 +#define TK_TABLE_SUFFIX 93 +#define TK_NK_COLON 94 +#define TK_MAX_SPEED 95 +#define TK_START 96 +#define TK_TIMESTAMP 97 +#define TK_END 98 +#define TK_TABLE 99 +#define TK_NK_LP 100 +#define TK_NK_RP 101 +#define TK_STABLE 102 +#define TK_ADD 103 +#define TK_COLUMN 104 +#define TK_MODIFY 105 +#define TK_RENAME 106 +#define TK_TAG 107 +#define TK_SET 108 +#define TK_NK_EQ 109 +#define TK_USING 110 +#define TK_TAGS 111 +#define TK_BOOL 112 +#define TK_TINYINT 113 +#define TK_SMALLINT 114 +#define TK_INT 115 +#define TK_INTEGER 116 +#define TK_BIGINT 117 +#define TK_FLOAT 118 +#define TK_DOUBLE 119 +#define TK_BINARY 120 +#define TK_NCHAR 121 +#define TK_UNSIGNED 122 +#define TK_JSON 123 +#define TK_VARCHAR 124 +#define TK_MEDIUMBLOB 125 +#define TK_BLOB 126 +#define TK_VARBINARY 127 +#define TK_DECIMAL 128 +#define TK_COMMENT 129 +#define TK_MAX_DELAY 130 +#define TK_WATERMARK 131 +#define TK_ROLLUP 132 +#define TK_TTL 133 +#define TK_SMA 134 +#define TK_DELETE_MARK 135 +#define TK_FIRST 136 +#define TK_LAST 137 +#define TK_SHOW 138 +#define TK_PRIVILEGES 139 +#define TK_DATABASES 140 +#define TK_TABLES 141 +#define TK_STABLES 142 +#define TK_MNODES 143 +#define TK_QNODES 144 +#define TK_FUNCTIONS 145 +#define TK_INDEXES 146 +#define TK_ACCOUNTS 147 +#define TK_APPS 148 +#define TK_CONNECTIONS 149 +#define TK_LICENCES 150 +#define TK_GRANTS 151 +#define TK_QUERIES 152 +#define TK_SCORES 153 +#define TK_TOPICS 154 +#define TK_VARIABLES 155 +#define TK_CLUSTER 156 +#define TK_BNODES 157 +#define TK_SNODES 158 +#define TK_TRANSACTIONS 159 +#define TK_DISTRIBUTED 160 +#define TK_CONSUMERS 161 +#define TK_SUBSCRIPTIONS 162 +#define TK_VNODES 163 +#define TK_ALIVE 164 +#define TK_LIKE 165 +#define TK_TBNAME 166 +#define TK_QTAGS 167 +#define TK_AS 168 +#define TK_INDEX 169 +#define TK_FUNCTION 170 +#define TK_INTERVAL 171 +#define TK_COUNT 172 +#define TK_LAST_ROW 173 +#define TK_TOPIC 174 +#define TK_META 175 +#define TK_CONSUMER 176 +#define TK_GROUP 177 +#define TK_DESC 178 +#define TK_DESCRIBE 179 +#define TK_RESET 180 +#define TK_QUERY 181 +#define TK_CACHE 182 +#define TK_EXPLAIN 183 +#define TK_ANALYZE 184 +#define TK_VERBOSE 185 +#define TK_NK_BOOL 186 +#define TK_RATIO 187 +#define TK_NK_FLOAT 188 +#define TK_OUTPUTTYPE 189 +#define TK_AGGREGATE 190 +#define TK_BUFSIZE 191 +#define TK_LANGUAGE 192 +#define TK_REPLACE 193 +#define TK_STREAM 194 +#define TK_INTO 195 +#define TK_TRIGGER 196 +#define TK_AT_ONCE 197 +#define TK_WINDOW_CLOSE 198 +#define TK_IGNORE 199 +#define TK_EXPIRED 200 +#define TK_FILL_HISTORY 201 +#define TK_UPDATE 202 +#define TK_SUBTABLE 203 +#define TK_KILL 204 +#define TK_CONNECTION 205 +#define TK_TRANSACTION 206 +#define TK_BALANCE 207 +#define TK_VGROUP 208 +#define TK_LEADER 209 +#define TK_MERGE 210 +#define TK_REDISTRIBUTE 211 +#define TK_SPLIT 212 +#define TK_DELETE 213 +#define TK_INSERT 214 +#define TK_NULL 215 +#define TK_NK_QUESTION 216 +#define TK_NK_ARROW 217 +#define TK_ROWTS 218 +#define TK_QSTART 219 +#define TK_QEND 220 +#define TK_QDURATION 221 +#define TK_WSTART 222 +#define TK_WEND 223 +#define TK_WDURATION 224 +#define TK_IROWTS 225 +#define TK_ISFILLED 226 +#define TK_CAST 227 +#define TK_NOW 228 +#define TK_TODAY 229 +#define TK_TIMEZONE 230 +#define TK_CLIENT_VERSION 231 +#define TK_SERVER_VERSION 232 +#define TK_SERVER_STATUS 233 +#define TK_CURRENT_USER 234 +#define TK_CASE 235 +#define TK_WHEN 236 +#define TK_THEN 237 +#define TK_ELSE 238 +#define TK_BETWEEN 239 +#define TK_IS 240 +#define TK_NK_LT 241 +#define TK_NK_GT 242 +#define TK_NK_LE 243 +#define TK_NK_GE 244 +#define TK_NK_NE 245 +#define TK_MATCH 246 +#define TK_NMATCH 247 +#define TK_CONTAINS 248 +#define TK_IN 249 +#define TK_JOIN 250 +#define TK_INNER 251 +#define TK_SELECT 252 +#define TK_DISTINCT 253 +#define TK_WHERE 254 +#define TK_PARTITION 255 +#define TK_BY 256 +#define TK_SESSION 257 +#define TK_STATE_WINDOW 258 +#define TK_EVENT_WINDOW 259 +#define TK_SLIDING 260 +#define TK_FILL 261 +#define TK_VALUE 262 +#define TK_VALUE_F 263 +#define TK_NONE 264 +#define TK_PREV 265 +#define TK_NULL_F 266 +#define TK_LINEAR 267 +#define TK_NEXT 268 +#define TK_HAVING 269 +#define TK_RANGE 270 +#define TK_EVERY 271 +#define TK_ORDER 272 +#define TK_SLIMIT 273 +#define TK_SOFFSET 274 +#define TK_LIMIT 275 +#define TK_OFFSET 276 +#define TK_ASC 277 +#define TK_NULLS 278 +#define TK_ABORT 279 +#define TK_AFTER 280 +#define TK_ATTACH 281 +#define TK_BEFORE 282 +#define TK_BEGIN 283 +#define TK_BITAND 284 +#define TK_BITNOT 285 +#define TK_BITOR 286 +#define TK_BLOCKS 287 +#define TK_CHANGE 288 +#define TK_COMMA 289 +#define TK_CONCAT 290 +#define TK_CONFLICT 291 +#define TK_COPY 292 +#define TK_DEFERRED 293 +#define TK_DELIMITERS 294 +#define TK_DETACH 295 +#define TK_DIVIDE 296 +#define TK_DOT 297 +#define TK_EACH 298 +#define TK_FAIL 299 +#define TK_FILE 300 +#define TK_FOR 301 +#define TK_GLOB 302 +#define TK_ID 303 +#define TK_IMMEDIATE 304 +#define TK_IMPORT 305 +#define TK_INITIALLY 306 +#define TK_INSTEAD 307 +#define TK_ISNULL 308 +#define TK_KEY 309 +#define TK_MODULES 310 +#define TK_NK_BITNOT 311 +#define TK_NK_SEMI 312 +#define TK_NOTNULL 313 +#define TK_OF 314 +#define TK_PLUS 315 +#define TK_PRIVILEGE 316 +#define TK_RAISE 317 +#define TK_RESTRICT 318 +#define TK_ROW 319 +#define TK_SEMI 320 +#define TK_STAR 321 +#define TK_STATEMENT 322 +#define TK_STRICT 323 +#define TK_STRING 324 +#define TK_TIMES 325 +#define TK_VALUES 326 +#define TK_VARIABLE 327 +#define TK_VIEW 328 +#define TK_WAL 329 + #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index ed7cbc8125..ce7d038d42 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -29,7 +29,6 @@ extern "C" { #define DS_BUF_FULL 2 #define DS_BUF_EMPTY 3 -struct SDataSink; struct SSDataBlock; typedef struct SDeleterRes { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 84f34e1426..c18fedc6eb 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "os.h" #include "executor.h" +#include "os.h" #include "query.h" #include "streamState.h" #include "tdatablock.h" @@ -206,14 +206,12 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { atomic_store_8(&queue->status, STREAM_QUEUE__FAILED); } -static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { - return queue->qItem; -} +static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } void* streamQueueNextItem(SStreamQueue* queue); SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type); -void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit); +void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit); SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit); @@ -272,7 +270,8 @@ typedef struct SStreamId { typedef struct SCheckpointInfo { int64_t id; - int64_t version; // offset in WAL + int64_t version; // offset in WAL + int64_t currentVer; // current offset in WAL, not serialize it } SCheckpointInfo; typedef struct SStreamStatus { @@ -335,18 +334,17 @@ struct SStreamTask { // meta typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pCheckpointDb; - SHashObj* pTasks; - void* ahandle; - TXN* txn; - FTaskExpand* expandFunc; - int32_t vgId; - SRWLatch lock; - int32_t walScan; - bool quit; + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pCheckpointDb; + SHashObj* pTasks; + void* ahandle; + TXN* txn; + FTaskExpand* expandFunc; + int32_t vgId; + SRWLatch lock; + int32_t walScan; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); @@ -359,10 +357,6 @@ void tFreeStreamTask(SStreamTask* pTask); int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem); bool tInputQueueIsFull(const SStreamTask* pTask); -static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); -} - typedef struct { SMsgHead head; int64_t streamId; @@ -538,9 +532,11 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); // int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); +void streamTaskInputFail(SStreamTask* pTask); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock); +bool streamTaskShouldStop(const SStreamStatus* pStatus); int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz); @@ -568,19 +564,19 @@ int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId); SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); void streamMetaClose(SStreamMeta* streamMeta); -int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); -int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); -int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); +int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); +int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); +int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); -int32_t streamMetaBegin(SStreamMeta* pMeta); -int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaRollBack(SStreamMeta* pMeta); -int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); +int32_t streamMetaBegin(SStreamMeta* pMeta); +int32_t streamMetaCommit(SStreamMeta* pMeta); +int32_t streamMetaRollBack(SStreamMeta* pMeta); +int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/include/util/tdef.h b/include/util/tdef.h index 9dd454bb68..a77663fad6 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -371,11 +371,11 @@ typedef enum ELogicConditionType { #define TSDB_MIN_STT_TRIGGER 1 #define TSDB_MAX_STT_TRIGGER 16 #define TSDB_DEFAULT_SST_TRIGGER 1 -#define TSDB_MIN_HASH_PREFIX 0 -#define TSDB_MAX_HASH_PREFIX 128 +#define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN) +#define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_PREFIX 0 -#define TSDB_MIN_HASH_SUFFIX 0 -#define TSDB_MAX_HASH_SUFFIX 128 +#define TSDB_MIN_HASH_SUFFIX (2 - TSDB_TABLE_NAME_LEN) +#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 diff --git a/include/util/tlog.h b/include/util/tlog.h index 0071b3d32c..541b7589b7 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -102,7 +102,6 @@ bool taosAssertRelease(bool condition); void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo); void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr *pFd); void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); -int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime); // clang-format off #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} diff --git a/include/util/tutil.h b/include/util/tutil.h index e0801e5295..e96c7a07d9 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -81,14 +81,22 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix, int32_t suffix) { - if (prefix == 0 && suffix == 0) { + if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) { return MurmurHash3_32(tbname, tblen); + } else if (prefix > 0 || suffix > 0) { + return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); } else { - if (tblen <= (prefix + suffix)) { - return MurmurHash3_32(tbname, tblen); - } else { - return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); + char tbName[TSDB_TABLE_FNAME_LEN]; + int32_t offset = 0; + if (prefix < 0) { + offset = -1 * prefix; + strncpy(tbName, tbname, offset); } + if (suffix < 0) { + strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix); + offset += -1 *suffix; + } + return MurmurHash3_32(tbName, offset); } } diff --git a/packaging/docker/bin/entrypoint.sh b/packaging/docker/bin/entrypoint.sh index f4be349c0d..f2811de7bd 100755 --- a/packaging/docker/bin/entrypoint.sh +++ b/packaging/docker/bin/entrypoint.sh @@ -42,8 +42,9 @@ if [ "$DISABLE_ADAPTER" = "0" ]; then done fi -# if has mnode ep set or the host is first ep or not for cluster, just start. -if [ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] || +# if dnode has been created or has mnode ep set or the host is first ep or not for cluster, just start. +if [ -f "$DATA_DIR/dnode/dnode.json" ] || + [ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] || [ "$TAOS_FQDN" = "$FIRST_EP_HOST" ]; then $@ # others will first wait the first ep ready. diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index b473f3b527..28dc770755 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -197,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : fi git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index e4df233d67..a590835257 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -338,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + tmp_pwd=`pwd` + cd ${install_dir}/connector + if [ ! -d taos-connector-jdbc ];then + git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + fi + cd taos-connector-jdbc + mvn clean package -Dmaven.test.skip=true + echo ${build_dir}/lib/ + cp target/*.jar ${build_dir}/lib/ + cd ${install_dir}/connector + rm -rf taos-connector-jdbc + cd ${tmp_pwd} + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 41f87379a9..46d44d7443 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -80,6 +80,7 @@ typedef struct { int64_t appId; // ctl int8_t threadStop; + int8_t quitByKill; TdThread thread; TdThreadMutex lock; // used when app init and cleanup SHashObj* appSummary; diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 92896e6f23..3982c0d9aa 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -70,7 +70,7 @@ extern "C" { #define VALUE_LEN 6 #define OTD_JSON_FIELDS_NUM 4 -#define MAX_RETRY_TIMES 100 +#define MAX_RETRY_TIMES 10 typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef enum { @@ -107,6 +107,7 @@ typedef struct { int32_t colsLen; int32_t timestampLen; + bool measureEscaped; SArray *colArray; } SSmlLineInfo; @@ -206,6 +207,19 @@ typedef struct { #define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) +#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } + extern int64_t smlFactorNS[3]; extern int64_t smlFactorS[3]; @@ -237,6 +251,7 @@ uint8_t smlGetTimestampLen(int64_t num); void clearColValArray(SArray* pCols); void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); +void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index c9c2e7a5f8..8d082ab60b 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -845,7 +845,12 @@ static void hbStopThread() { return; } - taosThreadJoin(clientHbMgr.thread, NULL); + // thread quit mode kill or inner exit from self-thread + if (clientHbMgr.quitByKill) { + taosThreadKill(clientHbMgr.thread, 0); + } else { + taosThreadJoin(clientHbMgr.thread, NULL); + } tscDebug("hb thread stopped"); } @@ -1037,3 +1042,8 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } + +// set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner +void taos_set_hb_quit(int8_t quitByKill) { + clientHbMgr.quitByKill = quitByKill; +} diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index d5f4cea92e..2f57bb5473 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -534,8 +534,9 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key); - return TSDB_CODE_TSC_INVALID_VALUE; + uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, + colField[*index].type, kv->type, kv->key); + return TSDB_CODE_SML_INVALID_DATA; } if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && @@ -765,8 +766,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); + char *measure = taosMemoryMalloc(superTableLen); + memcpy(measure, superTable, superTableLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, superTable, superTableLen); + memcpy(pName.tname, measure, superTableLen); + taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1049,7 +1054,7 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { // } // taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); - taosArrayDestroy(tag->tags); + taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); } @@ -1063,6 +1068,12 @@ void clearColValArray(SArray *pCols) { } } +void freeSSmlKv(void *data) { + SSmlKv *kv = (SSmlKv *)data; + if (kv->keyEscaped) taosMemoryFree((void *)(kv->key)); + if (kv->valueEscaped) taosMemoryFree((void *)(kv->value)); +} + void smlDestroyInfo(SSmlHandle *info) { if (!info) return; qDestroyQuery(info->pQuery); @@ -1098,11 +1109,11 @@ void smlDestroyInfo(SSmlHandle *info) { } taosArrayDestroy(info->valueJsonArray); - taosArrayDestroy(info->preLineTagKV); + taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { - taosArrayDestroy(info->lines[i].colArray); + taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); if (info->parseJsonByLib) { taosMemoryFree(info->lines[i].tags); } @@ -1165,6 +1176,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + terrno = 0; taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { taosHashCleanup(kvHash); @@ -1242,6 +1254,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); + terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; @@ -1305,9 +1318,15 @@ static int32_t smlInsertData(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat); + int measureLen = tableData->sTableNameLen; + char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); + memcpy(measure, tableData->sTableName, tableData->sTableNameLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, - info->ttl, info->msgBuf.buf, info->msgBuf.len); + (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, + info->msgBuf.len); + taosMemoryFree(measure); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; @@ -1421,14 +1440,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char char cTmp = 0; // for print tmp if is raw if (info->isRawLine) { - cTmp = tmp[len - 1]; - tmp[len - 1] = '\0'; + cTmp = tmp[len]; + tmp[len] = '\0'; } uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, info->isRawLine, numLines, info->protocol, len, tmp); if (info->isRawLine) { - tmp[len - 1] = cTmp; + tmp[len] = cTmp; } if (info->protocol == TSDB_SML_LINE_PROTOCOL) { @@ -1450,6 +1469,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { + tmp[len] = '\0'; uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } @@ -1495,8 +1515,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL do { code = smlModifyDBSchemas(info); - if (code == 0) break; - taosMsleep(500); + if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA) break; + taosMsleep(100); uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 335e3a1dc7..1732473c11 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -21,32 +21,33 @@ #include "clientSml.h" // comma , -// #define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) // space -// #define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) // equal = -// #define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) // quote " -// #define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) -#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) +//#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) // SLASH -// #define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) -#define IS_SLASH_LETTER(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || \ - *(sql) == SLASH)) // (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || - // IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) +#define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) \ + (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH)) -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) +#define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL)) -#define PROCESS_SLASH(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER(key + i)) { \ +#define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_FIELD_VALUE(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } + +#define PROCESS_SLASH_IN_TAG_FIELD_KEY(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(key + i)) { \ MOVE_FORWARD_ONE(key + i, keyLen - i); \ - i--; \ keyLen--; \ } \ } @@ -151,7 +152,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + char* measure = currElement->measure; + int measureLen = currElement->measureLen; + if(currElement->measureEscaped){ + measure = (char*)taosMemoryMalloc(currElement->measureLen); + memcpy(measure, currElement->measure, currElement->measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if(currElement->measureEscaped){ + taosMemoryFree(measure); + } if (pTableMeta == NULL) { info->dataFormat = false; info->reRun = true; @@ -171,17 +182,18 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->maxTagKVs = (*tmp)->tags; } } - taosArrayClear(preLineKV); + taosArrayClearEx(preLineKV, freeSSmlKv); while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { break; } - bool hasSlash = false; // parse key const char *key = *sql; size_t keyLen = 0; + bool keyEscaped = false; + size_t keyLenEscaped = 0; while (*sql < sqlEnd) { if (unlikely(IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); @@ -192,16 +204,14 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin (*sql)++; break; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + keyLenEscaped++; + keyEscaped = true; } (*sql)++; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(key, keyLen) - } - if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -209,7 +219,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin // parse value const char *value = *sql; size_t valueLen = 0; - hasSlash = false; + bool valueEscaped = false; + size_t valueLenEscaped = 0; while (*sql < sqlEnd) { // parse value if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { @@ -219,8 +230,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + valueLenEscaped++; + valueEscaped = true; } (*sql)++; @@ -232,15 +244,24 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(value, valueLen) - } - - if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + if (keyEscaped){ + char *tmp = (char*)taosMemoryMalloc(keyLen); + memcpy(tmp, key, keyLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); + key = tmp; + } + if (valueEscaped){ + char *tmp = (char*)taosMemoryMalloc(valueLen); + memcpy(tmp, value, valueLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); + value = tmp; + } + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; + taosArrayPush(preLineKV, &kv); if (info->dataFormat) { if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { info->dataFormat = false; @@ -266,7 +287,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->needModifySchema = true; } } - taosArrayPush(preLineKV, &kv); cnt++; if (IS_SPACE(*sql)) { @@ -285,6 +305,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_OUT_OF_MEMORY; } tinfo->tags = taosArrayDup(preLineKV, NULL); + for(size_t i = 0; i < taosArrayGetSize(preLineKV); i++){ + SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i); + if(kv->keyEscaped)kv->key = NULL; + if(kv->valueEscaped)kv->value = NULL; + } smlSetCTableName(tinfo); tinfo->uid = info->uid++; @@ -321,7 +346,17 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + char* measure = currElement->measure; + int measureLen = currElement->measureLen; + if(currElement->measureEscaped){ + measure = (char*)taosMemoryMalloc(currElement->measureLen); + memcpy(measure, currElement->measure, currElement->measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if(currElement->measureEscaped){ + taosMemoryFree(measure); + } if (pTableMeta == NULL) { info->dataFormat = false; info->reRun = true; @@ -352,10 +387,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin break; } - bool hasSlash = false; // parse key const char *key = *sql; size_t keyLen = 0; + bool keyEscaped = false; + size_t keyLenEscaped = 0; while (*sql < sqlEnd) { if (unlikely(IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); @@ -366,16 +402,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin (*sql)++; break; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + keyLenEscaped++; + keyEscaped = true; } (*sql)++; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(key, keyLen) - } - if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -383,11 +417,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin // parse value const char *value = *sql; size_t valueLen = 0; - hasSlash = false; - bool isInQuote = false; + bool valueEscaped = false; + size_t valueLenEscaped = 0; + bool isInQuote = false; + const char *escapeChar = NULL; while (*sql < sqlEnd) { // parse value - if (unlikely(IS_QUOTE(*sql))) { + if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) { isInQuote = !isInQuote; (*sql)++; continue; @@ -395,13 +431,12 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (!isInQuote) { if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { break; - } else if (unlikely(IS_EQUAL(*sql))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; } } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) { + escapeChar = *sql; + valueEscaped = true; + valueLenEscaped++; } (*sql)++; @@ -416,9 +451,6 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(value, valueLen) - } SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; int32_t ret = smlParseValue(&kv, &info->msgBuf); @@ -427,11 +459,28 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return ret; } + if (keyEscaped){ + char *tmp = (char*)taosMemoryMalloc(kv.keyLen); + memcpy(tmp, key, kv.keyLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); + kv.key = tmp; + kv.keyEscaped = keyEscaped; + } + + if (valueEscaped){ + char *tmp = (char*)taosMemoryMalloc(kv.length); + memcpy(tmp, kv.value, kv.length); + PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); + kv.value = tmp; + kv.valueEscaped = valueEscaped; + } + if (info->dataFormat) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } // bind data @@ -440,22 +489,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin uDebug("smlBuildCol error, retry"); info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } if (cnt >= taosArrayGetSize(info->masColKVs)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt); if (kv.type != maxKV->type) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } if (unlikely(!IS_SAME_KEY)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } @@ -463,6 +516,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin maxKV->length = kv.length; info->needModifySchema = true; } + freeSSmlKv(&kv); } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); @@ -487,10 +541,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->measure = sql; // parse measure + size_t measureLenEscaped = 0; while (sql < sqlEnd) { - if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { - MOVE_FORWARD_ONE(sql, sqlEnd - sql); - sqlEnd--; + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) { + elements->measureEscaped = true; + measureLenEscaped++; + sql++; continue; } if (unlikely(IS_COMMA(sql))) { @@ -503,7 +559,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine sql++; } elements->measureLen = sql - elements->measure; - if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -581,7 +637,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, + .keyEscaped = false, + .valueEscaped = false}; if (info->dataFormat) { uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts); ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ceca06e309..16a4f55840 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); - if (topic[0] != '`') { - strtolower(topic, src); - } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } @@ -1243,9 +1240,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pParam); if (code != 0) { - tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, - vgId, epoch, tstrerror(code), requestId); - if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); @@ -1267,6 +1261,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosWriteQitem(tmq->mqueue, pRspWrapper); } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert taosMsleep(500); + } else{ + tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, + vgId, epoch, tstrerror(code), requestId); } goto CREATE_MSG_FAIL; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 76911e229a..1578b8b607 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -50,8 +50,9 @@ TEST(testCase, smlParseInfluxString_Test) { int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); - ASSERT_EQ(elements.measureLen, strlen(",st")); - ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3")); + ASSERT_EQ(elements.measureLen, strlen("\\,st")); + ASSERT_EQ(elements.measureEscaped, true); + ASSERT_EQ(elements.measureTagsLen, strlen("\\,st,t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); @@ -204,7 +205,26 @@ TEST(testCase, smlParseCols_Error_Test) { "st,t=1 c=-3.402823466e+39u64 1626006833639000000", "st,t=1 c=-339u64 1626006833639000000", "st,t=1 c=18446744073709551616u64 1626006833639000000", - "st,t=1 c=1=2 1626006833639000000"}; + "st,t=1 c=1=2 1626006833639000000,", + // escape error test + // measure comma,space + "s,t,t=1 c=1 1626006833639000000,", + "s t,t=1 c=1 1626006833639000000,", + //tag key comma,equal,space + "st,t,t=1 c=2 1626006833639000000,", + "st,t=t=1 c=2 1626006833639000000,", + "st,t t=1 c=2 1626006833639000000,", + //tag value comma,equal,space + "st,tt=a,a c=2 1626006833639000000,", + "st,t=t=a a c=2 1626006833639000000,", + "st,t t=a=a c=2 1626006833639000000,", + //field key comma,equal,space + "st,tt=aa c,1=2 1626006833639000000,", + "st,tt=aa c=1=2 1626006833639000000,", + "st,tt=aa c 1=2 1626006833639000000,", + //field value double quote,slash + "st,tt=aa c=\"a\"a\" 1626006833639000000,", + }; SSmlHandle *info = smlBuildSmlInfo(NULL); info->protocol = TSDB_SML_LINE_PROTOCOL; @@ -256,16 +276,18 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); - ASSERT_EQ(kv->length, 17); - ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); + ASSERT_EQ(kv->length, 18); + ASSERT_EQ(kv->keyEscaped, true); + ASSERT_EQ(kv->valueEscaped, false); + ASSERT_EQ(strncasecmp(kv->value, "pass\\,it ", 9), 0); // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 8); - ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); + ASSERT_EQ(kv->length, 9); + ASSERT_EQ(strncasecmp(kv->value, "ii\\=sd", 5), 0); // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d6ab974c6c..f379084cf5 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2503,9 +2503,11 @@ _exit: int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { int32_t code = 0; - ASSERT(pColData->type == pBind->buffer_type); - - if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type + if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { + ASSERT(pColData->type == pBind->buffer_type); + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3e320f952b..81b9b47e5c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -198,7 +198,7 @@ int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; int32_t tsStreamCheckpointTickInterval = 1; int32_t tsTtlUnit = 86400; -int32_t tsTtlPushInterval = 86400; +int32_t tsTtlPushInterval = 3600; int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 59afce1bbb..c195f5387c 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tmisce.h" +#include "tjson.h" #include "tglobal.h" #include "tlog.h" #include "tname.h" @@ -87,3 +88,63 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet) { return ep; } + +int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { + SJson* pJson = tjsonCreateObject(); + if (pJson == NULL) return -1; + char tmp[4096] = {0}; + + tjsonAddDoubleToObject(pJson, "reportVersion", 1); + + tjsonAddIntegerToObject(pJson, "clusterId", clusterId); + tjsonAddIntegerToObject(pJson, "startTime", startTime); + + // Do NOT invoke the taosGetFqdn here. + // this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked + // environment. The lock operation by taosGetFqdn may cause this program deadlock. + tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn); + + tjsonAddIntegerToObject(pJson, "pid", taosGetPId()); + + taosGetAppName(tmp, NULL); + tjsonAddStringToObject(pJson, "appName", tmp); + + if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { + tjsonAddStringToObject(pJson, "os", tmp); + } + + float numOfCores = 0; + if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { + tjsonAddStringToObject(pJson, "cpuModel", tmp); + tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); + } else { + tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); + } + + snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); + tjsonAddStringToObject(pJson, "memory", tmp); + + tjsonAddStringToObject(pJson, "version", version); + tjsonAddStringToObject(pJson, "buildInfo", buildinfo); + tjsonAddStringToObject(pJson, "gitInfo", gitinfo); + + tjsonAddIntegerToObject(pJson, "crashSig", signum); + tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()); + +#ifdef _TD_DARWIN_64 + taosLogTraceToBuf(tmp, sizeof(tmp), 4); +#elif !defined(WINDOWS) + taosLogTraceToBuf(tmp, sizeof(tmp), 3); +#else + taosLogTraceToBuf(tmp, sizeof(tmp), 8); +#endif + + tjsonAddStringToObject(pJson, "stackInfo", tmp); + + char* pCont = tjsonToString(pJson); + tjsonDelete(pJson); + + *pMsg = pCont; + + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 0244a4fd6e..16e7ffc536 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -119,6 +119,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) pVnode->pFetchQ->threadId); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + tqNotifyClose(pVnode->pImpl->pTq); dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ); while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10); @@ -141,7 +142,6 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) dInfo("vgId:%d, vnode is closed", pVnode->vgId); if (commitAndRemoveWal) { - char path[TSDB_FILENAME_LEN] = {0}; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP); dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path); tfsRmdir(pMgmt->pTfs, path); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 3a4f06f6fa..fcd314d2ae 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -137,12 +137,12 @@ typedef enum { } EDndReason; typedef enum { - CONSUMER_UPDATE__TOUCH = 1, + CONSUMER_UPDATE__TOUCH = 1, // rebalance req do not need change consume topic CONSUMER_UPDATE__ADD, CONSUMER_UPDATE__REMOVE, CONSUMER_UPDATE__LOST, CONSUMER_UPDATE__RECOVER, - CONSUMER_UPDATE__MODIFY, + CONSUMER_UPDATE__MODIFY, // subscribe req need change consume topic } ECsmUpdateType; typedef struct { @@ -624,7 +624,7 @@ typedef struct { SArray* rebVgs; // SArray SArray* newConsumers; // SArray SArray* removedConsumers; // SArray - SArray* touchedConsumers; // SArray + SArray* modifyConsumers; // SArray SMqSubscribeObj* pSub; SMqSubActionLogEntry* pLogEntry; } SMqRebOutputObj; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 1b146506a2..65a2fa72a2 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -247,7 +247,6 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { static void freeRebalanceItem(void *param) { SMqRebInfo *pInfo = param; - taosArrayDestroy(pInfo->lostConsumers); taosArrayDestroy(pInfo->newConsumers); taosArrayDestroy(pInfo->removedConsumers); } @@ -335,7 +334,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } taosRUnLockLatch(&pConsumer->lock); - } else if (status == MQ_CONSUMER_STATUS__MODIFY) { + } else { taosRLockLatch(&pConsumer->lock); int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); @@ -356,8 +355,6 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } taosRUnLockLatch(&pConsumer->lock); - } else { - // do nothing } mndReleaseConsumer(pMnode, pConsumer); @@ -917,34 +914,22 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, taosWLockLatch(&pOldConsumer->lock); if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { - /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ - /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ + SArray *tmp = pOldConsumer->rebNewTopics; + pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; + pNewConsumer->rebNewTopics = tmp; - // this new consumer has identical topics with one existed consumers. - if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { - pOldConsumer->status = MQ_CONSUMER_STATUS__READY; - } else { - SArray *tmp = pOldConsumer->rebNewTopics; - pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; - pNewConsumer->rebNewTopics = tmp; + tmp = pOldConsumer->rebRemovedTopics; + pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; + pNewConsumer->rebRemovedTopics = tmp; - tmp = pOldConsumer->rebRemovedTopics; - pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; - pNewConsumer->rebRemovedTopics = tmp; + tmp = pOldConsumer->assignedTopics; + pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; + pNewConsumer->assignedTopics = tmp; - tmp = pOldConsumer->assignedTopics; - pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; - pNewConsumer->assignedTopics = tmp; - - pOldConsumer->subscribeTime = pNewConsumer->upTime; - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; - } + pOldConsumer->subscribeTime = pNewConsumer->upTime; + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { - /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ - /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ - int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); - /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/ for (int32_t i = 0; i < sz; i++) { char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i)); taosArrayPush(pOldConsumer->rebRemovedTopics, &topic); @@ -958,9 +943,6 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status), pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { - /*A(taosArrayGetSize(pOldConsumer->currentTopics) == 0);*/ - /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ - int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); @@ -976,7 +958,6 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->rebalanceTime = pNewConsumer->upTime; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { - ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); // not exist in current topic @@ -1015,15 +996,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { - /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/ - /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/ char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); -#if 0 - for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { - char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); - A(strcmp(topic, removedTopic) != 0); - } -#endif // remove from removed topic removeFromRemoveTopicList(pOldConsumer, removedTopic); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index df22034068..23b2b9d7c6 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -373,6 +373,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1; if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; + if ((pCfg->hashSuffix * pCfg->hashPrefix) < 0) return -1; + if ((pCfg->hashPrefix + pCfg->hashSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) return -1; if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1; @@ -409,8 +411,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD; if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; - if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; - if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; } @@ -553,6 +553,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, int32_t dbLen = strlen(dbObj.name) + 1; mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen); dbObj.cfg.hashPrefix += dbLen; + } else if (dbObj.cfg.hashPrefix < 0) { + int32_t dbLen = strlen(dbObj.name) + 1; + mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix - dbLen); + dbObj.cfg.hashPrefix -= dbLen; } SVgObj *pVgroups = NULL; @@ -1437,7 +1441,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } -static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, SRpcMsg *pReq) { +static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; @@ -1459,7 +1463,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, SRpcMsg *pReq) { pHead->vgId = htonl(pVgroup->vgId); tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), contLen, &trimReq); - SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen, .info = pReq->info}; + SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { @@ -1495,7 +1499,7 @@ static int32_t mndProcessTrimDbReq(SRpcMsg *pReq) { goto _OVER; } - code = mndTrimDb(pMnode, pDb, pReq); + code = mndTrimDb(pMnode, pDb); _OVER: if (code != 0) { @@ -1788,6 +1792,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, int16_t hashPrefix = pDb->cfg.hashPrefix; if (hashPrefix > 0) { hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1; + } else if (hashPrefix < 0) { + hashPrefix = pDb->cfg.hashPrefix + strlen(pDb->name) + 1; } colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 49921c9a1a..015c497de1 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -269,7 +269,7 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", pSubKey, pVgEp->vgId); + mInfo("sub:%s mq re-balance addUnassignedVgroups vgId:%d from unassigned", pSubKey, pVgEp->vgId); } } @@ -289,9 +289,9 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); - // all old consumers still existing are touched - // TODO optimize: touch only consumer whose vgs changed - taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId); + // all old consumers still existing need to be modified + // TODO optimize: modify only consumer whose vgs changed + taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId); if (consumerVgNum > minVgCnt) { if (imbCnt < imbConsumerNum) { if (consumerVgNum == minVgCnt + 1) { @@ -339,13 +339,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR mInfo("sub:%s mq re-balance %d vgroups, existed consumers:%d, added:%d, removed:%d", pSubKey, totalVgNum, pInput->oldConsumerNum, numOfAdded, numOfRemoved); - // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg + // 1. build temporary hash(vgId -> SMqRebOutputVg) to store vg that need to be assigned SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - // 2. check and get actual removed consumers, put their vg into hash + // 2. check and get actual removed consumers, put their vg into pHash doRemoveExistedConsumers(pOutput, pHash, pInput); - // 3. if previously no consumer, there are vgs not assigned + // 3. if previously no consumer, there are vgs not assigned, put these vg into pHash addUnassignedVgroups(pOutput, pHash); // 4. calc vg number of each consumer @@ -364,19 +364,17 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR mInfo("sub:%s no consumer subscribe this topic", pSubKey); } - // 5. first scan: remove vgroups from te consumers, who have more vgroups than the threashold value that is - // minVgCnt, and then put them into the recycled hash list + // 5. remove vgroups from consumers who have more vgroups than the threshold value(minVgCnt or minVgCnt + 1), and then another vg into pHash transferVgroupsForConsumers(pOutput, pHash, minVgCnt, imbConsumerNum); // 6. add new consumer into sub doAddNewConsumers(pOutput, pInput); - // 7. second scan: find consumer do not have enough vgroups, extract from temporary hash and assign to them - // All related vg should be put into rebVgs SMqRebOutputVg *pRebVg = NULL; void *pRemovedIter = NULL; void *pIter = NULL; + // 7. extract bgroups from pHash and assign to consumers that do not have enough vgroups while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) { @@ -390,68 +388,52 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR // iter hash and find one vg pRemovedIter = taosHashIterate(pHash, pRemovedIter); if (pRemovedIter == NULL) { - mError("sub:%s removed iter is null", pSubKey); + mError("sub:%s removed iter is null, never can reach hear", pSubKey); break; } pRebVg = (SMqRebOutputVg *)pRemovedIter; - // push - taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; - taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId, - pConsumerEp->consumerId); + taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); + mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } - // 7. handle unassigned vg - if (taosHashGetSize(pOutput->pSub->consumerHash) != 0) { - // if has consumer, assign all left vg - while (1) { - SMqConsumerEp *pConsumerEp = NULL; + while (1) { + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + if (pIter == NULL) { + break; + } + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + + if (taosArrayGetSize(pConsumerEp->vgs) == minVgCnt) { pRemovedIter = taosHashIterate(pHash, pRemovedIter); if (pRemovedIter == NULL) { - if (pIter != NULL) { - taosHashCancelIterate(pOutput->pSub->consumerHash, pIter); - pIter = NULL; - } + mInfo("sub:%s removed iter is null", pSubKey); break; } - while (1) { - pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); - pConsumerEp = (SMqConsumerEp *)pIter; - if (taosArrayGetSize(pConsumerEp->vgs) == minVgCnt) { - break; - } - } pRebVg = (SMqRebOutputVg *)pRemovedIter; - taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; - if (pRebVg->newConsumerId == pRebVg->oldConsumerId) { - mInfo("mq rebalance: skip vg %d for same consumer:0x%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId, - pConsumerEp->consumerId); - continue; - } - taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId, - pConsumerEp->consumerId); + taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); + mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } - } else { - // if all consumer is removed, put all vg into unassigned - pIter = NULL; - SMqRebOutputVg *pRebOutput = NULL; - while (1) { - pIter = taosHashIterate(pHash, pIter); - if (pIter == NULL) { - break; - } + } - pRebOutput = (SMqRebOutputVg *)pIter; + // All assigned vg should be put into pOutput->rebVgs + if(pRemovedIter != NULL){ + mError("sub:%s error pRemovedIter should be NULL", pSubKey); + } + while (1) { + pRemovedIter = taosHashIterate(pHash, pRemovedIter); + if (pRemovedIter == NULL) { + break; + } + SMqRebOutputVg* pRebOutput = (SMqRebOutputVg *)pRemovedIter; + taosArrayPush(pOutput->rebVgs, pRebOutput); + if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed, put all vg into unassigned taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); - taosArrayPush(pOutput->rebVgs, pRebOutput); - mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", pSubKey, pRebOutput->pVgEp->vgId); } } @@ -462,19 +444,18 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, pSubKey, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } - { - pIter = NULL; - while (1) { - pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); - if (pIter == NULL) break; - SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", pSubKey, pConsumerEp->consumerId, sz); - for (int32_t i = 0; i < sz; i++) { - SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); - mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, pSubKey, pVgEp->vgId, - pConsumerEp->consumerId); - } + + pIter = NULL; + while (1) { + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + int32_t sz = taosArrayGetSize(pConsumerEp->vgs); + mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", pSubKey, pConsumerEp->consumerId, sz); + for (int32_t i = 0; i < sz; i++) { + SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); + mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, pSubKey, pVgEp->vgId, + pConsumerEp->consumerId); } } @@ -515,9 +496,9 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu // 3. commit log: consumer to update status and epoch // 3.1 set touched consumer - int32_t consumerNum = taosArrayGetSize(pOutput->touchedConsumers); + int32_t consumerNum = taosArrayGetSize(pOutput->modifyConsumers); for (int32_t i = 0; i < consumerNum; i++) { - int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->touchedConsumers, i); + int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->modifyConsumers, i); SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH; @@ -597,15 +578,15 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqDoRebalanceMsg *pReq = pMsg->pCont; void *pIter = NULL; - bool rebalanceOnce = false; // to ensure only once. +// bool rebalanceOnce = false; // to ensure only once. mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction. while (1) { - if (rebalanceOnce) { - break; - } +// if (rebalanceOnce) { +// break; +// } pIter = taosHashIterate(pReq->rebSubHash, pIter); if (pIter == NULL) { @@ -617,7 +598,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMqRebOutputObj rebOutput = {0}; rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t)); rebOutput.removedConsumers = taosArrayInit(0, sizeof(int64_t)); - rebOutput.touchedConsumers = taosArrayInit(0, sizeof(int64_t)); + rebOutput.modifyConsumers = taosArrayInit(0, sizeof(int64_t)); rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg)); SMqRebInfo *pRebInfo = (SMqRebInfo *)pIter; @@ -653,13 +634,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mndReleaseTopic(pMnode, pTopic); rebInput.oldConsumerNum = 0; - mInfo("topic:%s has no consumers sub yet", topic); + mInfo("sub topic:%s has no consumers sub yet", pRebInfo->key); } else { taosRLockLatch(&pSub->lock); rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash); rebOutput.pSub = tCloneSubscribeObj(pSub); taosRUnLockLatch(&pSub->lock); - mInfo("topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum); + mInfo("sub topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum); mndReleaseSubscribe(pMnode, pSub); } @@ -675,13 +656,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { } taosArrayDestroy(rebOutput.newConsumers); - taosArrayDestroy(rebOutput.touchedConsumers); + taosArrayDestroy(rebOutput.modifyConsumers); taosArrayDestroy(rebOutput.removedConsumers); taosArrayDestroy(rebOutput.rebVgs); tDeleteSubscribeObj(rebOutput.pSub); taosMemoryFree(rebOutput.pSub); - rebalanceOnce = true; +// taosSsleep(100); +// rebalanceOnce = true; } // reset flag diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 569c78a68c..f1cee6395b 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -196,13 +196,13 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SSdbRow *pOldRow = *ppOldRow; pOldRow->status = pRaw->status; sdbPrintOper(pSdb, pOldRow, "update"); - sdbUnLock(pSdb, type); int32_t code = 0; SdbUpdateFp updateFp = pSdb->updateFps[type]; if (updateFp != NULL) { code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } + sdbUnLock(pSdb, type); // sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pNewRow, false); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index c007f84790..acc0d29382 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -179,16 +179,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqStreamTasksScanWal(STQ* pTq); // tq util -void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId); +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); -int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); -void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver); -void saveOffsetForAllTasks(STQ* pTq, int64_t ver); -void initOffsetForAllRestoreTasks(STQ* pTq); -int32_t transferToWalReadTask(SStreamMeta* pStreamMeta, SArray* pTaskList); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2576fce998..33e331faa8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -190,6 +190,7 @@ int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); int tqInit(); void tqCleanUp(); STQ* tqOpen(const char* path, SVnode* pVnode); +void tqNotifyClose(STQ*); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8892291727..8c541e288a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -18,7 +18,9 @@ // 0: not init // 1: already inited // 2: wait to be inited or cleaup -#define WAL_READ_TASKS_ID (-1) +#define WAL_READ_TASKS_ID (-1) + +static int32_t tqInitialize(STQ* pTq); int32_t tqInit() { int8_t old; @@ -109,25 +111,32 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); + tqInitialize(pTq); + return pTq; +} + +int32_t tqInitialize(STQ* pTq) { if (tqMetaOpen(pTq) < 0) { - return NULL; + return -1; } pTq->pOffsetStore = tqOffsetOpen(pTq); if (pTq->pOffsetStore == NULL) { - return NULL; + return -1; } - pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); + pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); if (pTq->pStreamMeta == NULL) { - return NULL; + return -1; } - if (streamLoadTasks(pTq->pStreamMeta, walGetCommittedVer(pVnode->pWal)) < 0) { - return NULL; + // the version is kept in task's meta data + // todo check if this version is required or not + if (streamLoadTasks(pTq->pStreamMeta, walGetCommittedVer(pTq->pVnode->pWal)) < 0) { + return -1; } - return pTq; + return 0; } void tqClose(STQ* pTq) { @@ -145,6 +154,31 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq); } +void tqNotifyClose(STQ* pTq) { + if (pTq != NULL) { + taosWLockLatch(&pTq->pStreamMeta->lock); + + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr); + pTask->status.taskStatus = TASK_STATUS__STOP; + + int64_t st = taosGetTimestampMs(); + qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + int64_t el = taosGetTimestampMs() - st; + tqDebug("vgId:%d s-task:%s is closed in %" PRId64 "ms", pTq->pStreamMeta->vgId, pTask->id.idStr, el); + } + + taosWUnLockLatch(&pTq->pStreamMeta->lock); + } +} + static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type) { int32_t len = 0; @@ -548,12 +582,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { - // todo extract method - char buf[128] = {0}; - sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId); - int32_t vgId = TD_VID(pTq->pVnode); - pTask->id.idStr = taosStrdup(buf); + pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId); pTask->refCnt = 1; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(512 << 10); @@ -568,6 +598,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->pMsgCb = &pTq->pVnode->msgCb; pTask->pMeta = pTq->pStreamMeta; pTask->chkInfo.version = ver; + pTask->chkInfo.currentVer = ver; // expand executor if (pTask->fillHistory) { @@ -596,8 +627,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { return -1; } - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); - SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); + SReadHandle mgHandle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); if (pTask->exec.pExecutor == NULL) { @@ -623,7 +654,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { SSchemaWrapper* pschemaWrapper = pTask->tbSink.pSchemaWrapper; pTask->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1); - if(pTask->tbSink.pTSchema == NULL) { + if (pTask->tbSink.pTSchema == NULL) { return -1; } } @@ -633,8 +664,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } streamSetupTrigger(pTask); - tqInfo("vgId:%d expand stream task, s-task:%s, ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr, - pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); + tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId, + pTask->id.idStr, pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); + + // next valid version will add one + pTask->chkInfo.version += 1; return 0; } @@ -692,7 +726,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { tEncodeSStreamTaskCheckRsp(&encoder, &rsp); tEncoderClear(&encoder); - SRpcMsg rspMsg = { .code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info }; + SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info}; tmsgSendRsp(&rspMsg); return 0; } @@ -750,7 +784,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms tDecoderClear(&decoder); - // 2.save task + // 2.save task, use the newest commit version as the initial start version of stream task. code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask); if (code < 0) { tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr, @@ -958,14 +992,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { int32_t* pRef = taosMemoryMalloc(sizeof(int32_t)); *pRef = 1; + taosWLockLatch(&pTq->pStreamMeta->lock); + void* pIter = NULL; while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; + if (pIter == NULL) { + break; + } - qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver); + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + qDebug("s-task:%s delete req enqueue, ver: %" PRId64, pTask->id.idStr, ver); if (!failed) { SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); @@ -975,15 +1016,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { atomic_add_fetch_32(pRefBlock->dataRef, 1); if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->id.taskId); - atomic_sub_fetch_32(pRef, 1); taosFreeQitem(pRefBlock); continue; } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->id.taskId); + qError("s-task:%s stream task launch failed", pTask->id.idStr); continue; } @@ -992,8 +1031,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } } + taosWUnLockLatch(&pTq->pStreamMeta->lock); + int32_t ref = atomic_sub_fetch_32(pRef, 1); - /*A(ref >= 0);*/ if (ref == 0) { blockDataDestroy(pDelBlock); taosMemoryFree(pRef); @@ -1024,23 +1064,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } blockDataDestroy(pDelBlock); #endif - return 0; } -static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit, - const char* key, int64_t ver) { - doSaveTaskOffset(pOffsetStore, key, ver); - int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver); - - // remove the offset, if all functions are completed successfully. - if (code == TSDB_CODE_SUCCESS) { - tqOffsetDelete(pOffsetStore, key); - } - - return code; -} - int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { #if 0 void* pIter = NULL; @@ -1122,8 +1148,8 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr); streamProcessRunReq(pTask); } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) { - tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, - pTask->id.idStr, pTask->chkInfo.version); + tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, pTask->id.idStr, + pTask->chkInfo.version); streamProcessRunReq(pTask); } else { tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); @@ -1149,11 +1175,12 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); if (pTask) { - SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchReq(pTask, &req, &rsp, exec); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { + tDeleteStreamDispatchReq(&req); return -1; } } @@ -1190,12 +1217,13 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { int32_t taskId = req.dstTaskId; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask) { - SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessRetrieveReq(pTask, &req, &rsp); streamMetaReleaseTask(pTq->pStreamMeta, pTask); tDeleteStreamRetrieveReq(&req); return 0; } else { + tDeleteStreamRetrieveReq(&req); return -1; } } @@ -1226,7 +1254,7 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask) { - SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchReq(pTask, &req, &rsp, false); streamMetaReleaseTask(pTq->pStreamMeta, pTask); rpcFreeCont(pMsg->pCont); @@ -1243,7 +1271,7 @@ FAIL: SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); if (pRspHead == NULL) { - SRpcMsg rsp = { .code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info }; + SRpcMsg rsp = {.code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info}; tqDebug("send dispatch error rsp, code: %x", code); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); @@ -1276,6 +1304,13 @@ int32_t tqStartStreamTasks(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; taosWLockLatch(&pMeta->lock); + int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks); + if (numOfTasks == 0) { + tqInfo("vgId:%d no stream tasks exists", vgId); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return 0; + } + pMeta->walScan += 1; if (pMeta->walScan > 1) { @@ -1287,16 +1322,12 @@ int32_t tqStartStreamTasks(STQ* pTq) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr(terrno)); + tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr()); taosWUnLockLatch(&pTq->pStreamMeta->lock); return -1; } - int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks); - tqDebug("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks); - initOffsetForAllRestoreTasks(pTq); - pRunReq->head.vgId = vgId; pRunReq->streamId = 0; pRunReq->taskId = WAL_READ_TASKS_ID; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 25ab7209d2..2cda12c0e1 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1023,6 +1023,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } // update the table list handle for each stream scanner/wal reader + taosWLockLatch(&pTq->pStreamMeta->lock); while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) { @@ -1039,5 +1040,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index f96573702d..7109e0acc9 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -15,25 +15,23 @@ #include "tq.h" -static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle); -static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList); +static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle); // this function should be executed by stream threads. // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure // will not stop eventually. -int tqStreamTasksScanWal(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); +int32_t tqStreamTasksScanWal(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; - int64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampMs(); while (1) { int32_t scan = pMeta->walScan; tqDebug("vgId:%d continue check if data in wal are available, scan:%d", vgId, scan); - ASSERT(scan >= 1); // check all restore tasks bool shouldIdle = true; - streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &shouldIdle); + createStreamRunReq(pTq->pStreamMeta, &shouldIdle); int32_t times = 0; @@ -51,50 +49,19 @@ int tqStreamTasksScanWal(STQ* pTq) { taosWUnLockLatch(&pMeta->lock); tqDebug("vgId:%d scan wal for stream tasks for %d times", vgId, times); - } else { - tqDebug("vgId:%d no idle, scan wal for stream tasks for %d times", vgId, pMeta->walScan); - ASSERT(pMeta->walScan >= 1); } } int64_t el = (taosGetTimestampMs() - st); - tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%"PRId64" ms", vgId, el); - - // restore wal scan flag -// atomic_store_8(&pTq->pStreamMeta->walScan, 0); + tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%" PRId64 " ms", vgId, el); return 0; } -//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) { -// int32_t numOfTask = taosArrayGetSize(pTaskList); -// if (numOfTask <= 0) { -// return TSDB_CODE_SUCCESS; -// } -// -// // todo: add lock -// for (int32_t i = 0; i < numOfTask; ++i) { -// SStreamTask* pTask = taosArrayGetP(pTaskList, i); -// tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64, -// pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id); -// taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); -// -// // NOTE: do not change the following order -// atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); -// taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES); -// } -// -// return TSDB_CODE_SUCCESS; -//} - -int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) { +static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) { + SArray* pTaskIdList = taosArrayInit(numOfTasks, sizeof(int32_t)); void* pIter = NULL; - int32_t vgId = pStreamMeta->vgId; - - *pScanIdle = true; - - bool allWalChecked = true; - tqDebug("vgId:%d start to check wal to extract new submit block", vgId); + taosWLockLatch(&pStreamMeta->lock); while (1) { pIter = taosHashIterate(pStreamMeta->pTasks, pIter); if (pIter == NULL) { @@ -102,44 +69,71 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto } SStreamTask* pTask = *(SStreamTask**)pIter; + taosArrayPush(pTaskIdList, &pTask->id.taskId); + } + + taosWUnLockLatch(&pStreamMeta->lock); + return pTaskIdList; +} + +int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { + *pScanIdle = true; + bool noNewDataInWal = true; + int32_t vgId = pStreamMeta->vgId; + + int32_t numOfTasks = taosHashGetSize(pStreamMeta->pTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks); + SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks); + + // update the new task number + numOfTasks = taosArrayGetSize(pTaskIdList); + for (int32_t i = 0; i < numOfTasks; ++i) { + int32_t* pTaskId = taosArrayGet(pTaskIdList, i); + SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId); + if (pTask == NULL) { + continue; + } + + int32_t status = pTask->status.taskStatus; if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + tqDebug("s-task:%s not source task, no need to start", pTask->id.idStr); + streamMetaReleaseTask(pStreamMeta, pTask); continue; } - if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || - pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, - pTask->status.taskStatus); + if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE || + status == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, status); + streamMetaReleaseTask(pStreamMeta, pTask); continue; } - // check if offset value exists - char key[128] = {0}; - createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - if (tInputQueueIsFull(pTask)) { tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr); + streamMetaReleaseTask(pStreamMeta, pTask); continue; } *pScanIdle = false; - // check if offset value exists - STqOffset* pOffset = tqOffsetRead(pOffsetStore, key); - ASSERT(pOffset != NULL); - // seek the stored version and extract data from WAL - int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version); + int32_t code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + streamMetaReleaseTask(pStreamMeta, pTask); continue; } // append the data for the stream - tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr); + tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); SPackedData packData = {0}; code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); if (code != TSDB_CODE_SUCCESS) { // failed, continue + streamMetaReleaseTask(pStreamMeta, pTask); continue; } @@ -147,28 +141,31 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto if (p == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr); + streamMetaReleaseTask(pStreamMeta, pTask); continue; } - allWalChecked = false; + noNewDataInWal = false; - tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr); code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); if (code == TSDB_CODE_SUCCESS) { - pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader); + pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, - pOffset->val.version); + pTask->chkInfo.currentVer); } else { - // do nothing + tqError("s-task:%s append input queue failed, ver:%" PRId64, pTask->id.idStr, pTask->chkInfo.currentVer); } streamDataSubmitDestroy(p); taosFreeQitem(p); + streamMetaReleaseTask(pStreamMeta, pTask); } - if (allWalChecked) { + // all wal are checked, and no new data available in wal. + if (noNewDataInWal) { *pScanIdle = true; } + + taosArrayDestroy(pTaskIdList); return 0; } - diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 4c37e1052f..017b479c1b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -19,19 +19,10 @@ static int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp); -// stream_task:stream_id:task_id -void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) { - int32_t n = 12; - char* p = dst; - - memcpy(p, "stream_task:", n); - p += n; - - int32_t inc = tintToHex(streamId, p); - p += inc; - - *(p++) = ':'; - tintToHex(taskId, p); +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { + char buf[128] = {0}; + sprintf(buf, "0x%" PRIx64 "-%d", streamId, taskId); + return taosStrdup(buf); } int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) { @@ -49,75 +40,6 @@ int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueI return TSDB_CODE_SUCCESS; } -void initOffsetForAllRestoreTasks(STQ* pTq) { - void* pIter = NULL; - - while(1) { - pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) { - break; - } - - SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) { - continue; - } - - if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus); - continue; - } - - char key[128] = {0}; - createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); - if (pOffset == NULL) { - doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version); - } - } -} - -void saveOffsetForAllTasks(STQ* pTq, int64_t ver) { - void* pIter = NULL; - - while(1) { - pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) { - break; - } - - SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) { - continue; - } - - if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, - pTask->status.taskStatus); - continue; - } - - char key[128] = {0}; - createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); - if (pOffset == NULL) { - doSaveTaskOffset(pTq->pOffsetStore, key, ver); - } - } -} - -void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) { - STqOffset offset = {0}; - tqOffsetResetToLog(&offset.val, ver); - - tstrncpy(offset.subKey, pKey, tListLen(offset.subKey)); - - // keep the offset info in the offset store - tqOffsetWrite(pOffsetStore, &offset); -} - static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { pRsp->reqOffset = pReq->reqOffset; @@ -144,6 +66,21 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { + if (pRsp->blockData != NULL) { + pRsp->blockData = taosArrayDestroy(pRsp->blockData); + } + + if (pRsp->blockDataLen != NULL) { + pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); + } + + if (pRsp->blockTbName != NULL) { + pRsp->blockTbName = taosArrayDestroy(pRsp->blockTbName); + } + + if (pRsp->blockSchema != NULL) { + pRsp->blockSchema = taosArrayDestroy(pRsp->blockSchema); + } return -1; } @@ -277,6 +214,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { + tDeleteSTaosxRsp(&taosxRsp); return -1; } @@ -352,6 +290,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tDeleteSTaosxRsp(&taosxRsp); return code; } + code = 0; taosMemoryFreeClear(pCkHead); tDeleteSTaosxRsp(&taosxRsp); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 89686c3d33..5bd41dd86f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1881,8 +1881,8 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_ return pReader->pSchema; } - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); - if (pReader->pSchema == NULL) { + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); + if (code != TSDB_CODE_SUCCESS || pReader->pSchema == NULL) { tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); } @@ -1890,9 +1890,15 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_ } static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + int32_t code = 0; + // always set the newest schema version in pReader->pSchema if (pReader->pSchema == NULL) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); + code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } } if (pReader->pSchema && sversion == pReader->pSchema->version) { @@ -1905,7 +1911,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* } STSchema* ptr = NULL; - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); + code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -1969,7 +1975,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* // DESC: mem -----> imem -----> last block -----> file block if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { - init = true; + init = true; // todo check if pReader->pSchema is null or not int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2014,6 +2020,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2222,6 +2232,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (pSchema == NULL) { return code; } + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); if (piSchema == NULL) { return code; @@ -3843,11 +3854,8 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - - code = tsdbRowMergerInit(&merge, pReader->pSchema, ¤t, pTSchema); + STSchema* ps = (pReader->pSchema != NULL)? pReader->pSchema:pTSchema; + code = tsdbRowMergerInit(&merge, ps, ¤t, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3891,7 +3899,14 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return terrno; + } if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); @@ -4000,10 +4015,11 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT int64_t uid = pScanInfo->uid; int32_t code = TSDB_CODE_SUCCESS; - int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); + if (pSchema == NULL) { + return terrno; + } SColVal colVal = {0}; int32_t i = 0, j = 0; @@ -4235,8 +4251,8 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { } static void freeSchemaFunc(void* param) { - void* p = *(void**)param; - taosMemoryFree(p); + void **p = (void **)param; + taosMemoryFreeClear(*p); } // ====================================== EXPOSED APIs ====================================== @@ -5187,8 +5203,6 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { } int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { - int32_t sversion = 1; - SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); int32_t code = metaGetTableEntryByUidCache(&mr, uid); @@ -5200,6 +5214,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 *suid = 0; + // only child table and ordinary table is allowed, super table is not allowed. if (mr.me.type == TSDB_CHILD_TABLE) { tDecoderClear(&mr.coder); *suid = mr.me.ctbEntry.suid; @@ -5209,9 +5224,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 metaReaderClear(&mr); return terrno; } - sversion = mr.me.stbEntry.schemaRow.version; - } else if (mr.me.type == TSDB_NORMAL_TABLE) { - sversion = mr.me.ntbEntry.schemaRow.version; + } else if (mr.me.type == TSDB_NORMAL_TABLE) { // do nothing } else { terrno = TSDB_CODE_INVALID_PARA; metaReaderClear(&mr); @@ -5219,9 +5232,10 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 } metaReaderClear(&mr); - *pSchema = metaGetTbTSchema(pVnode->pMeta, uid, sversion, 1); - return TSDB_CODE_SUCCESS; + // get the newest table schema version + code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema); + return code; } int32_t tsdbTakeReadSnap(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index fb7f2901a1..500e9cb52d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -547,7 +547,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) walApplyVer(pVnode->pWal, commitIdx); pVnode->restored = true; - vInfo("vgId:%d, sync restore finished", pVnode->config.vgId); + vInfo("vgId:%d, sync restore finished, start to restore stream tasks by replay wal", pVnode->config.vgId); // start to restore all stream tasks tqStartStreamTasks(pVnode->pTq); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 58c43829cf..6efdd8d8eb 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -278,7 +278,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch char* retentions = buildRetension(pCfg->pRetensions); int32_t dbFNameLen = strlen(dbFName); - int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0; + int32_t hashPrefix = 0; + if (pCfg->hashPrefix > 0) { + hashPrefix = pCfg->hashPrefix - dbFNameLen - 1; + } else if (pCfg->hashPrefix < 0) { + hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; + } len += sprintf( buf2 + VARSTR_HEADER_SIZE, diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index de36151af9..3925de2764 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -109,6 +109,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList); uint64_t tableListGetSuid(const STableListInfo* pTableList); STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index); int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex); +void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psuid, uint64_t* uid, int32_t* type); size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 0fe8ed0a33..ae1e7743a5 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -429,7 +429,6 @@ typedef struct STimeWindowAggSupp { } STimeWindowAggSupp; typedef struct SStreamScanInfo { - uint64_t tableUid; // queried super table uid SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SExprSupp tbnameCalSup; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index dc3bf83a91..d693faf7f1 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -240,6 +240,7 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); if (NULL == deleter) { + taosMemoryFree(pParam); code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index e9c46843c0..90d740bebd 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -408,6 +408,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat void* pParam) { SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); if (NULL == inserter) { + taosMemoryFree(pParam); terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 7d318786ba..f61fd1ae01 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -27,15 +27,21 @@ #include "executorimpl.h" #include "tcompression.h" +typedef struct STableListIdInfo { + uint64_t suid; + uint64_t uid; + int32_t tableType; +} STableListIdInfo; + // If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly // The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups struct STableListInfo { - bool oneTableForEachGroup; - int32_t numOfOuputGroups; // the data block will be generated one by one - int32_t* groupOffset; // keep the offset value for each group in the tableList - SArray* pTableList; - SHashObj* map; // speedup acquire the tableQueryInfo by table uid - uint64_t suid; + bool oneTableForEachGroup; + int32_t numOfOuputGroups; // the data block will be generated one by one + int32_t* groupOffset; // keep the offset value for each group in the tableList + SArray* pTableList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid + STableListIdInfo idInfo; // this maybe the super table or ordinary table }; typedef struct tagFilterAssist { @@ -470,7 +476,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis } // int64_t stt = taosGetTimestampUs(); - code = metaGetTableTags(metaHandle, pTableListInfo->suid, pUidTagList); + code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -953,7 +959,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN FilterCondType condType = checkTagCond(pTagCond); - int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->suid, pUidTagList, pTagCond); + int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond); if (filter == 0) { // tbname in filter is activated, do nothing and return taosArrayClear(pUidList); @@ -966,12 +972,12 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN terrno = 0; } else { if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { - code = metaGetTableTagsByUids(metaHandle, pListInfo->suid, pUidTagList); + code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList); } else { - code = metaGetTableTags(metaHandle, pListInfo->suid, pUidTagList); + code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList); } if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->suid); + qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid); terrno = code; goto end; } @@ -1025,15 +1031,17 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, int32_t code = TSDB_CODE_SUCCESS; size_t numOfTables = 0; - pListInfo->suid = pScanNode->suid; + pListInfo->idInfo.suid = pScanNode->suid; + pListInfo->idInfo.tableType = pScanNode->tableType; + SArray* pUidList = taosArrayInit(8, sizeof(uint64_t)); SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { + pListInfo->idInfo.uid = pScanNode->uid; if (metaIsTableExist(metaHandle, pScanNode->uid)) { taosArrayPush(pUidList, &pScanNode->uid); } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { goto _end; @@ -1794,7 +1802,9 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) { return taosArrayGetSize(pTableList->pTableList); } -uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->suid; } +uint64_t tableListGetSuid(const STableListInfo* pTableList) { + return pTableList->idInfo.suid; +} STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { if (taosArrayGetSize(pTableList->pTableList) == 0) { @@ -1819,6 +1829,12 @@ int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t st return -1; } +void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psuid, uint64_t* uid, int32_t* type) { + *psuid = pTableList->idInfo.suid; + *uid = pTableList->idInfo.uid; + *type = pTableList->idInfo.tableType; +} + uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid)); ASSERT(pTableList->map != NULL && slot != NULL); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6e3a7d8725..546cd18cda 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,6 +14,7 @@ */ #include "executor.h" +#include #include "executorimpl.h" #include "planner.h" #include "tdatablock.h" @@ -327,6 +328,13 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S return qa; } + STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; + + uint64_t suid = 0; + uint64_t uid = 0; + int32_t type = 0; + tableListGetSourceTableInfo(pTableScanInfo->base.pTableListInfo, &suid, &uid, &type); + // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); @@ -341,9 +349,21 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S tDecoderClear(&mr.coder); - // TODO handle ntb case - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { + if (mr.me.type == TSDB_SUPER_TABLE) { continue; + } else { + if (type == TSDB_SUPER_TABLE) { + // this new created child table does not belong to the scanned super table. + if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != suid) { + continue; + } + } else { // ordinary table + // In case that the scanned target table is an ordinary table. When replay the WAL during restore the vnode, we + // should check all newly created ordinary table to make sure that this table isn't the destination table. + if (mr.me.uid != uid) { + continue; + } + } } if (pScanInfo->pTagCond != NULL) { @@ -382,7 +402,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); + SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id); int32_t numOfQualifiedTables = taosArrayGetSize(qa); qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); @@ -497,10 +517,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, goto _error; } + // pSinkParam has been freed during create sinker. code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pSinkParam); - } } qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); @@ -1090,6 +1108,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanBaseInfo->dataReader = NULL; // let's seek to the next version in wal file + int64_t firstVer = walGetFirstVer(pInfo->tqReader->pWalReader->pWal); + if (pOffset->version + 1 < firstVer){ + pOffset->version = firstVer - 1; + } + if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); return -1; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 446bf0ef10..397b339bba 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -691,7 +691,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } uint32_t status = 0; - int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); + code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2437,7 +2437,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } pInfo->readHandle = *pHandle; - pInfo->tableUid = pScanPhyNode->uid; pTaskInfo->streamInfo.snapshotVer = pHandle->version; pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 1abe678ac6..c78e6002cd 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1482,11 +1482,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { pInfo->pIdx->init = 1; SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); return blk; - } else if (flt == -2) { - qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); - return NULL; - } else if (flt == -1) { - // not idx + } else if ((flt == -1) || (flt == -2)) { qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); } } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index f198277cdc..2d11410818 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2529,6 +2529,20 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } while (1) { + if (isTaskKilled(pTaskInfo)) { + + if (pInfo->pUpdated != NULL) { + pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated); + } + + if (pInfo->pUpdatedMap != NULL) { + tSimpleHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; + } + + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { pOperator->status = OP_RES_TO_RETURN; @@ -2605,6 +2619,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { taosArrayPush(pInfo->pUpdated, pIte); } + tSimpleHashCleanup(pInfo->pUpdatedMap); pInfo->pUpdatedMap = NULL; taosArraySort(pInfo->pUpdated, winPosCmprImpl); diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 9d11d7b376..f23b4d3e87 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -79,6 +79,26 @@ ENDIF () target_link_libraries( udf1 PUBLIC os ${LINK_JEMALLOC}) + +add_library(udf1_dup STATIC MODULE test/udf1_dup.c) +target_include_directories( + udf1_dup + PUBLIC + "${TD_SOURCE_DIR}/include/libs/function" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/client" + "${TD_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udf1_dup jemalloc) +ENDIF () + +target_link_libraries( + udf1_dup PUBLIC os ${LINK_JEMALLOC}) + add_library(udf2 STATIC MODULE test/udf2.c) target_include_directories( udf2 @@ -99,6 +119,26 @@ target_link_libraries( udf2 PUBLIC os ${LINK_JEMALLOC} ) +add_library(udf2_dup STATIC MODULE test/udf2_dup.c) +target_include_directories( + udf2_dup + PUBLIC + "${TD_SOURCE_DIR}/include/libs/function" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/client" + "${TD_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udf2_dup jemalloc) +ENDIF () + +target_link_libraries( + udf2_dup PUBLIC os ${LINK_JEMALLOC} +) + #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) add_executable(udfd src/udfd.c) target_include_directories( diff --git a/source/libs/function/test/udf1_dup.c b/source/libs/function/test/udf1_dup.c new file mode 100644 index 0000000000..c251192da3 --- /dev/null +++ b/source/libs/function/test/udf1_dup.c @@ -0,0 +1,42 @@ +#include +#include +#include +#ifdef LINUX +#include +#endif +#ifdef WINDOWS +#include +#endif +#include "taosudf.h" + + +DLL_EXPORT int32_t udf1_dup_init() { return 0; } + +DLL_EXPORT int32_t udf1_dup_destroy() { return 0; } + +DLL_EXPORT int32_t udf1_dup(SUdfDataBlock *block, SUdfColumn *resultCol) { + SUdfColumnData *resultData = &resultCol->colData; + for (int32_t i = 0; i < block->numOfRows; ++i) { + int j = 0; + for (; j < block->numOfCols; ++j) { + if (udfColDataIsNull(block->udfCols[j], i)) { + udfColDataSetNull(resultCol, i); + break; + } + } + if (j == block->numOfCols) { + int32_t luckyNum = 2; + udfColDataSet(resultCol, i, (char *)&luckyNum, false); + } + } + // to simulate actual processing delay by udf +#ifdef LINUX + usleep(1 * 1000); // usleep takes sleep time in us (1 millionth of a second) +#endif +#ifdef WINDOWS + Sleep(1); +#endif + resultData->numOfRows = block->numOfRows; + return 0; +} + diff --git a/source/libs/function/test/udf2_dup.c b/source/libs/function/test/udf2_dup.c new file mode 100644 index 0000000000..1a98190823 --- /dev/null +++ b/source/libs/function/test/udf2_dup.c @@ -0,0 +1,78 @@ +#include +#include +#include +#include + +#include "taosudf.h" + +DLL_EXPORT int32_t udf2_dup_init() { return 0; } + +DLL_EXPORT int32_t udf2_dup_destroy() { return 0; } + +DLL_EXPORT int32_t udf2_dup_start(SUdfInterBuf* buf) { + *(int64_t*)(buf->buf) = 0; + buf->bufLen = sizeof(double); + buf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t udf2_dup(SUdfDataBlock* block, SUdfInterBuf* interBuf, SUdfInterBuf* newInterBuf) { + double sumSquares = 0; + if (interBuf->numOfResult == 1) { + sumSquares = *(double*)interBuf->buf; + } + int8_t numNotNull = 0; + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + for (int32_t i = 0; i < block->numOfCols; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { + SUdfColumn* col = block->udfCols[i]; + if (udfColDataIsNull(col, j)) { + continue; + } + switch (col->colMeta.type) { + case TSDB_DATA_TYPE_INT: { + char* cell = udfColDataGetData(col, j); + int32_t num = *(int32_t*)cell; + sumSquares += (double)num * num; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + char* cell = udfColDataGetData(col, j); + double num = *(double*)cell; + sumSquares += num * num; + break; + } + default: + break; + } + ++numNotNull; + } + } + + *(double*)(newInterBuf->buf) = sumSquares; + newInterBuf->bufLen = sizeof(double); + + if (interBuf->numOfResult == 0 && numNotNull == 0) { + newInterBuf->numOfResult = 0; + } else { + newInterBuf->numOfResult = 1; + } + return 0; +} + +DLL_EXPORT int32_t udf2_dup_finish(SUdfInterBuf* buf, SUdfInterBuf* resultData) { + if (buf->numOfResult == 0) { + resultData->numOfResult = 0; + return 0; + } + double sumSquares = *(double*)(buf->buf); + *(double*)(resultData->buf) = sqrt(sumSquares) + 100; + resultData->bufLen = sizeof(double); + resultData->numOfResult = 1; + return 0; +} diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y old mode 100644 new mode 100755 index d7a6baaffe..e64fb0f3fa --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -221,8 +221,8 @@ db_options(A) ::= db_options(B) WAL_RETENTION_SIZE NK_MINUS(D) NK_INTEGER(C). db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); } db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); } db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); } -db_options(A) ::= db_options(B) TABLE_PREFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, &C); } -db_options(A) ::= db_options(B) TABLE_SUFFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, &C); } +db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); } +db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2afe34c1f7..5a47ed731d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1024,12 +1024,28 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED case DB_OPTION_STT_TRIGGER: pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; - case DB_OPTION_TABLE_PREFIX: - pDbOptions->tablePrefix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + case DB_OPTION_TABLE_PREFIX: { + SValueNode *pNode = (SValueNode *)pVal; + if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) { + pDbOptions->tablePrefix = taosStr2Int32(pNode->literal, NULL, 10); + } else { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_prefix data type"); + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; + } + nodesDestroyNode((SNode*)pNode); break; - case DB_OPTION_TABLE_SUFFIX: - pDbOptions->tableSuffix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + } + case DB_OPTION_TABLE_SUFFIX:{ + SValueNode *pNode = (SValueNode *)pVal; + if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) { + pDbOptions->tableSuffix = taosStr2Int32(pNode->literal, NULL, 10); + } else { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_suffix data type"); + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; + } + nodesDestroyNode((SNode*)pNode); break; + } default: break; } diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 01a635e4b2..922a0f45ff 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -251,7 +251,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in goto _return; } - if (bind[c].buffer_type != pColSchema->type) { + if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); goto _return; } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 312584994f..49da87dc16 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -692,7 +692,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC len = tGetToken(&str[*i + t0.n + 1], &type); // only id and string are valid - if ((TK_NK_STRING != t0.type) && (TK_NK_ID != t0.type)) { + if (((TK_NK_STRING != t0.type) && (TK_NK_ID != t0.type)) || ((TK_NK_STRING != type) && (TK_NK_ID != type))) { t0.type = TK_NK_ILLEGAL; t0.n = 0; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 25e92a55ec..4c8cfa435a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4184,6 +4184,34 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return TSDB_CODE_SUCCESS; } + +static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbPrefix, int32_t tbSuffix) { + if (tbPrefix < TSDB_MIN_HASH_PREFIX || tbPrefix > TSDB_MAX_HASH_PREFIX) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option table_prefix: %d valid range: [%d, %d]", tbPrefix, + TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX); + } + + if (tbSuffix < TSDB_MIN_HASH_SUFFIX || tbSuffix > TSDB_MAX_HASH_SUFFIX) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option table_suffix: %d valid range: [%d, %d]", tbSuffix, + TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX); + } + + if ((tbPrefix * tbSuffix) < 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option table_prefix & table_suffix: mixed usage not allowed"); + } + + if ((tbPrefix + tbSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option table_prefix & table_suffix: exceed max table name length"); + } + + return TSDB_CODE_SUCCESS; +} + + static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t daysPerFile = pOptions->daysPerFile; int64_t daysToKeep0 = pOptions->keep[0]; @@ -4291,10 +4319,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX); - } - if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "tableSuffix", pOptions->tableSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX); + code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix); } if (TSDB_CODE_SUCCESS == code) { code = checkOptionsDependency(pCxt, pDbName, pOptions); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index a38e2368dd..fbd14086ff 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -142,6 +142,7 @@ typedef union { #define YYFALLBACK 1 #define YYNSTATE 762 #define YYNRULE 583 +#define YYNRULE_WITH_ACTION 583 #define YYNTOKEN 330 #define YY_MAX_SHIFT 761 #define YY_MIN_SHIFTREDUCE 1136 @@ -217,6 +218,7 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ +<<<<<<< HEAD #define YY_ACTTAB_COUNT (3030) static const YYACTIONTYPE yy_action[] = { /* 0 */ 2116, 209, 2010, 1881, 432, 502, 1883, 1765, 431, 2102, @@ -831,11 +833,637 @@ static const YYCODETYPE yy_lookahead[] = { #define YY_SHIFT_COUNT (761) #define YY_SHIFT_MIN (0) #define YY_SHIFT_MAX (2185) +======= +#define YY_ACTTAB_COUNT (2904) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 2116, 2010, 1881, 396, 434, 168, 636, 1734, 433, 1989, + /* 10 */ 672, 162, 46, 44, 1646, 1723, 2008, 642, 374, 1894, + /* 20 */ 393, 506, 1495, 440, 39, 38, 1942, 503, 45, 43, + /* 30 */ 42, 41, 40, 1576, 1791, 1493, 123, 2134, 1520, 122, + /* 40 */ 121, 120, 119, 118, 117, 116, 115, 114, 2279, 2084, + /* 50 */ 593, 671, 593, 2275, 194, 2275, 39, 38, 167, 1571, + /* 60 */ 45, 43, 42, 41, 40, 19, 1833, 340, 2281, 186, + /* 70 */ 2281, 186, 1501, 2276, 619, 2276, 619, 45, 43, 42, + /* 80 */ 41, 40, 2115, 501, 654, 2151, 502, 1758, 169, 2117, + /* 90 */ 675, 2119, 2120, 670, 182, 665, 1520, 758, 36, 298, + /* 100 */ 15, 735, 734, 733, 732, 405, 1931, 731, 730, 144, + /* 110 */ 725, 724, 723, 722, 721, 720, 719, 157, 715, 714, + /* 120 */ 713, 404, 403, 710, 709, 708, 175, 174, 594, 2241, + /* 130 */ 509, 654, 1320, 502, 1758, 123, 1578, 1579, 122, 121, + /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 697, 696, + /* 150 */ 695, 1315, 694, 1317, 1318, 693, 690, 179, 1326, 687, + /* 160 */ 1328, 1329, 684, 681, 49, 640, 1551, 1561, 654, 2219, + /* 170 */ 655, 1892, 1577, 1580, 1720, 39, 38, 360, 1993, 45, + /* 180 */ 43, 42, 41, 40, 1409, 1410, 1496, 1523, 1494, 133, + /* 190 */ 655, 1892, 387, 62, 1715, 2216, 539, 39, 38, 280, + /* 200 */ 165, 45, 43, 42, 41, 40, 39, 38, 1894, 191, + /* 210 */ 45, 43, 42, 41, 40, 1499, 1500, 1868, 1550, 1553, + /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, 1569, + /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 46, 44, 1722, 179, + /* 240 */ 62, 343, 93, 1518, 393, 408, 1495, 62, 49, 407, + /* 250 */ 471, 2116, 2280, 485, 352, 2275, 484, 1576, 1989, 1493, + /* 260 */ 1994, 672, 132, 131, 130, 129, 128, 127, 126, 125, + /* 270 */ 124, 2279, 454, 1522, 486, 2276, 2278, 456, 432, 396, + /* 280 */ 431, 707, 2280, 1571, 1275, 2275, 211, 165, 2134, 19, + /* 290 */ 504, 593, 1765, 1714, 2275, 1894, 1501, 1274, 1605, 1685, + /* 300 */ 2084, 2279, 671, 196, 2010, 2276, 2277, 430, 618, 2281, + /* 310 */ 186, 2275, 518, 2116, 2276, 619, 386, 621, 66, 2007, + /* 320 */ 642, 758, 361, 672, 15, 1767, 617, 186, 1264, 655, + /* 330 */ 1892, 2276, 619, 2115, 444, 257, 2151, 655, 1892, 110, + /* 340 */ 2117, 675, 2119, 2120, 670, 189, 665, 1521, 133, 143, + /* 350 */ 2134, 150, 2175, 2204, 1606, 544, 55, 389, 2200, 487, + /* 360 */ 1578, 1579, 2084, 482, 671, 1266, 476, 475, 474, 473, + /* 370 */ 470, 469, 468, 467, 466, 462, 461, 460, 459, 342, + /* 380 */ 451, 450, 449, 613, 446, 445, 359, 1650, 1520, 1521, + /* 390 */ 1551, 1561, 189, 1520, 545, 2115, 1577, 1580, 2151, 189, + /* 400 */ 189, 110, 2117, 675, 2119, 2120, 670, 707, 665, 1191, + /* 410 */ 1496, 1190, 1494, 2295, 62, 2204, 1262, 630, 140, 389, + /* 420 */ 2200, 279, 50, 608, 641, 630, 140, 1745, 35, 391, + /* 430 */ 1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499, + /* 440 */ 1500, 1192, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, + /* 450 */ 1560, 667, 663, 1569, 1570, 1572, 1573, 1574, 1575, 2, + /* 460 */ 12, 46, 44, 1354, 1355, 1495, 402, 401, 62, 393, + /* 470 */ 1279, 1495, 557, 556, 555, 516, 2084, 2003, 1493, 547, + /* 480 */ 137, 551, 1576, 1278, 1493, 550, 2280, 2116, 32, 1502, + /* 490 */ 549, 554, 368, 367, 39, 38, 548, 672, 45, 43, + /* 500 */ 42, 41, 40, 1522, 1523, 1426, 1427, 2116, 1571, 614, + /* 510 */ 609, 602, 1674, 699, 19, 1501, 1935, 669, 185, 2212, + /* 520 */ 2213, 1501, 138, 2217, 2134, 632, 184, 2212, 2213, 398, + /* 530 */ 138, 2217, 1937, 1939, 2051, 1191, 2084, 1190, 671, 1552, + /* 540 */ 758, 1425, 1428, 1519, 2134, 489, 758, 39, 38, 15, + /* 550 */ 228, 45, 43, 42, 41, 40, 2084, 153, 671, 605, + /* 560 */ 604, 1672, 1673, 1675, 1676, 1677, 189, 1192, 166, 2115, + /* 570 */ 1744, 580, 2151, 318, 189, 170, 2117, 675, 2119, 2120, + /* 580 */ 670, 246, 665, 28, 1708, 1578, 1579, 316, 73, 2115, + /* 590 */ 12, 72, 2151, 2094, 562, 334, 2117, 675, 2119, 2120, + /* 600 */ 670, 668, 665, 656, 2169, 655, 1892, 2102, 107, 572, + /* 610 */ 209, 497, 495, 492, 54, 1551, 1561, 2098, 612, 2084, + /* 620 */ 189, 1577, 1580, 242, 438, 141, 620, 2296, 213, 1496, + /* 630 */ 478, 1494, 504, 1884, 1765, 1496, 1794, 1494, 165, 565, + /* 640 */ 1944, 1944, 1505, 641, 559, 2134, 1895, 358, 373, 241, + /* 650 */ 62, 279, 195, 2100, 376, 1942, 1942, 1501, 1499, 1500, + /* 660 */ 1169, 289, 290, 665, 1499, 1500, 288, 1550, 1553, 1554, + /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, 1569, 1570, + /* 680 */ 1572, 1573, 1574, 1575, 2, 46, 44, 1581, 109, 70, + /* 690 */ 202, 201, 69, 393, 639, 1495, 2003, 2116, 611, 1171, + /* 700 */ 1974, 1174, 1175, 557, 556, 555, 1576, 633, 1493, 365, + /* 710 */ 547, 137, 551, 477, 641, 698, 550, 618, 655, 1892, + /* 720 */ 2275, 549, 554, 368, 367, 630, 140, 548, 81, 80, + /* 730 */ 437, 1643, 1571, 193, 2134, 617, 186, 439, 1938, 1939, + /* 740 */ 2276, 619, 717, 244, 1944, 1501, 2084, 243, 671, 655, + /* 750 */ 1892, 383, 14, 13, 341, 655, 1892, 423, 1877, 1942, + /* 760 */ 421, 417, 413, 410, 430, 650, 1743, 2003, 448, 1944, + /* 770 */ 758, 106, 2077, 47, 463, 366, 388, 364, 363, 2115, + /* 780 */ 541, 103, 2151, 2116, 1942, 110, 2117, 675, 2119, 2120, + /* 790 */ 670, 1944, 665, 672, 1869, 655, 1892, 183, 397, 2204, + /* 800 */ 101, 543, 189, 389, 2200, 542, 1942, 90, 347, 1578, + /* 810 */ 1579, 372, 87, 573, 464, 2084, 188, 2078, 593, 1735, + /* 820 */ 2134, 2275, 553, 552, 2230, 1885, 277, 2212, 629, 362, + /* 830 */ 134, 628, 2084, 2275, 671, 1742, 2281, 186, 1887, 1551, + /* 840 */ 1561, 2276, 619, 630, 140, 1577, 1580, 245, 617, 186, + /* 850 */ 655, 1892, 1619, 2276, 619, 458, 1586, 2219, 12, 1496, + /* 860 */ 10, 1494, 1520, 593, 457, 2115, 2275, 1741, 2151, 517, + /* 870 */ 1879, 111, 2117, 675, 2119, 2120, 670, 657, 665, 2176, + /* 880 */ 622, 2281, 186, 2215, 2084, 2204, 2276, 619, 1499, 1500, + /* 890 */ 2201, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, + /* 900 */ 667, 663, 1569, 1570, 1572, 1573, 1574, 1575, 2, 46, + /* 910 */ 44, 659, 1523, 2176, 1468, 1469, 2084, 393, 427, 1495, + /* 920 */ 52, 2116, 3, 705, 155, 154, 702, 701, 700, 152, + /* 930 */ 1576, 633, 1493, 39, 38, 655, 1892, 45, 43, 42, + /* 940 */ 41, 40, 429, 425, 187, 2212, 2213, 2116, 138, 2217, + /* 950 */ 198, 1875, 655, 1892, 1889, 578, 1571, 672, 2134, 2238, + /* 960 */ 705, 155, 154, 702, 701, 700, 152, 399, 1520, 1501, + /* 970 */ 2084, 247, 671, 655, 1892, 165, 42, 41, 40, 2219, + /* 980 */ 1642, 39, 38, 1894, 2134, 45, 43, 42, 41, 40, + /* 990 */ 84, 543, 589, 83, 758, 542, 2084, 47, 671, 655, + /* 1000 */ 1892, 593, 1740, 2115, 2275, 2214, 2151, 2116, 1552, 110, + /* 1010 */ 2117, 675, 2119, 2120, 670, 2067, 665, 672, 634, 2281, + /* 1020 */ 186, 183, 9, 2204, 2276, 619, 1739, 389, 2200, 2115, + /* 1030 */ 1867, 87, 2151, 1578, 1579, 110, 2117, 675, 2119, 2120, + /* 1040 */ 670, 2094, 665, 254, 2134, 1738, 571, 2295, 2231, 2204, + /* 1050 */ 146, 2084, 135, 389, 2200, 1883, 2084, 1888, 671, 569, + /* 1060 */ 256, 567, 1989, 1551, 1561, 2098, 34, 655, 1892, 1577, + /* 1070 */ 1580, 1607, 39, 38, 1662, 2084, 45, 43, 42, 41, + /* 1080 */ 40, 729, 727, 1496, 1896, 1494, 638, 655, 1892, 2115, + /* 1090 */ 655, 1892, 2151, 164, 2084, 169, 2117, 675, 2119, 2120, + /* 1100 */ 670, 2100, 665, 1174, 1175, 312, 293, 200, 1921, 652, + /* 1110 */ 623, 665, 1499, 1500, 1552, 1550, 1553, 1554, 1555, 1556, + /* 1120 */ 1557, 1558, 1559, 1560, 667, 663, 1569, 1570, 1572, 1573, + /* 1130 */ 1574, 1575, 2, 46, 44, 1737, 2242, 2116, 655, 1892, + /* 1140 */ 1736, 393, 1733, 1495, 2094, 33, 1732, 672, 1731, 2251, + /* 1150 */ 655, 1892, 655, 1892, 1576, 1612, 1493, 653, 2102, 705, + /* 1160 */ 155, 154, 702, 701, 700, 152, 1730, 142, 2098, 299, + /* 1170 */ 2175, 400, 2224, 1639, 2134, 1944, 703, 1177, 704, 1935, + /* 1180 */ 1571, 1935, 1729, 1519, 2084, 2094, 2084, 1728, 671, 2084, + /* 1190 */ 1943, 2084, 2094, 1501, 1870, 2084, 625, 2084, 1727, 2102, + /* 1200 */ 1726, 1725, 2070, 251, 2100, 377, 2103, 718, 234, 2098, + /* 1210 */ 1854, 232, 74, 592, 665, 2084, 2098, 236, 758, 2115, + /* 1220 */ 235, 15, 2151, 2116, 422, 110, 2117, 675, 2119, 2120, + /* 1230 */ 670, 2084, 665, 672, 1639, 600, 2084, 2295, 148, 2204, + /* 1240 */ 153, 441, 546, 389, 2200, 2100, 390, 2084, 1781, 2084, + /* 1250 */ 2084, 415, 2100, 238, 442, 665, 237, 1578, 1579, 1774, + /* 1260 */ 2134, 82, 665, 240, 1260, 153, 239, 1772, 711, 575, + /* 1270 */ 558, 574, 2084, 662, 671, 64, 255, 64, 261, 1717, + /* 1280 */ 1718, 560, 14, 13, 666, 2105, 1834, 1551, 1561, 563, + /* 1290 */ 1240, 2244, 274, 1577, 1580, 227, 268, 1463, 606, 1832, + /* 1300 */ 1504, 1503, 712, 1831, 153, 2115, 48, 1496, 2151, 1494, + /* 1310 */ 2135, 110, 2117, 675, 2119, 2120, 670, 286, 665, 91, + /* 1320 */ 71, 151, 1466, 2295, 1238, 2204, 406, 64, 48, 389, + /* 1330 */ 2200, 1998, 1671, 53, 1670, 263, 1499, 1500, 2107, 1550, + /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, + /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 2116, 402, 401, + /* 1360 */ 1221, 637, 153, 1423, 48, 679, 1768, 672, 1509, 2269, + /* 1370 */ 151, 1764, 153, 1759, 291, 1932, 2234, 647, 295, 1576, + /* 1380 */ 136, 1502, 631, 2116, 1613, 1562, 276, 151, 273, 1, + /* 1390 */ 5, 409, 356, 672, 2134, 2223, 414, 1222, 1446, 306, + /* 1400 */ 199, 443, 1523, 1999, 480, 1571, 2084, 447, 671, 452, + /* 1410 */ 1518, 490, 465, 1991, 479, 472, 753, 491, 1501, 1305, + /* 1420 */ 2134, 311, 1332, 1597, 481, 488, 203, 1336, 626, 1343, + /* 1430 */ 2116, 204, 2084, 493, 671, 494, 206, 1341, 496, 2115, + /* 1440 */ 672, 498, 2151, 661, 156, 110, 2117, 675, 2119, 2120, + /* 1450 */ 670, 2116, 665, 1507, 1506, 1524, 4, 2295, 499, 2204, + /* 1460 */ 507, 672, 1526, 389, 2200, 2115, 500, 2134, 2151, 508, + /* 1470 */ 510, 110, 2117, 675, 2119, 2120, 670, 1521, 665, 2084, + /* 1480 */ 1525, 671, 214, 2295, 511, 2204, 512, 216, 2134, 389, + /* 1490 */ 2200, 1527, 513, 1194, 219, 515, 221, 85, 86, 519, + /* 1500 */ 2084, 536, 671, 225, 538, 537, 112, 346, 540, 1882, + /* 1510 */ 231, 1878, 2115, 2060, 577, 2151, 2057, 2056, 170, 2117, + /* 1520 */ 675, 2119, 2120, 670, 579, 665, 233, 89, 149, 307, + /* 1530 */ 158, 159, 1510, 2115, 1505, 248, 2151, 1880, 1876, 110, + /* 1540 */ 2117, 675, 2119, 2120, 670, 160, 665, 161, 583, 252, + /* 1550 */ 582, 2179, 1453, 2204, 584, 2116, 597, 389, 2200, 607, + /* 1560 */ 587, 1513, 1515, 250, 2250, 672, 590, 645, 259, 588, + /* 1570 */ 2297, 2235, 2245, 603, 663, 1569, 1570, 1572, 1573, 1574, + /* 1580 */ 1575, 262, 2249, 8, 616, 378, 610, 2226, 598, 596, + /* 1590 */ 595, 267, 2134, 379, 139, 272, 1639, 2298, 627, 1522, + /* 1600 */ 624, 2220, 382, 635, 2084, 269, 671, 1528, 2004, 308, + /* 1610 */ 643, 281, 96, 309, 644, 648, 98, 2018, 2017, 2016, + /* 1620 */ 649, 2116, 173, 385, 270, 1893, 271, 100, 61, 102, + /* 1630 */ 2185, 672, 1936, 313, 1855, 310, 754, 2115, 302, 755, + /* 1640 */ 2151, 757, 348, 110, 2117, 675, 2119, 2120, 670, 2116, + /* 1650 */ 665, 2274, 677, 275, 337, 2177, 51, 2204, 2134, 672, + /* 1660 */ 322, 389, 2200, 336, 326, 317, 2076, 2075, 349, 2074, + /* 1670 */ 2084, 315, 671, 78, 2071, 411, 412, 1486, 1487, 192, + /* 1680 */ 416, 2116, 2069, 418, 419, 420, 2134, 2068, 357, 2066, + /* 1690 */ 424, 672, 426, 2064, 428, 79, 1449, 1448, 2084, 2065, + /* 1700 */ 671, 2030, 2029, 2115, 2028, 435, 2151, 436, 2027, 110, + /* 1710 */ 2117, 675, 2119, 2120, 670, 2026, 665, 1982, 2134, 1400, + /* 1720 */ 1981, 658, 1979, 2204, 1978, 145, 1977, 389, 2200, 1980, + /* 1730 */ 2084, 2115, 671, 1976, 2151, 1975, 1973, 111, 2117, 675, + /* 1740 */ 2119, 2120, 670, 1972, 665, 1971, 197, 453, 1970, 455, + /* 1750 */ 1984, 2204, 2116, 1969, 1968, 2203, 2200, 483, 147, 1954, + /* 1760 */ 1953, 1952, 672, 2115, 1967, 1966, 2151, 1965, 1964, 111, + /* 1770 */ 2117, 675, 2119, 2120, 670, 1963, 665, 1962, 1961, 1960, + /* 1780 */ 2116, 1959, 1958, 2204, 1957, 1956, 1955, 660, 2200, 2134, + /* 1790 */ 672, 1983, 1951, 1950, 1402, 1949, 1948, 1947, 1946, 2116, + /* 1800 */ 1945, 2084, 1276, 671, 344, 1280, 345, 1797, 205, 669, + /* 1810 */ 1796, 1272, 1795, 1793, 207, 2116, 1754, 2134, 180, 1176, + /* 1820 */ 2104, 208, 1753, 2047, 2037, 672, 76, 2025, 77, 2084, + /* 1830 */ 210, 671, 2024, 218, 673, 220, 2134, 2151, 2002, 1871, + /* 1840 */ 111, 2117, 675, 2119, 2120, 670, 181, 665, 2084, 505, + /* 1850 */ 671, 212, 2134, 1792, 2204, 1790, 1214, 384, 351, 2200, + /* 1860 */ 520, 1788, 2115, 521, 2084, 2151, 671, 522, 328, 2117, + /* 1870 */ 675, 2119, 2120, 670, 526, 665, 2116, 524, 525, 1786, + /* 1880 */ 528, 2115, 529, 530, 2151, 1784, 672, 334, 2117, 675, + /* 1890 */ 2119, 2120, 670, 532, 665, 533, 2170, 2115, 581, 2116, + /* 1900 */ 2151, 534, 1771, 335, 2117, 675, 2119, 2120, 670, 672, + /* 1910 */ 665, 615, 1770, 2134, 1750, 1873, 761, 63, 392, 1348, + /* 1920 */ 1347, 1872, 230, 1263, 1261, 2084, 1259, 671, 1258, 1257, + /* 1930 */ 305, 1256, 1250, 1782, 1255, 369, 2134, 561, 1252, 726, + /* 1940 */ 1251, 394, 1249, 1775, 728, 370, 178, 1773, 2084, 371, + /* 1950 */ 671, 564, 751, 747, 743, 739, 303, 566, 2115, 1749, + /* 1960 */ 1748, 2151, 1747, 568, 335, 2117, 675, 2119, 2120, 670, + /* 1970 */ 570, 665, 113, 1473, 1475, 1472, 1477, 27, 2046, 1455, + /* 1980 */ 67, 2115, 2036, 1457, 2151, 56, 585, 335, 2117, 675, + /* 1990 */ 2119, 2120, 670, 2116, 665, 163, 108, 2023, 586, 296, + /* 2000 */ 2021, 253, 1459, 672, 375, 2280, 29, 20, 17, 591, + /* 2010 */ 1687, 258, 59, 6, 7, 599, 601, 60, 265, 260, + /* 2020 */ 1669, 31, 266, 2105, 171, 2116, 264, 21, 65, 30, + /* 2030 */ 2134, 651, 1661, 1707, 92, 672, 1708, 22, 1702, 278, + /* 2040 */ 1701, 380, 2084, 2116, 671, 1706, 1705, 381, 176, 2022, + /* 2050 */ 1636, 1635, 58, 672, 2020, 2019, 2001, 18, 94, 95, + /* 2060 */ 284, 646, 2134, 2000, 103, 97, 283, 23, 57, 297, + /* 2070 */ 285, 282, 1667, 294, 2084, 576, 671, 24, 2151, 287, + /* 2080 */ 2134, 330, 2117, 675, 2119, 2120, 670, 292, 665, 68, + /* 2090 */ 2116, 249, 2084, 11, 671, 99, 1588, 1587, 13, 1511, + /* 2100 */ 672, 1543, 1598, 678, 177, 190, 395, 2115, 1566, 2154, + /* 2110 */ 2151, 682, 1325, 319, 2117, 675, 2119, 2120, 670, 664, + /* 2120 */ 665, 685, 1564, 37, 16, 2115, 676, 2134, 2151, 1563, + /* 2130 */ 25, 320, 2117, 675, 2119, 2120, 670, 2116, 665, 2084, + /* 2140 */ 1535, 671, 674, 26, 688, 1333, 680, 672, 1330, 1327, + /* 2150 */ 683, 686, 691, 1321, 1319, 300, 1324, 1323, 1322, 2116, + /* 2160 */ 689, 692, 104, 1342, 105, 75, 1338, 1212, 706, 672, + /* 2170 */ 1244, 1243, 2115, 301, 2134, 2151, 1242, 1241, 321, 2117, + /* 2180 */ 675, 2119, 2120, 670, 2116, 665, 2084, 1239, 671, 1237, + /* 2190 */ 1236, 1235, 1270, 716, 672, 1233, 2134, 1232, 1231, 1230, + /* 2200 */ 1229, 1228, 1227, 1267, 1265, 1224, 2116, 1223, 2084, 1220, + /* 2210 */ 671, 1219, 1218, 1217, 1789, 736, 672, 737, 738, 2115, + /* 2220 */ 1787, 2134, 2151, 740, 741, 327, 2117, 675, 2119, 2120, + /* 2230 */ 670, 742, 665, 2084, 1785, 671, 744, 746, 745, 1783, + /* 2240 */ 748, 2115, 749, 2134, 2151, 750, 1769, 331, 2117, 675, + /* 2250 */ 2119, 2120, 670, 752, 665, 2084, 1166, 671, 304, 756, + /* 2260 */ 1721, 1721, 1497, 314, 759, 760, 2115, 1746, 1721, 2151, + /* 2270 */ 2116, 1721, 323, 2117, 675, 2119, 2120, 670, 1721, 665, + /* 2280 */ 672, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 1721, + /* 2290 */ 2116, 2151, 1721, 1721, 332, 2117, 675, 2119, 2120, 670, + /* 2300 */ 672, 665, 1721, 1721, 1721, 2116, 1721, 2134, 1721, 1721, + /* 2310 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 2084, + /* 2320 */ 1721, 671, 1721, 1721, 1721, 1721, 1721, 2134, 1721, 1721, + /* 2330 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 2084, + /* 2340 */ 1721, 671, 2134, 1721, 1721, 1721, 1721, 672, 1721, 1721, + /* 2350 */ 1721, 1721, 2115, 1721, 2084, 2151, 671, 1721, 324, 2117, + /* 2360 */ 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, 1721, + /* 2370 */ 1721, 1721, 2115, 1721, 2134, 2151, 1721, 1721, 333, 2117, + /* 2380 */ 675, 2119, 2120, 670, 1721, 665, 2084, 2115, 671, 1721, + /* 2390 */ 2151, 1721, 1721, 325, 2117, 675, 2119, 2120, 670, 1721, + /* 2400 */ 665, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, + /* 2410 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 2115, + /* 2420 */ 1721, 1721, 2151, 1721, 2116, 338, 2117, 675, 2119, 2120, + /* 2430 */ 670, 1721, 665, 1721, 672, 1721, 1721, 1721, 1721, 1721, + /* 2440 */ 1721, 1721, 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2450 */ 1721, 1721, 2116, 1721, 2084, 1721, 671, 1721, 1721, 1721, + /* 2460 */ 1721, 2134, 672, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2470 */ 1721, 2116, 1721, 2084, 1721, 671, 1721, 1721, 1721, 1721, + /* 2480 */ 1721, 672, 1721, 1721, 1721, 1721, 1721, 2115, 1721, 2134, + /* 2490 */ 2151, 1721, 1721, 339, 2117, 675, 2119, 2120, 670, 1721, + /* 2500 */ 665, 2084, 1721, 671, 1721, 1721, 2115, 1721, 2134, 2151, + /* 2510 */ 1721, 1721, 2128, 2117, 675, 2119, 2120, 670, 2116, 665, + /* 2520 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 672, 1721, + /* 2530 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 2151, 1721, 1721, + /* 2540 */ 2127, 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, + /* 2550 */ 1721, 1721, 1721, 2115, 1721, 2134, 2151, 1721, 1721, 2126, + /* 2560 */ 2117, 675, 2119, 2120, 670, 1721, 665, 2084, 1721, 671, + /* 2570 */ 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, + /* 2580 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 1721, + /* 2590 */ 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2600 */ 2115, 672, 1721, 2151, 1721, 1721, 353, 2117, 675, 2119, + /* 2610 */ 2120, 670, 2134, 665, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2620 */ 1721, 1721, 1721, 1721, 2084, 1721, 671, 1721, 2134, 1721, + /* 2630 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2640 */ 2084, 2116, 671, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2650 */ 1721, 672, 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, + /* 2660 */ 2151, 2116, 1721, 354, 2117, 675, 2119, 2120, 670, 1721, + /* 2670 */ 665, 672, 1721, 2115, 1721, 1721, 2151, 1721, 2134, 350, + /* 2680 */ 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, + /* 2690 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 2134, 1721, + /* 2700 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2710 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 672, 1721, + /* 2720 */ 1721, 1721, 1721, 2115, 1721, 1721, 2151, 1721, 1721, 355, + /* 2730 */ 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, + /* 2740 */ 1721, 1721, 229, 673, 1721, 2134, 2151, 1721, 1721, 330, + /* 2750 */ 2117, 675, 2119, 2120, 670, 1721, 665, 2084, 172, 671, + /* 2760 */ 1721, 1721, 1721, 1721, 535, 531, 527, 523, 226, 1721, + /* 2770 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2780 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2790 */ 2115, 1721, 1721, 2151, 1721, 1721, 329, 2117, 675, 2119, + /* 2800 */ 2120, 670, 1721, 665, 1721, 1721, 1721, 1721, 88, 1721, + /* 2810 */ 1721, 224, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2820 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2830 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2840 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2850 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2860 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2870 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 223, 217, + /* 2880 */ 1721, 1721, 1721, 222, 1721, 514, 1721, 1721, 1721, 1721, + /* 2890 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2900 */ 1721, 1721, 1721, 215, +}; +static const YYCODETYPE yy_lookahead[] = { + /* 0 */ 333, 384, 371, 362, 401, 332, 401, 334, 405, 343, + /* 10 */ 343, 370, 12, 13, 14, 0, 399, 400, 377, 378, + /* 20 */ 20, 14, 22, 342, 8, 9, 385, 20, 12, 13, + /* 30 */ 14, 15, 16, 33, 0, 35, 21, 370, 20, 24, + /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 3, 382, + /* 50 */ 447, 384, 447, 450, 388, 450, 8, 9, 351, 59, + /* 60 */ 12, 13, 14, 15, 16, 65, 359, 386, 465, 466, + /* 70 */ 465, 466, 72, 470, 471, 470, 471, 12, 13, 14, + /* 80 */ 15, 16, 415, 337, 20, 418, 340, 341, 421, 422, + /* 90 */ 423, 424, 425, 426, 369, 428, 20, 97, 436, 437, + /* 100 */ 100, 67, 68, 69, 70, 71, 381, 73, 74, 75, + /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, + /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 461, 462, + /* 130 */ 337, 20, 97, 340, 341, 21, 136, 137, 24, 25, + /* 140 */ 26, 27, 28, 29, 30, 31, 32, 112, 113, 114, + /* 150 */ 115, 116, 117, 118, 119, 120, 121, 370, 123, 124, + /* 160 */ 125, 126, 127, 128, 100, 20, 166, 167, 20, 420, + /* 170 */ 342, 343, 172, 173, 330, 8, 9, 390, 391, 12, + /* 180 */ 13, 14, 15, 16, 166, 167, 186, 20, 188, 361, + /* 190 */ 342, 343, 362, 100, 178, 446, 368, 8, 9, 59, + /* 200 */ 370, 12, 13, 14, 15, 16, 8, 9, 378, 361, + /* 210 */ 12, 13, 14, 15, 16, 215, 216, 0, 218, 219, + /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 0, 370, + /* 240 */ 100, 18, 102, 20, 20, 401, 22, 100, 100, 405, + /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 343, 35, + /* 260 */ 391, 343, 24, 25, 26, 27, 28, 29, 30, 31, + /* 270 */ 32, 466, 49, 20, 51, 470, 471, 54, 185, 362, + /* 280 */ 187, 64, 447, 59, 22, 450, 338, 370, 370, 65, + /* 290 */ 342, 447, 344, 277, 450, 378, 72, 35, 109, 101, + /* 300 */ 382, 466, 384, 388, 384, 470, 471, 214, 447, 465, + /* 310 */ 466, 450, 64, 333, 470, 471, 396, 272, 4, 399, + /* 320 */ 400, 97, 99, 343, 100, 345, 465, 466, 35, 342, + /* 330 */ 343, 470, 471, 415, 111, 168, 418, 342, 343, 421, + /* 340 */ 422, 423, 424, 425, 426, 252, 428, 20, 361, 431, + /* 350 */ 370, 433, 434, 435, 165, 368, 361, 439, 440, 97, + /* 360 */ 136, 137, 382, 140, 384, 72, 143, 144, 145, 146, + /* 370 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, + /* 380 */ 157, 158, 159, 20, 161, 162, 163, 14, 20, 20, + /* 390 */ 166, 167, 252, 20, 13, 415, 172, 173, 418, 252, + /* 400 */ 252, 421, 422, 423, 424, 425, 426, 64, 428, 20, + /* 410 */ 186, 22, 188, 433, 100, 435, 35, 342, 343, 439, + /* 420 */ 440, 168, 100, 171, 342, 342, 343, 333, 239, 240, + /* 430 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 215, + /* 440 */ 216, 52, 218, 219, 220, 221, 222, 223, 224, 225, + /* 450 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, + /* 460 */ 236, 12, 13, 136, 137, 22, 12, 13, 100, 20, + /* 470 */ 22, 22, 67, 68, 69, 393, 382, 395, 35, 74, + /* 480 */ 75, 76, 33, 35, 35, 80, 3, 333, 2, 35, + /* 490 */ 85, 86, 87, 88, 8, 9, 91, 343, 12, 13, + /* 500 */ 14, 15, 16, 20, 20, 136, 137, 333, 59, 257, + /* 510 */ 258, 259, 215, 379, 65, 72, 382, 343, 443, 444, + /* 520 */ 445, 72, 447, 448, 370, 442, 443, 444, 445, 380, + /* 530 */ 447, 448, 383, 384, 366, 20, 382, 22, 384, 166, + /* 540 */ 97, 172, 173, 20, 370, 97, 97, 8, 9, 100, + /* 550 */ 35, 12, 13, 14, 15, 16, 382, 44, 384, 262, + /* 560 */ 263, 264, 265, 266, 267, 268, 252, 52, 18, 415, + /* 570 */ 333, 111, 418, 23, 252, 421, 422, 423, 424, 425, + /* 580 */ 426, 413, 428, 44, 101, 136, 137, 37, 38, 415, + /* 590 */ 236, 41, 418, 358, 4, 421, 422, 423, 424, 425, + /* 600 */ 426, 427, 428, 429, 430, 342, 343, 372, 348, 19, + /* 610 */ 60, 61, 62, 63, 101, 166, 167, 382, 343, 382, + /* 620 */ 252, 172, 173, 33, 361, 365, 472, 473, 338, 186, + /* 630 */ 81, 188, 342, 373, 344, 186, 0, 188, 370, 49, + /* 640 */ 370, 370, 188, 342, 54, 370, 378, 377, 377, 59, + /* 650 */ 100, 168, 168, 418, 419, 385, 385, 72, 215, 216, + /* 660 */ 4, 130, 131, 428, 215, 216, 135, 218, 219, 220, + /* 670 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 99, + /* 690 */ 141, 142, 102, 20, 393, 22, 395, 333, 423, 43, + /* 700 */ 0, 45, 46, 67, 68, 69, 33, 343, 35, 37, + /* 710 */ 74, 75, 76, 164, 342, 111, 80, 447, 342, 343, + /* 720 */ 450, 85, 86, 87, 88, 342, 343, 91, 178, 179, + /* 730 */ 180, 4, 59, 183, 370, 465, 466, 361, 383, 384, + /* 740 */ 470, 471, 72, 131, 370, 72, 382, 135, 384, 342, + /* 750 */ 343, 377, 1, 2, 204, 342, 343, 207, 371, 385, + /* 760 */ 210, 211, 212, 213, 214, 393, 333, 395, 361, 370, + /* 770 */ 97, 100, 401, 100, 361, 103, 377, 105, 106, 415, + /* 780 */ 108, 110, 418, 333, 385, 421, 422, 423, 424, 425, + /* 790 */ 426, 370, 428, 343, 0, 342, 343, 433, 377, 435, + /* 800 */ 348, 129, 252, 439, 440, 133, 385, 195, 196, 136, + /* 810 */ 137, 199, 350, 201, 361, 382, 452, 401, 447, 334, + /* 820 */ 370, 450, 355, 356, 460, 373, 443, 444, 445, 367, + /* 830 */ 447, 448, 382, 450, 384, 333, 465, 466, 376, 166, + /* 840 */ 167, 470, 471, 342, 343, 172, 173, 130, 465, 466, + /* 850 */ 342, 343, 101, 470, 471, 155, 14, 420, 236, 186, + /* 860 */ 238, 188, 20, 447, 164, 415, 450, 333, 418, 361, + /* 870 */ 371, 421, 422, 423, 424, 425, 426, 432, 428, 434, + /* 880 */ 44, 465, 466, 446, 382, 435, 470, 471, 215, 216, + /* 890 */ 440, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 12, + /* 910 */ 13, 432, 20, 434, 197, 198, 382, 20, 181, 22, + /* 920 */ 42, 333, 44, 129, 130, 131, 132, 133, 134, 135, + /* 930 */ 33, 343, 35, 8, 9, 342, 343, 12, 13, 14, + /* 940 */ 15, 16, 205, 206, 443, 444, 445, 333, 447, 448, + /* 950 */ 59, 371, 342, 343, 361, 401, 59, 343, 370, 345, + /* 960 */ 129, 130, 131, 132, 133, 134, 135, 362, 20, 72, + /* 970 */ 382, 361, 384, 342, 343, 370, 14, 15, 16, 420, + /* 980 */ 253, 8, 9, 378, 370, 12, 13, 14, 15, 16, + /* 990 */ 99, 129, 361, 102, 97, 133, 382, 100, 384, 342, + /* 1000 */ 343, 447, 333, 415, 450, 446, 418, 333, 166, 421, + /* 1010 */ 422, 423, 424, 425, 426, 0, 428, 343, 361, 465, + /* 1020 */ 466, 433, 39, 435, 470, 471, 333, 439, 440, 415, + /* 1030 */ 0, 350, 418, 136, 137, 421, 422, 423, 424, 425, + /* 1040 */ 426, 358, 428, 406, 370, 333, 21, 433, 460, 435, + /* 1050 */ 42, 382, 44, 439, 440, 372, 382, 376, 384, 34, + /* 1060 */ 168, 36, 343, 166, 167, 382, 2, 342, 343, 172, + /* 1070 */ 173, 165, 8, 9, 101, 382, 12, 13, 14, 15, + /* 1080 */ 16, 355, 356, 186, 371, 188, 361, 342, 343, 415, + /* 1090 */ 342, 343, 418, 168, 382, 421, 422, 423, 424, 425, + /* 1100 */ 426, 418, 428, 45, 46, 363, 361, 388, 366, 361, + /* 1110 */ 274, 428, 215, 216, 166, 218, 219, 220, 221, 222, + /* 1120 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, + /* 1130 */ 233, 234, 235, 12, 13, 333, 462, 333, 342, 343, + /* 1140 */ 333, 20, 333, 22, 358, 239, 333, 343, 333, 345, + /* 1150 */ 342, 343, 342, 343, 33, 249, 35, 361, 372, 129, + /* 1160 */ 130, 131, 132, 133, 134, 135, 333, 431, 382, 361, + /* 1170 */ 434, 361, 250, 251, 370, 370, 379, 14, 379, 382, + /* 1180 */ 59, 382, 333, 20, 382, 358, 382, 333, 384, 382, + /* 1190 */ 385, 382, 358, 72, 0, 382, 44, 382, 333, 372, + /* 1200 */ 333, 333, 0, 371, 418, 419, 372, 357, 104, 382, + /* 1210 */ 360, 107, 111, 48, 428, 382, 382, 104, 97, 415, + /* 1220 */ 107, 100, 418, 333, 209, 421, 422, 423, 424, 425, + /* 1230 */ 426, 382, 428, 343, 251, 345, 382, 433, 44, 435, + /* 1240 */ 44, 22, 13, 439, 440, 418, 419, 382, 0, 382, + /* 1250 */ 382, 49, 418, 104, 35, 428, 107, 136, 137, 0, + /* 1260 */ 370, 160, 428, 104, 35, 44, 107, 0, 13, 200, + /* 1270 */ 22, 202, 382, 65, 384, 44, 59, 44, 44, 136, + /* 1280 */ 137, 22, 1, 2, 371, 47, 359, 166, 167, 22, + /* 1290 */ 35, 392, 474, 172, 173, 346, 457, 101, 463, 358, + /* 1300 */ 35, 35, 13, 358, 44, 415, 44, 186, 418, 188, + /* 1310 */ 370, 421, 422, 423, 424, 425, 426, 44, 428, 102, + /* 1320 */ 44, 44, 101, 433, 35, 435, 346, 44, 44, 439, + /* 1330 */ 440, 392, 101, 168, 101, 101, 215, 216, 100, 218, + /* 1340 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 333, 12, 13, + /* 1360 */ 35, 101, 44, 101, 44, 44, 0, 343, 22, 345, + /* 1370 */ 44, 343, 44, 341, 101, 381, 392, 101, 101, 33, + /* 1380 */ 44, 35, 449, 333, 101, 101, 467, 44, 441, 451, + /* 1390 */ 254, 417, 416, 343, 370, 345, 49, 72, 184, 403, + /* 1400 */ 42, 389, 20, 392, 165, 59, 382, 389, 384, 387, + /* 1410 */ 20, 98, 342, 342, 387, 389, 50, 354, 72, 101, + /* 1420 */ 370, 101, 101, 215, 387, 96, 353, 101, 276, 101, + /* 1430 */ 333, 342, 382, 95, 384, 352, 342, 101, 342, 415, + /* 1440 */ 343, 342, 418, 97, 101, 421, 422, 423, 424, 425, + /* 1450 */ 426, 333, 428, 188, 188, 20, 48, 433, 335, 435, + /* 1460 */ 335, 343, 20, 439, 440, 415, 339, 370, 418, 339, + /* 1470 */ 410, 421, 422, 423, 424, 425, 426, 20, 428, 382, + /* 1480 */ 20, 384, 350, 433, 384, 435, 344, 350, 370, 439, + /* 1490 */ 440, 20, 402, 53, 350, 344, 350, 350, 350, 342, + /* 1500 */ 382, 347, 384, 350, 335, 347, 342, 335, 370, 370, + /* 1510 */ 370, 370, 415, 382, 203, 418, 382, 382, 421, 422, + /* 1520 */ 423, 424, 425, 426, 414, 428, 370, 100, 412, 410, + /* 1530 */ 370, 370, 186, 415, 188, 348, 418, 370, 370, 421, + /* 1540 */ 422, 423, 424, 425, 426, 370, 428, 370, 192, 348, + /* 1550 */ 191, 433, 190, 435, 409, 333, 382, 439, 440, 261, + /* 1560 */ 384, 215, 216, 408, 456, 343, 342, 260, 397, 407, + /* 1570 */ 473, 392, 392, 382, 228, 229, 230, 231, 232, 233, + /* 1580 */ 234, 397, 456, 269, 177, 382, 382, 459, 271, 270, + /* 1590 */ 255, 458, 370, 278, 343, 417, 251, 475, 275, 20, + /* 1600 */ 273, 420, 344, 342, 382, 455, 384, 20, 395, 397, + /* 1610 */ 382, 348, 348, 397, 382, 170, 348, 382, 382, 382, + /* 1620 */ 394, 333, 456, 382, 454, 343, 453, 348, 100, 100, + /* 1630 */ 438, 343, 382, 342, 360, 366, 36, 415, 348, 336, + /* 1640 */ 418, 335, 398, 421, 422, 423, 424, 425, 426, 333, + /* 1650 */ 428, 469, 374, 468, 411, 433, 404, 435, 370, 343, + /* 1660 */ 364, 439, 440, 364, 364, 331, 0, 0, 398, 0, + /* 1670 */ 382, 349, 384, 42, 0, 35, 208, 35, 35, 35, + /* 1680 */ 208, 333, 0, 35, 35, 208, 370, 0, 208, 0, + /* 1690 */ 35, 343, 22, 0, 35, 195, 188, 186, 382, 0, + /* 1700 */ 384, 0, 0, 415, 0, 182, 418, 181, 0, 421, + /* 1710 */ 422, 423, 424, 425, 426, 0, 428, 0, 370, 47, + /* 1720 */ 0, 433, 0, 435, 0, 42, 0, 439, 440, 0, + /* 1730 */ 382, 415, 384, 0, 418, 0, 0, 421, 422, 423, + /* 1740 */ 424, 425, 426, 0, 428, 0, 155, 35, 0, 155, + /* 1750 */ 0, 435, 333, 0, 0, 439, 440, 139, 42, 0, + /* 1760 */ 0, 0, 343, 415, 0, 0, 418, 0, 0, 421, + /* 1770 */ 422, 423, 424, 425, 426, 0, 428, 0, 0, 0, + /* 1780 */ 333, 0, 0, 435, 0, 0, 0, 439, 440, 370, + /* 1790 */ 343, 0, 0, 0, 22, 0, 0, 0, 0, 333, + /* 1800 */ 0, 382, 22, 384, 48, 22, 48, 0, 59, 343, + /* 1810 */ 0, 35, 0, 0, 59, 333, 0, 370, 44, 14, + /* 1820 */ 47, 59, 0, 0, 0, 343, 39, 0, 39, 382, + /* 1830 */ 42, 384, 0, 39, 415, 177, 370, 418, 0, 0, + /* 1840 */ 421, 422, 423, 424, 425, 426, 47, 428, 382, 47, + /* 1850 */ 384, 40, 370, 0, 435, 0, 66, 375, 439, 440, + /* 1860 */ 35, 0, 415, 49, 382, 418, 384, 39, 421, 422, + /* 1870 */ 423, 424, 425, 426, 39, 428, 333, 35, 49, 0, + /* 1880 */ 35, 415, 49, 39, 418, 0, 343, 421, 422, 423, + /* 1890 */ 424, 425, 426, 35, 428, 49, 430, 415, 1, 333, + /* 1900 */ 418, 39, 0, 421, 422, 423, 424, 425, 426, 343, + /* 1910 */ 428, 464, 0, 370, 0, 0, 19, 109, 375, 35, + /* 1920 */ 22, 0, 107, 35, 35, 382, 35, 384, 35, 35, + /* 1930 */ 33, 35, 22, 0, 35, 22, 370, 51, 35, 44, + /* 1940 */ 35, 375, 35, 0, 44, 22, 49, 0, 382, 22, + /* 1950 */ 384, 35, 55, 56, 57, 58, 59, 35, 415, 0, + /* 1960 */ 0, 418, 0, 35, 421, 422, 423, 424, 425, 426, + /* 1970 */ 22, 428, 20, 35, 35, 35, 101, 100, 0, 35, + /* 1980 */ 100, 415, 0, 22, 418, 168, 22, 421, 422, 423, + /* 1990 */ 424, 425, 426, 333, 428, 189, 99, 0, 168, 102, + /* 2000 */ 0, 170, 193, 343, 168, 3, 100, 44, 256, 175, + /* 2010 */ 101, 100, 44, 48, 48, 98, 96, 44, 44, 101, + /* 2020 */ 101, 44, 47, 47, 100, 333, 100, 256, 3, 100, + /* 2030 */ 370, 134, 101, 101, 100, 343, 101, 44, 35, 47, + /* 2040 */ 35, 35, 382, 333, 384, 35, 35, 35, 47, 0, + /* 2050 */ 101, 101, 44, 343, 0, 0, 0, 256, 100, 39, + /* 2060 */ 47, 171, 370, 0, 110, 39, 169, 100, 250, 47, + /* 2070 */ 101, 174, 101, 169, 382, 415, 384, 44, 418, 100, + /* 2080 */ 370, 421, 422, 423, 424, 425, 426, 100, 428, 100, + /* 2090 */ 333, 194, 382, 237, 384, 100, 98, 98, 2, 22, + /* 2100 */ 343, 22, 215, 35, 47, 47, 35, 415, 101, 100, + /* 2110 */ 418, 35, 122, 421, 422, 423, 424, 425, 426, 100, + /* 2120 */ 428, 35, 101, 100, 100, 415, 111, 370, 418, 101, + /* 2130 */ 100, 421, 422, 423, 424, 425, 426, 333, 428, 382, + /* 2140 */ 101, 384, 217, 100, 35, 101, 100, 343, 101, 101, + /* 2150 */ 100, 100, 35, 101, 101, 44, 122, 122, 122, 333, + /* 2160 */ 100, 100, 100, 35, 100, 100, 22, 66, 65, 343, + /* 2170 */ 35, 35, 415, 44, 370, 418, 35, 35, 421, 422, + /* 2180 */ 423, 424, 425, 426, 333, 428, 382, 35, 384, 35, + /* 2190 */ 35, 35, 72, 94, 343, 35, 370, 35, 35, 22, + /* 2200 */ 35, 35, 35, 72, 35, 35, 333, 35, 382, 35, + /* 2210 */ 384, 35, 22, 35, 0, 35, 343, 49, 39, 415, + /* 2220 */ 0, 370, 418, 35, 49, 421, 422, 423, 424, 425, + /* 2230 */ 426, 39, 428, 382, 0, 384, 35, 39, 49, 0, + /* 2240 */ 35, 415, 49, 370, 418, 39, 0, 421, 422, 423, + /* 2250 */ 424, 425, 426, 35, 428, 382, 35, 384, 22, 21, + /* 2260 */ 476, 476, 22, 22, 21, 20, 415, 0, 476, 418, + /* 2270 */ 333, 476, 421, 422, 423, 424, 425, 426, 476, 428, + /* 2280 */ 343, 476, 476, 476, 476, 476, 476, 476, 415, 476, + /* 2290 */ 333, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2300 */ 343, 428, 476, 476, 476, 333, 476, 370, 476, 476, + /* 2310 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 382, + /* 2320 */ 476, 384, 476, 476, 476, 476, 476, 370, 476, 476, + /* 2330 */ 476, 476, 476, 476, 476, 476, 476, 333, 476, 382, + /* 2340 */ 476, 384, 370, 476, 476, 476, 476, 343, 476, 476, + /* 2350 */ 476, 476, 415, 476, 382, 418, 384, 476, 421, 422, + /* 2360 */ 423, 424, 425, 426, 476, 428, 476, 476, 476, 476, + /* 2370 */ 476, 476, 415, 476, 370, 418, 476, 476, 421, 422, + /* 2380 */ 423, 424, 425, 426, 476, 428, 382, 415, 384, 476, + /* 2390 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2400 */ 428, 476, 476, 476, 476, 333, 476, 476, 476, 476, + /* 2410 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 415, + /* 2420 */ 476, 476, 418, 476, 333, 421, 422, 423, 424, 425, + /* 2430 */ 426, 476, 428, 476, 343, 476, 476, 476, 476, 476, + /* 2440 */ 476, 476, 370, 476, 476, 476, 476, 476, 476, 476, + /* 2450 */ 476, 476, 333, 476, 382, 476, 384, 476, 476, 476, + /* 2460 */ 476, 370, 343, 476, 476, 476, 476, 476, 476, 476, + /* 2470 */ 476, 333, 476, 382, 476, 384, 476, 476, 476, 476, + /* 2480 */ 476, 343, 476, 476, 476, 476, 476, 415, 476, 370, + /* 2490 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2500 */ 428, 382, 476, 384, 476, 476, 415, 476, 370, 418, + /* 2510 */ 476, 476, 421, 422, 423, 424, 425, 426, 333, 428, + /* 2520 */ 382, 476, 384, 476, 476, 476, 476, 476, 343, 476, + /* 2530 */ 476, 476, 476, 476, 415, 476, 476, 418, 476, 476, + /* 2540 */ 421, 422, 423, 424, 425, 426, 476, 428, 476, 476, + /* 2550 */ 476, 476, 476, 415, 476, 370, 418, 476, 476, 421, + /* 2560 */ 422, 423, 424, 425, 426, 476, 428, 382, 476, 384, + /* 2570 */ 476, 476, 476, 476, 476, 333, 476, 476, 476, 476, + /* 2580 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 476, + /* 2590 */ 476, 333, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2600 */ 415, 343, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2610 */ 425, 426, 370, 428, 476, 476, 476, 476, 476, 476, + /* 2620 */ 476, 476, 476, 476, 382, 476, 384, 476, 370, 476, + /* 2630 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2640 */ 382, 333, 384, 476, 476, 476, 476, 476, 476, 476, + /* 2650 */ 476, 343, 476, 476, 476, 476, 476, 415, 476, 476, + /* 2660 */ 418, 333, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2670 */ 428, 343, 476, 415, 476, 476, 418, 476, 370, 421, + /* 2680 */ 422, 423, 424, 425, 426, 476, 428, 476, 476, 476, + /* 2690 */ 382, 476, 384, 476, 476, 476, 476, 476, 370, 476, + /* 2700 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2710 */ 382, 476, 384, 476, 476, 476, 476, 476, 343, 476, + /* 2720 */ 476, 476, 476, 415, 476, 476, 418, 476, 476, 421, + /* 2730 */ 422, 423, 424, 425, 426, 476, 428, 476, 476, 476, + /* 2740 */ 476, 476, 33, 415, 476, 370, 418, 476, 476, 421, + /* 2750 */ 422, 423, 424, 425, 426, 476, 428, 382, 49, 384, + /* 2760 */ 476, 476, 476, 476, 55, 56, 57, 58, 59, 476, + /* 2770 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2780 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2790 */ 415, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2800 */ 425, 426, 476, 428, 476, 476, 476, 476, 99, 476, + /* 2810 */ 476, 102, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2820 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2830 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2840 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2850 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2860 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2870 */ 476, 476, 476, 476, 476, 476, 476, 476, 169, 170, + /* 2880 */ 476, 476, 476, 174, 476, 176, 476, 476, 476, 476, + /* 2890 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2900 */ 476, 476, 476, 194, 476, 476, 476, 476, 476, 476, + /* 2910 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2920 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2930 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2940 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2950 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2960 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2970 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2980 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2990 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3000 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3010 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3020 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3030 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3040 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3050 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3060 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3070 */ 476, 476, 476, 330, 330, 330, 330, 330, 330, 330, + /* 3080 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3090 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3100 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3110 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3120 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3130 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3140 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3150 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3160 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3170 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3180 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3190 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3200 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3210 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3220 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3230 */ 330, 330, 330, 330, +}; +#define YY_SHIFT_COUNT (761) +#define YY_SHIFT_MIN (0) +#define YY_SHIFT_MAX (2709) +>>>>>>> fix/rocksdbState static const unsigned short int yy_shift_ofst[] = { /* 0 */ 550, 0, 224, 0, 449, 449, 449, 449, 449, 449, /* 10 */ 449, 449, 449, 449, 449, 449, 673, 897, 897, 1121, /* 20 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, /* 30 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, +<<<<<<< HEAD /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, /* 50 */ 897, 452, 457, 142, 145, 147, 30, 107, 30, 145, /* 60 */ 145, 1347, 30, 1347, 1347, 501, 30, 56, 447, 344, @@ -946,6 +1574,118 @@ static const short yy_reduce_ofst[] = { /* 290 */ 1210, 1173, 1220, 1227, 1236, 1239, 1150, 1221, 1214, 1237, /* 300 */ 1256, 1249, 1268, 1266, 1202, 1196, 1211, 1213, 1244, 1251, /* 310 */ 1252, 1262, 1281, +======= + /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 148, + /* 50 */ 368, 93, 64, 140, 147, 322, 147, 64, 64, 1346, + /* 60 */ 1346, 1346, 147, 1346, 1346, 314, 147, 76, 369, 111, + /* 70 */ 111, 369, 656, 656, 18, 327, 7, 7, 111, 111, + /* 80 */ 111, 111, 111, 111, 111, 145, 111, 111, 248, 76, + /* 90 */ 111, 111, 363, 111, 76, 111, 145, 111, 145, 76, + /* 100 */ 111, 111, 76, 111, 76, 76, 76, 111, 343, 223, + /* 110 */ 189, 189, 405, 114, 443, 443, 443, 443, 443, 443, + /* 120 */ 443, 443, 443, 443, 443, 443, 443, 443, 443, 443, + /* 130 */ 443, 443, 443, 672, 483, 18, 327, 293, 253, 253, + /* 140 */ 253, 217, 622, 622, 293, 523, 523, 523, 248, 460, + /* 150 */ 354, 76, 585, 76, 585, 585, 604, 670, 35, 35, + /* 160 */ 35, 35, 35, 35, 35, 35, 1897, 636, 15, 167, + /* 170 */ 16, 297, 515, 252, 454, 454, 373, 842, 389, 484, + /* 180 */ 1058, 1163, 862, 892, 922, 983, 45, 922, 878, 727, + /* 190 */ 948, 1136, 1347, 1214, 1358, 1382, 1358, 1239, 1390, 1390, + /* 200 */ 1358, 1239, 1239, 1313, 1329, 1390, 1338, 1390, 1390, 1390, + /* 210 */ 1435, 1408, 1435, 1408, 1442, 248, 1457, 248, 1460, 1471, + /* 220 */ 248, 1460, 248, 248, 248, 1390, 248, 1440, 1440, 1435, + /* 230 */ 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, + /* 240 */ 76, 1390, 1435, 585, 585, 585, 1311, 1427, 1442, 343, + /* 250 */ 1356, 1359, 1457, 343, 1362, 1390, 1382, 1382, 585, 1298, + /* 260 */ 1307, 585, 1298, 1307, 585, 585, 76, 1314, 1407, 1298, + /* 270 */ 1317, 1319, 1335, 1136, 1315, 1323, 1327, 1345, 523, 1579, + /* 280 */ 1390, 1460, 343, 343, 1587, 1307, 585, 585, 585, 585, + /* 290 */ 585, 1307, 585, 1445, 343, 604, 343, 523, 1528, 1529, + /* 300 */ 585, 670, 1390, 343, 1600, 1435, 2904, 2904, 2904, 2904, + /* 310 */ 2904, 2904, 2904, 2904, 2904, 34, 2709, 238, 590, 198, + /* 320 */ 539, 973, 794, 486, 1064, 925, 1030, 48, 48, 48, + /* 330 */ 48, 48, 48, 48, 48, 48, 831, 612, 65, 65, + /* 340 */ 549, 737, 700, 891, 262, 448, 1025, 717, 531, 531, + /* 350 */ 962, 751, 906, 962, 962, 962, 1202, 1015, 513, 1219, + /* 360 */ 1008, 1101, 1194, 1104, 1113, 1149, 1159, 381, 1229, 1248, + /* 370 */ 1259, 1267, 1069, 1196, 1221, 1217, 1231, 1233, 1234, 1143, + /* 380 */ 836, 1152, 1165, 1260, 1262, 1273, 1276, 1277, 1318, 1281, + /* 390 */ 1283, 1208, 1284, 1238, 1320, 1321, 1326, 1328, 1336, 1343, + /* 400 */ 671, 1265, 1266, 1255, 1289, 1325, 1366, 1666, 1667, 1669, + /* 410 */ 1631, 1674, 1640, 1468, 1642, 1643, 1644, 1472, 1682, 1648, + /* 420 */ 1649, 1477, 1687, 1480, 1689, 1655, 1699, 1670, 1693, 1659, + /* 430 */ 1500, 1508, 1511, 1701, 1702, 1704, 1523, 1526, 1708, 1715, + /* 440 */ 1672, 1717, 1720, 1722, 1683, 1724, 1726, 1729, 1733, 1735, + /* 450 */ 1736, 1743, 1745, 1591, 1712, 1748, 1594, 1750, 1753, 1754, + /* 460 */ 1764, 1765, 1767, 1768, 1775, 1777, 1778, 1779, 1781, 1782, + /* 470 */ 1784, 1785, 1786, 1716, 1759, 1760, 1761, 1791, 1792, 1793, + /* 480 */ 1772, 1795, 1796, 1797, 1618, 1798, 1800, 1780, 1756, 1783, + /* 490 */ 1758, 1807, 1749, 1776, 1810, 1755, 1812, 1762, 1813, 1816, + /* 500 */ 1788, 1787, 1774, 1773, 1799, 1805, 1802, 1822, 1811, 1789, + /* 510 */ 1823, 1824, 1827, 1794, 1658, 1832, 1838, 1839, 1790, 1853, + /* 520 */ 1855, 1825, 1814, 1828, 1861, 1842, 1829, 1835, 1879, 1845, + /* 530 */ 1833, 1844, 1885, 1858, 1846, 1862, 1902, 1912, 1914, 1915, + /* 540 */ 1808, 1815, 1884, 1898, 1921, 1888, 1889, 1891, 1893, 1894, + /* 550 */ 1896, 1899, 1895, 1900, 1903, 1905, 1910, 1907, 1933, 1913, + /* 560 */ 1943, 1923, 1886, 1947, 1927, 1916, 1959, 1922, 1960, 1928, + /* 570 */ 1962, 1948, 1952, 1938, 1939, 1940, 1875, 1877, 1978, 1817, + /* 580 */ 1880, 1809, 1944, 1961, 1982, 1806, 1964, 1830, 1831, 1997, + /* 590 */ 2000, 1836, 1834, 2002, 1963, 1752, 1906, 1909, 1911, 1965, + /* 600 */ 1917, 1966, 1920, 1918, 1968, 1973, 1919, 1924, 1926, 1929, + /* 610 */ 1931, 1974, 1975, 1976, 1934, 1977, 1771, 1932, 1935, 2025, + /* 620 */ 1993, 1801, 2003, 2005, 2006, 2010, 2011, 2012, 1949, 1950, + /* 630 */ 1992, 1818, 2008, 2001, 2049, 2054, 2055, 2056, 1958, 2020, + /* 640 */ 1773, 2013, 1967, 1969, 1971, 1979, 1987, 1890, 1989, 2063, + /* 650 */ 2026, 1904, 1995, 1954, 1773, 2022, 2033, 1998, 1856, 1999, + /* 660 */ 2096, 2077, 1887, 2009, 2007, 2019, 2021, 2023, 2028, 2057, + /* 670 */ 2024, 2030, 2058, 2039, 2079, 1925, 2043, 2015, 2044, 2068, + /* 680 */ 2071, 2046, 2047, 2076, 2050, 2048, 2086, 2051, 2052, 2109, + /* 690 */ 2060, 2053, 2117, 2061, 1990, 2034, 2035, 2036, 2062, 2111, + /* 700 */ 2064, 2128, 2065, 2111, 2111, 2144, 2101, 2103, 2135, 2136, + /* 710 */ 2141, 2142, 2152, 2154, 2155, 2156, 2120, 2099, 2129, 2160, + /* 720 */ 2162, 2163, 2177, 2165, 2166, 2167, 2131, 1895, 2169, 1900, + /* 730 */ 2170, 2172, 2174, 2176, 2190, 2178, 2214, 2180, 2168, 2179, + /* 740 */ 2220, 2188, 2175, 2192, 2234, 2201, 2189, 2198, 2239, 2205, + /* 750 */ 2193, 2206, 2246, 2218, 2221, 2267, 2236, 2238, 2240, 2241, + /* 760 */ 2243, 2245, +}; +#define YY_REDUCE_COUNT (314) +#define YY_REDUCE_MIN (-397) +#define YY_REDUCE_MAX (2375) +static const short yy_reduce_ofst[] = { + /* 0 */ -156, 364, -82, 588, -20, 614, 804, 890, 1024, 1050, + /* 10 */ 1118, 1222, 1288, 1316, 1348, 1419, 174, -333, 154, 450, + /* 20 */ 674, 1447, 1097, 1482, 1466, 1543, 1566, 1660, 1692, 1710, + /* 30 */ 1757, 1804, 1826, 1851, 1873, 1937, 1957, 1972, 2004, 2072, + /* 40 */ 2091, 2119, 2138, 2185, 2242, 2258, 2308, 2328, 2375, 383, + /* 50 */ 270, -397, 83, -395, 371, 416, 554, 75, 501, 235, + /* 60 */ 786, 827, -139, 683, 834, -195, -165, -359, -80, -172, + /* 70 */ -13, -383, -254, -207, -213, 149, -52, 290, -152, -5, + /* 80 */ 263, 376, 407, 413, 453, 82, 508, 593, 462, 271, + /* 90 */ 610, 631, 275, 657, 374, 725, 301, 745, 372, -170, + /* 100 */ 748, 796, 399, 808, -83, 421, 605, 810, 260, -319, + /* 110 */ -338, -338, -293, -327, 94, 237, 433, 502, 534, 669, + /* 120 */ 693, 712, 802, 807, 809, 813, 815, 833, 849, 854, + /* 130 */ 865, 867, 868, -275, -251, -131, 355, 467, -251, 437, + /* 140 */ 559, 452, 445, 479, 726, -334, -85, 719, 681, 168, + /* 150 */ 736, 268, 134, 805, 797, 799, 742, 850, -369, 387, + /* 160 */ 499, 580, 713, 832, 913, 713, 637, 927, 485, 899, + /* 170 */ 818, 835, 949, 839, 941, 945, 940, 940, 980, 939, + /* 180 */ 1032, 1028, 994, 984, 933, 933, 919, 933, 947, 938, + /* 190 */ 940, 974, 976, 996, 1012, 1011, 1018, 1022, 1070, 1071, + /* 200 */ 1026, 1027, 1037, 1063, 1073, 1089, 1083, 1094, 1096, 1099, + /* 210 */ 1123, 1127, 1125, 1130, 1060, 1132, 1100, 1137, 1142, 1090, + /* 220 */ 1144, 1151, 1146, 1147, 1148, 1157, 1153, 1154, 1158, 1169, + /* 230 */ 1138, 1139, 1140, 1141, 1156, 1160, 1161, 1167, 1168, 1175, + /* 240 */ 1177, 1164, 1172, 1131, 1134, 1135, 1110, 1116, 1119, 1187, + /* 250 */ 1145, 1155, 1176, 1201, 1162, 1224, 1179, 1180, 1174, 1108, + /* 260 */ 1171, 1191, 1126, 1184, 1203, 1204, 940, 1128, 1133, 1166, + /* 270 */ 1150, 1170, 1173, 1178, 1122, 1182, 1185, 933, 1251, 1181, + /* 280 */ 1261, 1258, 1263, 1264, 1213, 1212, 1228, 1232, 1235, 1236, + /* 290 */ 1237, 1216, 1241, 1226, 1268, 1269, 1279, 1282, 1192, 1278, + /* 300 */ 1250, 1274, 1291, 1290, 1303, 1306, 1252, 1243, 1244, 1270, + /* 310 */ 1296, 1299, 1300, 1322, 1334, +>>>>>>> fix/rocksdbState }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, @@ -965,28 +1705,28 @@ static const YYACTIONTYPE yy_default[] = { /* 140 */ 2218, 1799, 2178, 2178, 1719, 1719, 1719, 1719, 1801, 2050, /* 150 */ 1719, 1719, 1719, 1719, 1719, 1719, 1920, 1719, 1719, 1719, /* 160 */ 1719, 1719, 1944, 1719, 1719, 1719, 2044, 1719, 1719, 2243, - /* 170 */ 2299, 1719, 1719, 2246, 1719, 1719, 1719, 1997, 1719, 1719, - /* 180 */ 1874, 2233, 2210, 2224, 2283, 2211, 2208, 2227, 1719, 2237, - /* 190 */ 1719, 2031, 1990, 1719, 1990, 1987, 1719, 1719, 1990, 1987, - /* 200 */ 1987, 1863, 1859, 1719, 1857, 1719, 1719, 1719, 1719, 1766, - /* 210 */ 1719, 1766, 1719, 1801, 1719, 1801, 1719, 1719, 1801, 1719, - /* 220 */ 1801, 1801, 1801, 1719, 1801, 1779, 1779, 1719, 1719, 1719, + /* 170 */ 2299, 1719, 1719, 2246, 1719, 1719, 1719, 1719, 1719, 1997, + /* 180 */ 1719, 1719, 1874, 2233, 2210, 2224, 2283, 2211, 2208, 2227, + /* 190 */ 1719, 2237, 1719, 2031, 1990, 1719, 1990, 1987, 1719, 1719, + /* 200 */ 1990, 1987, 1987, 1863, 1859, 1719, 1857, 1719, 1719, 1719, + /* 210 */ 1719, 1766, 1719, 1766, 1719, 1801, 1719, 1801, 1719, 1719, + /* 220 */ 1801, 1719, 1801, 1801, 1801, 1719, 1801, 1779, 1779, 1719, /* 230 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 240 */ 1719, 1719, 1719, 1719, 2062, 2048, 1719, 1799, 2042, 2040, - /* 250 */ 1719, 1799, 2038, 1719, 1719, 1719, 1719, 2254, 2252, 1719, - /* 260 */ 2254, 2252, 1719, 1719, 1719, 2268, 2264, 2254, 2272, 2270, - /* 270 */ 2239, 2237, 2302, 2289, 2285, 2224, 1719, 1719, 1719, 1719, - /* 280 */ 1799, 1799, 1719, 2252, 1719, 1719, 1719, 1719, 1719, 2252, - /* 290 */ 1719, 1719, 1799, 1719, 1799, 1719, 1719, 1890, 1719, 1719, - /* 300 */ 1719, 1799, 1751, 1719, 2033, 2053, 2015, 2015, 1923, 1923, - /* 310 */ 1923, 1802, 1724, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 320 */ 1719, 1719, 1719, 1719, 1719, 2267, 2266, 2133, 1719, 2182, - /* 330 */ 2181, 2180, 2171, 2132, 1886, 1719, 2131, 2130, 1719, 1719, - /* 340 */ 1719, 1719, 1719, 1719, 1719, 1719, 2006, 2005, 2124, 1719, - /* 350 */ 1719, 2125, 2123, 2122, 1719, 1719, 1719, 1719, 1719, 1719, + /* 240 */ 1719, 1719, 1719, 1719, 1719, 1719, 2062, 2048, 1719, 1799, + /* 250 */ 2042, 2040, 1719, 1799, 2038, 1719, 1719, 1719, 1719, 2254, + /* 260 */ 2252, 1719, 2254, 2252, 1719, 1719, 1719, 2268, 2264, 2254, + /* 270 */ 2272, 2270, 2239, 2237, 2302, 2289, 2285, 2224, 1719, 1719, + /* 280 */ 1719, 1719, 1799, 1799, 1719, 2252, 1719, 1719, 1719, 1719, + /* 290 */ 1719, 2252, 1719, 1719, 1799, 1719, 1799, 1719, 1719, 1890, + /* 300 */ 1719, 1719, 1719, 1799, 1751, 1719, 2033, 2053, 2015, 2015, + /* 310 */ 1923, 1923, 1923, 1802, 1724, 1719, 1719, 1719, 1719, 1719, + /* 320 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2267, 2266, 2133, + /* 330 */ 1719, 2182, 2181, 2180, 2171, 2132, 1886, 1719, 2131, 2130, + /* 340 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2006, 2005, + /* 350 */ 2124, 1719, 1719, 2125, 2123, 2122, 1719, 1719, 1719, 1719, /* 360 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 370 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2286, 2290, - /* 380 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2207, 1719, 1719, + /* 370 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 380 */ 2286, 2290, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2207, /* 390 */ 1719, 1719, 1719, 2106, 1719, 1719, 1719, 1719, 1719, 1719, /* 400 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 410 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, @@ -998,27 +1738,27 @@ static const YYACTIONTYPE yy_default[] = { /* 470 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 480 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 490 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 500 */ 1756, 2111, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 500 */ 1719, 1719, 1756, 2111, 1719, 1719, 1719, 1719, 1719, 1719, /* 510 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 520 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 530 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 540 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 550 */ 1840, 1839, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 550 */ 1719, 1719, 1840, 1839, 1719, 1719, 1719, 1719, 1719, 1719, /* 560 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 570 */ 1719, 1719, 1719, 1719, 2115, 1719, 1719, 1719, 1719, 1719, + /* 570 */ 1719, 1719, 1719, 1719, 1719, 1719, 2115, 1719, 1719, 1719, /* 580 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 590 */ 1719, 2282, 2240, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 590 */ 1719, 1719, 1719, 2282, 2240, 1719, 1719, 1719, 1719, 1719, /* 600 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 610 */ 1719, 2106, 1719, 2265, 1719, 1719, 2280, 1719, 2284, 1719, - /* 620 */ 1719, 1719, 1719, 1719, 1719, 1719, 2217, 2213, 1719, 1719, - /* 630 */ 2209, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2114, 1719, - /* 640 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 650 */ 1719, 1719, 2105, 1719, 2168, 1719, 1719, 1719, 2202, 1719, - /* 660 */ 1719, 2153, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 670 */ 1719, 2115, 1719, 2118, 1719, 1719, 1719, 1719, 1719, 1917, - /* 680 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 690 */ 1719, 1719, 1902, 1900, 1899, 1898, 1719, 1930, 1719, 1719, - /* 700 */ 1719, 1926, 1925, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 610 */ 1719, 1719, 1719, 2106, 1719, 2265, 1719, 1719, 2280, 1719, + /* 620 */ 2284, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2217, 2213, + /* 630 */ 1719, 1719, 2209, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 640 */ 2114, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 650 */ 1719, 1719, 1719, 1719, 2105, 1719, 2168, 1719, 1719, 1719, + /* 660 */ 2202, 1719, 1719, 2153, 1719, 1719, 1719, 1719, 1719, 1719, + /* 670 */ 1719, 1719, 1719, 2115, 1719, 2118, 1719, 1719, 1719, 1719, + /* 680 */ 1719, 1917, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 690 */ 1719, 1719, 1719, 1719, 1902, 1900, 1899, 1898, 1719, 1930, + /* 700 */ 1719, 1719, 1719, 1926, 1925, 1719, 1719, 1719, 1719, 1719, /* 710 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1820, 1719, /* 720 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1812, 1719, 1811, /* 730 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, @@ -1819,66 +2559,66 @@ static const char *const yyTokenName[] = { /* 355 */ "integer_list", /* 356 */ "variable_list", /* 357 */ "retention_list", - /* 358 */ "alter_db_option", - /* 359 */ "retention", - /* 360 */ "full_table_name", - /* 361 */ "column_def_list", - /* 362 */ "tags_def_opt", - /* 363 */ "table_options", - /* 364 */ "multi_create_clause", - /* 365 */ "tags_def", - /* 366 */ "multi_drop_clause", - /* 367 */ "alter_table_clause", - /* 368 */ "alter_table_options", - /* 369 */ "column_name", - /* 370 */ "type_name", - /* 371 */ "signed_literal", - /* 372 */ "create_subtable_clause", - /* 373 */ "specific_cols_opt", - /* 374 */ "expression_list", - /* 375 */ "drop_table_clause", - /* 376 */ "col_name_list", - /* 377 */ "column_def", - /* 378 */ "duration_list", - /* 379 */ "rollup_func_list", - /* 380 */ "alter_table_option", - /* 381 */ "duration_literal", - /* 382 */ "rollup_func_name", - /* 383 */ "function_name", - /* 384 */ "col_name", - /* 385 */ "db_name_cond_opt", - /* 386 */ "like_pattern_opt", - /* 387 */ "table_name_cond", - /* 388 */ "from_db_opt", - /* 389 */ "tag_list_opt", - /* 390 */ "tag_item", - /* 391 */ "column_alias", - /* 392 */ "full_index_name", - /* 393 */ "index_options", - /* 394 */ "index_name", - /* 395 */ "func_list", - /* 396 */ "sliding_opt", - /* 397 */ "sma_stream_opt", - /* 398 */ "func", - /* 399 */ "sma_func_name", - /* 400 */ "query_or_subquery", - /* 401 */ "cgroup_name", - /* 402 */ "analyze_opt", - /* 403 */ "explain_options", - /* 404 */ "insert_query", - /* 405 */ "or_replace_opt", - /* 406 */ "agg_func_opt", - /* 407 */ "bufsize_opt", - /* 408 */ "language_opt", - /* 409 */ "stream_name", - /* 410 */ "stream_options", - /* 411 */ "col_list_opt", - /* 412 */ "tag_def_or_ref_opt", - /* 413 */ "subtable_opt", - /* 414 */ "expression", - /* 415 */ "dnode_list", - /* 416 */ "where_clause_opt", - /* 417 */ "signed", + /* 358 */ "signed", + /* 359 */ "alter_db_option", + /* 360 */ "retention", + /* 361 */ "full_table_name", + /* 362 */ "column_def_list", + /* 363 */ "tags_def_opt", + /* 364 */ "table_options", + /* 365 */ "multi_create_clause", + /* 366 */ "tags_def", + /* 367 */ "multi_drop_clause", + /* 368 */ "alter_table_clause", + /* 369 */ "alter_table_options", + /* 370 */ "column_name", + /* 371 */ "type_name", + /* 372 */ "signed_literal", + /* 373 */ "create_subtable_clause", + /* 374 */ "specific_cols_opt", + /* 375 */ "expression_list", + /* 376 */ "drop_table_clause", + /* 377 */ "col_name_list", + /* 378 */ "column_def", + /* 379 */ "duration_list", + /* 380 */ "rollup_func_list", + /* 381 */ "alter_table_option", + /* 382 */ "duration_literal", + /* 383 */ "rollup_func_name", + /* 384 */ "function_name", + /* 385 */ "col_name", + /* 386 */ "db_name_cond_opt", + /* 387 */ "like_pattern_opt", + /* 388 */ "table_name_cond", + /* 389 */ "from_db_opt", + /* 390 */ "tag_list_opt", + /* 391 */ "tag_item", + /* 392 */ "column_alias", + /* 393 */ "full_index_name", + /* 394 */ "index_options", + /* 395 */ "index_name", + /* 396 */ "func_list", + /* 397 */ "sliding_opt", + /* 398 */ "sma_stream_opt", + /* 399 */ "func", + /* 400 */ "sma_func_name", + /* 401 */ "query_or_subquery", + /* 402 */ "cgroup_name", + /* 403 */ "analyze_opt", + /* 404 */ "explain_options", + /* 405 */ "insert_query", + /* 406 */ "or_replace_opt", + /* 407 */ "agg_func_opt", + /* 408 */ "bufsize_opt", + /* 409 */ "language_opt", + /* 410 */ "stream_name", + /* 411 */ "stream_options", + /* 412 */ "col_list_opt", + /* 413 */ "tag_def_or_ref_opt", + /* 414 */ "subtable_opt", + /* 415 */ "expression", + /* 416 */ "dnode_list", + /* 417 */ "where_clause_opt", /* 418 */ "literal_func", /* 419 */ "literal_list", /* 420 */ "table_alias", @@ -2053,8 +2793,8 @@ static const char *const yyRuleName[] = { /* 106 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", /* 107 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", /* 108 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", - /* 109 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", - /* 110 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", + /* 109 */ "db_options ::= db_options TABLE_PREFIX signed", + /* 110 */ "db_options ::= db_options TABLE_SUFFIX signed", /* 111 */ "alter_db_options ::= alter_db_option", /* 112 */ "alter_db_options ::= alter_db_options alter_db_option", /* 113 */ "alter_db_option ::= BUFFER NK_INTEGER", @@ -2661,36 +3401,36 @@ static void yy_destructor( case 351: /* alter_db_options */ case 353: /* start_opt */ case 354: /* end_opt */ - case 359: /* retention */ - case 360: /* full_table_name */ - case 363: /* table_options */ - case 367: /* alter_table_clause */ - case 368: /* alter_table_options */ - case 371: /* signed_literal */ - case 372: /* create_subtable_clause */ - case 375: /* drop_table_clause */ - case 377: /* column_def */ - case 381: /* duration_literal */ - case 382: /* rollup_func_name */ - case 384: /* col_name */ - case 385: /* db_name_cond_opt */ - case 386: /* like_pattern_opt */ - case 387: /* table_name_cond */ - case 388: /* from_db_opt */ - case 390: /* tag_item */ - case 392: /* full_index_name */ - case 393: /* index_options */ - case 396: /* sliding_opt */ - case 397: /* sma_stream_opt */ - case 398: /* func */ - case 400: /* query_or_subquery */ - case 403: /* explain_options */ - case 404: /* insert_query */ - case 410: /* stream_options */ - case 413: /* subtable_opt */ - case 414: /* expression */ - case 416: /* where_clause_opt */ - case 417: /* signed */ + case 358: /* signed */ + case 360: /* retention */ + case 361: /* full_table_name */ + case 364: /* table_options */ + case 368: /* alter_table_clause */ + case 369: /* alter_table_options */ + case 372: /* signed_literal */ + case 373: /* create_subtable_clause */ + case 376: /* drop_table_clause */ + case 378: /* column_def */ + case 382: /* duration_literal */ + case 383: /* rollup_func_name */ + case 385: /* col_name */ + case 386: /* db_name_cond_opt */ + case 387: /* like_pattern_opt */ + case 388: /* table_name_cond */ + case 389: /* from_db_opt */ + case 391: /* tag_item */ + case 393: /* full_index_name */ + case 394: /* index_options */ + case 397: /* sliding_opt */ + case 398: /* sma_stream_opt */ + case 399: /* func */ + case 401: /* query_or_subquery */ + case 404: /* explain_options */ + case 405: /* insert_query */ + case 411: /* stream_options */ + case 414: /* subtable_opt */ + case 415: /* expression */ + case 417: /* where_clause_opt */ case 418: /* literal_func */ case 421: /* expr_or_subquery */ case 422: /* pseudo_column */ @@ -2735,7 +3475,7 @@ static void yy_destructor( case 332: /* alter_account_options */ case 334: /* alter_account_option */ case 352: /* speed_opt */ - case 407: /* bufsize_opt */ + case 408: /* bufsize_opt */ { } @@ -2745,14 +3485,14 @@ static void yy_destructor( case 343: /* table_name */ case 344: /* topic_name */ case 346: /* dnode_endpoint */ - case 369: /* column_name */ - case 383: /* function_name */ - case 391: /* column_alias */ - case 394: /* index_name */ - case 399: /* sma_func_name */ - case 401: /* cgroup_name */ - case 408: /* language_opt */ - case 409: /* stream_name */ + case 370: /* column_name */ + case 384: /* function_name */ + case 392: /* column_alias */ + case 395: /* index_name */ + case 400: /* sma_func_name */ + case 402: /* cgroup_name */ + case 409: /* language_opt */ + case 410: /* stream_name */ case 420: /* table_alias */ case 426: /* star_func */ case 428: /* noarg_func */ @@ -2781,9 +3521,9 @@ static void yy_destructor( case 347: /* force_opt */ case 348: /* not_exists_opt */ case 350: /* exists_opt */ - case 402: /* analyze_opt */ - case 405: /* or_replace_opt */ - case 406: /* agg_func_opt */ + case 403: /* analyze_opt */ + case 406: /* or_replace_opt */ + case 407: /* agg_func_opt */ case 451: /* set_quantifier_opt */ { @@ -2792,21 +3532,21 @@ static void yy_destructor( case 355: /* integer_list */ case 356: /* variable_list */ case 357: /* retention_list */ - case 361: /* column_def_list */ - case 362: /* tags_def_opt */ - case 364: /* multi_create_clause */ - case 365: /* tags_def */ - case 366: /* multi_drop_clause */ - case 373: /* specific_cols_opt */ - case 374: /* expression_list */ - case 376: /* col_name_list */ - case 378: /* duration_list */ - case 379: /* rollup_func_list */ - case 389: /* tag_list_opt */ - case 395: /* func_list */ - case 411: /* col_list_opt */ - case 412: /* tag_def_or_ref_opt */ - case 415: /* dnode_list */ + case 362: /* column_def_list */ + case 363: /* tags_def_opt */ + case 365: /* multi_create_clause */ + case 366: /* tags_def */ + case 367: /* multi_drop_clause */ + case 374: /* specific_cols_opt */ + case 375: /* expression_list */ + case 377: /* col_name_list */ + case 379: /* duration_list */ + case 380: /* rollup_func_list */ + case 390: /* tag_list_opt */ + case 396: /* func_list */ + case 412: /* col_list_opt */ + case 413: /* tag_def_or_ref_opt */ + case 416: /* dnode_list */ case 419: /* literal_list */ case 427: /* star_func_para_list */ case 429: /* other_para_list */ @@ -2822,13 +3562,13 @@ static void yy_destructor( nodesDestroyList((yypminor->yy432)); } break; - case 358: /* alter_db_option */ - case 380: /* alter_table_option */ + case 359: /* alter_db_option */ + case 381: /* alter_table_option */ { } break; - case 370: /* type_name */ + case 371: /* type_name */ { } @@ -2982,15 +3722,18 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -3005,16 +3748,8 @@ static YYACTIONTYPE yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -3028,6 +3763,7 @@ static YYACTIONTYPE yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ + assert( i>=0 && i>>>>>> fix/rocksdbState }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3767,14 +5680,17 @@ static YYACTIONTYPE yy_reduce( yymsp = yypParser->yytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; + yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -4207,12 +6123,12 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 109: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + case 109: /* db_options ::= db_options TABLE_PREFIX signed */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy448); } yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 110: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + case 110: /* db_options ::= db_options TABLE_SUFFIX signed */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy448); } yymsp[-2].minor.yy448 = yylhsminor.yy448; break; case 111: /* alter_db_options ::= alter_db_option */ @@ -5468,9 +7384,9 @@ static YYACTIONTYPE yy_reduce( break; /********** End reduce actions ************************************************/ }; - assert( yyruleno 0; } static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) { diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 609976bf65..b7a1a68adf 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -16,7 +16,7 @@ #include "streamInc.h" #include "ttimer.h" -#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 2000 +#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 3000 int32_t streamInit() { int8_t old; @@ -356,4 +356,8 @@ void* streamQueueNextItem(SStreamQueue* queue) { } return streamQueueCurItem(queue); } +} + +void streamTaskInputFail(SStreamTask* pTask) { + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); } \ No newline at end of file diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index a9f6d29bf5..549374ed94 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -230,23 +230,21 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) { - goto FAIL; + rpcFreeCont(buf); + return code; } + tEncoderClear(&encoder); msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; msg.msgType = TDMT_STREAM_TASK_CHECK; - qDebug("dispatch from s-task:%s to downstream s-task:%"PRIx64":%d node %d: check msg", pTask->id.idStr, - pReq->streamId, pReq->downstreamTaskId, nodeId); + qDebug("dispatch from s-task:%s to downstream s-task:%" PRIx64 ":%d node %d: check msg", pTask->id.idStr, + pReq->streamId, pReq->downstreamTaskId, nodeId); tmsgSendReq(pEpSet, &msg); - return 0; -FAIL: - if (buf) rpcFreeCont(buf); - return code; } int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 67f51d3896..a3cdeb170a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -17,6 +17,11 @@ #define STREAM_EXEC_MAX_BATCH_NUM 1024 +bool streamTaskShouldStop(const SStreamStatus* pStatus) { + int32_t status = atomic_load_8((int8_t*) &pStatus->taskStatus); + return (status == TASK_STATUS__STOP) || (status == TASK_STATUS__DROPPING); +} + static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { int32_t code = TSDB_CODE_SUCCESS; void* pExecutor = pTask->exec.pExecutor; @@ -66,7 +71,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* // pExecutor while (1) { - if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + if (streamTaskShouldStop(&pTask->status)) { return 0; } @@ -106,7 +111,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* continue; } - qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId); + qDebug("s-task:%s (child %d) executed and get block", pTask->id.idStr, pTask->selfChildId); SSDataBlock block = {0}; assignOneDataBlock(&block, output); @@ -136,7 +141,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t batchCnt = 0; while (1) { - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { + if (streamTaskShouldStop(&pTask->status)) { taosArrayDestroy(pRes); return 0; } @@ -195,7 +200,10 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { qDebug("task %d scan exec dispatch block num %d", pTask->id.taskId, batchCnt); streamDispatch(pTask); } - if (finished) break; + + if (finished) { + break; + } } return 0; } @@ -274,7 +282,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { } } - if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + if (streamTaskShouldStop(&pTask->status)) { if (pInput) { streamFreeQitem(pInput); } @@ -305,7 +313,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { ", checkPoint id:%" PRId64 " -> %" PRId64, pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId); - pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId}; + pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer}; taosWLockLatch(&pTask->pMeta->lock); streamMetaSaveTask(pTask->pMeta, pTask); @@ -374,7 +382,7 @@ int32_t streamTryExec(SStreamTask* pTask) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); qDebug("s-task:%s exec completed", pTask->id.idStr); - if (!taosQueueEmpty(pTask->inputQueue->queue)) { + if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status))) { streamSchedExec(pTask); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 2d6af92204..63141d6219 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -85,11 +85,6 @@ void streamMetaClose(SStreamMeta* pMeta) { tdbClose(pMeta->db); void* pIter = NULL; -// while(pMeta->walScan) { -// qDebug("wait stream daemon quit"); -// taosMsleep(100); -// } - while (1) { pIter = taosHashIterate(pMeta->pTasks, pIter); if (pIter == NULL) { @@ -103,7 +98,6 @@ void streamMetaClose(SStreamMeta* pMeta) { } tFreeStreamTask(pTask); - /*streamMetaReleaseTask(pMeta, pTask);*/ } taosHashCleanup(pMeta->pTasks); @@ -177,10 +171,12 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { // add to the ready tasks hash map, not the restored tasks hash map int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { + tFreeStreamTask(pTask); return -1; } if (streamMetaSaveTask(pMeta, pTask) < 0) { + tFreeStreamTask(pTask); return -1; } @@ -196,10 +192,12 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { taosRLockLatch(&pMeta->lock); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) { - atomic_add_fetch_32(&(*ppTask)->refCnt, 1); - taosRUnLockLatch(&pMeta->lock); - return *ppTask; + if (ppTask != NULL) { + if (!streamTaskShouldStop(&(*ppTask)->status)) { + atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + taosRUnLockLatch(&pMeta->lock); + return *ppTask; + } } taosRUnLockLatch(&pMeta->lock); @@ -210,7 +208,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) { int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1); ASSERT(left >= 0); if (left == 0) { - ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING); + ASSERT(streamTaskShouldStop(&pTask->status)); tFreeStreamTask(pTask); } } @@ -221,11 +219,8 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask* pTask = *ppTask; taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn); - /*if (pTask->timer) { - * taosTmrStop(pTask->timer);*/ - /*pTask->timer = NULL;*/ - /*}*/ - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); + + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__STOP); taosWLockLatch(&pMeta->lock); streamMetaReleaseTask(pMeta, pTask); @@ -297,8 +292,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tDecodeStreamTask(&decoder, pTask); tDecoderClear(&decoder); - // todo set correct initial version. - if (pMeta->expandFunc(pMeta->ahandle, pTask, 0) < 0) { + if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 03afc0692d..55c745e417 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -102,8 +102,10 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp .downstreamNodeId = pRsp->downstreamNodeId, .childId = pRsp->childId, }; - qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->id.taskId, pTask->nodeId, + + qDebug("s-task:%s at node %d check downstream task %d at node %d (recheck)", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -116,6 +118,7 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp } } } + return 0; } @@ -158,9 +161,10 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* } else { ASSERT(0); } - } else { + } else { // not ready, it should wait for at least 100ms and then retry streamRecheckOneDownstream(pTask, pRsp); } + return 0; } diff --git a/source/libs/sync/inc/syncCommit.h b/source/libs/sync/inc/syncCommit.h index 7d638a7336..07b4702b1b 100644 --- a/source/libs/sync/inc/syncCommit.h +++ b/source/libs/sync/inc/syncCommit.h @@ -48,8 +48,6 @@ extern "C" { void syncOneReplicaAdvance(SSyncNode* pSyncNode); void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode); -bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index); -bool syncAgree(SSyncNode* pSyncNode, SyncIndex index); bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index); int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex); diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index a55fd7ead3..04456b2454 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -55,7 +55,6 @@ int32_t syncNodeReplicateReset(SSyncNode* pSyncNode, SRaftId* pDestId); int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode); int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg); -int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 9ab545075c..deae4b0b3f 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -89,17 +89,6 @@ // /\ UNCHANGED <> // -SSyncRaftEntry* syncBuildRaftEntryFromAppendEntries(const SyncAppendEntries* pMsg) { - SSyncRaftEntry* pEntry = taosMemoryMalloc(pMsg->dataLen); - if (pEntry == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - (void)memcpy(pEntry, pMsg->data, pMsg->dataLen); - ASSERT(pEntry->bytes == pMsg->dataLen); - return pEntry; -} - int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncAppendEntries* pMsg = pRpcMsg->pCont; SRpcMsg rpcRsp = {0}; @@ -146,7 +135,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { goto _IGNORE; } - pEntry = syncBuildRaftEntryFromAppendEntries(pMsg); + pEntry = syncEntryBuildFromAppendEntries(pMsg); if (pEntry == NULL) { sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr()); goto _IGNORE; diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 2501b4df8b..01f1f00c8b 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -44,22 +44,6 @@ // /\ UNCHANGED <> // -bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index) { - // I am leader, I agree - if (syncUtilSameId(pRaftId, &(pSyncNode->myRaftId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - return true; - } - - // follower agree - SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, pRaftId); - if (matchIndex >= index) { - return true; - } - - // not agree - return false; -} - static inline int64_t syncNodeAbs64(int64_t a, int64_t b) { ASSERT(a >= 0); ASSERT(b >= 0); @@ -85,19 +69,6 @@ bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index) { return count >= pNode->quorum; } -bool syncAgree(SSyncNode* pNode, SyncIndex index) { - int agreeCount = 0; - for (int i = 0; i < pNode->replicaNum; ++i) { - if (syncAgreeIndex(pNode, &(pNode->replicasId[i]), index)) { - ++agreeCount; - } - if (agreeCount >= pNode->quorum) { - return true; - } - } - return false; -} - int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex) { SyncIndex lastVer = ths->pLogStore->syncLogLastIndex(ths->pLogStore); commitIndex = TMAX(commitIndex, ths->commitIndex); diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 3e63e2fb8e..8f42780eb9 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -64,10 +64,13 @@ SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, Syn } SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) { - SSyncRaftEntry* pEntry = syncEntryBuild((int32_t)(pMsg->dataLen)); - if (pEntry == NULL) return NULL; - + SSyncRaftEntry* pEntry = taosMemoryMalloc(pMsg->dataLen); + if (pEntry == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } memcpy(pEntry, pMsg->data, pMsg->dataLen); + ASSERT(pEntry->bytes == pMsg->dataLen); return pEntry; } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 8ac9a860e3..2776225a39 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -46,8 +46,6 @@ // mdest |-> j]) // /\ UNCHANGED <> -int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg); - int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); @@ -86,20 +84,6 @@ int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftI return 0; } -int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) { - int32_t ret = 0; - SyncAppendEntries* pMsg = pRpcMsg->pCont; - - if (syncNodeNeedSendAppendEntries(pSyncNode, destRaftId, pMsg)) { - ret = syncNodeSendAppendEntries(pSyncNode, destRaftId, pRpcMsg); - } else { - sNTrace(pSyncNode, "do not repcate to dnode:%d for index:%" PRId64, DID(destRaftId), pMsg->prevLogIndex + 1); - rpcFreeCont(pRpcMsg->pCont); - } - - return ret; -} - int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcMsg* pMsg) { return syncNodeSendMsgById(destId, pSyncNode, pMsg); } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 7d2c8aa4e5..2b2a0daf7b 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -1008,7 +1008,7 @@ int32_t taosGetFqdn(char *fqdn) { // hints.ai_family = AF_INET; strcpy(fqdn, hostname); strcpy(fqdn + strlen(hostname), ".local"); -#else // __APPLE__ +#else // linux struct addrinfo hints = {0}; struct addrinfo *result = NULL; hints.ai_flags = AI_CANONNAME; @@ -1020,7 +1020,7 @@ int32_t taosGetFqdn(char *fqdn) { } strcpy(fqdn, result->ai_canonname); freeaddrinfo(result); -#endif // __APPLE__ +#endif // linux return 0; } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a3d3c399ab..2a18f420cd 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -17,7 +17,6 @@ #include "tlog.h" #include "os.h" #include "tconfig.h" -#include "tutil.h" #include "tjson.h" #include "tglobal.h" @@ -781,65 +780,6 @@ bool taosAssertDebug(bool condition, const char *file, int32_t line, const char return true; } -int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { - SJson* pJson = tjsonCreateObject(); - if (pJson == NULL) return -1; - char tmp[4096] = {0}; - - tjsonAddDoubleToObject(pJson, "reportVersion", 1); - - tjsonAddIntegerToObject(pJson, "clusterId", clusterId); - tjsonAddIntegerToObject(pJson, "startTime", startTime); - - taosGetFqdn(tmp); - tjsonAddStringToObject(pJson, "fqdn", tmp); - - tjsonAddIntegerToObject(pJson, "pid", taosGetPId()); - - taosGetAppName(tmp, NULL); - tjsonAddStringToObject(pJson, "appName", tmp); - - if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { - tjsonAddStringToObject(pJson, "os", tmp); - } - - float numOfCores = 0; - if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { - tjsonAddStringToObject(pJson, "cpuModel", tmp); - tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); - } else { - tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); - } - - snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); - tjsonAddStringToObject(pJson, "memory", tmp); - - tjsonAddStringToObject(pJson, "version", version); - tjsonAddStringToObject(pJson, "buildInfo", buildinfo); - tjsonAddStringToObject(pJson, "gitInfo", gitinfo); - - tjsonAddIntegerToObject(pJson, "crashSig", signum); - tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()); - -#ifdef _TD_DARWIN_64 - taosLogTraceToBuf(tmp, sizeof(tmp), 4); -#elif !defined(WINDOWS) - taosLogTraceToBuf(tmp, sizeof(tmp), 3); -#else - taosLogTraceToBuf(tmp, sizeof(tmp), 8); -#endif - - tjsonAddStringToObject(pJson, "stackInfo", tmp); - - char* pCont = tjsonToString(pJson); - tjsonDelete(pJson); - - *pMsg = pCont; - - return TSDB_CODE_SUCCESS; -} - - void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo) { const char *flags = "UTL FATAL "; ELogLevel level = DEBUG_FATAL; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index a49ff0cd5b..6edee27c05 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -217,8 +217,8 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem int32_t queueNum = taosGetQueueNumber(pool->qset); int32_t curWorkerNum = taosArrayGetSize(pool->workers); - int32_t dstWorkerNum = ceil(queueNum * pool->ratio); - if (dstWorkerNum < 2) dstWorkerNum = 2; + int32_t dstWorkerNum = ceilf(queueNum * pool->ratio); + if (dstWorkerNum < 1) dstWorkerNum = 1; // spawn a thread to process queue while (curWorkerNum < dstWorkerNum) { diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json index 9b99521f52..078688bb7e 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb3-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json index 7b87919a6d..e808c9d3f3 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb4-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json index baf0384e46..0054d985ee 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb2-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json index f683cc016b..bed3598bfe 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -55,7 +55,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb1-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f97c42b9ba..f1e0131e88 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1162,6 +1162,7 @@ ,,y,script,./test.sh -f tsim/db/show_create_table.sim ,,y,script,./test.sh -f tsim/db/tables.sim ,,y,script,./test.sh -f tsim/db/taosdlog.sim +,,y,script,./test.sh -f tsim/db/table_prefix_suffix.sim ,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim ,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim ,,y,script,./test.sh -f tsim/dnode/balance1.sim diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 8af997a060..59f3e6e51e 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,8 +68,13 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ +<<<<<<< HEAD --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=false -DPORTABLE=true -DWITH_GFLAGS=false;make -j || exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ +======= + -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=true;make -j || exit 1" +>>>>>>> fix/rocksdbState if [[ -d ${WORKDIR}/debugNoSan ]] ;then echo "delete ${WORKDIR}/debugNoSan" diff --git a/tests/script/tsim/db/table_prefix_suffix.sim b/tests/script/tsim/db/table_prefix_suffix.sim new file mode 100644 index 0000000000..1b483d7df7 --- /dev/null +++ b/tests/script/tsim/db/table_prefix_suffix.sim @@ -0,0 +1,182 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database db1 vgroups 5 TABLE_PREFIX 1 TABLE_SUFFIX 2; +sql use db1; +sql create table atb1aa (ts timestamp, f1 int); +sql create table btb1bb (ts timestamp, f1 int); +sql create table ctb1cc (ts timestamp, f1 int); +sql create table dtb1dd (ts timestamp, f1 int); +sql create table atb2aa (ts timestamp, f1 int); +sql create table btb2bb (ts timestamp, f1 int); +sql create table ctb2cc (ts timestamp, f1 int); +sql create table dtb2dd (ts timestamp, f1 int); +sql create table etb2ee (ts timestamp, f1 int); +sql show create database db1; +sql select count(*) a from information_schema.ins_tables where db_name='db1' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db1; + +sql drop database if exists db2; +sql create database db2 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX -2; +sql use db2; +sql create table taaa11 (ts timestamp, f1 int); +sql create table tbbb11 (ts timestamp, f1 int); +sql create table tccc11 (ts timestamp, f1 int); +sql create table tddd11 (ts timestamp, f1 int); +sql create table taaa22 (ts timestamp, f1 int); +sql create table tbbb22 (ts timestamp, f1 int); +sql create table tccc22 (ts timestamp, f1 int); +sql create table tddd22 (ts timestamp, f1 int); +sql create table teee22 (ts timestamp, f1 int); +sql show create database db2; +sql select count(*) a from information_schema.ins_tables where db_name='db2' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db2; + +sql drop database if exists db3; +sql create database db3 vgroups 5 TABLE_PREFIX -1; +sql use db3; +sql create table taaa11 (ts timestamp, f1 int); +sql create table tbbb11 (ts timestamp, f1 int); +sql create table tccc11 (ts timestamp, f1 int); +sql create table tddd11 (ts timestamp, f1 int); +sql create table zaaa22 (ts timestamp, f1 int); +sql create table zbbb22 (ts timestamp, f1 int); +sql create table zccc22 (ts timestamp, f1 int); +sql create table zddd22 (ts timestamp, f1 int); +sql create table zeee22 (ts timestamp, f1 int); +sql show create database db3; +sql select count(*) a from information_schema.ins_tables where db_name='db3' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db3; + +sql drop database if exists db4; +sql create database db4 vgroups 5 TABLE_SUFFIX -2; +sql use db4; +sql create table taaa11 (ts timestamp, f1 int); +sql create table tbbb11 (ts timestamp, f1 int); +sql create table tccc11 (ts timestamp, f1 int); +sql create table tddd11 (ts timestamp, f1 int); +sql create table zaaa22 (ts timestamp, f1 int); +sql create table zbbb22 (ts timestamp, f1 int); +sql create table zccc22 (ts timestamp, f1 int); +sql create table zddd22 (ts timestamp, f1 int); +sql create table zeee22 (ts timestamp, f1 int); +sql show create database db4; +sql select count(*) a from information_schema.ins_tables where db_name='db4' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db4; + +sql drop database if exists db5; +sql create database db5 vgroups 5 TABLE_PREFIX 1; +sql use db5; +sql create table taaa11 (ts timestamp, f1 int); +sql create table baaa11 (ts timestamp, f1 int); +sql create table caaa11 (ts timestamp, f1 int); +sql create table daaa11 (ts timestamp, f1 int); +sql create table faaa11 (ts timestamp, f1 int); +sql create table gbbb11 (ts timestamp, f1 int); +sql create table hbbb11 (ts timestamp, f1 int); +sql create table ibbb11 (ts timestamp, f1 int); +sql create table jbbb11 (ts timestamp, f1 int); +sql show create database db5; +sql select count(*) a from information_schema.ins_tables where db_name='db5' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db5; + +sql drop database if exists db6; +sql create database db6 vgroups 5 TABLE_SUFFIX 2; +sql use db6; +sql create table taaa11 (ts timestamp, f1 int); +sql create table taaa12 (ts timestamp, f1 int); +sql create table taaa13 (ts timestamp, f1 int); +sql create table taaa14 (ts timestamp, f1 int); +sql create table tbbb23 (ts timestamp, f1 int); +sql create table tbbb24 (ts timestamp, f1 int); +sql create table tbbb31 (ts timestamp, f1 int); +sql create table tbbb32 (ts timestamp, f1 int); +sql create table tbbb33 (ts timestamp, f1 int); +sql show create database db6; +sql select count(*) a from information_schema.ins_tables where db_name='db6' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db6; + +sql drop database if exists db7; +sql create database db7 vgroups 5 TABLE_PREFIX -100 TABLE_SUFFIX -92; +sql use db7; +sql create table taaa11 (ts timestamp, f1 int); +sql create table taaa12 (ts timestamp, f1 int); +sql create table taaa13 (ts timestamp, f1 int); +sql create table tbbb21 (ts timestamp, f1 int); +sql create table tbbb22 (ts timestamp, f1 int); +sql create table tbbb23 (ts timestamp, f1 int); +sql create table tbbb24 (ts timestamp, f1 int); +sql create table tccc31 (ts timestamp, f1 int); +sql create table tccc32 (ts timestamp, f1 int); +sql create table tccc33 (ts timestamp, f1 int); +sql create table tddd24 (ts timestamp, f1 int); +sql create table tddd31 (ts timestamp, f1 int); +sql create table tddd32 (ts timestamp, f1 int); +sql create table tddd33 (ts timestamp, f1 int); +sql show create database db7; +sql select count(*) a from information_schema.ins_tables where db_name='db7' group by vgroup_id having(count(*) > 0) order by a; +sql drop database if exists db7; + +sql_error create database db8 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX 2; +sql_error create database db8 vgroups 5 TABLE_PREFIX 191 TABLE_SUFFIX 192; +sql_error create database db8 vgroups 5 TABLE_PREFIX -192 TABLE_SUFFIX -191; +sql_error create database db8 vgroups 5 TABLE_PREFIX 100 TABLE_SUFFIX 92; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim index c676f2b1e0..849aeb2ac5 100644 --- a/tests/script/tsim/query/sys_tbname.sim +++ b/tests/script/tsim/query/sys_tbname.sim @@ -109,5 +109,26 @@ if $rows != 5000 then return -1 endi +sql create database d1; +sql create stable d1.st1 (ts timestamp, f int) tags(t int); +sql create stable d1.st2 (ts timestamp, f int) tags(t int); +sql create table d1.ct1 using d1.st1 tags(1); +sql create table d1.ct2 using d1.st2 tags(2); +sql create database d2; +sql create stable d2.st1(ts timestamp, f int) tags(t int); +sql create stable d2.st2(ts timestamp, f int) tags(t int); +sql create table d2.ct1 using d2.st1 tags(1); +sql create table d2.ct2 using d2.st2 tags(2); + +sql create database d3; +sql create stable d3.st1(ts timestamp, f int) tags(t int); +sql create stable d3.st2(ts timestamp, f int) tags(t int); +sql create table d3.ct1 using d3.st1 tags(1); +sql create table d3.ct2 using d3.st2 tags(2); +sql select count(*), stable_name, db_name from information_schema.ins_tables where db_name != 'd2' group by stable_name,db_name +print $rows +if $rows != 9 then + return -1 +endi #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/tableCount.sim b/tests/script/tsim/query/tableCount.sim index d8d9bb9b03..ac5e23273a 100644 --- a/tests/script/tsim/query/tableCount.sim +++ b/tests/script/tsim/query/tableCount.sim @@ -104,4 +104,9 @@ if $data62 != 5 then return -1 endi +sql select distinct db_name from information_schema.ins_tables; +print $rows +if $rows != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index 61cdb61ae0..3a1663df4d 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -37,7 +37,7 @@ if $loop_count == 20 then endi if $rows != 4 then - print =====rows=$rows, expect 4 + print =====rows=$rows expect 4 goto loop0 endi diff --git a/tests/system-test/0-others/TS-3131.tsql b/tests/system-test/0-others/TS-3131.tsql new file mode 100644 index 0000000000..6d4b4c53f4 --- /dev/null +++ b/tests/system-test/0-others/TS-3131.tsql @@ -0,0 +1,71 @@ +drop database if exists d0; +create database d0 replica 1 keep 365 minRows 100 maxRows 4096 comp 2 vgroups 2 precision 'ms'; +use d0; +create table if not exists almlog (starttime timestamp,endtime timestamp,durationtime int, alarmno int, alarmtext nchar(256),isactive nchar(64)) tags (mcid nchar(16)); +create table if not exists mplog (starttime timestamp,mpid int, paravalue nchar(256),mptype nchar(32)) tags (mcid nchar(16)); +create table if not exists mdlog (starttime timestamp,endtime timestamp,durationtime int, statuscode int, npcgmname nchar(256),attr int) tags (mcid nchar(16)); +create table if not exists nrglog (updatetime timestamp,energyvalue double,enerygyincrease double) tags (mcid nchar(16),enerygytype nchar(16)); + +create table almlog_m201 using almlog tags("m201"); +create table almlog_m0103 using almlog tags("m0103"); +create table almlog_m0103_20031 using almlog tags("m0103"); +create table almlog_m0103_20032 using almlog tags("m0103"); +create table almlog_m0103_3003 using almlog tags("m0103"); +create table almlog_m0103_20033 using almlog tags("m0103"); +create table almlog_m0103_30031 using almlog tags("m0103"); +create table almlog_m0201 using almlog tags("m0201"); +create table almlog_m0102 using almlog tags("m0102"); +create table almlog_m0101 using almlog tags("m0101"); +create table almlog_m1002 using almlog tags("m1002"); + +create table mplog_m0204_4 using mplog tags("m0204"); +create table mplog_m0204_5 using mplog tags("m0204"); +create table mplog_m0204_6 using mplog tags("m0204"); +create table mplog_m0204_12 using mplog tags("m0204"); +create table mplog_m0204 using mplog tags("m0204"); +create table mplog_m201 using mplog tags("m201"); +create table mplog_m0102 using mplog tags("m0102"); +create table mplog_m1101 using mplog tags("m1101"); + +create table mdlog_m0102 using mplog tags("m0102"); +create table mdlog_m0504 using mplog tags("m0504"); +create table mdlog_m0505 using mplog tags("m0505"); +create table mdlog_m0507 using mplog tags("m0507"); +create table mdlog_m1002 using mplog tags("m1002"); +create table mdlog_m3201 using mplog tags("m3201"); +create table mdlog_m0201 using mplog tags("m0201"); +create table mdlog_m1102 using mplog tags("m1102"); +create table mdlog_m201 using mplog tags("m201"); + +create table nrglog_m201_electricvalue1 using nrglog tags("m201","electricValue1"); +create table nrglog_m201_oilvalue1 using nrglog tags("m201","oilValue1"); +create table nrglog_m201_gasvalue1 using nrglog tags("m201","gasValue1"); +create table nrglog_m201_watervalue1 using nrglog tags("m201","waterValue1"); +create table nrglog_m0101_oilvalue1 using nrglog tags("m0101","oilValue1"); +create table nrglog_m0101_watervalue1 using nrglog tags("m0101","waterValue1"); +create table nrglog_m0102_gasvalue1 using nrglog tags("m0102","gasValue1"); +create table nrglog_m1903 using nrglog tags("m1903",NULL); +create table nrglog_m2802 using nrglog tags("m2802",NULL); +create table nrglog_m2101 using nrglog tags("m2101",NULL); +create table nrglog_m0102 using nrglog tags("m0102",NULL); +create table nrglog_m0101_electricvalue1 using nrglog tags("m0101","electricValue1"); +create table nrglog_m0101_gasvalue1 using nrglog tags("m0101","gasValue1"); +create table nrglog_m0102_electricvalue1 using nrglog tags("m0102","electricValue1"); +create table nrglog_m0102_oilvalue1 using nrglog tags("m0102","oilValue1"); +create table nrglog_m0102_watervalue1 using nrglog tags("m0102","waterValue1"); + + +insert into almlog_m0103 values(now,now+1s,10,0,'','dismissed'); +insert into almlog_m0103_20031 values(now,now+1s,10,20031,'','dismissed'); +insert into almlog_m0103_20032 values(now,now+1s,10,20032,'','dismissed'); +insert into almlog_m0103_3003 values(now,now+1s,10,3003,'','dismissed'); +insert into almlog_m0103_20033 values(now,now+1s,10,20033,'','dismissed'); +insert into almlog_m0103_30031 values(now,now+1s,10,30031,'','dismissed'); + +flush database d0; + +show table tags from almlog; + +select *,tbname from d0.almlog where mcid='m0103'; + +select table_name from information_schema.ins_tables where db_name='d0'; diff --git a/tests/system-test/0-others/compa4096.json b/tests/system-test/0-others/compa4096.json index 5cc5d2084d..5e203ded45 100644 --- a/tests/system-test/0-others/compa4096.json +++ b/tests/system-test/0-others/compa4096.json @@ -53,18 +53,8 @@ "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", - "columns": [ - { - "type": "INT", - "count": 4094 - } - ], - "tags": [ - { - "type": "TINYINT", - "count": 1 - } - ] + "columns": [{ "type": "INT","count": 4093}], + "tags": [{"type": "TINYINT", "count": 1},{"type": "NCHAR","count": 1}] } ] } diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 1922ebfb2f..33d1dac4b5 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -98,9 +98,12 @@ class TDTestCase: def buildTaosd(self,bPath): # os.system(f"mv {bPath}/build_bak {bPath}/build ") - os.system(f" cd {bPath} ") - + os.system(f" cd {bPath} ") + def is_list_same_as_ordered_list(self,unordered_list, ordered_list): + sorted_list = sorted(unordered_list) + return sorted_list == ordered_list + def run(self): scriptsPath = os.path.dirname(os.path.realpath(__file__)) distro_id = distro.id() @@ -146,6 +149,8 @@ class TDTestCase: tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") + cmd = f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taos -h localhost ;" if os.system(cmd) == 0: raise Exception("failed to execute system command. cmd: %s" % cmd) @@ -220,6 +225,17 @@ class TDTestCase: tdLog.exit("%s(%d) failed" % args) tdsql.query("show streams;") tdsql.checkRows(2) + tdsql.query("select *,tbname from d0.almlog where mcid='m0103';") + tdsql.checkRows(6) + expectList = [0,3003,20031,20032,20033,30031] + resultList = [] + for i in range(6): + resultList.append(tdsql.queryResult[i][3]) + print(resultList) + if self.is_list_same_as_ordered_list(resultList,expectList): + print("The unordered list is the same as the ordered list.") + else: + tdlog.error("The unordered list is not the same as the ordered list.") tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index f467e802ac..ee1a0ef5b3 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -47,17 +47,27 @@ class TDTestCase: if platform.system().lower() == 'windows': self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1_dup = subprocess.Popen('(for /r %s %%i in ("udf1_dup.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2_dup = subprocess.Popen('(for /r %s %%i in ("udf2_dup.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") if (not tdDnodes.dnodes[0].remoteIP == ""): tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1_dup.so',projPath+"\\debug\\build\\lib\\") tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2_dup.so',projPath+"\\debug\\build\\lib\\") self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf1_dup = self.libudf1_dup.replace('udf1_dup.dll','libudf1_dup.so') self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + self.libudf2_dup = self.libudf2_dup.replace('udf2_dup.dll','libudf2_dup.so') else: self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1_dup = subprocess.Popen('find %s -name "libudf1_dup.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2_dup = subprocess.Popen('find %s -name "libudf2_dup.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf1_dup = self.libudf1_dup.replace('\r','').replace('\n','') self.libudf2 = self.libudf2.replace('\r','').replace('\n','') + self.libudf2_dup = self.libudf2_dup.replace('\r','').replace('\n','') def prepare_data(self): @@ -174,10 +184,12 @@ class TDTestCase: # create scalar functions tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) + tdSql.execute("create function udf1_dup as '%s' outputtype int;"%self.libudf1_dup) # create aggregate functions tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + tdSql.execute("create aggregate function udf2_dup as '%s' outputtype double bufSize 8;"%self.libudf2_dup) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -188,6 +200,13 @@ class TDTestCase: # scalar functions + # udf1_dup + tdSql.query("select udf1(num1) ,udf1_dup(num1) from tb") + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,2) + tdSql.checkData(2,0,1) + tdSql.checkData(2,1,2) + tdSql.execute("use db ") tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") tdSql.checkData(0,0,None) @@ -238,6 +257,10 @@ class TDTestCase: # aggregate functions + tdSql.query("select udf2(num1) ,udf2_dup(num1) from tb") + val = tdSql.queryResult[0][0] + 100 + tdSql.checkData(0,1,val) + tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb") tdSql.checkData(0,0,15.362291496) tdSql.checkData(0,1,10000949.553189287) diff --git a/tests/system-test/1-insert/delete_childtable.py b/tests/system-test/1-insert/delete_childtable.py index e3144edb45..a12f884981 100644 --- a/tests/system-test/1-insert/delete_childtable.py +++ b/tests/system-test/1-insert/delete_childtable.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.dbname = 'db_test' self.setsql = TDSetSql() self.stbname = 'stb' diff --git a/tests/system-test/1-insert/delete_stable.py b/tests/system-test/1-insert/delete_stable.py index 313b6ce731..8ebe7b6692 100644 --- a/tests/system-test/1-insert/delete_stable.py +++ b/tests/system-test/1-insert/delete_stable.py @@ -29,6 +29,9 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db_test' + self.ns_dbname = 'ns_test' + self.us_dbname = 'us_test' + self.ms_dbname = 'ms_test' self.setsql = TDSetSql() self.stbname = 'stb' self.ntbname = 'ntb' @@ -220,11 +223,45 @@ class TDTestCase: tdSql.query(f'select {func}(*) from {self.stbname}') tdSql.execute(f'drop table {self.stbname}') tdSql.execute(f'drop database {self.dbname}') + + def precision_now_check(self): + for dbname in [self.ms_dbname, self.us_dbname, self.ns_dbname]: + self.ts = 1537146000000 + if dbname == self.us_dbname: + self.ts = int(self.ts*1000) + precision = "us" + elif dbname == self.ns_dbname: + precision = "ns" + self.ts = int(self.ts*1000000) + else: + precision = "ms" + self.ts = int(self.ts) + tdSql.execute(f'drop database if exists {dbname}') + tdSql.execute(f'create database if not exists {dbname} precision "{precision}"') + tdSql.execute(f'use {dbname}') + self.base_data = { + 'tinyint':self.tinyint_val + } + self.column_dict = { + 'col1': 'tinyint' + } + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table if not exists {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table if not exists {self.stbname}_{i} using {self.stbname} tags(1)') + self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) + tdSql.query(f'select * from {self.stbname}') + tdSql.checkEqual(tdSql.queryRows, self.tbnum*self.rowNum) + tdSql.execute(f'delete from {self.stbname} where ts < now()') + tdSql.query(f'select * from {self.stbname}') + tdSql.checkEqual(tdSql.queryRows, 0) + def run(self): self.delete_data_stb() tdDnodes.stoptaosd(1) tdDnodes.starttaosd(1) self.delete_data_stb() + self.precision_now_check() def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 7ee5fce5a8..1867dc54cb 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -111,7 +111,7 @@ class TDTestCase: topicFromStb1 = 'topic_UpperCase_stb1' # queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -148,7 +148,7 @@ class TDTestCase: tmqCom.checkFileContent(consumerId, queryString) - tdSql.query("drop topic %s"%topicFromStb1) + tdSql.query("drop topic `%s`"%topicFromStb1) tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self): @@ -196,7 +196,7 @@ class TDTestCase: topicFromStb1 = 'topic_UpperCase_stb1' queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -242,7 +242,7 @@ class TDTestCase: # tmqCom.checkFileContent(consumerId, queryString) - tdSql.query("drop topic %s"%topicFromStb1) + tdSql.query("drop topic `%s`"%topicFromStb1) tdLog.printNoPrefix("======== test case 2 end ...... ") diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 71b7fdef5d..67cc60d196 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -111,7 +111,7 @@ class TDTestCase: topicFromStb1 = 'topic_UpperCase_stb1' queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -148,7 +148,7 @@ class TDTestCase: # tmqCom.checkFileContent(consumerId, queryString) - tdSql.query("drop topic %s"%topicFromStb1) + tdSql.query("drop topic `%s`"%topicFromStb1) tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self): @@ -196,7 +196,7 @@ class TDTestCase: topicFromStb1 = 'topic_UpperCase_stb1' # queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -244,7 +244,7 @@ class TDTestCase: # tmqCom.checkFileContent(consumerId, queryString) - tdSql.query("drop topic %s"%topicFromStb1) + tdSql.query("drop topic `%s`"%topicFromStb1) tdLog.printNoPrefix("======== test case 2 end ...... ") diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index f82f89f1b9..1de9b62bcd 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -82,7 +82,7 @@ class TDTestCase: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) if tdSql.getRows() == 2 : - print(tdSql.getData(0, 1), tdSql.getData(1, 1)) + tdLog.info("row[0][1]: %d, row[1][1]: %d"%(tdSql.getData(0, 1), tdSql.getData(1, 1))) if tdSql.getData(1, 1) == 1: break time.sleep(0.1) @@ -122,6 +122,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tdLog.info("start create tables......") tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) @@ -137,11 +138,11 @@ class TDTestCase: tsql.execute(sql) event.set() - tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + tdLog.info("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) return def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): - tdLog.debug("start to insert data ............") + tdLog.info("start to insert data ............") tsql.execute("use %s" %dbName) pre_insert = "insert into " sql = pre_insert @@ -163,7 +164,7 @@ class TDTestCase: if sql != pre_insert: #print("insert sql:%s"%sql) tsql.execute(sql) - tdLog.debug("insert data ............ [OK]") + tdLog.info("insert data ............ [OK]") return def prepareEnv(self, **parameterDict): @@ -226,7 +227,7 @@ class TDTestCase: event.wait() tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 20 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -286,7 +287,7 @@ class TDTestCase: prepareEnvThread.start() tdLog.info("create topics from db") - topicName1 = 'topic_db1' + topicName1 = 'topic_db11' tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) consumerId = 0 diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index d146dca449..f05f600f27 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -42,7 +42,7 @@ class TDTestCase: 'showRow': 1} topicNameList = ['topic1', 'topic2'] - expectRowsList = [] + queryRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) tdSql.query(queryString) - expectRowsList.append(tdSql.getRows()) + queryRowsList.append(tdSql.getRows()) # create one stb2 paraDict["stbName"] = 'stb2' @@ -77,7 +77,7 @@ class TDTestCase: tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) # tdSql.query(queryString) - # expectRowsList.append(tdSql.getRows()) + # queryRowsList.append(tdSql.getRows()) # init consume info, and start tmq_sim, then check consume result tdLog.info("insert consume info to consume processor") @@ -99,7 +99,8 @@ class TDTestCase: pThread = tmqCom.asyncInsertData(paraDict) tdLog.info("wait consumer commit notify") - tmqCom.getStartCommitNotifyFromTmqsim(rows=4) + # tmqCom.getStartCommitNotifyFromTmqsim(rows=4) + tmqCom.getStartConsumeNotifyFromTmqsim(rows=2) tdLog.info("pkill one consume processor") tmqCom.stopTmqSimProcess('tmq_sim_new') @@ -109,19 +110,21 @@ class TDTestCase: tdLog.info("wait the consume result") expectRows = 2 resultList = tmqCom.selectConsumeResult(expectRows) - actTotalRows = 0 + actConsumTotalRows = 0 for i in range(len(resultList)): - actTotalRows += resultList[i] + actConsumTotalRows += resultList[i] + + tdLog.info("act consumer1 rows: %d, consumer2 rows: %d"%(resultList[0], resultList[1])) tdSql.query(queryString) - expectRowsList.append(tdSql.getRows()) - expectTotalRows = 0 - for i in range(len(expectRowsList)): - expectTotalRows += expectRowsList[i] + queryRowsList.append(tdSql.getRows()) + queryTotalRows = 0 + for i in range(len(queryRowsList)): + queryTotalRows += queryRowsList[i] - tdLog.info("act consume rows: %d, expect consume rows: %d"%(actTotalRows, expectTotalRows)) - if expectTotalRows <= resultList[0]: - tdLog.info("act consume rows: %d should >= expect consume rows: %d"%(actTotalRows, expectTotalRows)) + tdLog.info("act consume rows: %d, query consume rows: %d"%(actConsumTotalRows, queryTotalRows)) + if actConsumTotalRows < queryTotalRows: + tdLog.info("act consume rows: %d should >= query consume rows: %d"%(actConsumTotalRows, queryTotalRows)) tdLog.exit("0 tmq consume rows error!") # time.sleep(10) @@ -130,9 +133,95 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 20, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic3', 'topic4'] + queryRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + queryRowsList.append(tdSql.getRows()) + + # create one stb2 + paraDict["stbName"] = 'stb2' + # queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + queryRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + paraDict["rowsPerTbl"] = 5000 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = "%s,%s"%(topicNameList[0],topicNameList[1]) + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:3000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 1") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("start consume processor 2") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'],'cdb',0,1) + + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + actConsumTotalRows = 0 + for i in range(len(resultList)): + actConsumTotalRows += resultList[i] + + tdLog.info("act consumer1 rows: %d, consumer2 rows: %d"%(resultList[0], resultList[1])) + + queryTotalRows = 0 + for i in range(len(queryRowsList)): + queryTotalRows += queryRowsList[i] + + tdLog.info("act consume rows: %d, query consume rows: %d"%(actConsumTotalRows, queryTotalRows)) + if actConsumTotalRows < queryTotalRows: + tdLog.info("act consume rows: %d should >= query consume rows: %d"%(actConsumTotalRows, queryTotalRows)) + tdLog.exit("0 tmq consume rows error!") + + # time.sleep(10) + # for i in range(len(topicNameList)): + # tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + def run(self): tdSql.prepare() self.tmqCase1() + self.tmqCase2() def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index a94747e574..0b252a7334 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -82,7 +82,7 @@ class TDTestCase: tdLog.info("create topics from db") topicName1 = 'UpperCasetopic_%s'%(self.paraDict['dbName']) - tdSql.execute("create topic %s as database %s" %(topicName1, self.paraDict['dbName'])) + tdSql.execute("create topic `%s` as database %s" %(topicName1, self.paraDict['dbName'])) topicList = topicName1 + ',' +topicName1 keyList = '%s,%s,%s,%s'%(self.groupId,self.autoCommit,self.autoCommitInterval,self.autoOffset) @@ -113,7 +113,7 @@ class TDTestCase: tdLog.exit("tmq consume rows error!") time.sleep(10) - tdSql.query("drop topic %s"%topicName1) + tdSql.query("drop topic `%s`"%topicName1) tdLog.printNoPrefix("======== test case 1 end ...... ") diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 910b067d4e..5ac32eaad9 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -58,7 +58,6 @@ int32_t shellRunSingleCommand(char *command) { } if (shellRegexMatch(command, "^[ \t]*(quit|q|exit)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { - shellWriteHistory(); return -1; } @@ -887,7 +886,6 @@ void shellWriteHistory() { } i = (i + 1) % SHELL_MAX_HISTORY_SIZE; } - taosFsyncFile(pFile); taosCloseFile(&pFile); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index bc5809ffe8..795621dfdd 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -83,6 +83,9 @@ int main(int argc, char *argv[]) { #endif taos_init(); + // kill heart-beat thread when quit + taos_set_hb_quit(1); + if (shell.args.is_dump_config) { shellDumpConfig(); taos_cleanup(); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 755ab55625..21964403ea 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -895,6 +895,63 @@ int smlProcess_18784_Test() { return code; } +int sml_escape_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use db_escape"); + taos_free_result(pRes); + + const char *sql[] = { + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_used=176059i,total=1076048383523889174i 1661943960000000000", + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_f\\\\ree=\"\\\"id,= ei\\\\\\f\" 1661943960000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 1); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from `d,i= s\\k\"`"); //check stable name + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 5); + printf("fieldNum:%d\n", fieldNum); + + int numFields = taos_num_fields(pRes); + TAOS_FIELD *fields = taos_fetch_fields(pRes); + ASSERT(numFields == 5); + ASSERT(strncmp(fields[1].name, "inode\"i,= s_used", sizeof("inode\"i,= s_used") - 1) == 0); + ASSERT(strncmp(fields[2].name, "total", sizeof("total") - 1) == 0); + ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\\\ree", sizeof("inode\"i,= s_f\\\\ree") - 1) == 0); + ASSERT(strncmp(fields[4].name, "dev\"i,= ce", sizeof("dev\"i,= ce") - 1) == 0); + + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + int64_t used = *(int64_t *)row[1]; + int64_t total = *(int64_t *)row[2]; + + if (rowIndex == 0) { + ASSERT(ts == 1661943960000); + ASSERT(used == 176059); + ASSERT(total == 1076048383523889174); + ASSERT(strncmp(row[3], "\"id,= ei\\\\f", sizeof("\"id,= ei\\\\f") - 1) == 0); + ASSERT(strncmp(row[4], "s\"i,= dc", sizeof("s\"i,= dc") - 1) == 0); + + } + rowIndex++; + } + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_19221_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -961,6 +1018,55 @@ int sml_ts2164_Test() { return code; } + +int sml_ts3116_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = + taos_query(taos, "DROP DATABASE IF EXISTS ts3116"); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts3116 BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + taos_free_result(pRes); + + char *sql = { + "meters,location=la,groupid=ca current=11.8,voltage=221", + }; + + pRes = taos_query(taos, "use ts3116"); + taos_free_result(pRes); + int32_t totalRows = 0; + char *tmp = (char *)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql, strlen(sql)); + totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taosMemoryFree(tmp); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + + char *sql1 = { + "meters,location=la,groupid=ca\\=3 current=11.8,voltage=221\nmeters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + }; + + pRes = taos_query(taos, "use ts3116"); + taos_free_result(pRes); + + tmp = (char *)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql1, strlen(sql1)); + totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taosMemoryFree(tmp); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_td22898_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1195,6 +1301,10 @@ int main(int argc, char *argv[]) { } int ret = 0; + ret = sml_escape_Test(); + ASSERT(!ret); + ret = sml_ts3116_Test(); + ASSERT(!ret); ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file ASSERT(!ret); // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){