xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into lihui3.0/test

This commit is contained in:
plum-lihui 2023-04-26 16:11:07 +08:00
parent 96abcfe876 a7f405d4d4
commit 81a4e02e84
114 changed files with 7946 additions and 2917 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 0681d8b
GIT_TAG 4378702
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
include/common/tcommon.h
View File

@ -341,6 +341,8 @@ typedef struct {
float f;
};
size_t length;
bool keyEscaped;
bool valueEscaped;
} SSmlKv;
#define QUERY_ASC_FORWARD_STEP 1

1
include/common/tmsg.h
View File

@ -689,6 +689,7 @@ typedef struct {
int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
int32_t tDeserializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
void tFreeSAlterUserReq(SAlterUserReq* pReq);
typedef struct {
char user[TSDB_USER_LEN];

659
include/common/ttokendef.h
View File

@ -16,335 +16,336 @@
#ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_USER 33
#define TK_ENABLE 34
#define TK_NK_INTEGER 35
#define TK_SYSINFO 36
#define TK_DROP 37
#define TK_GRANT 38
#define TK_ON 39
#define TK_TO 40
#define TK_REVOKE 41
#define TK_FROM 42
#define TK_SUBSCRIBE 43
#define TK_NK_COMMA 44
#define TK_READ 45
#define TK_WRITE 46
#define TK_NK_DOT 47
#define TK_WITH 48
#define TK_DNODE 49
#define TK_PORT 50
#define TK_DNODES 51
#define TK_NK_IPTOKEN 52
#define TK_FORCE 53
#define TK_LOCAL 54
#define TK_QNODE 55
#define TK_BNODE 56
#define TK_SNODE 57
#define TK_MNODE 58
#define TK_DATABASE 59
#define TK_USE 60
#define TK_FLUSH 61
#define TK_TRIM 62
#define TK_COMPACT 63
#define TK_IF 64
#define TK_NOT 65
#define TK_EXISTS 66
#define TK_BUFFER 67
#define TK_CACHEMODEL 68
#define TK_CACHESIZE 69
#define TK_COMP 70
#define TK_DURATION 71
#define TK_NK_VARIABLE 72
#define TK_MAXROWS 73
#define TK_MINROWS 74
#define TK_KEEP 75
#define TK_PAGES 76
#define TK_PAGESIZE 77
#define TK_TSDB_PAGESIZE 78
#define TK_PRECISION 79
#define TK_REPLICA 80
#define TK_VGROUPS 81
#define TK_SINGLE_STABLE 82
#define TK_RETENTIONS 83
#define TK_SCHEMALESS 84
#define TK_WAL_LEVEL 85
#define TK_WAL_FSYNC_PERIOD 86
#define TK_WAL_RETENTION_PERIOD 87
#define TK_WAL_RETENTION_SIZE 88
#define TK_WAL_ROLL_PERIOD 89
#define TK_WAL_SEGMENT_SIZE 90
#define TK_STT_TRIGGER 91
#define TK_TABLE_PREFIX 92
#define TK_TABLE_SUFFIX 93
#define TK_NK_COLON 94
#define TK_MAX_SPEED 95
#define TK_START 96
#define TK_TIMESTAMP 97
#define TK_END 98
#define TK_TABLE 99
#define TK_NK_LP 100
#define TK_NK_RP 101
#define TK_STABLE 102
#define TK_ADD 103
#define TK_COLUMN 104
#define TK_MODIFY 105
#define TK_RENAME 106
#define TK_TAG 107
#define TK_SET 108
#define TK_NK_EQ 109
#define TK_USING 110
#define TK_TAGS 111
#define TK_BOOL 112
#define TK_TINYINT 113
#define TK_SMALLINT 114
#define TK_INT 115
#define TK_INTEGER 116
#define TK_BIGINT 117
#define TK_FLOAT 118
#define TK_DOUBLE 119
#define TK_BINARY 120
#define TK_NCHAR 121
#define TK_UNSIGNED 122
#define TK_JSON 123
#define TK_VARCHAR 124
#define TK_MEDIUMBLOB 125
#define TK_BLOB 126
#define TK_VARBINARY 127
#define TK_DECIMAL 128
#define TK_COMMENT 129
#define TK_MAX_DELAY 130
#define TK_WATERMARK 131
#define TK_ROLLUP 132
#define TK_TTL 133
#define TK_SMA 134
#define TK_DELETE_MARK 135
#define TK_FIRST 136
#define TK_LAST 137
#define TK_SHOW 138
#define TK_PRIVILEGES 139
#define TK_DATABASES 140
#define TK_TABLES 141
#define TK_STABLES 142
#define TK_MNODES 143
#define TK_QNODES 144
#define TK_FUNCTIONS 145
#define TK_INDEXES 146
#define TK_ACCOUNTS 147
#define TK_APPS 148
#define TK_CONNECTIONS 149
#define TK_LICENCES 150
#define TK_GRANTS 151
#define TK_QUERIES 152
#define TK_SCORES 153
#define TK_TOPICS 154
#define TK_VARIABLES 155
#define TK_CLUSTER 156
#define TK_BNODES 157
#define TK_SNODES 158
#define TK_TRANSACTIONS 159
#define TK_DISTRIBUTED 160
#define TK_CONSUMERS 161
#define TK_SUBSCRIPTIONS 162
#define TK_VNODES 163
#define TK_ALIVE 164
#define TK_LIKE 165
#define TK_TBNAME 166
#define TK_QTAGS 167
#define TK_AS 168
#define TK_INDEX 169
#define TK_FUNCTION 170
#define TK_INTERVAL 171
#define TK_COUNT 172
#define TK_LAST_ROW 173
#define TK_TOPIC 174
#define TK_META 175
#define TK_CONSUMER 176
#define TK_GROUP 177
#define TK_DESC 178
#define TK_DESCRIBE 179
#define TK_RESET 180
#define TK_QUERY 181
#define TK_CACHE 182
#define TK_EXPLAIN 183
#define TK_ANALYZE 184
#define TK_VERBOSE 185
#define TK_NK_BOOL 186
#define TK_RATIO 187
#define TK_NK_FLOAT 188
#define TK_OUTPUTTYPE 189
#define TK_AGGREGATE 190
#define TK_BUFSIZE 191
#define TK_LANGUAGE 192
#define TK_REPLACE 193
#define TK_STREAM 194
#define TK_INTO 195
#define TK_TRIGGER 196
#define TK_AT_ONCE 197
#define TK_WINDOW_CLOSE 198
#define TK_IGNORE 199
#define TK_EXPIRED 200
#define TK_FILL_HISTORY 201
#define TK_UPDATE 202
#define TK_SUBTABLE 203
#define TK_KILL 204
#define TK_CONNECTION 205
#define TK_TRANSACTION 206
#define TK_BALANCE 207
#define TK_VGROUP 208
#define TK_LEADER 209
#define TK_MERGE 210
#define TK_REDISTRIBUTE 211
#define TK_SPLIT 212
#define TK_DELETE 213
#define TK_INSERT 214
#define TK_NULL 215
#define TK_NK_QUESTION 216
#define TK_NK_ARROW 217
#define TK_ROWTS 218
#define TK_QSTART 219
#define TK_QEND 220
#define TK_QDURATION 221
#define TK_WSTART 222
#define TK_WEND 223
#define TK_WDURATION 224
#define TK_IROWTS 225
#define TK_ISFILLED 226
#define TK_CAST 227
#define TK_NOW 228
#define TK_TODAY 229
#define TK_TIMEZONE 230
#define TK_CLIENT_VERSION 231
#define TK_SERVER_VERSION 232
#define TK_SERVER_STATUS 233
#define TK_CURRENT_USER 234
#define TK_CASE 235
#define TK_WHEN 236
#define TK_THEN 237
#define TK_ELSE 238
#define TK_BETWEEN 239
#define TK_IS 240
#define TK_NK_LT 241
#define TK_NK_GT 242
#define TK_NK_LE 243
#define TK_NK_GE 244
#define TK_NK_NE 245
#define TK_MATCH 246
#define TK_NMATCH 247
#define TK_CONTAINS 248
#define TK_IN 249
#define TK_JOIN 250
#define TK_INNER 251
#define TK_SELECT 252
#define TK_DISTINCT 253
#define TK_WHERE 254
#define TK_PARTITION 255
#define TK_BY 256
#define TK_SESSION 257
#define TK_STATE_WINDOW 258
#define TK_EVENT_WINDOW 259
#define TK_SLIDING 260
#define TK_FILL 261
#define TK_VALUE 262
#define TK_VALUE_F 263
#define TK_NONE 264
#define TK_PREV 265
#define TK_NULL_F 266
#define TK_LINEAR 267
#define TK_NEXT 268
#define TK_HAVING 269
#define TK_RANGE 270
#define TK_EVERY 271
#define TK_ORDER 272
#define TK_SLIMIT 273
#define TK_SOFFSET 274
#define TK_LIMIT 275
#define TK_OFFSET 276
#define TK_ASC 277
#define TK_NULLS 278
#define TK_ABORT 279
#define TK_AFTER 280
#define TK_ATTACH 281
#define TK_BEFORE 282
#define TK_BEGIN 283
#define TK_BITAND 284
#define TK_BITNOT 285
#define TK_BITOR 286
#define TK_BLOCKS 287
#define TK_CHANGE 288
#define TK_COMMA 289
#define TK_CONCAT 290
#define TK_CONFLICT 291
#define TK_COPY 292
#define TK_DEFERRED 293
#define TK_DELIMITERS 294
#define TK_DETACH 295
#define TK_DIVIDE 296
#define TK_DOT 297
#define TK_EACH 298
#define TK_FAIL 299
#define TK_FILE 300
#define TK_FOR 301
#define TK_GLOB 302
#define TK_ID 303
#define TK_IMMEDIATE 304
#define TK_IMPORT 305
#define TK_INITIALLY 306
#define TK_INSTEAD 307
#define TK_ISNULL 308
#define TK_KEY 309
#define TK_MODULES 310
#define TK_NK_BITNOT 311
#define TK_NK_SEMI 312
#define TK_NOTNULL 313
#define TK_OF 314
#define TK_PLUS 315
#define TK_PRIVILEGE 316
#define TK_RAISE 317
#define TK_RESTRICT 318
#define TK_ROW 319
#define TK_SEMI 320
#define TK_STAR 321
#define TK_STATEMENT 322
#define TK_STRICT 323
#define TK_STRING 324
#define TK_TIMES 325
#define TK_VALUES 326
#define TK_VARIABLE 327
#define TK_VIEW 328
#define TK_WAL 329
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_USER 33
#define TK_ENABLE 34
#define TK_NK_INTEGER 35
#define TK_SYSINFO 36
#define TK_DROP 37
#define TK_GRANT 38
#define TK_ON 39
#define TK_TO 40
#define TK_REVOKE 41
#define TK_FROM 42
#define TK_SUBSCRIBE 43
#define TK_NK_COMMA 44
#define TK_READ 45
#define TK_WRITE 46
#define TK_NK_DOT 47
#define TK_WITH 48
#define TK_DNODE 49
#define TK_PORT 50
#define TK_DNODES 51
#define TK_NK_IPTOKEN 52
#define TK_FORCE 53
#define TK_LOCAL 54
#define TK_QNODE 55
#define TK_BNODE 56
#define TK_SNODE 57
#define TK_MNODE 58
#define TK_DATABASE 59
#define TK_USE 60
#define TK_FLUSH 61
#define TK_TRIM 62
#define TK_COMPACT 63
#define TK_IF 64
#define TK_NOT 65
#define TK_EXISTS 66
#define TK_BUFFER 67
#define TK_CACHEMODEL 68
#define TK_CACHESIZE 69
#define TK_COMP 70
#define TK_DURATION 71
#define TK_NK_VARIABLE 72
#define TK_MAXROWS 73
#define TK_MINROWS 74
#define TK_KEEP 75
#define TK_PAGES 76
#define TK_PAGESIZE 77
#define TK_TSDB_PAGESIZE 78
#define TK_PRECISION 79
#define TK_REPLICA 80
#define TK_VGROUPS 81
#define TK_SINGLE_STABLE 82
#define TK_RETENTIONS 83
#define TK_SCHEMALESS 84
#define TK_WAL_LEVEL 85
#define TK_WAL_FSYNC_PERIOD 86
#define TK_WAL_RETENTION_PERIOD 87
#define TK_WAL_RETENTION_SIZE 88
#define TK_WAL_ROLL_PERIOD 89
#define TK_WAL_SEGMENT_SIZE 90
#define TK_STT_TRIGGER 91
#define TK_TABLE_PREFIX 92
#define TK_TABLE_SUFFIX 93
#define TK_NK_COLON 94
#define TK_MAX_SPEED 95
#define TK_START 96
#define TK_TIMESTAMP 97
#define TK_END 98
#define TK_TABLE 99
#define TK_NK_LP 100
#define TK_NK_RP 101
#define TK_STABLE 102
#define TK_ADD 103
#define TK_COLUMN 104
#define TK_MODIFY 105
#define TK_RENAME 106
#define TK_TAG 107
#define TK_SET 108
#define TK_NK_EQ 109
#define TK_USING 110
#define TK_TAGS 111
#define TK_BOOL 112
#define TK_TINYINT 113
#define TK_SMALLINT 114
#define TK_INT 115
#define TK_INTEGER 116
#define TK_BIGINT 117
#define TK_FLOAT 118
#define TK_DOUBLE 119
#define TK_BINARY 120
#define TK_NCHAR 121
#define TK_UNSIGNED 122
#define TK_JSON 123
#define TK_VARCHAR 124
#define TK_MEDIUMBLOB 125
#define TK_BLOB 126
#define TK_VARBINARY 127
#define TK_DECIMAL 128
#define TK_COMMENT 129
#define TK_MAX_DELAY 130
#define TK_WATERMARK 131
#define TK_ROLLUP 132
#define TK_TTL 133
#define TK_SMA 134
#define TK_DELETE_MARK 135
#define TK_FIRST 136
#define TK_LAST 137
#define TK_SHOW 138
#define TK_PRIVILEGES 139
#define TK_DATABASES 140
#define TK_TABLES 141
#define TK_STABLES 142
#define TK_MNODES 143
#define TK_QNODES 144
#define TK_FUNCTIONS 145
#define TK_INDEXES 146
#define TK_ACCOUNTS 147
#define TK_APPS 148
#define TK_CONNECTIONS 149
#define TK_LICENCES 150
#define TK_GRANTS 151
#define TK_QUERIES 152
#define TK_SCORES 153
#define TK_TOPICS 154
#define TK_VARIABLES 155
#define TK_CLUSTER 156
#define TK_BNODES 157
#define TK_SNODES 158
#define TK_TRANSACTIONS 159
#define TK_DISTRIBUTED 160
#define TK_CONSUMERS 161
#define TK_SUBSCRIPTIONS 162
#define TK_VNODES 163
#define TK_ALIVE 164
#define TK_LIKE 165
#define TK_TBNAME 166
#define TK_QTAGS 167
#define TK_AS 168
#define TK_INDEX 169
#define TK_FUNCTION 170
#define TK_INTERVAL 171
#define TK_COUNT 172
#define TK_LAST_ROW 173
#define TK_TOPIC 174
#define TK_META 175
#define TK_CONSUMER 176
#define TK_GROUP 177
#define TK_DESC 178
#define TK_DESCRIBE 179
#define TK_RESET 180
#define TK_QUERY 181
#define TK_CACHE 182
#define TK_EXPLAIN 183
#define TK_ANALYZE 184
#define TK_VERBOSE 185
#define TK_NK_BOOL 186
#define TK_RATIO 187
#define TK_NK_FLOAT 188
#define TK_OUTPUTTYPE 189
#define TK_AGGREGATE 190
#define TK_BUFSIZE 191
#define TK_LANGUAGE 192
#define TK_REPLACE 193
#define TK_STREAM 194
#define TK_INTO 195
#define TK_TRIGGER 196
#define TK_AT_ONCE 197
#define TK_WINDOW_CLOSE 198
#define TK_IGNORE 199
#define TK_EXPIRED 200
#define TK_FILL_HISTORY 201
#define TK_UPDATE 202
#define TK_SUBTABLE 203
#define TK_KILL 204
#define TK_CONNECTION 205
#define TK_TRANSACTION 206
#define TK_BALANCE 207
#define TK_VGROUP 208
#define TK_LEADER 209
#define TK_MERGE 210
#define TK_REDISTRIBUTE 211
#define TK_SPLIT 212
#define TK_DELETE 213
#define TK_INSERT 214
#define TK_NULL 215
#define TK_NK_QUESTION 216
#define TK_NK_ARROW 217
#define TK_ROWTS 218
#define TK_QSTART 219
#define TK_QEND 220
#define TK_QDURATION 221
#define TK_WSTART 222
#define TK_WEND 223
#define TK_WDURATION 224
#define TK_IROWTS 225
#define TK_ISFILLED 226
#define TK_CAST 227
#define TK_NOW 228
#define TK_TODAY 229
#define TK_TIMEZONE 230
#define TK_CLIENT_VERSION 231
#define TK_SERVER_VERSION 232
#define TK_SERVER_STATUS 233
#define TK_CURRENT_USER 234
#define TK_CASE 235
#define TK_WHEN 236
#define TK_THEN 237
#define TK_ELSE 238
#define TK_BETWEEN 239
#define TK_IS 240
#define TK_NK_LT 241
#define TK_NK_GT 242
#define TK_NK_LE 243
#define TK_NK_GE 244
#define TK_NK_NE 245
#define TK_MATCH 246
#define TK_NMATCH 247
#define TK_CONTAINS 248
#define TK_IN 249
#define TK_JOIN 250
#define TK_INNER 251
#define TK_SELECT 252
#define TK_DISTINCT 253
#define TK_WHERE 254
#define TK_PARTITION 255
#define TK_BY 256
#define TK_SESSION 257
#define TK_STATE_WINDOW 258
#define TK_EVENT_WINDOW 259
#define TK_SLIDING 260
#define TK_FILL 261
#define TK_VALUE 262
#define TK_VALUE_F 263
#define TK_NONE 264
#define TK_PREV 265
#define TK_NULL_F 266
#define TK_LINEAR 267
#define TK_NEXT 268
#define TK_HAVING 269
#define TK_RANGE 270
#define TK_EVERY 271
#define TK_ORDER 272
#define TK_SLIMIT 273
#define TK_SOFFSET 274
#define TK_LIMIT 275
#define TK_OFFSET 276
#define TK_ASC 277
#define TK_NULLS 278
#define TK_ABORT 279
#define TK_AFTER 280
#define TK_ATTACH 281
#define TK_BEFORE 282
#define TK_BEGIN 283
#define TK_BITAND 284
#define TK_BITNOT 285
#define TK_BITOR 286
#define TK_BLOCKS 287
#define TK_CHANGE 288
#define TK_COMMA 289
#define TK_CONCAT 290
#define TK_CONFLICT 291
#define TK_COPY 292
#define TK_DEFERRED 293
#define TK_DELIMITERS 294
#define TK_DETACH 295
#define TK_DIVIDE 296
#define TK_DOT 297
#define TK_EACH 298
#define TK_FAIL 299
#define TK_FILE 300
#define TK_FOR 301
#define TK_GLOB 302
#define TK_ID 303
#define TK_IMMEDIATE 304
#define TK_IMPORT 305
#define TK_INITIALLY 306
#define TK_INSTEAD 307
#define TK_ISNULL 308
#define TK_KEY 309
#define TK_MODULES 310
#define TK_NK_BITNOT 311
#define TK_NK_SEMI 312
#define TK_NOTNULL 313
#define TK_OF 314
#define TK_PLUS 315
#define TK_PRIVILEGE 316
#define TK_RAISE 317
#define TK_RESTRICT 318
#define TK_ROW 319
#define TK_SEMI 320
#define TK_STAR 321
#define TK_STATEMENT 322
#define TK_STRICT 323
#define TK_STRING 324
#define TK_TIMES 325
#define TK_VALUES 326
#define TK_VARIABLE 327
#define TK_VIEW 328
#define TK_WAL 329
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

8
include/libs/catalog/catalog.h
View File

@ -82,6 +82,7 @@ typedef struct SCatalogReq {
SArray* pUser; // element is SUserAuthInfo
SArray* pTableIndex; // element is SNAME
SArray* pTableCfg; // element is SNAME
SArray* pTableTag; // element is SNAME
bool qNodeRequired; // valid qnode
bool dNodeRequired; // valid dnode
bool svrVerRequired;
@ -105,6 +106,7 @@ typedef struct SMetaData {
SArray* pUser; // pRes = SUserAuthRes*
SArray* pQnodeList; // pRes = SArray<SQueryNodeLoad>*
SArray* pTableCfg; // pRes = STableCfg*
SArray* pTableTag; // pRes = SArray<STagVal>*
SArray* pDnodeList; // pRes = SArray<SEpSet>*
SMetaRes* pSvrVer; // pRes = char*
} SMetaData;
@ -122,8 +124,8 @@ typedef struct SSTableVersion {
char stbName[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
uint64_t suid;
int16_t sversion;
int16_t tversion;
int32_t sversion;
int32_t tversion;
int32_t smaVer;
} SSTableVersion;
@ -312,6 +314,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char*
int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg);
int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp);

2
include/libs/nodes/querynodes.h
View File

@ -379,6 +379,8 @@ typedef struct SVnodeModifyOpStmt {
SName usingTableName;
const char* pBoundCols;
struct STableMeta* pTableMeta;
SNode* pTagCond;
SArray* pTableTag;
SHashObj* pVgroupsHashObj;
SHashObj* pTableBlockHashObj; // SHashObj<tuid, STableDataCxt*>
SHashObj* pSubTableHashObj;

4
include/libs/qcom/query.h
View File

@ -116,8 +116,8 @@ typedef struct STableMeta {
// if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta
// info
int16_t sversion;
int16_t tversion;
int32_t sversion;
int32_t tversion;
STableComInfo tableInfo;
SSchema schema[];
} STableMeta;

2
include/libs/stream/tstream.h
View File

@ -273,6 +273,7 @@ typedef struct SStreamId {
typedef struct SCheckpointInfo {
int64_t id;
int64_t version; // offset in WAL
int64_t currentVer;// current offset in WAL, not serialize it
} SCheckpointInfo;
typedef struct SStreamStatus {
@ -537,6 +538,7 @@ void streamTaskInputFail(SStreamTask* pTask);
int32_t streamTryExec(SStreamTask* pTask);
int32_t streamSchedExec(SStreamTask* pTask);
int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock);
bool streamTaskShouldStop(const SStreamStatus* pStatus);
int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz);

8
include/util/tdef.h
View File

@ -368,11 +368,11 @@ typedef enum ELogicConditionType {
#define TSDB_MIN_STT_TRIGGER 1
#define TSDB_MAX_STT_TRIGGER 16
#define TSDB_DEFAULT_SST_TRIGGER 1
#define TSDB_MIN_HASH_PREFIX 0
#define TSDB_MAX_HASH_PREFIX 128
#define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN)
#define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_PREFIX 0
#define TSDB_MIN_HASH_SUFFIX 0
#define TSDB_MAX_HASH_SUFFIX 128
#define TSDB_MIN_HASH_SUFFIX (2 - TSDB_TABLE_NAME_LEN)
#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_SUFFIX 0
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1

20
include/util/tutil.h
View File

@ -81,14 +81,22 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
int32_t suffix) {
if (prefix == 0 && suffix == 0) {
if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) {
return MurmurHash3_32(tbname, tblen);
} else if (prefix > 0 || suffix > 0) {
return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
} else {
if (tblen <= (prefix + suffix)) {
return MurmurHash3_32(tbname, tblen);
} else {
return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
char tbName[TSDB_TABLE_FNAME_LEN];
int32_t offset = 0;
if (prefix < 0) {
offset = -1 * prefix;
strncpy(tbName, tbname, offset);
}
if (suffix < 0) {
strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix);
offset += -1 *suffix;
}
return MurmurHash3_32(tbName, offset);
}
}
@ -98,6 +106,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
goto LABEL; \
}
#define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL)
#ifdef __cplusplus
}
#endif

37
packaging/deb/DEBIAN/preinst
View File

@ -26,6 +26,38 @@ if pidof taosd &> /dev/null; then
sleep 1
fi
# Stop adapter service if running
if pidof taosadapter &> /dev/null; then
if pidof systemd &> /dev/null; then
${csudo}systemctl stop taosadapter || :
elif $(which service &> /dev/null); then
${csudo}service taosadapter stop || :
else
pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
fi
echo "Stop taosadapter service success!"
sleep 1
fi
# Stop keeper service if running
if pidof taoskeeper &> /dev/null; then
if pidof systemd &> /dev/null; then
${csudo}systemctl stop taoskeeper || :
elif $(which service &> /dev/null); then
${csudo}service taoskeeper stop || :
else
pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
fi
echo "Stop taoskeeper service success!"
sleep 1
fi
# if taos.cfg already softlink, remove it
cfg_install_dir="/etc/taos"
install_main_dir="/usr/local/taos"
@ -41,6 +73,11 @@ if [ -f "${install_main_dir}/taosadapter.service" ]; then
${csudo}rm -f ${install_main_dir}/cfg/taosadapter.service || :
fi
if [ -f "${install_main_dir}/taoskeeper.toml" ]; then
${csudo}rm -f ${install_main_dir}/cfg/taoskeeper.toml || :
fi
# there can not libtaos.so*, otherwise ln -s error
${csudo}rm -f ${install_main_dir}/driver/libtaos.* || :
[ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || :

1
packaging/deb/DEBIAN/prerm
View File

@ -32,6 +32,7 @@ else
${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${cfg_link_dir}/* || :
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :

26
packaging/deb/makedeb.sh
View File

@ -44,8 +44,31 @@ mkdir -p ${pkg_dir}${install_home_path}/include
#mkdir -p ${pkg_dir}${install_home_path}/init.d
mkdir -p ${pkg_dir}${install_home_path}/script
# download taoskeeper and build
if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then
arch=amd64
elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then
arch=386
elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then
arch=arm
elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then
arch=arm64
else
arch=$cpuType
fi
echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper"
echo "$top_dir=${top_dir}"
taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper`
echo "taoskeeper_binary: ${taoskeeper_binary}"
# copy config files
cp $(dirname ${taoskeeper_binary})/config/taoskeeper.toml ${pkg_dir}${install_home_path}/cfg
cp $(dirname ${taoskeeper_binary})/taoskeeper.service ${pkg_dir}${install_home_path}/cfg
cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg
cp ${compile_dir}/../packaging/cfg/taosd.service ${pkg_dir}${install_home_path}/cfg
if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then
cp ${compile_dir}/test/cfg/taosadapter.toml ${pkg_dir}${install_home_path}/cfg || :
fi
@ -53,6 +76,7 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || :
fi
cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin
#cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d
cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script
@ -143,6 +167,7 @@ else
exit 1
fi
rm -rf ${pkg_dir}/build-taoskeeper
# make deb package
dpkg -b ${pkg_dir} $debname
echo "make deb package success!"
@ -150,4 +175,5 @@ echo "make deb package success!"
cp ${pkg_dir}/*.deb ${output_dir}
# clean temp dir
rm -rf ${pkg_dir}

24
packaging/rpm/makerpm.sh
View File

@ -35,14 +35,16 @@ function cp_rpm_package() {
local cur_dir
cd $1
cur_dir=$(pwd)
echo "cp_rpm_package cd: ${cur_dir}"
for dirlist in "$(ls ${cur_dir})"; do
if test -d ${dirlist}; then
cd ${dirlist}
echo 'cp_rpm_package ${cur_dir}/${dirlist}'
cp_rpm_package ${cur_dir}/${dirlist}
cd ..
fi
if test -e ${dirlist}; then
echo "${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm"
cp ${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm
fi
done
@ -54,6 +56,25 @@ fi
${csudo}mkdir -p ${pkg_dir}
cd ${pkg_dir}
# download taoskeeper and build
if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then
arch=amd64
elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then
arch=386
elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then
arch=arm
elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then
arch=arm64
else
arch=$cpuType
fi
cd ${top_dir}
echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper"
taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper`
echo "taoskeeper_binary: ${taoskeeper_binary}"
cd ${package_dir}
${csudo}mkdir -p BUILD BUILDROOT RPMS SOURCES SPECS SRPMS
${csudo}rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_dir}" --define="_compiledir ${compile_dir}" -bb ${spec_file}
@ -85,3 +106,4 @@ mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/${rpmname}
cd ..
${csudo}rm -rf ${pkg_dir}
rm -rf ${top_dir}/build-taoskeeper

21
packaging/rpm/tdengine.spec
View File

@ -3,6 +3,7 @@
%define cfg_install_dir /etc/taos
%define __strip /bin/true
%global __python /usr/bin/python3
%global _build_id_links none
Name: tdengine
Version: %{_version}
@ -62,6 +63,15 @@ fi
if [ -f %{_compiledir}/test/cfg/taosadapter.service ]; then
cp %{_compiledir}/test/cfg/taosadapter.service %{buildroot}%{homepath}/cfg
fi
if [ -f %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml ]; then
cp %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml %{buildroot}%{homepath}/cfg ||:
fi
if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper.service ]; then
cp %{_compiledir}/../build-taoskeeper/taoskeeper.service %{buildroot}%{homepath}/cfg ||:
fi
#cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d
cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script
cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/script
@ -73,8 +83,12 @@ cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin
if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper ]; then
cp %{_compiledir}/../build-taoskeeper/taoskeeper %{buildroot}%{homepath}/bin
fi
if [ -f %{_compiledir}/build/bin/taosadapter ]; then
cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin ||:
cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin
fi
cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver
[ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||:
@ -119,7 +133,9 @@ if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then
cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig
fi
fi
ls -al %{buildroot}%{homepath}/bin
tree -L 5
echo "==============================copying files done"
#Scripts executed before installation
%pre
if [ -f /var/lib/taos/dnode/dnodeCfg.json ]; then
@ -196,6 +212,7 @@ if [ $1 -eq 0 ];then
${csudo}rm -f ${bin_link_dir}/taosd || :
${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${cfg_link_dir}/* || :
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :

3
packaging/tools/makeclient.sh
View File

@ -197,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then
mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
if [ "$osType" != "Darwin" ]; then
[ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l)
[ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
fi
git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go
rm -rf ${install_dir}/connector/go/.git ||:

15
packaging/tools/makepkg.sh
View File

@ -338,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
[ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
tmp_pwd=`pwd`
cd ${install_dir}/connector
if [ ! -d taos-connector-jdbc ];then
git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||:
fi
cd taos-connector-jdbc
mvn clean package -Dmaven.test.skip=true
echo ${build_dir}/lib/
cp target/*.jar ${build_dir}/lib/
cd ${install_dir}/connector
rm -rf taos-connector-jdbc
cd ${tmp_pwd}
jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l)
[ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go
rm -rf ${install_dir}/connector/go/.git ||:

39
packaging/tools/post.sh
View File

@ -436,7 +436,7 @@ function local_fqdn_check() {
function install_taosadapter_config() {
if [ ! -f "${cfg_install_dir}/taosadapter.toml" ]; then
[ ! -d %{cfg_install_dir} ] &&
[ ! -d ${cfg_install_dir} ] &&
${csudo}${csudo}mkdir -p ${cfg_install_dir}
[ -f ${cfg_dir}/taosadapter.toml ] && ${csudo}cp ${cfg_dir}/taosadapter.toml ${cfg_install_dir}
[ -f ${cfg_install_dir}/taosadapter.toml ] &&
@ -451,19 +451,26 @@ function install_taosadapter_config() {
}
function install_taoskeeper_config() {
if [ ! -f "${cfg_install_dir}/keeper.toml" ]; then
[ ! -d %{cfg_install_dir} ] &&
${csudo}${csudo}mkdir -p ${cfg_install_dir}
[ -f ${cfg_dir}/keeper.toml ] && ${csudo}cp ${cfg_dir}/keeper.toml ${cfg_install_dir}
[ -f ${cfg_install_dir}/keeper.toml ] &&
${csudo}chmod 644 ${cfg_install_dir}/keeper.toml
# if new environment without taoskeeper
if [[ ! -f "${cfg_install_dir}/keeper.toml" ]] && [[ ! -f "${cfg_install_dir}/taoskeeper.toml" ]]; then
[ ! -d ${cfg_install_dir} ] && ${csudo}${csudo}mkdir -p ${cfg_install_dir}
[ -f ${cfg_dir}/taoskeeper.toml ] && ${csudo}cp ${cfg_dir}/taoskeeper.toml ${cfg_install_dir}
[ -f ${cfg_install_dir}/taoskeeper.toml ] &&
${csudo}chmod 644 ${cfg_install_dir}/taoskeeper.toml
fi
# if old machine with taoskeeper.toml file
if [ -f ${cfg_install_dir}/taoskeeper.toml ]; then
${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new
fi
[ -f ${cfg_dir}/keeper.toml ] &&
${csudo}mv ${cfg_dir}/keeper.toml ${cfg_dir}/keeper.toml.new
if [ -f ${cfg_install_dir}/keeper.toml ]; then
echo "The file keeper.toml will be renamed to taoskeeper.toml"
${csudo}mv ${cfg_install_dir}/keeper.toml ${cfg_install_dir}/taoskeeper.toml
${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new
fi
[ -f ${cfg_install_dir}/keeper.toml ] &&
${csudo}ln -s ${cfg_install_dir}/keeper.toml ${cfg_dir}
[ -f ${cfg_install_dir}/taoskeeper.toml ] &&
${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir}
}
function install_config() {
@ -655,6 +662,15 @@ function install_taosadapter_service() {
fi
}
function install_taoskeeper_service() {
if ((${service_mod}==0)); then
[ -f ${script_dir}/../cfg/taoskeeper.service ] &&\
${csudo}cp ${script_dir}/../cfg/taoskeeper.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
fi
}
function install_service() {
log_print "start install service"
if [ "$osType" != "Darwin" ]; then
@ -732,6 +748,7 @@ function install_TDengine() {
install_taosadapter_config
install_taoskeeper_config
install_taosadapter_service
install_taoskeeper_service
install_service
install_app

17
packaging/tools/preun.sh
View File

@ -17,7 +17,7 @@ cfg_link_dir="/usr/local/taos/cfg"
service_config_dir="/etc/systemd/system"
taos_service_name="taosd"
taoskeeper_service_name="taoskeeper"
csudo=""
if command -v sudo > /dev/null; then
csudo="sudo "
@ -57,6 +57,13 @@ function kill_taosd() {
fi
}
function kill_taoskeeper() {
pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function clean_service_on_systemd() {
taosadapter_service_config="${service_config_dir}/taosadapter.service"
if systemctl is-active --quiet taosadapter; then
@ -76,6 +83,12 @@ function clean_service_on_systemd() {
[ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config}
taoskeeper_service_config="${service_config_dir}/${taoskeeper_service_name}.service"
if systemctl is-active --quiet ${taoskeeper_service_name}; then
echo "TDengine taoskeeper is running, stopping it..."
${csudo}systemctl stop ${taoskeeper_service_name} &> /dev/null || echo &> /dev/null
fi
[ -f ${taoskeeper_service_config} ] && ${csudo}rm -f ${taoskeeper_service_config}
}
function clean_service_on_sysvinit() {
@ -111,6 +124,7 @@ function clean_service() {
# must manual stop taosd
kill_taosadapter
kill_taosd
kill_taoskeeper
fi
}
@ -124,6 +138,7 @@ ${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${cfg_link_dir}/*.new || :
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :

17
source/client/inc/clientSml.h
View File

@ -70,7 +70,7 @@ extern "C" {
#define VALUE_LEN 6
#define OTD_JSON_FIELDS_NUM 4
#define MAX_RETRY_TIMES 100
#define MAX_RETRY_TIMES 10
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
typedef enum {
@ -107,6 +107,7 @@ typedef struct {
int32_t colsLen;
int32_t timestampLen;
bool measureEscaped;
SArray *colArray;
} SSmlLineInfo;
@ -206,6 +207,19 @@ typedef struct {
#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \
if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \
MOVE_FORWARD_ONE(key + i, keyLen - i); \
keyLen--; \
} \
}
extern int64_t smlFactorNS[3];
extern int64_t smlFactorS[3];
@ -237,6 +251,7 @@ uint8_t smlGetTimestampLen(int64_t num);
void clearColValArray(SArray* pCols);
void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag);
void freeSSmlKv(void* data);
int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseJSON(SSmlHandle *info, char *payload);

36
source/client/src/clientMain.c
View File

@ -108,7 +108,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
if (pass == NULL) {
pass = TSDB_DEFAULT_PASS;
}
STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY);
if (pObj) {
int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t));
@ -359,11 +359,11 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
case TSDB_DATA_TYPE_NCHAR: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (fields[i].type == TSDB_DATA_TYPE_BINARY) {
if(ASSERT(charLen <= fields[i].bytes && charLen >= 0)){
if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) {
tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
}
} else {
if(ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)){
if (ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)) {
tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
}
}
@ -705,16 +705,16 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) {
int code = TSDB_CODE_SUCCESS;
taosThreadMutexLock(&pTscObj->mutex);
if(database == NULL || len <= 0){
if(required != NULL) *required = strlen(pTscObj->db) + 1;
if (database == NULL || len <= 0) {
if (required != NULL) *required = strlen(pTscObj->db) + 1;
terrno = TSDB_CODE_INVALID_PARA;
code = -1;
}else if(len < strlen(pTscObj->db) + 1){
} else if (len < strlen(pTscObj->db) + 1) {
tstrncpy(database, pTscObj->db, len);
if(required) *required = strlen(pTscObj->db) + 1;
if (required) *required = strlen(pTscObj->db) + 1;
terrno = TSDB_CODE_INVALID_PARA;
code = -1;
}else{
} else {
strcpy(database, pTscObj->db);
code = 0;
}
@ -741,6 +741,7 @@ static void destoryCatalogReq(SCatalogReq *pCatalogReq) {
taosArrayDestroy(pCatalogReq->pUser);
taosArrayDestroy(pCatalogReq->pTableIndex);
taosArrayDestroy(pCatalogReq->pTableCfg);
taosArrayDestroy(pCatalogReq->pTableTag);
taosMemoryFree(pCatalogReq);
}
@ -975,8 +976,10 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
if (TSDB_CODE_SUCCESS == code) {
pRequest->stmtType = pRequest->pQuery->pRoot->type;
phaseAsyncQuery(pWrapper);
} else {
code = phaseAsyncQuery(pWrapper);
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
@ -1042,11 +1045,11 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
}
void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
if(ASSERT(res != NULL && fp != NULL)){
if (ASSERT(res != NULL && fp != NULL)) {
tscError("taos_fetch_rows_a invalid paras");
return;
}
if(ASSERT(TD_RES_QUERY(res))){
if (ASSERT(TD_RES_QUERY(res))) {
tscError("taos_fetch_rows_a res is NULL");
return;
}
@ -1092,11 +1095,11 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
}
void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
if(ASSERT(res != NULL && fp != NULL)){
if (ASSERT(res != NULL && fp != NULL)) {
tscError("taos_fetch_rows_a invalid paras");
return;
}
if(ASSERT(TD_RES_QUERY(res))){
if (ASSERT(TD_RES_QUERY(res))) {
tscError("taos_fetch_rows_a res is NULL");
return;
}
@ -1111,11 +1114,11 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
}
const void *taos_get_raw_block(TAOS_RES *res) {
if(ASSERT(res != NULL)){
if (ASSERT(res != NULL)) {
tscError("taos_fetch_rows_a invalid paras");
return NULL;
}
if(ASSERT(TD_RES_QUERY(res))){
if (ASSERT(TD_RES_QUERY(res))) {
tscError("taos_fetch_rows_a res is NULL");
return NULL;
}
@ -1273,7 +1276,6 @@ _return:
return code;
}
int taos_load_table_info(TAOS *taos, const char *tableNameList) {
if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED;

44
source/client/src/clientSml.c
View File

@ -535,7 +535,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
if (index) {
if (colField[*index].type != kv->type) {
uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key);
return TSDB_CODE_TSC_INVALID_VALUE;
return TSDB_CODE_SML_INVALID_DATA;
}
if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR &&
@ -765,8 +765,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
size_t superTableLen = 0;
void *superTable = taosHashGetKey(tmp, &superTableLen);
char* measure = taosMemoryMalloc(superTableLen);
memcpy(measure, superTable, superTableLen);
PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen);
memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
memcpy(pName.tname, superTable, superTableLen);
memcpy(pName.tname, measure, superTableLen);
taosMemoryFree(measure);
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
@ -1049,7 +1053,7 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
// }
// taosMemoryFree(tag->key);
taosArrayDestroy(tag->cols);
taosArrayDestroy(tag->tags);
taosArrayDestroyEx(tag->tags, freeSSmlKv);
taosMemoryFree(tag);
}
@ -1063,6 +1067,12 @@ void clearColValArray(SArray *pCols) {
}
}
void freeSSmlKv(void* data){
SSmlKv *kv = (SSmlKv*)data;
if(kv->keyEscaped) taosMemoryFree((void*)(kv->key));
if(kv->valueEscaped) taosMemoryFree((void*)(kv->value));
}
void smlDestroyInfo(SSmlHandle *info) {
if (!info) return;
qDestroyQuery(info->pQuery);
@ -1098,11 +1108,11 @@ void smlDestroyInfo(SSmlHandle *info) {
}
taosArrayDestroy(info->valueJsonArray);
taosArrayDestroy(info->preLineTagKV);
taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv);
if (!info->dataFormat) {
for (int i = 0; i < info->lineNum; i++) {
taosArrayDestroy(info->lines[i].colArray);
taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
if (info->parseJsonByLib) {
taosMemoryFree(info->lines[i].tags);
}
@ -1165,8 +1175,10 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
}
for (size_t i = 0; i < taosArrayGetSize(cols); i++) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
terrno = 0;
taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
if (terrno == TSDB_CODE_DUP_KEY) {
taosHashCleanup(kvHash);
return terrno;
}
}
@ -1240,12 +1252,13 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
info->lineNum);
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
terrno = 0;
smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
if (terrno == TSDB_CODE_DUP_KEY) {
return terrno;
}
smlInsertMeta(meta->colHash, meta->cols, elements->colArray);
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
}
}
uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
@ -1304,9 +1317,15 @@ static int32_t smlInsertData(SSmlHandle *info) {
uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname,
tableData->uid, info->dataFormat);
int measureLen = tableData->sTableNameLen;
char* measure = (char*)taosMemoryMalloc(tableData->sTableNameLen);
memcpy(measure, tableData->sTableName, tableData->sTableNameLen);
PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
(*pMeta)->tableMeta, tableData->childTableName, measure, measureLen,
info->ttl, info->msgBuf.buf, info->msgBuf.len);
taosMemoryFree(measure);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
return code;
@ -1420,14 +1439,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
char cTmp = 0; // for print tmp if is raw
if (info->isRawLine) {
cTmp = tmp[len - 1];
tmp[len - 1] = '\0';
cTmp = tmp[len];
tmp[len] = '\0';
}
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
info->isRawLine, numLines, info->protocol, len, tmp);
if (info->isRawLine) {
tmp[len - 1] = cTmp;
tmp[len] = cTmp;
}
if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
@ -1449,6 +1468,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
}
if (code != TSDB_CODE_SUCCESS) {
tmp[len] = '\0';
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
return code;
}
@ -1494,8 +1514,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
do {
code = smlModifyDBSchemas(info);
if (code == 0) break;
taosMsleep(500);
if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA) break;
taosMsleep(100);
uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum);
} while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES);

174
source/client/src/clientSmlLine.c
View File

@ -21,32 +21,33 @@
#include "clientSml.h"
// comma ,
// #define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH)
#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
// space
// #define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH)
#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
// equal =
// #define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH)
#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
// quote "
// #define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH)
#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
//#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
// SLASH
// #define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH)
#define IS_SLASH_LETTER(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || \
*(sql) == SLASH)) // (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) ||
// IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql))
#define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) \
(*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
#define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL))
#define PROCESS_SLASH(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \
if (IS_SLASH_LETTER(key + i)) { \
#define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \
if (IS_SLASH_LETTER_IN_FIELD_VALUE(key + i)) { \
MOVE_FORWARD_ONE(key + i, keyLen - i); \
keyLen--; \
} \
}
#define PROCESS_SLASH_IN_TAG_FIELD_KEY(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(key + i)) { \
MOVE_FORWARD_ONE(key + i, keyLen - i); \
i--; \
keyLen--; \
} \
}
@ -151,7 +152,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
SSmlSTableMeta *sMeta = NULL;
if (unlikely(tmp == NULL)) {
STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
char* measure = currElement->measure;
int measureLen = currElement->measureLen;
if(currElement->measureEscaped){
measure = (char*)taosMemoryMalloc(currElement->measureLen);
memcpy(measure, currElement->measure, currElement->measureLen);
PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
}
STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
if(currElement->measureEscaped){
taosMemoryFree(measure);
}
if (pTableMeta == NULL) {
info->dataFormat = false;
info->reRun = true;
@ -171,17 +182,18 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
info->maxTagKVs = (*tmp)->tags;
}
}
taosArrayClear(preLineKV);
taosArrayClearEx(preLineKV, freeSSmlKv);
while (*sql < sqlEnd) {
if (unlikely(IS_SPACE(*sql))) {
break;
}
bool hasSlash = false;
// parse key
const char *key = *sql;
size_t keyLen = 0;
bool keyEscaped = false;
size_t keyLenEscaped = 0;
while (*sql < sqlEnd) {
if (unlikely(IS_COMMA(*sql))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
@ -192,16 +204,14 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
(*sql)++;
break;
}
if (!hasSlash) {
hasSlash = (*(*sql) == SLASH);
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
keyLenEscaped++;
keyEscaped = true;
}
(*sql)++;
}
if (unlikely(hasSlash)) {
PROCESS_SLASH(key, keyLen)
}
if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
@ -209,7 +219,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
// parse value
const char *value = *sql;
size_t valueLen = 0;
hasSlash = false;
bool valueEscaped = false;
size_t valueLenEscaped = 0;
while (*sql < sqlEnd) {
// parse value
if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
@ -219,8 +230,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return TSDB_CODE_SML_INVALID_DATA;
}
if (!hasSlash) {
hasSlash = (*(*sql) == SLASH);
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
valueLenEscaped++;
valueEscaped = true;
}
(*sql)++;
@ -232,15 +244,24 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(hasSlash)) {
PROCESS_SLASH(value, valueLen)
}
if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen};
if (keyEscaped){
char *tmp = (char*)taosMemoryMalloc(keyLen);
memcpy(tmp, key, keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen);
key = tmp;
}
if (valueEscaped){
char *tmp = (char*)taosMemoryMalloc(valueLen);
memcpy(tmp, value, valueLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen);
value = tmp;
}
SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped};
taosArrayPush(preLineKV, &kv);
if (info->dataFormat) {
if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
info->dataFormat = false;
@ -266,7 +287,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
info->needModifySchema = true;
}
}
taosArrayPush(preLineKV, &kv);
cnt++;
if (IS_SPACE(*sql)) {
@ -285,6 +305,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return TSDB_CODE_OUT_OF_MEMORY;
}
tinfo->tags = taosArrayDup(preLineKV, NULL);
for(size_t i = 0; i < taosArrayGetSize(preLineKV); i++){
SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
if(kv->keyEscaped)kv->key = NULL;
if(kv->valueEscaped)kv->value = NULL;
}
smlSetCTableName(tinfo);
tinfo->uid = info->uid++;
@ -321,7 +346,17 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
SSmlSTableMeta **tmp =
(SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
if (unlikely(tmp == NULL)) {
STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
char* measure = currElement->measure;
int measureLen = currElement->measureLen;
if(currElement->measureEscaped){
measure = (char*)taosMemoryMalloc(currElement->measureLen);
memcpy(measure, currElement->measure, currElement->measureLen);
PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
}
STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
if(currElement->measureEscaped){
taosMemoryFree(measure);
}
if (pTableMeta == NULL) {
info->dataFormat = false;
info->reRun = true;
@ -352,10 +387,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
break;
}
bool hasSlash = false;
// parse key
const char *key = *sql;
size_t keyLen = 0;
bool keyEscaped = false;
size_t keyLenEscaped = 0;
while (*sql < sqlEnd) {
if (unlikely(IS_COMMA(*sql))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
@ -366,16 +402,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
(*sql)++;
break;
}
if (!hasSlash) {
hasSlash = (*(*sql) == SLASH);
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
keyLenEscaped++;
keyEscaped = true;
}
(*sql)++;
}
if (unlikely(hasSlash)) {
PROCESS_SLASH(key, keyLen)
}
if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
@ -383,11 +417,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
// parse value
const char *value = *sql;
size_t valueLen = 0;
hasSlash = false;
bool isInQuote = false;
bool valueEscaped = false;
size_t valueLenEscaped = 0;
bool isInQuote = false;
const char *escapeChar = NULL;
while (*sql < sqlEnd) {
// parse value
if (unlikely(IS_QUOTE(*sql))) {
if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) {
isInQuote = !isInQuote;
(*sql)++;
continue;
@ -395,13 +431,12 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
if (!isInQuote) {
if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
break;
} else if (unlikely(IS_EQUAL(*sql))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
return TSDB_CODE_SML_INVALID_DATA;
}
}
if (!hasSlash) {
hasSlash = (*(*sql) == SLASH);
if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) {
escapeChar = *sql;
valueEscaped = true;
valueLenEscaped++;
}
(*sql)++;
@ -416,9 +451,6 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(hasSlash)) {
PROCESS_SLASH(value, valueLen)
}
SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen};
int32_t ret = smlParseValue(&kv, &info->msgBuf);
@ -427,11 +459,28 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return ret;
}
if (keyEscaped){
char *tmp = (char*)taosMemoryMalloc(kv.keyLen);
memcpy(tmp, key, kv.keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen);
kv.key = tmp;
kv.keyEscaped = keyEscaped;
}
if (valueEscaped){
char *tmp = (char*)taosMemoryMalloc(kv.length);
memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
kv.value = tmp;
kv.valueEscaped = valueEscaped;
}
if (info->dataFormat) {
// cnt begin 0, add ts so + 2
if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
info->dataFormat = false;
info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS;
}
// bind data
@ -440,22 +489,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
uDebug("smlBuildCol error, retry");
info->dataFormat = false;
info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS;
}
if (cnt >= taosArrayGetSize(info->masColKVs)) {
info->dataFormat = false;
info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS;
}
SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt);
if (kv.type != maxKV->type) {
info->dataFormat = false;
info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS;
}
if (unlikely(!IS_SAME_KEY)) {
info->dataFormat = false;
info->reRun = true;
freeSSmlKv(&kv);
return TSDB_CODE_SUCCESS;
}
@ -463,6 +516,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
maxKV->length = kv.length;
info->needModifySchema = true;
}
freeSSmlKv(&kv);
} else {
if (currElement->colArray == NULL) {
currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@ -487,10 +541,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
elements->measure = sql;
// parse measure
size_t measureLenEscaped = 0;
while (sql < sqlEnd) {
if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) {
MOVE_FORWARD_ONE(sql, sqlEnd - sql);
sqlEnd--;
if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) {
elements->measureEscaped = true;
measureLenEscaped++;
sql++;
continue;
}
if (unlikely(IS_COMMA(sql))) {
@ -503,7 +559,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
sql++;
}
elements->measureLen = sql - elements->measure;
if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) {
if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
@ -581,7 +637,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
.keyLen = TS_LEN,
.type = TSDB_DATA_TYPE_TIMESTAMP,
.i = ts,
.length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
.length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
.keyEscaped = false,
.valueEscaped = false};
if (info->dataFormat) {
uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);

15
source/client/src/clientTmq.c
View File

@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
SArray* container = &list->container;
if (src == NULL || src[0] == 0) return -1;
char* topic = taosStrdup(src);
if (topic[0] != '`') {
strtolower(topic, src);
}
if (taosArrayPush(container, &topic) == NULL) return -1;
return 0;
}
@ -1243,9 +1240,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pParam);
if (code != 0) {
tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
vgId, epoch, tstrerror(code), requestId);
if (pMsg->pData) taosMemoryFree(pMsg->pData);
if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet);
@ -1267,6 +1261,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosWriteQitem(tmq->mqueue, pRspWrapper);
} else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert
taosMsleep(500);
} else{
tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
vgId, epoch, tstrerror(code), requestId);
}
goto CREATE_MSG_FAIL;
@ -1667,11 +1664,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
return handleErrorBeforePoll(pVg, pTmq);
}
sendInfo->msgInfo = (SDataBuf){
.pData = msg,
.len = msgSize,
.handle = NULL,
};
sendInfo->msgInfo = (SDataBuf){ .pData = msg, .len = msgSize, .handle = NULL };
sendInfo->requestId = req.reqId;
sendInfo->requestObjRefId = 0;

36
source/client/test/smlTest.cpp
View File

@ -50,8 +50,9 @@ TEST(testCase, smlParseInfluxString_Test) {
int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql);
ASSERT_EQ(elements.measureLen, strlen(",st"));
ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3"));
ASSERT_EQ(elements.measureLen, strlen("\\,st"));
ASSERT_EQ(elements.measureEscaped, true);
ASSERT_EQ(elements.measureTagsLen, strlen("\\,st,t1=3,t2=4,t3=t3"));
ASSERT_EQ(elements.tags, sql + elements.measureLen + 1);
ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3"));
@ -204,7 +205,26 @@ TEST(testCase, smlParseCols_Error_Test) {
"st,t=1 c=-3.402823466e+39u64 1626006833639000000",
"st,t=1 c=-339u64 1626006833639000000",
"st,t=1 c=18446744073709551616u64 1626006833639000000",
"st,t=1 c=1=2 1626006833639000000"};
"st,t=1 c=1=2 1626006833639000000,",
// escape error test
// measure comma,space
"s,t,t=1 c=1 1626006833639000000,",
"s t,t=1 c=1 1626006833639000000,",
//tag key comma,equal,space
"st,t,t=1 c=2 1626006833639000000,",
"st,t=t=1 c=2 1626006833639000000,",
"st,t t=1 c=2 1626006833639000000,",
//tag value comma,equal,space
"st,tt=a,a c=2 1626006833639000000,",
"st,t=t=a a c=2 1626006833639000000,",
"st,t t=a=a c=2 1626006833639000000,",
//field key comma,equal,space
"st,tt=aa c,1=2 1626006833639000000,",
"st,tt=aa c=1=2 1626006833639000000,",
"st,tt=aa c 1=2 1626006833639000000,",
//field value double quote,slash
"st,tt=aa c=\"a\"a\" 1626006833639000000,",
};
SSmlHandle *info = smlBuildSmlInfo(NULL);
info->protocol = TSDB_SML_LINE_PROTOCOL;
@ -256,16 +276,18 @@ TEST(testCase, smlParseCols_Test) {
ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
ASSERT_EQ(kv->length, 17);
ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0);
ASSERT_EQ(kv->length, 18);
ASSERT_EQ(kv->keyEscaped, true);
ASSERT_EQ(kv->valueEscaped, false);
ASSERT_EQ(strncasecmp(kv->value, "pass\\,it ", 9), 0);
// nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 2);
ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 8);
ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0);
ASSERT_EQ(kv->length, 9);
ASSERT_EQ(strncasecmp(kv->value, "ii\\=sd", 5), 0);
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 3);

2
source/common/src/tglobal.c
View File

@ -198,7 +198,7 @@ int32_t tsTransPullupInterval = 2;
int32_t tsMqRebalanceInterval = 2;
int32_t tsStreamCheckpointTickInterval = 1;
int32_t tsTtlUnit = 86400;
int32_t tsTtlPushInterval = 86400;
int32_t tsTtlPushInterval = 3600;
int32_t tsGrantHBInterval = 60;
int32_t tsUptimeInterval = 300; // seconds
char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits

9
source/common/src/tmsg.c
View File

@ -1409,6 +1409,8 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
return 0;
}
void tFreeSAlterUserReq(SAlterUserReq *pReq) { taosMemoryFreeClear(pReq->tagCond); }
int32_t tSerializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@ -1635,6 +1637,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
int32_t ref = 0;
if (tDecodeI32(pDecoder, &ref) < 0) return -1;
taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref));
taosMemoryFree(key);
}
}
@ -1831,7 +1834,6 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1;
}
if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1;
tEndEncode(&encoder);
@ -1876,7 +1878,6 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1;
} else {
@ -2053,12 +2054,12 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp
if (pRsp->pFuncExtraInfos == NULL) return -1;
if (tDecodeIsEnd(&decoder)) {
for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) {
SFuncExtraInfo extraInfo = { 0 };
SFuncExtraInfo extraInfo = {0};
taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo);
}
} else {
for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) {
SFuncExtraInfo extraInfo = { 0 };
SFuncExtraInfo extraInfo = {0};
if (tDecodeI32(&decoder, &extraInfo.funcVersion) < 0) return -1;
if (tDecodeI64(&decoder, &extraInfo.funcCreatedTime) < 0) return -1;
taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo);

2
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -119,6 +119,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
pVnode->pFetchQ->threadId);
while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
tqNotifyClose(pVnode->pImpl->pTq);
dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ);
while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10);
@ -141,7 +142,6 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
dInfo("vgId:%d, vnode is closed", pVnode->vgId);
if (commitAndRemoveWal) {
char path[TSDB_FILENAME_LEN] = {0};
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP);
dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path);
tfsRmdir(pMgmt->pTfs, path);

4
source/dnode/mgmt/node_mgmt/src/dmNodes.c
View File

@ -132,11 +132,15 @@ int32_t dmRunDnode(SDnode *pDnode) {
int32_t count = 0;
if (dmOpenNodes(pDnode) != 0) {
dError("failed to open nodes since %s", terrstr());
dmCloseNodes(pDnode);
return -1;
}
if (dmStartNodes(pDnode) != 0) {
dError("failed to start nodes since %s", terrstr());
dmSetStatus(pDnode, DND_STAT_STOPPED);
dmStopNodes(pDnode);
dmCloseNodes(pDnode);
return -1;
}

10
source/dnode/mnode/impl/src/mndDb.c
View File

@ -373,6 +373,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
if ((pCfg->hashSuffix * pCfg->hashPrefix) < 0) return -1;
if ((pCfg->hashPrefix + pCfg->hashSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) return -1;
if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
@ -409,8 +411,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD;
if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
}
@ -553,6 +553,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
int32_t dbLen = strlen(dbObj.name) + 1;
mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen);
dbObj.cfg.hashPrefix += dbLen;
} else if (dbObj.cfg.hashPrefix < 0) {
int32_t dbLen = strlen(dbObj.name) + 1;
mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix - dbLen);
dbObj.cfg.hashPrefix -= dbLen;
}
SVgObj *pVgroups = NULL;
@ -1788,6 +1792,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
int16_t hashPrefix = pDb->cfg.hashPrefix;
if (hashPrefix > 0) {
hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1;
} else if (hashPrefix < 0) {
hashPrefix = pDb->cfg.hashPrefix + strlen(pDb->name) + 1;
}
colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false);

6
source/dnode/mnode/impl/src/mndMain.c
View File

@ -124,11 +124,7 @@ static void mndCalMqRebalance(SMnode *pMnode) {
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) {
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_TMQ_TIMER,
.pCont = pReq,
.contLen = contLen,
};
SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen };
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
}
}

2
source/dnode/mnode/impl/src/mndUser.c
View File

@ -390,6 +390,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &ref, _OVER);
taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref));
taosMemoryFree(key);
}
}
@ -956,6 +957,7 @@ _OVER:
mError("user:%s, failed to alter since %s", alterReq.user, terrstr());
}
tFreeSAlterUserReq(&alterReq);
mndReleaseUser(pMnode, pOperUser);
mndReleaseUser(pMnode, pUser);
mndUserFreeObj(&newUser);

1
source/dnode/vnode/inc/vnode.h
View File

@ -162,7 +162,6 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType);
#endif
// tsdb
// typedef struct STsdb STsdb;
typedef struct STsdbReader STsdbReader;
#define TSDB_DEFAULT_STT_FILE 8

7
source/dnode/vnode/src/inc/tq.h
View File

@ -180,16 +180,9 @@ int32_t tqStreamTasksScanWal(STQ* pTq);
// tq util
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId);
int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver);
int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset);
int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver);
void saveOffsetForAllTasks(STQ* pTq, int64_t ver);
void initOffsetForAllRestoreTasks(STQ* pTq);
int32_t transferToWalReadTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
#ifdef __cplusplus
}
#endif

38
source/dnode/vnode/src/inc/tsdb.h
View File

@ -16,6 +16,7 @@
#ifndef _TD_VNODE_TSDB_H_
#define _TD_VNODE_TSDB_H_
#include "tsimplehash.h"
#include "vnodeInt.h"
#ifdef __cplusplus
@ -125,11 +126,13 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter);
// SRowMerger
int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema);
int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
void tsdbRowMergerClear(SRowMerger *pMerger);
// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow);
int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow);
int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema);
void tsdbRowMergerClear_rv(SRowMerger* pMerger);
void tsdbRowMergerCleanup_rv(SRowMerger* pMerger);
// TABLEID
int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
// TSDBKEY
@ -224,7 +227,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward,
void *tsdbTbDataIterDestroy(STbDataIter *pIter);
void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
bool tsdbTbDataIterNext(STbDataIter *pIter);
void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum);
void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum);
// STbData
int32_t tsdbGetNRowsInTbData(STbData *pTbData);
@ -322,8 +325,9 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData,
#define TSDB_STT_FILE_DATA_ITER 2
#define TSDB_TOMB_FILE_DATA_ITER 3
#define TSDB_FILTER_FLAG_BY_VERSION 0x1
#define TSDB_FILTER_FLAG_BY_TABLEID 0x2
#define TSDB_FILTER_FLAG_BY_VERSION 0x1
#define TSDB_FILTER_FLAG_BY_TABLEID 0x2
#define TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE 0x4
#define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn)))
/* open */
@ -705,7 +709,6 @@ typedef struct SSttBlockLoadInfo {
typedef struct SMergeTree {
int8_t backward;
SRBTree rbt;
SArray *pIterList;
SLDataIter *pIter;
bool destroyLoadInfo;
SSttBlockLoadInfo *pLoadInfo;
@ -751,13 +754,29 @@ struct SDiskDataBuilder {
SBlkInfo bi;
};
typedef struct SLDataIter {
SRBTreeNode node;
SSttBlk *pSttBlk;
SDataFReader *pReader;
int32_t iStt;
int8_t backward;
int32_t iSttBlk;
int32_t iRow;
SRowInfo rInfo;
uint64_t uid;
STimeWindow timeWindow;
SVersionRange verRange;
SSttBlockLoadInfo *pBlockLoadInfo;
bool ignoreEarlierTs;
} SLDataIter;
#define tMergeTreeGetRow(_t) (&((_t)->pIter->rInfo.row))
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
bool destroyLoadInfo, const char *idStr, bool strictTimeRange);
bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter);
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
bool tMergeTreeNext(SMergeTree *pMTree);
bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree);
SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt);
@ -782,6 +801,7 @@ typedef struct SCacheRowsReader {
STableKeyInfo *pTableList; // table id list
int32_t numOfTables;
SSttBlockLoadInfo *pLoadInfo;
SLDataIter *pDataIter;
STsdbReadSnap *pReadSnap;
SDataFReader *pDataFReader;
SDataFReader *pDataFReaderLast;

1
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -190,6 +190,7 @@ int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
int tqInit();
void tqCleanUp();
STQ* tqOpen(const char* path, SVnode* pVnode);
void tqNotifyClose(STQ*);
void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp,

7
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -187,23 +187,24 @@ _err:
int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
int32_t line = 0;
SMeta* pMeta = pWriter->pMeta;
SMetaEntry metaEntry = {0};
SDecoder* pDecoder = &(SDecoder){0};
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
code = metaDecodeEntry(pDecoder, &metaEntry);
if (code) goto _err;
VND_CHECK_CODE(code, line, _err);
code = metaHandleEntry(pMeta, &metaEntry);
if (code) goto _err;
VND_CHECK_CODE(code, line, _err);
tDecoderClear(pDecoder);
return code;
_err:
tDecoderClear(pDecoder);
metaError("vgId:%d, vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line);
return code;
}

52
source/dnode/vnode/src/meta/metaTable.c
View File

@ -936,8 +936,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
int tLen = 0;
if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
version = ((SUidIdxVal *)tData)[0].version;
STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version};
STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version};
if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
SDecoder tdc = {0};
SMetaEntry stbEntry = {0};
@ -1029,7 +1028,7 @@ int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " ctime:%" PRId64, TD_VID(pMeta->pVnode),
pME->version, pME->uid, ctimeKey.ctime);
return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
return tdbTbUpsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
}
int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
@ -1044,7 +1043,7 @@ int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
if (metaBuildNColIdxKey(&ncolKey, pME) < 0) {
return 0;
}
return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn);
return tdbTbUpsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn);
}
int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
@ -1878,24 +1877,24 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
}
static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn);
return tdbTbUpsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn);
}
static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
}
static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
STtlIdxKey ttlKey = {0};
metaBuildTtlIdxKey(&ttlKey, pME);
if (ttlKey.dtime == 0) return 0;
return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
return tdbTbUpsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
}
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags,
return tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags,
((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn);
}
@ -2065,49 +2064,66 @@ _exit:
}
int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
int32_t code = 0;
int32_t line = 0;
metaWLock(pMeta);
// save to table.db
if (metaSaveToTbDb(pMeta, pME) < 0) goto _err;
code = metaSaveToTbDb(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
// update uid.idx
if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err;
code = metaUpdateUidIdx(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
// update name.idx
if (metaUpdateNameIdx(pMeta, pME) < 0) goto _err;
code = metaUpdateNameIdx(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
if (pME->type == TSDB_CHILD_TABLE) {
// update ctb.idx
if (metaUpdateCtbIdx(pMeta, pME) < 0) goto _err;
code = metaUpdateCtbIdx(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
// update tag.idx
if (metaUpdateTagIdx(pMeta, pME) < 0) goto _err;
code = metaUpdateTagIdx(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
} else {
// update schema.db
if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err;
code = metaSaveToSkmDb(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
if (pME->type == TSDB_SUPER_TABLE) {
if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err;
code = metaUpdateSuidIdx(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
}
}
if (metaUpdateCtimeIdx(pMeta, pME) < 0) goto _err;
code = metaUpdateCtimeIdx(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
if (pME->type == TSDB_NORMAL_TABLE) {
if (metaUpdateNcolIdx(pMeta, pME) < 0) goto _err;
code = metaUpdateNcolIdx(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
}
if (pME->type != TSDB_SUPER_TABLE) {
if (metaUpdateTtlIdx(pMeta, pME) < 0) goto _err;
code = metaUpdateTtlIdx(pMeta, pME);
VND_CHECK_CODE(code, line, _err);
}
metaULock(pMeta);
metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode),
pME->version, pME->uid, pME->name);
return 0;
_err:
metaULock(pMeta);
metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64 ", name:%s",
TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid, pME->name);
return -1;
}
// refactor later
void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; }
void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; }

66
source/dnode/vnode/src/tq/tq.c
View File

@ -154,6 +154,31 @@ void tqClose(STQ* pTq) {
taosMemoryFree(pTq);
}
void tqNotifyClose(STQ* pTq) {
if (pTq != NULL) {
taosWLockLatch(&pTq->pStreamMeta->lock);
void* pIter = NULL;
while (1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) {
break;
}
SStreamTask* pTask = *(SStreamTask**)pIter;
tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr);
pTask->status.taskStatus = TASK_STATUS__STOP;
int64_t st = taosGetTimestampMs();
qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS);
int64_t el = taosGetTimestampMs() - st;
tqDebug("vgId:%d s-task:%s is closed in %" PRId64 "ms", pTq->pStreamMeta->vgId, pTask->id.idStr, el);
}
taosWUnLockLatch(&pTq->pStreamMeta->lock);
}
}
static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch,
int64_t consumerId, int32_t type) {
int32_t len = 0;
@ -573,6 +598,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask->pMsgCb = &pTq->pVnode->msgCb;
pTask->pMeta = pTq->pStreamMeta;
pTask->chkInfo.version = ver;
pTask->chkInfo.currentVer = ver;
// expand executor
if (pTask->fillHistory) {
@ -966,14 +992,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
int32_t* pRef = taosMemoryMalloc(sizeof(int32_t));
*pRef = 1;
taosWLockLatch(&pTq->pStreamMeta->lock);
void* pIter = NULL;
while (1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) break;
SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue;
if (pIter == NULL) {
break;
}
qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver);
SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
continue;
}
qDebug("s-task:%s delete req enqueue, ver: %" PRId64, pTask->id.idStr, ver);
if (!failed) {
SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);
@ -983,15 +1016,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
atomic_add_fetch_32(pRefBlock->dataRef, 1);
if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
qError("stream task input del failed, task id %d", pTask->id.taskId);
atomic_sub_fetch_32(pRef, 1);
taosFreeQitem(pRefBlock);
continue;
}
if (streamSchedExec(pTask) < 0) {
qError("stream task launch failed, task id %d", pTask->id.taskId);
qError("s-task:%s stream task launch failed", pTask->id.idStr);
continue;
}
@ -1000,8 +1031,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
}
}
taosWUnLockLatch(&pTq->pStreamMeta->lock);
int32_t ref = atomic_sub_fetch_32(pRef, 1);
/*A(ref >= 0);*/
if (ref == 0) {
blockDataDestroy(pDelBlock);
taosMemoryFree(pRef);
@ -1032,23 +1064,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
}
blockDataDestroy(pDelBlock);
#endif
return 0;
}
static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit,
const char* key, int64_t ver) {
doSaveTaskOffset(pOffsetStore, key, ver);
int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver);
// remove the offset, if all functions are completed successfully.
if (code == TSDB_CODE_SUCCESS) {
tqOffsetDelete(pOffsetStore, key);
}
return code;
}
int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
#if 0
void* pIter = NULL;
@ -1309,9 +1327,7 @@ int32_t tqStartStreamTasks(STQ* pTq) {
return -1;
}
tqInfo("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks);
initOffsetForAllRestoreTasks(pTq);
tqDebug("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks);
pRunReq->head.vgId = vgId;
pRunReq->streamId = 0;
pRunReq->taskId = WAL_READ_TASKS_ID;

3
source/dnode/vnode/src/tq/tqRead.c
View File

@ -1023,6 +1023,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
}
// update the table list handle for each stream scanner/wal reader
taosWLockLatch(&pTq->pStreamMeta->lock);
while (1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) {
@ -1039,5 +1040,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
}
}
taosWUnLockLatch(&pTq->pStreamMeta->lock);
return 0;
}

114
source/dnode/vnode/src/tq/tqRestore.c
View File

@ -15,13 +15,12 @@
#include "tq.h"
static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle);
static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle);
// this function should be executed by stream threads.
// there is a case that the WAL increases more fast than the restore procedure, and this restore procedure
// will not stop eventually.
int tqStreamTasksScanWal(STQ* pTq) {
int32_t tqStreamTasksScanWal(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta;
int64_t st = taosGetTimestampMs();
@ -32,7 +31,7 @@ int tqStreamTasksScanWal(STQ* pTq) {
// check all restore tasks
bool shouldIdle = true;
streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &shouldIdle);
createStreamRunReq(pTq->pStreamMeta, &shouldIdle);
int32_t times = 0;
@ -55,87 +54,86 @@ int tqStreamTasksScanWal(STQ* pTq) {
int64_t el = (taosGetTimestampMs() - st);
tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%"PRId64" ms", vgId, el);
// restore wal scan flag
// atomic_store_8(&pTq->pStreamMeta->walScan, 0);
return 0;
}
//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
// int32_t numOfTask = taosArrayGetSize(pTaskList);
// if (numOfTask <= 0) {
// return TSDB_CODE_SUCCESS;
// }
//
// // todo: add lock
// for (int32_t i = 0; i < numOfTask; ++i) {
// SStreamTask* pTask = taosArrayGetP(pTaskList, i);
// tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
// pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
// taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
//
// // NOTE: do not change the following order
// atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
// taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
// }
//
// return TSDB_CODE_SUCCESS;
//}
int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) {
static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) {
SArray* pTaskIdList = taosArrayInit(numOfTasks, sizeof(int32_t));
void* pIter = NULL;
int32_t vgId = pStreamMeta->vgId;
*pScanIdle = true;
bool allWalChecked = true;
tqDebug("vgId:%d start to check wal to extract new submit block", vgId);
while (1) {
taosWLockLatch(&pStreamMeta->lock);
while(1) {
pIter = taosHashIterate(pStreamMeta->pTasks, pIter);
if (pIter == NULL) {
break;
}
SStreamTask* pTask = *(SStreamTask**)pIter;
taosArrayPush(pTaskIdList, &pTask->id.taskId);
}
taosWUnLockLatch(&pStreamMeta->lock);
return pTaskIdList;
}
int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
*pScanIdle = true;
bool noNewDataInWal = true;
int32_t vgId = pStreamMeta->vgId;
int32_t numOfTasks = taosHashGetSize(pStreamMeta->pTasks);
if (numOfTasks == 0) {
return TSDB_CODE_SUCCESS;
}
tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks);
SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks);
// update the new task number
numOfTasks = taosArrayGetSize(pTaskIdList);
for (int32_t i = 0; i < numOfTasks; ++i) {
int32_t* pTaskId = taosArrayGet(pTaskIdList, i);
SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId);
if (pTask == NULL) {
continue;
}
int32_t status = pTask->status.taskStatus;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
tqDebug("s-task:%s not source task, no need to start", pTask->id.idStr);
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
}
if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE ||
pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
pTask->status.taskStatus);
if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE ||
status == TASK_STATUS__WAIT_DOWNSTREAM) {
tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, status);
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
}
// check if offset value exists
char key[128] = {0};
createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
if (tInputQueueIsFull(pTask)) {
tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr);
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
}
*pScanIdle = false;
// check if offset value exists
STqOffset* pOffset = tqOffsetRead(pOffsetStore, key);
ASSERT(pOffset != NULL);
// seek the stored version and extract data from WAL
int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version);
int32_t code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer);
if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
}
// append the data for the stream
tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr);
tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer);
SPackedData packData = {0};
code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData);
if (code != TSDB_CODE_SUCCESS) { // failed, continue
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
}
@ -143,28 +141,32 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto
if (p == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr);
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
}
allWalChecked = false;
noNewDataInWal = false;
tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr);
code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver);
if (code == TSDB_CODE_SUCCESS) {
pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader);
pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader);
tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
pOffset->val.version);
pTask->chkInfo.currentVer);
} else {
// do nothing
tqError("s-task:%s append input queue failed, ver:%"PRId64, pTask->id.idStr, pTask->chkInfo.currentVer);
}
streamDataSubmitDestroy(p);
taosFreeQitem(p);
streamMetaReleaseTask(pStreamMeta, pTask);
}
if (allWalChecked) {
// all wal are checked, and no new data available in wal.
if (noNewDataInWal) {
*pScanIdle = true;
}
taosArrayDestroy(pTaskIdList);
return 0;
}

84
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -25,21 +25,6 @@ char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) {
return taosStrdup(buf);
}
// stream_task:stream_id:task_id
void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) {
int32_t n = 12;
char* p = dst;
memcpy(p, "stream_task:", n);
p += n;
int32_t inc = tintToHex(streamId, p);
p += inc;
*(p++) = ':';
tintToHex(taskId, p);
}
int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) {
int32_t code = tAppendDataToInputQueue(pTask, pQueueItem);
if (code < 0) {
@ -55,75 +40,6 @@ int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueI
return TSDB_CODE_SUCCESS;
}
void initOffsetForAllRestoreTasks(STQ* pTq) {
void* pIter = NULL;
while(1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) {
break;
}
SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
continue;
}
if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus);
continue;
}
char key[128] = {0};
createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
if (pOffset == NULL) {
doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version);
}
}
}
void saveOffsetForAllTasks(STQ* pTq, int64_t ver) {
void* pIter = NULL;
while(1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
if (pIter == NULL) {
break;
}
SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
continue;
}
if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
pTask->status.taskStatus);
continue;
}
char key[128] = {0};
createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
if (pOffset == NULL) {
doSaveTaskOffset(pTq->pOffsetStore, key, ver);
}
}
}
void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) {
STqOffset offset = {0};
tqOffsetResetToLog(&offset.val, ver);
tstrncpy(offset.subKey, pKey, tListLen(offset.subKey));
// keep the offset info in the offset store
tqOffsetWrite(pOffsetStore, &offset);
}
static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) {
pRsp->reqOffset = pReq->reqOffset;

12
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -598,6 +598,7 @@ typedef struct {
SMergeTree mergeTree;
SMergeTree *pMergeTree;
SSttBlockLoadInfo *pLoadInfo;
SLDataIter* pDataIter;
int64_t lastTs;
} SFSLastNextRowIter;
@ -645,7 +646,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
}
tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
&(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true, state->pDataIter);
state->pMergeTree = &state->mergeTree;
state->state = SFSLASTNEXTROW_BLOCKROW;
}
@ -667,7 +668,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
state->state = SFSLASTNEXTROW_FILESET;
goto _next_fileset;
}
state->row = tMergeTreeGetRow(&state->mergeTree);
state->row = *tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row;
if (TSDBROW_TS(&state->row) <= state->lastTs) {
@ -1211,7 +1212,7 @@ typedef struct {
} CacheNextRowIter;
static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
SSttBlockLoadInfo *pLoadInfo, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader,
SSttBlockLoadInfo *pLoadInfo, SLDataIter* pLDataIter, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader,
SDataFReader **pDataFReaderLast, int64_t lastTs) {
int code = 0;
@ -1274,6 +1275,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter->fsLastState.pLoadInfo = pLoadInfo;
pIter->fsLastState.pDataFReader = pDataFReaderLast;
pIter->fsLastState.lastTs = lastTs;
pIter->fsLastState.pDataIter = pLDataIter;
pIter->fsState.state = SFSNEXTROW_FS;
pIter->fsState.pTsdb = pTsdb;
@ -1465,7 +1467,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
TSKEY lastRowTs = TSKEY_MAX;
CacheNextRowIter iter = {0};
nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader,
nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader,
&pr->pDataFReaderLast, pr->lastTs);
do {
@ -1622,7 +1624,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
TSKEY lastRowTs = TSKEY_MAX;
CacheNextRowIter iter = {0};
nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader,
nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader,
&pr->pDataFReaderLast, pr->lastTs);
do {

11
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -187,13 +187,21 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
}
}
int32_t numOfStt = ((SVnode*)pVnode)->config.sttTrigger;
SVnodeCfg* pCfg = &((SVnode*)pVnode)->config;
int32_t numOfStt = pCfg->sttTrigger;
p->pLoadInfo = tCreateLastBlockLoadInfo(p->pSchema, NULL, 0, numOfStt);
if (p->pLoadInfo == NULL) {
tsdbCacherowsReaderClose(p);
return TSDB_CODE_OUT_OF_MEMORY;
}
p->pDataIter = taosMemoryCalloc(pCfg->sttTrigger, sizeof(SLDataIter));
if (p->pDataIter == NULL) {
tsdbCacherowsReaderClose(p);
return TSDB_CODE_OUT_OF_MEMORY;
}
p->idstr = taosStrdup(idstr);
taosThreadMutexInit(&p->readerMutex, NULL);
@ -215,6 +223,7 @@ void* tsdbCacherowsReaderClose(void* pReader) {
taosMemoryFree(p->pSchema);
}
taosMemoryFreeClear(p->pDataIter);
taosMemoryFree(p->pCurrSchema);
destroyLastBlockLoadInfo(p->pLoadInfo);

61
source/dnode/vnode/src/tsdb/tsdbDataIter.c
View File

@ -14,6 +14,7 @@
*/
#include "tsdb.h"
#include "vnodeInt.h"
// STsdbDataIter2
/* open */
@ -202,13 +203,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
for (;;) {
while (pIter->dIter.iRow < pIter->dIter.bData.nRow) {
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
if (pFilterInfo->tbid.uid == pIter->dIter.bData.uid) {
pIter->dIter.iRow = pIter->dIter.bData.nRow;
continue;
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver ||
pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) {
@ -232,13 +226,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
// filter
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
if (tTABLEIDCmprFn(&pFilterInfo->tbid, &pIter->rowInfo) == 0) {
pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem;
continue;
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) {
pIter->dIter.iDataBlk++;
@ -262,13 +249,23 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) {
SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx);
if (pFilterInfo && (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID)) {
int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid);
if (c == 0) {
pIter->dIter.iBlockIdx++;
continue;
} else if (c < 0) {
ASSERT(0);
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid);
if (c == 0) {
pIter->dIter.iBlockIdx++;
continue;
} else if (c < 0) {
ASSERT(0);
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
SMetaInfo info;
if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pBlockIdx->uid, &info, NULL)) {
pIter->dIter.iBlockIdx++;
continue;
}
}
}
@ -304,14 +301,24 @@ static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* p
for (;;) {
while (pIter->sIter.iRow < pIter->sIter.bData.nRow) {
if (pFilterInfo) {
int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow];
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow];
if (pFilterInfo->tbid.uid == uid) {
pIter->sIter.iRow++;
continue;
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
if (pIter->rowInfo.uid != uid) {
SMetaInfo info;
if (metaGetInfo(pIter->sIter.pReader->pTsdb->pVnode->pMeta, uid, &info, NULL)) {
pIter->sIter.iRow++;
continue;
}
}
}
if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] ||
pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) {
@ -395,6 +402,16 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) {
SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx);
if (pFilterInfo) {
if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
SMetaInfo info;
if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pDelIdx->uid, &info, NULL)) {
pIter->tIter.iDelIdx++;
continue;
}
}
}
code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData);
TSDB_CHECK_CODE(code, lino, _exit);

5
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <util/tsimplehash.h>
#include "tsdb.h"
#define MEM_MIN_HASH 1024
@ -298,12 +299,12 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) {
return rowsNum;
}
void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) {
void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj* pTableMap, int64_t *rowsNum) {
taosRLockLatch(&pMemTable->latch);
for (int32_t i = 0; i < pMemTable->nBucket; ++i) {
STbData *pTbData = pMemTable->aBucket[i];
while (pTbData) {
void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid));
void* p = tSimpleHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid));
if (p == NULL) {
pTbData = pTbData->next;
continue;

109
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -16,22 +16,6 @@
#include "tsdb.h"
// SLDataIter =================================================
struct SLDataIter {
SRBTreeNode node;
SSttBlk *pSttBlk;
SDataFReader *pReader;
int32_t iStt;
int8_t backward;
int32_t iSttBlk;
int32_t iRow;
SRowInfo rInfo;
uint64_t uid;
STimeWindow timeWindow;
SVersionRange verRange;
SSttBlockLoadInfo *pBlockLoadInfo;
bool ignoreEarlierTs;
};
SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols,
int32_t numOfSttTrigger) {
SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(numOfSttTrigger, sizeof(SSttBlockLoadInfo));
@ -88,6 +72,10 @@ void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double
}
void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
if (pLoadInfo == NULL) {
return NULL;
}
for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) {
pLoadInfo[i].currentLoadBlockIndex = 1;
pLoadInfo[i].blockIndex[0] = -1;
@ -264,25 +252,21 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint
}
}
int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo,
const char *idStr, bool strictTimeRange) {
int32_t code = TSDB_CODE_SUCCESS;
*pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
if (*pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pIter->uid = uid;
pIter->pReader = pReader;
pIter->iStt = iStt;
pIter->backward = backward;
pIter->verRange.minVer = pRange->minVer;
pIter->verRange.maxVer = pRange->maxVer;
pIter->timeWindow.skey = pTimeWindow->skey;
pIter->timeWindow.ekey = pTimeWindow->ekey;
(*pIter)->uid = uid;
(*pIter)->pReader = pReader;
(*pIter)->iStt = iStt;
(*pIter)->backward = backward;
(*pIter)->verRange = *pRange;
(*pIter)->timeWindow = *pTimeWindow;
(*pIter)->pBlockLoadInfo = pBlockLoadInfo;
pIter->pBlockLoadInfo = pBlockLoadInfo;
if (!pBlockLoadInfo->sttBlockLoaded) {
int64_t st = taosGetTimestampUs();
@ -290,7 +274,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
if (code) {
goto _exit;
return code;
}
// only apply to the child tables, ordinary tables will not incur this filter procedure.
@ -306,7 +290,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
// no qualified stt block existed
taosArrayClear(pBlockLoadInfo->aSttBlk);
(*pIter)->iSttBlk = -1;
pIter->iSttBlk = -1;
double el = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
return code;
@ -339,31 +323,27 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
// find the start block
(*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
if ((*pIter)->iSttBlk != -1) {
(*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk);
(*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1;
pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
if (pIter->iSttBlk != -1) {
pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk);
pIter->iRow = (pIter->backward) ? pIter->pSttBlk->nRow : -1;
if ((!backward) && ((strictTimeRange && (*pIter)->pSttBlk->minKey >= (*pIter)->timeWindow.ekey) ||
(!strictTimeRange && (*pIter)->pSttBlk->minKey > (*pIter)->timeWindow.ekey))) {
(*pIter)->pSttBlk = NULL;
if ((!backward) && ((strictTimeRange && pIter->pSttBlk->minKey >= pIter->timeWindow.ekey) ||
(!strictTimeRange && pIter->pSttBlk->minKey > pIter->timeWindow.ekey))) {
pIter->pSttBlk = NULL;
}
if (backward && ((strictTimeRange && (*pIter)->pSttBlk->maxKey <= (*pIter)->timeWindow.skey) ||
(!strictTimeRange && (*pIter)->pSttBlk->maxKey < (*pIter)->timeWindow.skey))) {
(*pIter)->pSttBlk = NULL;
(*pIter)->ignoreEarlierTs = true;
if (backward && ((strictTimeRange && pIter->pSttBlk->maxKey <= pIter->timeWindow.skey) ||
(!strictTimeRange && pIter->pSttBlk->maxKey < pIter->timeWindow.skey))) {
pIter->pSttBlk = NULL;
pIter->ignoreEarlierTs = true;
}
}
return code;
_exit:
taosMemoryFree(*pIter);
return code;
}
void tLDataIterClose(SLDataIter *pIter) { taosMemoryFree(pIter); }
void tLDataIterClose(SLDataIter *pIter) { /*taosMemoryFree(pIter); */}
void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) {
int32_t step = pIter->backward ? -1 : 1;
@ -590,43 +570,38 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
bool destroyLoadInfo, const char *idStr, bool strictTimeRange) {
bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter) {
int32_t code = TSDB_CODE_SUCCESS;
pMTree->backward = backward;
pMTree->pIter = NULL;
pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
if (pMTree->pIterList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pMTree->idStr = idStr;
if (!pMTree->backward) { // asc
tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
} else { // desc
tRBTreeCreate(&pMTree->rbt, tLDataIterDescCmprFn);
}
int32_t code = TSDB_CODE_SUCCESS;
pMTree->pLoadInfo = pBlockLoadInfo;
pMTree->destroyLoadInfo = destroyLoadInfo;
pMTree->ignoreEarlierTs = false;
for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file
struct SLDataIter *pIter = NULL;
code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange,
memset(&pLDataIter[i], 0, sizeof(SLDataIter));
code = tLDataIterOpen(&pLDataIter[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange,
&pMTree->pLoadInfo[i], pMTree->idStr, strictTimeRange);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr);
bool hasVal = tLDataIterNextRow(&pLDataIter[i], pMTree->idStr);
if (hasVal) {
taosArrayPush(pMTree->pIterList, &pIter);
tMergeTreeAddIter(pMTree, pIter);
tMergeTreeAddIter(pMTree, &pLDataIter[i]);
} else {
if (!pMTree->ignoreEarlierTs) {
pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs;
pMTree->ignoreEarlierTs = pLDataIter[i].ignoreEarlierTs;
}
tLDataIterClose(pIter);
}
}
@ -674,18 +649,8 @@ bool tMergeTreeNext(SMergeTree *pMTree) {
return pMTree->pIter != NULL;
}
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; }
void tMergeTreeClose(SMergeTree *pMTree) {
size_t size = taosArrayGetSize(pMTree->pIterList);
for (int32_t i = 0; i < size; ++i) {
SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i);
tLDataIterClose(pIter);
}
pMTree->pIterList = taosArrayDestroy(pMTree->pIterList);
pMTree->pIter = NULL;
if (pMTree->destroyLoadInfo) {
pMTree->pLoadInfo = destroyLastBlockLoadInfo(pMTree->pLoadInfo);
pMTree->destroyLoadInfo = false;

624
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

File diff suppressed because it is too large Load Diff

26
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -70,10 +70,11 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) {
if (pReader->pIter) {
// iter to next with filter info (sver, ever)
code = tsdbDataIterNext2(pReader->pIter,
&(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag
.sver = pReader->sver,
.ever = pReader->ever});
code = tsdbDataIterNext2(
pReader->pIter,
&(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag
.sver = pReader->sver,
.ever = pReader->ever});
TSDB_CHECK_CODE(code, lino, _exit);
if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) {
@ -94,10 +95,11 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) {
if (pReader->pIter) {
// iter to valid row
code = tsdbDataIterNext2(pReader->pIter,
&(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag
.sver = pReader->sver,
.ever = pReader->ever});
code = tsdbDataIterNext2(
pReader->pIter,
&(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag
.sver = pReader->sver,
.ever = pReader->ever});
TSDB_CHECK_CODE(code, lino, _exit);
if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) {
@ -139,7 +141,8 @@ static int32_t tsdbSnapReadNextRow(STsdbSnapReader* pReader, SRowInfo** ppRowInf
int32_t lino = 0;
if (pReader->pIter) {
code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag
code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION |
TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag
.sver = pReader->sver,
.ever = pReader->ever});
TSDB_CHECK_CODE(code, lino, _exit);
@ -346,8 +349,9 @@ static int32_t tsdbSnapReadNextTombData(STsdbSnapReader* pReader, SDelInfo** ppD
int32_t lino = 0;
code = tsdbDataIterNext2(
pReader->pTIter,
&(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, .sver = pReader->sver, .ever = pReader->ever});
pReader->pTIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,
.sver = pReader->sver,
.ever = pReader->ever});
TSDB_CHECK_CODE(code, lino, _exit);
if (ppDelInfo) {

257
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -712,124 +712,163 @@ _exit:
int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
int32_t code = 0;
TSDBKEY key = TSDBROW_KEY(pRow);
SColVal *pColVal = &(SColVal){0};
SColVal * pColVal = &(SColVal){0};
STColumn *pTColumn;
int32_t iCol, jCol = 1;
if (NULL == pTSchema) {
pTSchema = pMerger->pTSchema;
}
ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol];
if (pTSchema->columns[jCol].colId < pTColumn->colId) {
++jCol;
--iCol;
continue;
} else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
continue;
}
if (taosArrayGetSize(pMerger->pArray) == 0) {
// ts
jCol = 0;
pTColumn = &pTSchema->columns[jCol++];
tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
if (key.version > pMerger->version) {
if (!COL_VAL_IS_NONE(pColVal)) {
if (IS_VAR_DATA_TYPE(pColVal->type)) {
SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
if (!COL_VAL_IS_NULL(pColVal)) {
code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
if (code) return code;
pTColVal->value.nData = pColVal->value.nData;
if (pTColVal->value.nData) {
memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData);
}
pTColVal->flag = 0;
} else {
tFree(pTColVal->value.pData);
taosArraySet(pMerger->pArray, iCol, pColVal);
}
} else {
taosArraySet(pMerger->pArray, iCol, pColVal);
}
}
} else if (key.version < pMerger->version) {
SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
if (code) return code;
tColVal->value.nData = pColVal->value.nData;
if (pColVal->value.nData) {
memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
}
tColVal->flag = 0;
} else {
taosArraySet(pMerger->pArray, iCol, pColVal);
}
}
} else {
ASSERT(0 && "dup versions not allowed");
}
}
pMerger->version = key.version;
return code;
}
/*
int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
int32_t code = 0;
TSDBKEY key = TSDBROW_KEY(pRow);
SColVal *pColVal = &(SColVal){0};
STColumn *pTColumn;
pMerger->pTSchema = pTSchema;
pMerger->version = key.version;
pMerger->pArray = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal));
if (pMerger->pArray == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
// ts
pTColumn = &pTSchema->columns[0];
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
// other
for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
uint8_t *pVal = pColVal->value.pData;
pColVal->value.pData = NULL;
code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
if (code) goto _exit;
if (pColVal->value.nData) {
memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
}
}
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
return code;
// goto _exit;
}
// other
for (iCol = 1; jCol < pTSchema->numOfCols && iCol < pMerger->pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol];
if (pTSchema->columns[jCol].colId < pTColumn->colId) {
++jCol;
--iCol;
continue;
} else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
continue;
}
tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
uint8_t *pVal = pColVal->value.pData;
pColVal->value.pData = NULL;
code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
if (code) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (pColVal->value.nData) {
memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
}
}
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol];
taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
}
pMerger->version = key.version;
return 0;
} else {
ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol];
if (pTSchema->columns[jCol].colId < pTColumn->colId) {
++jCol;
--iCol;
continue;
} else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
continue;
}
tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
if (key.version > pMerger->version) {
if (!COL_VAL_IS_NONE(pColVal)) {
if (IS_VAR_DATA_TYPE(pColVal->type)) {
SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
if (!COL_VAL_IS_NULL(pColVal)) {
code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
if (code) return code;
pTColVal->value.nData = pColVal->value.nData;
if (pTColVal->value.nData) {
memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData);
}
pTColVal->flag = 0;
} else {
tFree(pTColVal->value.pData);
taosArraySet(pMerger->pArray, iCol, pColVal);
}
} else {
taosArraySet(pMerger->pArray, iCol, pColVal);
}
}
} else if (key.version < pMerger->version) {
SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
if (code) return code;
tColVal->value.nData = pColVal->value.nData;
if (pColVal->value.nData) {
memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
}
tColVal->flag = 0;
} else {
taosArraySet(pMerger->pArray, iCol, pColVal);
}
}
} else {
ASSERT(0 && "dup versions not allowed");
}
}
pMerger->version = key.version;
return code;
}
}
int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) {
pMerger->pTSchema = pSchema;
pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal));
if (pMerger->pArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} else {
return TSDB_CODE_SUCCESS;
}
}
void tsdbRowMergerClear_rv(SRowMerger* pMerger) {
for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
if (IS_VAR_DATA_TYPE(pTColVal->type)) {
tFree(pTColVal->value.pData);
}
}
_exit:
return code;
taosArrayClear(pMerger->pArray);
}
*/
void tsdbRowMergerCleanup_rv(SRowMerger* pMerger) {
int32_t numOfCols = taosArrayGetSize(pMerger->pArray);
for (int32_t iCol = 1; iCol < numOfCols; iCol++) {
SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
if (IS_VAR_DATA_TYPE(pTColVal->type)) {
tFree(pTColVal->value.pData);
}
}
taosArrayDestroy(pMerger->pArray);
}
void tsdbRowMergerClear(SRowMerger *pMerger) {
for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
@ -1218,9 +1257,10 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
int32_t iColumn = 1;
STColumn *pTColumn = &pTSchema->columns[iColumn];
for (int32_t iCid = 0; iCid < nCid; iCid++) {
if (ASSERTS(pTColumn != NULL, "invalid input param")) {
code = TSDB_CODE_INVALID_PARA;
goto _exit;
// aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is newer
if (pTColumn == NULL) {
continue;
}
while (pTColumn->colId < aCid[iCid]) {
@ -1229,9 +1269,8 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
pTColumn = &pTSchema->columns[iColumn];
}
if (ASSERTS(pTColumn->colId == aCid[iCid], "invalid input param")) {
code = TSDB_CODE_INVALID_PARA;
goto _exit;
if (pTColumn->colId != aCid[iCid]) {
continue;
}
tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type,

1
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -1348,6 +1348,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
code = terrno;
goto _exit;
}
terrno = 0;
pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; // update uid if table exist for using below
}
}

19
source/libs/catalog/inc/catalogInt.h
View File

@ -58,6 +58,7 @@ typedef enum {
CTG_CI_OTHERTABLE_META,
CTG_CI_TBL_SMA,
CTG_CI_TBL_CFG,
CTG_CI_TBL_TAG,
CTG_CI_INDEX_INFO,
CTG_CI_USER,
CTG_CI_UDF,
@ -110,6 +111,7 @@ typedef enum {
CTG_TASK_GET_SVR_VER,
CTG_TASK_GET_TB_META_BATCH,
CTG_TASK_GET_TB_HASH_BATCH,
CTG_TASK_GET_TB_TAG,
} CTG_TASK_TYPE;
typedef enum {
@ -152,6 +154,11 @@ typedef struct SCtgTbCacheInfo {
int32_t tbType;
} SCtgTbCacheInfo;
typedef struct SCtgTbMetaParam {
SName* pName;
int32_t flag;
} SCtgTbMetaParam;
typedef struct SCtgTbMetaCtx {
SCtgTbCacheInfo tbInfo;
int32_t vgId;
@ -186,6 +193,11 @@ typedef struct SCtgTbCfgCtx {
SVgroupInfo* pVgInfo;
} SCtgTbCfgCtx;
typedef struct SCtgTbTagCtx {
SName* pName;
SVgroupInfo* pVgInfo;
} SCtgTbTagCtx;
typedef struct SCtgDbVgCtx {
char dbFName[TSDB_DB_FNAME_LEN];
} SCtgDbVgCtx;
@ -304,6 +316,7 @@ typedef struct SCtgJob {
catalogCallback userFp;
int32_t tbMetaNum;
int32_t tbHashNum;
int32_t tbTagNum;
int32_t dbVgNum;
int32_t udfNum;
int32_t qnodeNum;
@ -346,6 +359,7 @@ typedef struct SCtgSubRes {
struct SCtgTask {
CTG_TASK_TYPE type;
bool subTask;
int32_t taskId;
SCtgJob* pJob;
void* taskCtx;
@ -623,6 +637,7 @@ typedef struct SCtgCacheItemInfo {
#define CTG_FLAG_SYS_DB 0x8
#define CTG_FLAG_FORCE_UPDATE 0x10
#define CTG_FLAG_ONLY_CACHE 0x20
#define CTG_FLAG_SYNC_OP 0x40
#define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v))
@ -925,6 +940,10 @@ void ctgReleaseVgMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCach
void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache);
void ctgGetGlobalCacheStat(SCtgCacheStat* pStat);
int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res);
int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName);
int32_t ctgGetTbTagCb(SCtgTask* pTask);
int32_t ctgGetUserCb(SCtgTask* pTask);
extern SCatalogMgmt gCtgMgmt;
extern SCtgDebug gCTGDebug;

59
source/libs/catalog/src/catalog.c
View File

@ -208,7 +208,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx
}
while (true) {
CTG_ERR_JRET(ctgRefreshTbMeta(pCtg, pConn, ctx, &output, false));
CTG_ERR_JRET(ctgRefreshTbMeta(pCtg, pConn, ctx, &output, ctx->flag & CTG_FLAG_SYNC_OP));
if (CTG_IS_META_TABLE(output->metaType)) {
*pTableMeta = output->tbMeta;
@ -429,6 +429,48 @@ int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName,
CTG_RET(TSDB_CODE_SUCCESS);
}
int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pRes) {
SVgroupInfo vgroupInfo = {0};
STableCfg* pCfg = NULL;
int32_t code = 0;
CTG_ERR_RET(ctgGetTbHashVgroup(pCtg, pConn, pTableName, &vgroupInfo, NULL));
CTG_ERR_RET(ctgGetTableCfgFromVnode(pCtg, pConn, pTableName, &vgroupInfo, &pCfg, NULL));
if (NULL == pCfg->pTags || pCfg->tagsLen <= 0) {
ctgError("invalid tag in tbCfg rsp, pTags:%p, len:%d", pCfg->pTags, pCfg->tagsLen);
CTG_ERR_JRET(TSDB_CODE_INVALID_MSG);
}
SArray* pTagVals = NULL;
STag* pTag = (STag*)pCfg->pTags;
if (tTagIsJson(pTag)) {
pTagVals = taosArrayInit(1, sizeof(STagVal));
if (NULL == pTagVals) {
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
char* pJson = parseTagDatatoJson(pTag);
STagVal tagVal;
tagVal.cid = 0;
tagVal.type = TSDB_DATA_TYPE_JSON;
tagVal.pData = pJson;
tagVal.nData = strlen(pJson);
taosArrayPush(pTagVals, &tagVal);
} else {
CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals));
}
*pRes = pTagVals;
_return:
tFreeSTableCfgRsp((STableCfgRsp*)pCfg);
CTG_RET(code);
}
int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pVgList) {
STableMeta* tbMeta = NULL;
int32_t code = 0;
@ -1414,6 +1456,21 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pRes) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgGetTbTag(pCtg, pConn, (SName*)pTableName, pRes));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg) {
CTG_API_ENTER();

278
source/libs/catalog/src/ctgAsync.c
View File

@ -21,7 +21,8 @@
#include "trpc.h"
int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
SName* name = (SName*)param;
SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param;
SName* name = pParam->pName;
SCtgTask task = {0};
task.type = CTG_TASK_GET_TB_META;
@ -41,7 +42,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
}
memcpy(ctx->pName, name, sizeof(*name));
ctx->flag = CTG_FLAG_UNKNOWN_STB;
ctx->flag = pParam->flag | CTG_FLAG_UNKNOWN_STB;
taosArrayPush(pJob->pTasks, &task);
@ -386,6 +387,37 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
SName* name = (SName*)param;
SCtgTask task = {0};
task.type = CTG_TASK_GET_TB_TAG;
task.taskId = taskIdx;
task.pJob = pJob;
task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTagCtx));
if (NULL == task.taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgTbTagCtx* ctx = task.taskCtx;
ctx->pName = taosMemoryMalloc(sizeof(*name));
if (NULL == ctx->pName) {
taosMemoryFree(task.taskCtx);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(ctx->pName, name, sizeof(*name));
taosArrayPush(pJob->pTasks, &task);
qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx,
ctgTaskTypeStr(task.type), name->tname);
return TSDB_CODE_SUCCESS;
}
int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) {
SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@ -437,6 +469,15 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(name, dbFName);
taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN);
taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName));
}
for (int32_t i = 0; i < pJob->tbTagNum; ++i) {
SName* name = taosArrayGet(pReq->pTableTag, i);
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(name, dbFName);
taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN);
taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName));
}
char* dbFName = taosHashIterate(pDb, NULL);
@ -505,9 +546,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo);
int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex);
int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg);
int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag);
int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum +
userNum + dbInfoNum + tbIndexNum + tbCfgNum;
userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum;
*job = taosMemoryCalloc(1, sizeof(SCtgJob));
if (NULL == *job) {
@ -537,6 +579,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
pJob->tbIndexNum = tbIndexNum;
pJob->tbCfgNum = tbCfgNum;
pJob->svrVerNum = svrVerNum;
pJob->tbTagNum = tbTagNum;
#if CTG_BATCH_FETCH
pJob->pBatchs =
@ -604,6 +647,12 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL));
}
for (int32_t i = 0; i < tbTagNum; ++i) {
SName* name = taosArrayGet(pReq->pTableTag, i);
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL));
}
for (int32_t i = 0; i < indexNum; ++i) {
char* indexName = taosArrayGet(pReq->pIndex, i);
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_INDEX_INFO, indexName, NULL));
@ -650,6 +699,10 @@ _return:
}
int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableMeta) {
pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes));
@ -665,6 +718,10 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
}
int32_t ctgDumpTbMetasRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
pJob->jobRes.pTableMeta = pTask->res;
@ -673,6 +730,10 @@ int32_t ctgDumpTbMetasRes(SCtgTask* pTask) {
}
int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDbVgroup) {
pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, sizeof(SMetaRes));
@ -688,6 +749,10 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
}
int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableHash) {
pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes));
@ -703,6 +768,10 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
}
int32_t ctgDumpTbHashsRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
pJob->jobRes.pTableHash = pTask->res;
@ -711,9 +780,17 @@ int32_t ctgDumpTbHashsRes(SCtgTask* pTask) {
}
int32_t ctgDumpTbIndexRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableIndex) {
pJob->jobRes.pTableIndex = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes));
SArray* pRes = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes));
if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableIndex, NULL, pRes)) {
taosArrayDestroy(pRes);
}
if (NULL == pJob->jobRes.pTableIndex) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
@ -726,9 +803,17 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) {
}
int32_t ctgDumpTbCfgRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableCfg) {
pJob->jobRes.pTableCfg = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes));
SArray* pRes = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes));
if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableCfg, NULL, pRes)) {
taosArrayDestroy(pRes);
}
if (NULL == pJob->jobRes.pTableCfg) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
@ -740,7 +825,35 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpTbTagRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pTableTag) {
SArray* pRes = taosArrayInit(pJob->tbTagNum, sizeof(SMetaRes));
if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableTag, NULL, pRes)) {
taosArrayDestroy(pRes);
}
if (NULL == pJob->jobRes.pTableTag) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
}
SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
taosArrayPush(pJob->jobRes.pTableTag, &res);
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpIndexRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pIndex) {
pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SMetaRes));
@ -756,6 +869,10 @@ int32_t ctgDumpIndexRes(SCtgTask* pTask) {
}
int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pQnodeList) {
pJob->jobRes.pQnodeList = taosArrayInit(1, sizeof(SMetaRes));
@ -771,6 +888,10 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
}
int32_t ctgDumpDnodeRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDnodeList) {
pJob->jobRes.pDnodeList = taosArrayInit(1, sizeof(SMetaRes));
@ -786,6 +907,10 @@ int32_t ctgDumpDnodeRes(SCtgTask* pTask) {
}
int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDbCfg) {
pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SMetaRes));
@ -801,6 +926,10 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
}
int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pDbInfo) {
pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SMetaRes));
@ -816,6 +945,10 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
}
int32_t ctgDumpUdfRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pUdfList) {
pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SMetaRes));
@ -831,6 +964,10 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) {
}
int32_t ctgDumpUserRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pUser) {
pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(SMetaRes));
@ -846,6 +983,10 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) {
}
int32_t ctgDumpSvrVer(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
if (NULL == pJob->jobRes.pSvrVer) {
pJob->jobRes.pSvrVer = taosMemoryCalloc(1, sizeof(SMetaRes));
@ -1075,7 +1216,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf
STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out;
ctgUpdateTbMetaToCache(pCtg, pOut, false);
ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP);
if (CTG_IS_META_BOTH(pOut->metaType)) {
memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta));
@ -1473,6 +1614,49 @@ _return:
CTG_RET(code);
}
int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out;
if (NULL == pRsp->pTags || pRsp->tagsLen <= 0) {
ctgError("invalid tag in tbCfg rsp, pTags:%p, len:%d", pRsp->pTags, pRsp->tagsLen);
CTG_ERR_JRET(TSDB_CODE_INVALID_MSG);
}
SArray* pTagVals = NULL;
STag* pTag = (STag*)pRsp->pTags;
if (tTagIsJson(pTag)) {
pTagVals = taosArrayInit(1, sizeof(STagVal));
if (NULL == pTagVals) {
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
char* pJson = parseTagDatatoJson(pTag);
STagVal tagVal;
tagVal.cid = 0;
tagVal.type = TSDB_DATA_TYPE_JSON;
tagVal.pData = pJson;
tagVal.nData = strlen(pJson);
taosArrayPush(pTagVals, &tagVal);
} else {
CTG_ERR_JRET(tTagToValArray((const STag*)pRsp->pTags, &pTagVals));
}
pTask->res = pTagVals;
_return:
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
@ -1905,7 +2089,10 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) {
if (pCtx->tbType <= 0) {
CTG_ERR_JRET(ctgReadTbTypeFromCache(pCtg, dbFName, pCtx->pName->tname, &pCtx->tbType));
if (pCtx->tbType <= 0) {
CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, pCtx->pName));
SCtgTbMetaParam param;
param.pName = pCtx->pName;
param.flag = 0;
CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &param));
return TSDB_CODE_SUCCESS;
}
}
@ -1935,6 +2122,45 @@ _return:
CTG_RET(code);
}
int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) {
int32_t code = 0;
SCatalog* pCtg = pTask->pJob->pCtg;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx;
SArray* pRes = NULL;
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pCtx->pName, dbFName);
SCtgJob* pJob = pTask->pJob;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1);
if (NULL == pMsgCtx->pBatchs) {
pMsgCtx->pBatchs = pJob->pBatchs;
}
if (NULL == pCtx->pVgInfo) {
CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo));
if (NULL == pCtx->pVgInfo) {
CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbTagCb, dbFName));
return TSDB_CODE_SUCCESS;
}
}
CTG_CACHE_NHIT_INC(CTG_CI_TBL_TAG, 1);
CTG_ERR_JRET(ctgGetTableCfgFromVnode(pCtg, pConn, pCtx->pName, pCtx->pVgInfo, NULL, pTask));
return TSDB_CODE_SUCCESS;
_return:
if (CTG_TASK_LAUNCHED == pTask->status) {
ctgHandleTaskEnd(pTask, code);
}
CTG_RET(code);
}
int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
SRequestConnInfo* pConn = &pTask->pJob->conn;
@ -2077,6 +2303,8 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) {
if (inCache) {
pTask->res = rsp.pRawRes;
ctgTaskDebug("Final res got, pass:%d, pCond:%p", rsp.pRawRes->pass, rsp.pRawRes->pCond);
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
@ -2084,7 +2312,10 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) {
taosMemoryFreeClear(rsp.pRawRes);
if (rsp.metaNotExists) {
CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &pCtx->user.tbName));
SCtgTbMetaParam param;
param.pName = &pCtx->user.tbName;
param.flag = CTG_FLAG_SYNC_OP;
CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetUserCb, &param));
} else {
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask));
}
@ -2138,6 +2369,27 @@ _return:
CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code));
}
int32_t ctgGetTbTagCb(SCtgTask* pTask) {
int32_t code = 0;
CTG_ERR_JRET(pTask->subRes.code);
SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx;
SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res;
if (NULL == pCtx->pVgInfo) {
pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo));
CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo));
}
CTG_RET(ctgLaunchGetTbTagTask(pTask));
_return:
CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code));
}
int32_t ctgGetUserCb(SCtgTask* pTask) {
int32_t code = 0;
@ -2162,8 +2414,12 @@ int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) {
int32_t ctgCompTbMetaTasks(SCtgTask* pTask, void* param, bool* equal) {
SCtgTbMetaCtx* ctx = pTask->taskCtx;
SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param;
*equal = tNameTbNameEqual(ctx->pName, (SName*)param);
*equal = tNameTbNameEqual(ctx->pName, (SName*)pParam->pName);
if (*equal) {
ctx->flag |= pParam->flag;
}
return TSDB_CODE_SUCCESS;
}
@ -2197,6 +2453,7 @@ SCtgAsyncFps gCtgAsyncFps[] = {
{ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL},
{ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL},
{ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL},
{ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL},
};
int32_t ctgMakeAsyncRes(SCtgJob* pJob) {
@ -2284,6 +2541,9 @@ int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp,
}
SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId);
if (newTask) {
pSub->subTask = true;
}
CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask));

29
source/libs/catalog/src/ctgCache.c
View File

@ -703,7 +703,31 @@ _return:
CTG_RET(code);
}
int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName) {
*stbName = NULL;
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
ctgDebug("db %s not in cache", dbFName);
return TSDB_CODE_SUCCESS;
}
char *stb = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid));
if (NULL == stb) {
ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName);
return TSDB_CODE_SUCCESS;
}
*stbName = taosStrdup(stb);
taosHashRelease(dbCache->stbCache, stb);
return TSDB_CODE_SUCCESS;
}
int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) {
int32_t code = 0;
if (IS_SYS_DBNAME(pReq->tbName.dbname)) {
*inCache = true;
pRes->pRawRes->pass = true;
@ -728,7 +752,7 @@ int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache,
CTG_LOCK(CTG_READ, &pUser->lock);
memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth));
int32_t code = ctgChkSetAuthRes(pCtg, &req, pRes);
code = ctgChkSetAuthRes(pCtg, &req, pRes);
CTG_UNLOCK(CTG_READ, &pUser->lock);
CTG_ERR_JRET(code);
@ -742,8 +766,9 @@ _return:
*inCache = false;
CTG_CACHE_NHIT_INC(CTG_CI_USER, 1);
ctgDebug("Get user from cache failed, user:%s, metaNotExists:%d, code:%d", pReq->user, pRes->metaNotExists, code);
return TSDB_CODE_SUCCESS;
return code;
}
void ctgDequeue(SCtgCacheOperation **op) {

148
source/libs/catalog/src/ctgUtil.c
View File

@ -170,6 +170,9 @@ void ctgFreeSMetaData(SMetaData* pData) {
taosArrayDestroy(pData->pTableCfg);
pData->pTableCfg = NULL;
taosArrayDestroy(pData->pTableTag);
pData->pTableTag = NULL;
taosMemoryFreeClear(pData->pSvrVer);
}
@ -486,6 +489,18 @@ void ctgFreeBatchHash(void* hash) {
taosMemoryFreeClear(pRes->pRes);
}
void ctgFreeJsonTagVal(void* val) {
if (NULL == val) {
return;
}
STagVal* pVal = (STagVal*)val;
if (TSDB_DATA_TYPE_JSON == pVal->type) {
taosMemoryFree(pVal->pData);
}
}
void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
switch (type) {
case CTG_TASK_GET_QNODE:
@ -516,16 +531,32 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
}
break;
}
case CTG_TASK_GET_USER: {
if (*pRes) {
SUserAuthRes* pAuth = (SUserAuthRes*)*pRes;
nodesDestroyNode(pAuth->pCond);
taosMemoryFreeClear(*pRes);
}
break;
}
case CTG_TASK_GET_TB_HASH:
case CTG_TASK_GET_DB_INFO:
case CTG_TASK_GET_INDEX_INFO:
case CTG_TASK_GET_UDF:
case CTG_TASK_GET_USER:
case CTG_TASK_GET_SVR_VER:
case CTG_TASK_GET_TB_META: {
taosMemoryFreeClear(*pRes);
break;
}
case CTG_TASK_GET_TB_TAG: {
if (1 == taosArrayGetSize(*pRes)) {
taosArrayDestroyEx(*pRes, ctgFreeJsonTagVal);
} else {
taosArrayDestroy(*pRes);
}
*pRes = NULL;
break;
}
case CTG_TASK_GET_TB_META_BATCH: {
SArray* pArray = (SArray*)*pRes;
int32_t num = taosArrayGetSize(pArray);
@ -679,6 +710,13 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
taosMemoryFreeClear(pTask->taskCtx);
break;
}
case CTG_TASK_GET_TB_TAG: {
SCtgTbTagCtx* taskCtx = (SCtgTbTagCtx*)pTask->taskCtx;
taosMemoryFreeClear(taskCtx->pName);
taosMemoryFreeClear(taskCtx->pVgInfo);
taosMemoryFreeClear(taskCtx);
break;
}
case CTG_TASK_GET_DB_VGROUP:
case CTG_TASK_GET_DB_CFG:
case CTG_TASK_GET_DB_INFO:
@ -1336,57 +1374,75 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
STableMeta* pMeta = NULL;
SGetUserAuthRsp* pInfo = &req->authInfo;
SHashObj* pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs;
char* stbName = NULL;
char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(&req->pRawReq->tbName, tbFullName);
char* pCond = taosHashGet(pTbs, tbFullName, strlen(tbFullName));
if (pCond) {
if (strlen(pCond) > 1) {
CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond));
char tbFName[TSDB_TABLE_FNAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
tNameExtractFullName(&req->pRawReq->tbName, tbFName);
tNameGetFullDbName(&req->pRawReq->tbName, dbFName);
while (true) {
taosMemoryFreeClear(pMeta);
char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName));
if (pCond) {
if (strlen(pCond) > 1) {
CTG_ERR_JRET(nodesStringToNode(pCond, &res->pRawRes->pCond));
}
res->pRawRes->pass = true;
goto _return;
}
res->pRawRes->pass = true;
return TSDB_CODE_SUCCESS;
if (stbName) {
res->pRawRes->pass = false;
goto _return;
}
CTG_ERR_JRET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta));
if (NULL == pMeta) {
if (req->onlyCache) {
res->metaNotExists = true;
ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname);
goto _return;
}
SCtgTbMetaCtx ctx = {0};
ctx.pName = (SName*)&req->pRawReq->tbName;
ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_SYNC_OP;
CTG_ERR_JRET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta));
}
if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) {
res->pRawRes->pass = false;
goto _return;
}
if (TSDB_CHILD_TABLE == pMeta->tableType) {
CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, dbFName, pMeta->suid, &stbName));
if (NULL == stbName) {
if (req->onlyCache) {
res->metaNotExists = true;
ctgDebug("suid %" PRIu64 " name not in cache for auth", pMeta->suid);
goto _return;
}
continue;
}
sprintf(tbFName, "%s.%s", dbFName, stbName);
continue;
}
ctgError("Invalid table type %d for %s", pMeta->tableType, tbFName);
CTG_ERR_JRET(TSDB_CODE_INVALID_PARA);
}
res->pRawRes->pass = false;
// CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta));
// if (NULL == pMeta) {
// if (req->onlyCache) {
// res->metaNotExists = true;
// ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname);
// return TSDB_CODE_SUCCESS;
// }
// CTG_ERR_RET(catalogGetTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, &pMeta));
// }
// if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) {
// res->pRawRes->pass = false;
// goto _return;
// }
// if (TSDB_CHILD_TABLE == pMeta->tableType) {
// res->pRawRes->pass = true;
// /*
// char stbName[TSDB_TABLE_NAME_LEN] = {0};
// CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, pMeta->suid, stbName));
// if (0 == stbName[0]) {
// if (req->onlyCache) {
// res->notExists = true;
// return TSDB_CODE_SUCCESS;
// }
// CTG_ERR_RET(catalogRefreshTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, 0));
// }
// */
// }
_return:
taosMemoryFree(pMeta);
taosMemoryFree(stbName);
CTG_RET(code);
}
@ -1423,7 +1479,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) {
req->singleType = AUTH_TYPE_READ;
CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
if (pRes->pass) {
if (pRes->pass || res->metaNotExists) {
return TSDB_CODE_SUCCESS;
}
}
@ -1439,7 +1495,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) {
req->singleType = AUTH_TYPE_WRITE;
CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
if (pRes->pass) {
if (pRes->pass || res->metaNotExists) {
return TSDB_CODE_SUCCESS;
}
}

7
source/libs/command/src/command.c
View File

@ -278,7 +278,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
char* retentions = buildRetension(pCfg->pRetensions);
int32_t dbFNameLen = strlen(dbFName);
int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0;
int32_t hashPrefix = 0;
if (pCfg->hashPrefix > 0) {
hashPrefix = pCfg->hashPrefix - dbFNameLen - 1;
} else if (pCfg->hashPrefix < 0) {
hashPrefix = pCfg->hashPrefix + dbFNameLen + 1;
}
len += sprintf(
buf2 + VARSTR_HEADER_SIZE,

5
source/libs/executor/src/executor.c
View File

@ -1108,6 +1108,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pScanBaseInfo->dataReader = NULL;
// let's seek to the next version in wal file
int64_t firstVer = walGetFirstVer(pInfo->tqReader->pWalReader->pWal);
if (pOffset->version + 1 < firstVer){
pOffset->version = firstVer - 1;
}
if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
return -1;

6
source/libs/executor/src/sysscanoperator.c
View File

@ -1482,11 +1482,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
pInfo->pIdx->init = 1;
SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator);
return blk;
} else if (flt == -2) {
qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo));
return NULL;
} else if (flt == -1) {
// not idx
} else if ((flt == -1) || (flt == -2)) {
qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo));
}
} else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) {

15
source/libs/executor/src/timewindowoperator.c
View File

@ -2541,6 +2541,20 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
}
while (1) {
if (isTaskKilled(pTaskInfo)) {
if (pInfo->pUpdated != NULL) {
pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated);
}
if (pInfo->pUpdatedMap != NULL) {
tSimpleHashCleanup(pInfo->pUpdatedMap);
pInfo->pUpdatedMap = NULL;
}
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
pOperator->status = OP_RES_TO_RETURN;
@ -2635,6 +2649,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {
taosArrayPush(pInfo->pUpdated, pIte);
}
tSimpleHashCleanup(pInfo->pUpdatedMap);
pInfo->pUpdatedMap = NULL;
taosArraySort(pInfo->pUpdated, winKeyCmprImpl);

4
source/libs/executor/src/tsort.c
View File

@ -155,7 +155,7 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) {
int64_t fetchUs = 0, fetchNum = 0;
tsortClearOrderdSource(pSortHandle->pOrderedSource, &fetchUs, &fetchNum);
qError("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr);
qDebug("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr);
taosArrayDestroy(pSortHandle->pOrderedSource);
taosMemoryFreeClear(pSortHandle);
@ -316,7 +316,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32
}
int64_t et = taosGetTimestampUs();
qError("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr);
qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr);
}
return code;

40
source/libs/function/CMakeLists.txt
View File

@ -79,6 +79,26 @@ ENDIF ()
target_link_libraries(
udf1 PUBLIC os ${LINK_JEMALLOC})
add_library(udf1_dup STATIC MODULE test/udf1_dup.c)
target_include_directories(
udf1_dup
PUBLIC
"${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/client"
"${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEPENDENCIES(udf1_dup jemalloc)
ENDIF ()
target_link_libraries(
udf1_dup PUBLIC os ${LINK_JEMALLOC})
add_library(udf2 STATIC MODULE test/udf2.c)
target_include_directories(
udf2
@ -99,6 +119,26 @@ target_link_libraries(
udf2 PUBLIC os ${LINK_JEMALLOC}
)
add_library(udf2_dup STATIC MODULE test/udf2_dup.c)
target_include_directories(
udf2_dup
PUBLIC
"${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/client"
"${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEPENDENCIES(udf2_dup jemalloc)
ENDIF ()
target_link_libraries(
udf2_dup PUBLIC os ${LINK_JEMALLOC}
)
#SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin)
add_executable(udfd src/udfd.c)
target_include_directories(

42
source/libs/function/test/udf1_dup.c Normal file
View File

@ -0,0 +1,42 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef LINUX
#include <unistd.h>
#endif
#ifdef WINDOWS
#include <windows.h>
#endif
#include "taosudf.h"
DLL_EXPORT int32_t udf1_dup_init() { return 0; }
DLL_EXPORT int32_t udf1_dup_destroy() { return 0; }
DLL_EXPORT int32_t udf1_dup(SUdfDataBlock *block, SUdfColumn *resultCol) {
SUdfColumnData *resultData = &resultCol->colData;
for (int32_t i = 0; i < block->numOfRows; ++i) {
int j = 0;
for (; j < block->numOfCols; ++j) {
if (udfColDataIsNull(block->udfCols[j], i)) {
udfColDataSetNull(resultCol, i);
break;
}
}
if (j == block->numOfCols) {
int32_t luckyNum = 2;
udfColDataSet(resultCol, i, (char *)&luckyNum, false);
}
}
// to simulate actual processing delay by udf
#ifdef LINUX
usleep(1 * 1000); // usleep takes sleep time in us (1 millionth of a second)
#endif
#ifdef WINDOWS
Sleep(1);
#endif
resultData->numOfRows = block->numOfRows;
return 0;
}

78
source/libs/function/test/udf2_dup.c Normal file
View File

@ -0,0 +1,78 @@
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "taosudf.h"
DLL_EXPORT int32_t udf2_dup_init() { return 0; }
DLL_EXPORT int32_t udf2_dup_destroy() { return 0; }
DLL_EXPORT int32_t udf2_dup_start(SUdfInterBuf* buf) {
*(int64_t*)(buf->buf) = 0;
buf->bufLen = sizeof(double);
buf->numOfResult = 1;
return 0;
}
DLL_EXPORT int32_t udf2_dup(SUdfDataBlock* block, SUdfInterBuf* interBuf, SUdfInterBuf* newInterBuf) {
double sumSquares = 0;
if (interBuf->numOfResult == 1) {
sumSquares = *(double*)interBuf->buf;
}
int8_t numNotNull = 0;
for (int32_t i = 0; i < block->numOfCols; ++i) {
SUdfColumn* col = block->udfCols[i];
if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) {
return TSDB_CODE_UDF_INVALID_INPUT;
}
}
for (int32_t i = 0; i < block->numOfCols; ++i) {
for (int32_t j = 0; j < block->numOfRows; ++j) {
SUdfColumn* col = block->udfCols[i];
if (udfColDataIsNull(col, j)) {
continue;
}
switch (col->colMeta.type) {
case TSDB_DATA_TYPE_INT: {
char* cell = udfColDataGetData(col, j);
int32_t num = *(int32_t*)cell;
sumSquares += (double)num * num;
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
char* cell = udfColDataGetData(col, j);
double num = *(double*)cell;
sumSquares += num * num;
break;
}
default:
break;
}
++numNotNull;
}
}
*(double*)(newInterBuf->buf) = sumSquares;
newInterBuf->bufLen = sizeof(double);
if (interBuf->numOfResult == 0 && numNotNull == 0) {
newInterBuf->numOfResult = 0;
} else {
newInterBuf->numOfResult = 1;
}
return 0;
}
DLL_EXPORT int32_t udf2_dup_finish(SUdfInterBuf* buf, SUdfInterBuf* resultData) {
if (buf->numOfResult == 0) {
resultData->numOfResult = 0;
return 0;
}
double sumSquares = *(double*)(buf->buf);
*(double*)(resultData->buf) = sqrt(sumSquares) + 100;
resultData->bufLen = sizeof(double);
resultData->numOfResult = 1;
return 0;
}

185
source/libs/index/src/indexFilter.c
View File

@ -221,6 +221,71 @@ static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCt
param->status = SFLT_COARSE_INDEX;
return 0;
}
static int32_t sifNeedConvertCond(SNode *l, SNode *r) {
if (nodeType(l) != QUERY_NODE_COLUMN || nodeType(r) != QUERY_NODE_VALUE) {
return 0;
}
SColumnNode *c = (SColumnNode *)l;
SValueNode *v = (SValueNode *)r;
int32_t ctype = c->node.resType.type;
int32_t vtype = v->node.resType.type;
if (!IS_VAR_DATA_TYPE(ctype) && IS_VAR_DATA_TYPE(vtype)) {
return 1;
}
return 0;
}
static int32_t sifInitParamValByCol(SNode *r, SNode *l, SIFParam *param, SIFCtx *ctx) {
param->status = SFLT_COARSE_INDEX;
SColumnNode *cn = (SColumnNode *)r;
SValueNode *vn = (SValueNode *)l;
if (vn->typeData == TSDB_DATA_TYPE_NULL && (vn->literal == NULL || strlen(vn->literal) == 0)) {
param->status = SFLT_NOT_INDEX;
return 0;
}
SDataType *pType = &cn->node.resType;
int32_t type = pType->type;
SDataType *pVType = &vn->node.resType;
int32_t vtype = pVType->type;
char *pData = nodesGetValueFromNode(vn);
int32_t valLen = 0;
char **value = &param->condValue;
if (IS_VAR_DATA_TYPE(type)) {
int32_t dataLen = varDataTLen(pData);
if (type == TSDB_DATA_TYPE_JSON) {
if (*pData == TSDB_DATA_TYPE_NULL) {
dataLen = 0;
} else if (*pData == TSDB_DATA_TYPE_NCHAR) {
dataLen = varDataTLen(pData);
} else if (*pData == TSDB_DATA_TYPE_DOUBLE) {
dataLen = LONG_BYTES;
} else if (*pData == TSDB_DATA_TYPE_BOOL) {
dataLen = CHAR_BYTES;
}
dataLen += CHAR_BYTES;
}
valLen = dataLen;
} else {
valLen = pType->bytes;
}
char *tv = taosMemoryCalloc(1, valLen + 1);
if (tv == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(tv, pData, valLen);
*value = tv;
param->colId = -1;
param->colValType = (uint8_t)(vn->node.resType.type);
if (vn->literal != NULL && strlen(vn->literal) <= sizeof(param->colName)) {
memcpy(param->colName, vn->literal, strlen(vn->literal));
} else {
param->status = SFLT_NOT_INDEX;
}
return 0;
}
static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
param->status = SFLT_COARSE_INDEX;
switch (nodeType(node)) {
@ -317,8 +382,13 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
return TSDB_CODE_SUCCESS;
} else {
SIF_ERR_JRET(sifInitParam(node->pLeft, &paramList[0], ctx));
if (nParam > 1) {
// if (sifNeedConvertCond(node->pLeft, node->pRight)) {
// SIF_ERR_JRET(sifInitParamValByCol(node->pLeft, node->pRight, &paramList[1], ctx));
// } else {
SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
// }
// if (paramList[0].colValType == TSDB_DATA_TYPE_JSON &&
// ((SOperatorNode *)(node))->opType == OP_TYPE_JSON_CONTAINS) {
// return TSDB_CODE_OUT_OF_MEMORY;
@ -404,60 +474,149 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve
}
return NULL;
}
int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) {
// signed/unsigned/float
if (IS_SIGNED_NUMERIC_TYPE(type)) {
int64_t v = 0;
if (0 != toInteger(buf, len, 10, &v)) {
return -1;
}
if (type == TSDB_DATA_TYPE_BIGINT) {
*(int64_t *)val = v;
} else if (type == TSDB_DATA_TYPE_INT) {
*(int32_t *)val = v;
} else if (type == TSDB_DATA_TYPE_TINYINT) {
*(int8_t *)val = v;
} else if (type == TSDB_DATA_TYPE_SMALLINT) {
*(int16_t *)val = v;
}
} else if (IS_FLOAT_TYPE(type)) {
if (type == TSDB_DATA_TYPE_FLOAT) {
*(float *)val = taosStr2Float(buf, NULL);
} else {
*(double *)val = taosStr2Double(buf, NULL);
}
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
uint64_t v = 0;
if (0 != toUInteger(buf, len, 10, &v)) {
return -1;
}
if (type == TSDB_DATA_TYPE_UBIGINT) {
*(uint64_t *)val = v;
} else if (type == TSDB_DATA_TYPE_UINT) {
*(uint32_t *)val = v;
} else if (type == TSDB_DATA_TYPE_UTINYINT) {
*(uint8_t *)val = v;
} else if (type == TSDB_DATA_TYPE_USMALLINT) {
*(uint16_t *)val = v;
}
} else {
return -1;
}
return 0;
}
static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) {
int8_t ltype = left->colValType, rtype = right->colValType;
static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) {
int32_t code = 0;
int8_t ltype = left->colValType, rtype = right->colValType;
if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) {
return -1;
}
if (ltype == TSDB_DATA_TYPE_FLOAT) {
float f = 0;
SIF_DATA_CONVERT(rtype, right->condValue, f);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, f);
} else {
SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_FLOAT, &f));
}
typedata->f = f;
param->val = &typedata->f;
} else if (ltype == TSDB_DATA_TYPE_DOUBLE) {
double d = 0;
SIF_DATA_CONVERT(rtype, right->condValue, d);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, d);
} else {
SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_DOUBLE, &d));
}
typedata->d = d;
param->val = &typedata->d;
} else if (ltype == TSDB_DATA_TYPE_BIGINT) {
int64_t i64 = 0;
SIF_DATA_CONVERT(rtype, right->condValue, i64);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, i64);
} else {
SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_BIGINT, &i64));
}
typedata->i64 = i64;
param->val = &typedata->i64;
} else if (ltype == TSDB_DATA_TYPE_INT) {
int32_t i32 = 0;
SIF_DATA_CONVERT(rtype, right->condValue, i32);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, i32);
} else {
SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_INT, &i32));
}
typedata->i32 = i32;
param->val = &typedata->i32;
} else if (ltype == TSDB_DATA_TYPE_SMALLINT) {
int16_t i16 = 0;
SIF_DATA_CONVERT(rtype, right->condValue, i16);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, i16);
} else {
SIF_ERR_RET(
sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_SMALLINT, &i16));
}
typedata->i16 = i16;
param->val = &typedata->i16;
} else if (ltype == TSDB_DATA_TYPE_TINYINT) {
int8_t i8 = 0;
SIF_DATA_CONVERT(rtype, right->condValue, i8)
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, i8);
} else {
SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_TINYINT, &i8));
}
typedata->i8 = i8;
param->val = &typedata->i8;
} else if (ltype == TSDB_DATA_TYPE_UBIGINT) {
uint64_t u64 = 0;
SIF_DATA_CONVERT(rtype, right->condValue, u64);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, u64);
} else {
SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UBIGINT, &u64));
}
typedata->u64 = u64;
param->val = &typedata->u64;
} else if (ltype == TSDB_DATA_TYPE_UINT) {
uint32_t u32 = 0;
SIF_DATA_CONVERT(rtype, right->condValue, u32);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, u32);
} else {
SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UINT, &u32));
}
typedata->u32 = u32;
param->val = &typedata->u32;
} else if (ltype == TSDB_DATA_TYPE_USMALLINT) {
uint16_t u16 = 0;
SIF_DATA_CONVERT(rtype, right->condValue, u16);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, u16);
} else {
SIF_ERR_RET(
sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_USMALLINT, &u16));
}
typedata->u16 = u16;
param->val = &typedata->u16;
} else if (ltype == TSDB_DATA_TYPE_UTINYINT) {
uint8_t u8 = 0;
SIF_DATA_CONVERT(rtype, right->condValue, u8);
if (IS_NUMERIC_TYPE(rtype)) {
SIF_DATA_CONVERT(rtype, right->condValue, u8);
} else {
SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UTINYINT, &u8));
}
typedata->u8 = u8;
param->val = &typedata->u8;
}
return 0;
}
static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
int ret = 0;
@ -498,7 +657,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
param.val = buf;
}
} else {
sifSetFltParam(left, right, &typedata, &param);
if (sifSetFltParam(left, right, &typedata, &param) != 0) return -1;
}
ret = metaFilterTableIds(arg->metaEx, &param, output->result);
}

10
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -827,6 +827,8 @@ void nodesDestroyNode(SNode* pNode) {
SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode;
destroyVgDataBlockArray(pStmt->pDataBlocks);
taosMemoryFreeClear(pStmt->pTableMeta);
nodesDestroyNode(pStmt->pTagCond);
taosArrayDestroy(pStmt->pTableTag);
taosHashCleanup(pStmt->pVgroupsHashObj);
taosHashCleanup(pStmt->pSubTableHashObj);
taosHashCleanup(pStmt->pTableNameHashObj);
@ -953,8 +955,12 @@ void nodesDestroyNode(SNode* pNode) {
break;
case QUERY_NODE_SPLIT_VGROUP_STMT: // no pointer field
case QUERY_NODE_SYNCDB_STMT: // no pointer field
case QUERY_NODE_GRANT_STMT: // no pointer field
case QUERY_NODE_REVOKE_STMT: // no pointer field
break;
case QUERY_NODE_GRANT_STMT:
nodesDestroyNode(((SGrantStmt*)pNode)->pTagCond);
break;
case QUERY_NODE_REVOKE_STMT:
nodesDestroyNode(((SRevokeStmt*)pNode)->pTagCond);
break;
case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT:

4
source/libs/parser/inc/sql.y Normal file → Executable file
View File

@ -221,8 +221,8 @@ db_options(A) ::= db_options(B) WAL_RETENTION_SIZE NK_MINUS(D) NK_INTEGER(C).
db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); }
db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); }
db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, &C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); }

24
source/libs/parser/src/parAstCreater.c
View File

@ -1024,12 +1024,28 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
case DB_OPTION_STT_TRIGGER:
pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_TABLE_PREFIX:
pDbOptions->tablePrefix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
case DB_OPTION_TABLE_PREFIX: {
SValueNode *pNode = (SValueNode *)pVal;
if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
pDbOptions->tablePrefix = taosStr2Int32(pNode->literal, NULL, 10);
} else {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_prefix data type");
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
}
nodesDestroyNode((SNode*)pNode);
break;
case DB_OPTION_TABLE_SUFFIX:
pDbOptions->tableSuffix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
}
case DB_OPTION_TABLE_SUFFIX:{
SValueNode *pNode = (SValueNode *)pVal;
if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
pDbOptions->tableSuffix = taosStr2Int32(pNode->literal, NULL, 10);
} else {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_suffix data type");
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
}
nodesDestroyNode((SNode*)pNode);
break;
}
default:
break;
}

6
source/libs/parser/src/parAuthenticator.c
View File

@ -70,7 +70,7 @@ static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) {
return TSDB_CODE_SUCCESS == authQuery(pCxt, pStmt) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
}
static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) {
static int32_t mergeStableTagCond(SNode** pWhere, SNode* pTagCond) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (NULL == pLogicCond) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -78,7 +78,7 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) {
pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
pLogicCond->condType = LOGIC_COND_TYPE_AND;
int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, *pTagCond);
int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, pTagCond);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeAppend(&pLogicCond->pParameterList, *pWhere);
}
@ -106,7 +106,7 @@ static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) {
return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy);
}
return mergeStableTagCond(pWhere, &pTagCondCopy);
return mergeStableTagCond(pWhere, pTagCondCopy);
}
static EDealRes authSelectImpl(SNode* pNode, void* pContext) {

114
source/libs/parser/src/parInsertSql.c
View File

@ -53,6 +53,7 @@ typedef struct SInsertParseContext {
bool missCache;
bool usingDuplicateTable;
bool forceUpdate;
bool needTableTagVal;
} SInsertParseContext;
typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param);
@ -577,28 +578,39 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) {
if (NULL == pValue) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pValue->node.resType.type = pVal->type;
pValue->node.resType = ((SColumnNode*)*pNode)->node.resType;
nodesDestroyNode(*pNode);
*pNode = (SNode*)pValue;
switch (pVal->type) {
case TSDB_DATA_TYPE_BOOL:
pValue->datum.b = *(int8_t*)(&pVal->i64);
*(bool*)&pValue->typeData = pValue->datum.b;
break;
case TSDB_DATA_TYPE_TINYINT:
pValue->datum.i = *(int8_t*)(&pVal->i64);
*(int8_t*)&pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_SMALLINT:
pValue->datum.i = *(int16_t*)(&pVal->i64);
*(int16_t*)&pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_INT:
pValue->datum.i = *(int32_t*)(&pVal->i64);
*(int32_t*)&pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_BIGINT:
pValue->datum.i = pVal->i64;
pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_FLOAT:
pValue->datum.d = *(float*)(&pVal->i64);
*(float*)&pValue->typeData = pValue->datum.d;
break;
case TSDB_DATA_TYPE_DOUBLE:
pValue->datum.d = *(double*)(&pVal->i64);
*(double*)&pValue->typeData = pValue->datum.d;
break;
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_NCHAR:
@ -611,18 +623,23 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) {
break;
case TSDB_DATA_TYPE_TIMESTAMP:
pValue->datum.i = pVal->i64;
pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_UTINYINT:
pValue->datum.i = *(uint8_t*)(&pVal->i64);
*(uint8_t*)&pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_USMALLINT:
pValue->datum.i = *(uint16_t*)(&pVal->i64);
*(uint16_t*)&pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_UINT:
pValue->datum.i = *(uint32_t*)(&pVal->i64);
*(uint32_t*)&pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_UBIGINT:
pValue->datum.i = *(uint64_t*)(&pVal->i64);
*(uint64_t*)&pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_VARBINARY:
@ -667,16 +684,15 @@ static int32_t checkTagCondResult(SNode* pResult) {
: TSDB_CODE_PAR_PERMISSION_DENIED;
}
int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode* pCond) {
int32_t code = setTagVal(pTagVals, pTagName, pCond);
SNode* pNew = NULL;
static int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode** pCond) {
int32_t code = setTagVal(pTagVals, pTagName, *pCond);
if (TSDB_CODE_SUCCESS == code) {
code = scalarCalculateConstants(pCond, &pNew);
code = scalarCalculateConstants(*pCond, pCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkTagCondResult(pNew);
code = checkTagCondResult(*pCond);
}
nodesDestroyNode(pNew);
NODES_DESTORY_NODE(*pCond);
return code;
}
@ -716,6 +732,10 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt
}
}
if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pTagCond) {
code = checkSubtablePrivilege(pTagVals, pTagName, &pStmt->pTagCond);
}
if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) {
code = tTagNew(pTagVals, 1, false, &pTag);
}
@ -843,7 +863,7 @@ static void setUserAuthInfo(SParseContext* pCxt, SName* pTbName, SUserAuthInfo*
pInfo->type = AUTH_TYPE_WRITE;
}
static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) {
static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) {
int32_t code = TSDB_CODE_SUCCESS;
SUserAuthInfo authInfo = {0};
setUserAuthInfo(pCxt, pTbName, &authInfo);
@ -863,11 +883,28 @@ static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache)
*pMissCache = true;
} else if (!authRes.pass) {
code = TSDB_CODE_PAR_PERMISSION_DENIED;
} else if (NULL != authRes.pCond) {
*pTagCond = authRes.pCond;
}
}
return code;
}
static int32_t checkAuthForTable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, bool* pNeedTableTagVal) {
SNode* pTagCond = NULL;
int32_t code = checkAuth(pCxt, pTbName, pMissCache, &pTagCond);
if (TSDB_CODE_SUCCESS == code) {
*pNeedTableTagVal = ((*pMissCache) || (NULL != pTagCond));
*pMissCache = (NULL != pTagCond);
}
nodesDestroyNode(pTagCond);
return code;
}
static int32_t checkAuthForStable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) {
return checkAuth(pCxt, pTbName, pMissCache, pTagCond);
}
static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta,
bool* pMissCache) {
SParseContext* pComCxt = pCxt->pComCxt;
@ -970,7 +1007,7 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStm
return TSDB_CODE_SUCCESS;
}
int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache);
int32_t code = checkAuthForTable(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache, &pCxt->needTableTagVal);
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = getTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache);
}
@ -993,7 +1030,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt
return TSDB_CODE_SUCCESS;
}
int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache);
int32_t code = checkAuthForStable(pCxt->pComCxt, &pStmt->usingTableName, &pCxt->missCache, &pStmt->pTagCond);
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache);
}
@ -1606,6 +1643,8 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod
static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
insDestroyBoundColInfo(&pCxt->tags);
taosMemoryFreeClear(pStmt->pTableMeta);
nodesDestroyNode(pStmt->pTagCond);
taosArrayDestroy(pStmt->pTableTag);
tdDestroySVCreateTbReq(pStmt->pCreateTblReq);
taosMemoryFreeClear(pStmt->pCreateTblReq);
pCxt->missCache = false;
@ -1780,14 +1819,18 @@ static int32_t createInsertQuery(SInsertParseContext* pCxt, SQuery** pOutput) {
return code;
}
static int32_t checkAuthFromMetaData(const SArray* pUsers) {
static int32_t checkAuthFromMetaData(const SArray* pUsers, SNode** pTagCond) {
if (1 != taosArrayGetSize(pUsers)) {
return TSDB_CODE_FAILED;
}
SMetaRes* pRes = taosArrayGet(pUsers, 0);
if (TSDB_CODE_SUCCESS == pRes->code) {
return (*(bool*)pRes->pRes) ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED;
SUserAuthRes* pAuth = pRes->pRes;
if (NULL != pAuth->pCond) {
*pTagCond = nodesCloneNode(pAuth->pCond);
}
return pAuth->pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED;
}
return pRes->code;
}
@ -1826,9 +1869,40 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpS
sizeof(SVgroupInfo));
}
static int32_t buildTagNameFromMeta(STableMeta* pMeta, SArray** pTagName) {
*pTagName = taosArrayInit(pMeta->tableInfo.numOfTags, TSDB_COL_NAME_LEN);
if (NULL == *pTagName) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SSchema* pSchema = getTableTagSchema(pMeta);
for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) {
taosArrayPush(*pTagName, pSchema[i].name);
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkSubtablePrivilegeForTable(const SArray* pTables, SVnodeModifyOpStmt* pStmt) {
if (1 != taosArrayGetSize(pTables)) {
return TSDB_CODE_FAILED;
}
SMetaRes* pRes = taosArrayGet(pTables, 0);
if (TSDB_CODE_SUCCESS != pRes->code) {
return pRes->code;
}
SArray* pTagName = NULL;
int32_t code = buildTagNameFromMeta(pStmt->pTableMeta, &pTagName);
if (TSDB_CODE_SUCCESS == code) {
code = checkSubtablePrivilege((SArray*)pRes->pRes, pTagName, &pStmt->pTagCond);
}
taosArrayDestroy(pTagName);
return code;
}
static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData,
SVnodeModifyOpStmt* pStmt, bool isStb) {
int32_t code = checkAuthFromMetaData(pMetaData->pUser);
int32_t code = checkAuthFromMetaData(pMetaData->pUser, &pStmt->pTagCond);
if (TSDB_CODE_SUCCESS == code) {
code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta);
}
@ -1841,6 +1915,9 @@ static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMeta
if (TSDB_CODE_SUCCESS == code) {
code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb);
}
if (TSDB_CODE_SUCCESS == code && !isStb && NULL != pStmt->pTagCond) {
code = checkSubtablePrivilegeForTable(pMetaData->pTableTag, pStmt);
}
return code;
}
@ -1860,6 +1937,8 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) {
pCatalogReq->pTableHash = NULL;
taosArrayDestroy(pCatalogReq->pUser);
pCatalogReq->pUser = NULL;
taosArrayDestroy(pCatalogReq->pTableTag);
pCatalogReq->pTableTag = NULL;
}
static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData,
@ -2033,8 +2112,15 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray**
return TSDB_CODE_SUCCESS;
}
static int32_t buildInsertTableTagReq(SName* pName, SArray** pTables) { return buildInsertTableReq(pName, pTables); }
static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SCatalogReq* pCatalogReq) {
int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser);
int32_t code = buildInsertUserAuthReq(
pCxt->pComCxt->pUser, (0 == pStmt->usingTableName.type ? &pStmt->targetTableName : &pStmt->usingTableName),
&pCatalogReq->pUser);
if (TSDB_CODE_SUCCESS == code && pCxt->needTableTagVal) {
code = buildInsertTableTagReq(&pStmt->targetTableName, &pCatalogReq->pTableTag);
}
if (TSDB_CODE_SUCCESS == code) {
if (0 == pStmt->usingTableName.type) {
code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableMeta);

37
source/libs/parser/src/parTranslater.c
View File

@ -1310,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
}
static EDealRes haveVectorFunction(SNode* pNode, void* pContext) {
if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || isInterpPseudoColumnFunc(pNode)) {
if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) ||
isInterpPseudoColumnFunc(pNode)) {
*((bool*)pContext) = true;
return DEAL_RES_END;
}
@ -4177,6 +4178,34 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbPrefix, int32_t tbSuffix) {
if (tbPrefix < TSDB_MIN_HASH_PREFIX || tbPrefix > TSDB_MAX_HASH_PREFIX) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix: %d valid range: [%d, %d]", tbPrefix,
TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
}
if (tbSuffix < TSDB_MIN_HASH_SUFFIX || tbSuffix > TSDB_MAX_HASH_SUFFIX) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_suffix: %d valid range: [%d, %d]", tbSuffix,
TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
}
if ((tbPrefix * tbSuffix) < 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix & table_suffix: mixed usage not allowed");
}
if ((tbPrefix + tbSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix & table_suffix: exceed max table name length");
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
int32_t daysPerFile = pOptions->daysPerFile;
int64_t daysToKeep0 = pOptions->keep[0];
@ -4284,10 +4313,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "tableSuffix", pOptions->tableSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkOptionsDependency(pCxt, pDbName, pOptions);
@ -6617,6 +6643,7 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
if (TSDB_CODE_SUCCESS == code) {
code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
}
tFreeSAlterUserReq(&req);
return code;
}

3518
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

2
source/libs/planner/src/planOptimizer.c
View File

@ -2526,7 +2526,7 @@ static bool tbCntScanOptIsEligibleAggFuncs(SNodeList* pAggFuncs) {
return false;
}
}
return true;
return LIST_LENGTH(pAggFuncs) > 0;
}
static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) {

2
source/libs/qcom/src/querymsg.c
View File

@ -407,7 +407,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta *
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
}
qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d" PRIx64
qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d"
" tagNum %d colNum %d precision %d rowSize %d",
msg->tbName, pTableMeta->uid, pTableMeta->tableType, pTableMeta->vgId, msg->dbFName, msg->stbName,
pTableMeta->suid, pTableMeta->sversion, pTableMeta->tversion, pTableMeta->tableInfo.numOfTags,

26
source/libs/scalar/src/scalar.c
View File

@ -53,6 +53,7 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara
int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
colDataDestroy(pColumnData);
taosMemoryFree(pColumnData);
return terrno;
}
@ -1061,17 +1062,20 @@ int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) {
if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) {
if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) {
SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pLeft));
SCL_ERR_JRET(
sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode *)node->pLeft));
}
} else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) {
if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) {
if (SCL_IS_VAR_VALUE_NODE(node->pRight)) {
SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pRight));
SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight),
(SValueNode *)node->pRight));
} else if (QUERY_NODE_NODE_LIST == node->pRight->type) {
SNode* pNode;
FOREACH(pNode, ((SNodeListNode*)node->pRight)->pNodeList) {
SNode *pNode;
FOREACH(pNode, ((SNodeListNode *)node->pRight)->pNodeList) {
if (SCL_IS_VAR_VALUE_NODE(pNode)) {
SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode*)pNode));
SCL_ERR_JRET(
sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode *)pNode));
}
}
}
@ -1086,8 +1090,6 @@ _return:
return DEAL_RES_ERROR;
}
int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
int32_t code = 0;
@ -1096,19 +1098,20 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
}
if (SCL_IS_VAR_VALUE_NODE(node->pCase)) {
SNode* pNode;
SNode *pNode;
FOREACH(pNode, node->pWhenThenList) {
SExprNode *pExpr = (SExprNode *)((SWhenThenNode *)pNode)->pWhen;
if (TSDB_DATA_TYPE_TIMESTAMP == pExpr->resType.type) {
SCL_ERR_JRET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode*)node->pCase));
SCL_ERR_JRET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode *)node->pCase));
break;
}
}
} else if (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pCase)->resType.type) {
SNode* pNode;
SNode *pNode;
FOREACH(pNode, node->pWhenThenList) {
if (SCL_IS_VAR_VALUE_NODE(((SWhenThenNode *)pNode)->pWhen)) {
SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision, (SValueNode*)((SWhenThenNode *)pNode)->pWhen));
SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision,
(SValueNode *)((SWhenThenNode *)pNode)->pWhen));
}
}
}
@ -1271,7 +1274,6 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) {
return DEAL_RES_CONTINUE;
}
EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
SOperatorNode *node = (SOperatorNode *)*pNode;

4
source/libs/stream/src/stream.c
View File

@ -16,7 +16,7 @@
#include "streamInc.h"
#include "ttimer.h"
#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 100000
#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 3000
int32_t streamInit() {
int8_t old;
@ -52,7 +52,7 @@ void streamCleanUp() {
void streamSchedByTimer(void* param, void* tmrId) {
SStreamTask* pTask = (void*)param;
if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
if (streamTaskShouldStop(&pTask->status)) {
streamMetaReleaseTask(NULL, pTask);
return;
}

17
source/libs/stream/src/streamExec.c
View File

@ -17,6 +17,11 @@
#define STREAM_EXEC_MAX_BATCH_NUM 100
bool streamTaskShouldStop(const SStreamStatus* pStatus) {
int32_t status = atomic_load_8((int8_t*) &pStatus->taskStatus);
return (status == TASK_STATUS__STOP) || (status == TASK_STATUS__DROPPING);
}
static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
void* pExecutor = pTask->exec.pExecutor;
@ -66,7 +71,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
// pExecutor
while (1) {
if (pTask->status.taskStatus == TASK_STATUS__DROPPING) {
if (streamTaskShouldStop(&pTask->status)) {
return 0;
}
@ -106,7 +111,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
continue;
}
qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId);
qDebug("s-task:%s (child %d) executed and get block", pTask->id.idStr, pTask->selfChildId);
SSDataBlock block = {0};
assignOneDataBlock(&block, output);
@ -134,7 +139,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
int32_t batchCnt = 0;
while (1) {
if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
if (streamTaskShouldStop(&pTask->status)) {
taosArrayDestroy(pRes);
return 0;
}
@ -270,7 +275,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
}
}
if (pTask->status.taskStatus == TASK_STATUS__DROPPING) {
if (streamTaskShouldStop(&pTask->status)) {
if (pInput) {
streamFreeQitem(pInput);
}
@ -301,7 +306,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
", checkPoint id:%" PRId64 " -> %" PRId64,
pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId);
pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId};
pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer};
taosWLockLatch(&pTask->pMeta->lock);
streamMetaSaveTask(pTask->pMeta, pTask);
@ -368,7 +373,7 @@ int32_t streamTryExec(SStreamTask* pTask) {
atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
qDebug("s-task:%s exec completed", pTask->id.idStr);
if (!taosQueueEmpty(pTask->inputQueue->queue)) {
if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status))) {
streamSchedExec(pTask);
}
}

25
source/libs/stream/src/streamMeta.c
View File

@ -84,11 +84,6 @@ void streamMetaClose(SStreamMeta* pMeta) {
tdbClose(pMeta->db);
void* pIter = NULL;
// while(pMeta->walScan) {
// qDebug("wait stream daemon quit");
// taosMsleep(100);
// }
while (1) {
pIter = taosHashIterate(pMeta->pTasks, pIter);
if (pIter == NULL) {
@ -102,7 +97,6 @@ void streamMetaClose(SStreamMeta* pMeta) {
}
tFreeStreamTask(pTask);
/*streamMetaReleaseTask(pMeta, pTask);*/
}
taosHashCleanup(pMeta->pTasks);
@ -197,10 +191,12 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) {
taosRLockLatch(&pMeta->lock);
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) {
atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
taosRUnLockLatch(&pMeta->lock);
return *ppTask;
if (ppTask != NULL) {
if (!streamTaskShouldStop(&(*ppTask)->status)) {
atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
taosRUnLockLatch(&pMeta->lock);
return *ppTask;
}
}
taosRUnLockLatch(&pMeta->lock);
@ -211,7 +207,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1);
ASSERT(left >= 0);
if (left == 0) {
ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING);
ASSERT(streamTaskShouldStop(&pTask->status));
tFreeStreamTask(pTask);
}
}
@ -222,11 +218,8 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
SStreamTask* pTask = *ppTask;
taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn);
/*if (pTask->timer) {
* taosTmrStop(pTask->timer);*/
/*pTask->timer = NULL;*/
/*}*/
atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__STOP);
taosWLockLatch(&pMeta->lock);
streamMetaReleaseTask(pMeta, pTask);

5
source/os/src/osFile.c
View File

@ -538,10 +538,11 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
return -1;
}
struct stat fileStat;
#ifdef WINDOWS
int32_t code = _fstat(pFile->fd, &fileStat);
struct __stat64 fileStat;
int32_t code = _fstat64(pFile->fd, &fileStat);
#else
struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat);
#endif
if (code < 0) {

2
source/os/src/osLocale.c
View File

@ -171,7 +171,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) {
strcpy(outLocale, "en_US.UTF-8");
} else {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
// printf("locale not configured, set to system default:%s", outLocale);
printf("locale not configured, set to system default:%s\n", outLocale);
}
// if user does not specify the charset, extract it from locale

4
source/util/src/tlog.c
View File

@ -347,7 +347,6 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
char name[LOG_FILE_NAME_LEN + 50] = "\0";
int32_t logstat0_mtime, logstat1_mtime;
int32_t size;
tsLogObj.maxLines = maxLines;
tsLogObj.fileNum = maxFileNum;
@ -395,8 +394,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
size = (int32_t)filesize;
tsLogObj.lines = size / 60;
tsLogObj.lines = (int32_t)(filesize / 60);
taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END);

4
source/util/src/tsimplehash.c
View File

@ -361,10 +361,6 @@ int32_t tSimpleHashIterateRemove(SSHashObj *pHashObj, const void *key, size_t ke
return TSDB_CODE_SUCCESS;
}
static void destroyItems(void* pItem) {
taosMemoryFree(*(void**)pItem);
}
void tSimpleHashClear(SSHashObj *pHashObj) {
if (!pHashObj || taosHashTableEmpty(pHashObj)) {
return;

3
tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json
View File

@ -11,6 +11,7 @@
"confirm_parameter_prompt": "no",
"prepared_rand": 100,
"chinese": "no",
"escape_character": "yes",
"insert_interval": 0,
"num_of_records_per_req": 10,
"databases": [{
@ -29,7 +30,6 @@
"child_table_exists":"no",
"childtable_count": 8,
"childtable_prefix": "stb_",
"escape_character": "yes",
"auto_create_table": "yes",
"batch_create_tbl_num": 10,
"data_source": "rand",
@ -54,7 +54,6 @@
"child_table_exists":"no",
"childtable_count": 8,
"childtable_prefix": "stb3-2_",
"escape_character": "yes",
"auto_create_table": "yes",
"batch_create_tbl_num": 10,
"data_source": "rand",

3
tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json
View File

@ -11,6 +11,7 @@
"confirm_parameter_prompt": "no",
"prepared_rand": 100,
"chinese": "no",
"escape_character": "yes",
"insert_interval": 0,
"num_of_records_per_req": 10,
"databases": [{
@ -29,7 +30,6 @@
"child_table_exists":"no",
"childtable_count": 8,
"childtable_prefix": "stb_",
"escape_character": "yes",
"auto_create_table": "yes",
"batch_create_tbl_num": 10,
"data_source": "rand",
@ -54,7 +54,6 @@
"child_table_exists":"no",
"childtable_count": 8,
"childtable_prefix": "stb4-2_",
"escape_character": "yes",
"auto_create_table": "yes",
"batch_create_tbl_num": 10,
"data_source": "rand",

3
tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json
View File

@ -11,6 +11,7 @@
"confirm_parameter_prompt": "no",
"prepared_rand": 100,
"chinese": "no",
"escape_character": "yes",
"insert_interval": 0,
"num_of_records_per_req": 10,
"databases": [{
@ -29,7 +30,6 @@
"child_table_exists":"no",
"childtable_count": 8,
"childtable_prefix": "stb_",
"escape_character": "yes",
"auto_create_table": "yes",
"batch_create_tbl_num": 10,
"data_source": "rand",
@ -54,7 +54,6 @@
"child_table_exists":"no",
"childtable_count": 8,
"childtable_prefix": "stb2-2_",
"escape_character": "yes",
"auto_create_table": "yes",
"batch_create_tbl_num": 10,
"data_source": "rand",

3
tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json
View File

@ -11,6 +11,7 @@
"confirm_parameter_prompt": "no",
"prepared_rand": 100,
"chinese": "no",
"escape_character": "yes",
"insert_interval": 0,
"num_of_records_per_req": 10,
"databases": [{
@ -29,7 +30,6 @@
"child_table_exists":"no",
"childtable_count": 8,
"childtable_prefix": "stb_",
"escape_character": "yes",
"auto_create_table": "yes",
"batch_create_tbl_num": 10,
"data_source": "rand",
@ -55,7 +55,6 @@
"child_table_exists":"no",
"childtable_count": 8,
"childtable_prefix": "stb1-2_",
"escape_character": "yes",
"auto_create_table": "yes",
"batch_create_tbl_num": 10,
"data_source": "rand",

12
tests/parallel_test/cases.task
View File

@ -51,6 +51,16 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
@ -722,6 +732,7 @@
,,y,script,./test.sh -f tsim/user/privilege_db.sim
,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim
,,y,script,./test.sh -f tsim/user/privilege_topic.sim
,,y,script,./test.sh -f tsim/user/privilege_table.sim
,,y,script,./test.sh -f tsim/db/alter_option.sim
,,y,script,./test.sh -f tsim/db/alter_replica_31.sim
,,y,script,./test.sh -f tsim/db/basic1.sim
@ -746,6 +757,7 @@
,,y,script,./test.sh -f tsim/db/show_create_table.sim
,,y,script,./test.sh -f tsim/db/tables.sim
,,y,script,./test.sh -f tsim/db/taosdlog.sim
,,y,script,./test.sh -f tsim/db/table_prefix_suffix.sim
,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim
,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim
,,y,script,./test.sh -f tsim/dnode/balance1.sim

2
tests/parallel_test/run.sh
View File

@ -303,7 +303,7 @@ function run_thread() {
if [ ! -z "$corefile" ]; then
echo -e "\e[34m corefiles: $corefile \e[0m"
local build_dir=$log_dir/build_${hosts[index]}
local remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build"
local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build"
# if [ $ent -ne 0 ]; then
# remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build"
# fi

182
tests/script/tsim/db/table_prefix_suffix.sim Normal file
View File

@ -0,0 +1,182 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql drop database if exists db1;
sql create database db1 vgroups 5 TABLE_PREFIX 1 TABLE_SUFFIX 2;
sql use db1;
sql create table atb1aa (ts timestamp, f1 int);
sql create table btb1bb (ts timestamp, f1 int);
sql create table ctb1cc (ts timestamp, f1 int);
sql create table dtb1dd (ts timestamp, f1 int);
sql create table atb2aa (ts timestamp, f1 int);
sql create table btb2bb (ts timestamp, f1 int);
sql create table ctb2cc (ts timestamp, f1 int);
sql create table dtb2dd (ts timestamp, f1 int);
sql create table etb2ee (ts timestamp, f1 int);
sql show create database db1;
sql select count(*) a from information_schema.ins_tables where db_name='db1' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db1;
sql drop database if exists db2;
sql create database db2 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX -2;
sql use db2;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table taaa22 (ts timestamp, f1 int);
sql create table tbbb22 (ts timestamp, f1 int);
sql create table tccc22 (ts timestamp, f1 int);
sql create table tddd22 (ts timestamp, f1 int);
sql create table teee22 (ts timestamp, f1 int);
sql show create database db2;
sql select count(*) a from information_schema.ins_tables where db_name='db2' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db2;
sql drop database if exists db3;
sql create database db3 vgroups 5 TABLE_PREFIX -1;
sql use db3;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table zaaa22 (ts timestamp, f1 int);
sql create table zbbb22 (ts timestamp, f1 int);
sql create table zccc22 (ts timestamp, f1 int);
sql create table zddd22 (ts timestamp, f1 int);
sql create table zeee22 (ts timestamp, f1 int);
sql show create database db3;
sql select count(*) a from information_schema.ins_tables where db_name='db3' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db3;
sql drop database if exists db4;
sql create database db4 vgroups 5 TABLE_SUFFIX -2;
sql use db4;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table zaaa22 (ts timestamp, f1 int);
sql create table zbbb22 (ts timestamp, f1 int);
sql create table zccc22 (ts timestamp, f1 int);
sql create table zddd22 (ts timestamp, f1 int);
sql create table zeee22 (ts timestamp, f1 int);
sql show create database db4;
sql select count(*) a from information_schema.ins_tables where db_name='db4' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db4;
sql drop database if exists db5;
sql create database db5 vgroups 5 TABLE_PREFIX 1;
sql use db5;
sql create table taaa11 (ts timestamp, f1 int);
sql create table baaa11 (ts timestamp, f1 int);
sql create table caaa11 (ts timestamp, f1 int);
sql create table daaa11 (ts timestamp, f1 int);
sql create table faaa11 (ts timestamp, f1 int);
sql create table gbbb11 (ts timestamp, f1 int);
sql create table hbbb11 (ts timestamp, f1 int);
sql create table ibbb11 (ts timestamp, f1 int);
sql create table jbbb11 (ts timestamp, f1 int);
sql show create database db5;
sql select count(*) a from information_schema.ins_tables where db_name='db5' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db5;
sql drop database if exists db6;
sql create database db6 vgroups 5 TABLE_SUFFIX 2;
sql use db6;
sql create table taaa11 (ts timestamp, f1 int);
sql create table taaa12 (ts timestamp, f1 int);
sql create table taaa13 (ts timestamp, f1 int);
sql create table taaa14 (ts timestamp, f1 int);
sql create table tbbb23 (ts timestamp, f1 int);
sql create table tbbb24 (ts timestamp, f1 int);
sql create table tbbb31 (ts timestamp, f1 int);
sql create table tbbb32 (ts timestamp, f1 int);
sql create table tbbb33 (ts timestamp, f1 int);
sql show create database db6;
sql select count(*) a from information_schema.ins_tables where db_name='db6' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db6;
sql drop database if exists db7;
sql create database db7 vgroups 5 TABLE_PREFIX -100 TABLE_SUFFIX -92;
sql use db7;
sql create table taaa11 (ts timestamp, f1 int);
sql create table taaa12 (ts timestamp, f1 int);
sql create table taaa13 (ts timestamp, f1 int);
sql create table tbbb21 (ts timestamp, f1 int);
sql create table tbbb22 (ts timestamp, f1 int);
sql create table tbbb23 (ts timestamp, f1 int);
sql create table tbbb24 (ts timestamp, f1 int);
sql create table tccc31 (ts timestamp, f1 int);
sql create table tccc32 (ts timestamp, f1 int);
sql create table tccc33 (ts timestamp, f1 int);
sql create table tddd24 (ts timestamp, f1 int);
sql create table tddd31 (ts timestamp, f1 int);
sql create table tddd32 (ts timestamp, f1 int);
sql create table tddd33 (ts timestamp, f1 int);
sql show create database db7;
sql select count(*) a from information_schema.ins_tables where db_name='db7' group by vgroup_id having(count(*) > 0) order by a;
sql drop database if exists db7;
sql_error create database db8 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX 2;
sql_error create database db8 vgroups 5 TABLE_PREFIX 191 TABLE_SUFFIX 192;
sql_error create database db8 vgroups 5 TABLE_PREFIX -192 TABLE_SUFFIX -191;
sql_error create database db8 vgroups 5 TABLE_PREFIX 100 TABLE_SUFFIX 92;
system sh/exec.sh -n dnode1 -s stop -x SIGINT

21
tests/script/tsim/query/sys_tbname.sim
View File

@ -109,5 +109,26 @@ if $rows != 5000 then
return -1
endi
sql create database d1;
sql create stable d1.st1 (ts timestamp, f int) tags(t int);
sql create stable d1.st2 (ts timestamp, f int) tags(t int);
sql create table d1.ct1 using d1.st1 tags(1);
sql create table d1.ct2 using d1.st2 tags(2);
sql create database d2;
sql create stable d2.st1(ts timestamp, f int) tags(t int);
sql create stable d2.st2(ts timestamp, f int) tags(t int);
sql create table d2.ct1 using d2.st1 tags(1);
sql create table d2.ct2 using d2.st2 tags(2);
sql create database d3;
sql create stable d3.st1(ts timestamp, f int) tags(t int);
sql create stable d3.st2(ts timestamp, f int) tags(t int);
sql create table d3.ct1 using d3.st1 tags(1);
sql create table d3.ct2 using d3.st2 tags(2);
sql select count(*), stable_name, db_name from information_schema.ins_tables where db_name != 'd2' group by stable_name,db_name
print $rows
if $rows != 9 then
return -1
endi
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

5
tests/script/tsim/query/tableCount.sim
View File

@ -104,4 +104,9 @@ if $data62 != 5 then
return -1
endi
sql select distinct db_name from information_schema.ins_tables;
print $rows
if $rows != 4 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

2
tests/script/tsim/stream/basic1.sim
View File

@ -37,7 +37,7 @@ if $loop_count == 20 then
endi
if $rows != 4 then
print =====rows=$rows, expect 4
print =====rows=$rows expect 4
goto loop0
endi

11
tests/script/tsim/tag/bigint.sim
View File

@ -123,6 +123,17 @@ sql select * from $mt where tgcol = 1
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = '1'
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = "1"
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol <> 1
if $rows != 100 then
return -1

9
tests/script/tsim/tag/double.sim
View File

@ -123,6 +123,15 @@ sql select * from $mt where tgcol = 1
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = '1';
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = "1.0"
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol <> 1
if $rows != 100 then
return -1

10
tests/script/tsim/tag/float.sim
View File

@ -123,6 +123,16 @@ sql select * from $mt where tgcol = 1
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = "1.0"
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = "1"
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol <> 1
if $rows != 100 then
return -1

10
tests/script/tsim/tag/int.sim
View File

@ -123,6 +123,16 @@ sql select * from $mt where tgcol = 1
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = '1'
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = "1";
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol <> 1
if $rows != 100 then
return -1

12
tests/script/tsim/tag/int_float.sim
View File

@ -85,10 +85,22 @@ sql select * from $mt where tgcol <> 1
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = 1
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = '1'
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = "1"
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol <> 1
if $rows != 100 then
return -1

22
tests/script/tsim/tag/tinyint.sim
View File

@ -115,14 +115,36 @@ sql select * from $mt where tgcol = 0
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = '0'
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = "0"
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol <> 0
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = 1
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = "1"
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol = '1'
if $rows != 100 then
return -1
endi
sql select * from $mt where tgcol <> 1
if $rows != 100 then
return -1

302
tests/script/tsim/user/privilege_table.sim Normal file
View File

@ -0,0 +1,302 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== init env
sql drop database if exists test;
sql create database test vgroups 1;
sql use test;
sql create stable st1(ts timestamp, i int) tags(id int, loc varchar(20));
sql create table st1s1 using st1 tags(1, 'beijing');
sql create table st1s2 using st1 tags(2, 'shanghai');
sql insert into st1s1 values(now, 1) st1s2 values(now, 2);
sql create stable st2(ts timestamp, i int) tags(id int, loc varchar(20));
sql create table st2s1 using st2 tags(1, 'beijing');
sql create table st2s2 using st2 tags(2, 'shanghai');
sql insert into st2s1 values(now, 1) st2s2 values(now, 2);
sql create user wxy pass 'taosdata';
print =============== case 1: database unauthorized and table unauthorized
sql close
sql connect wxy
sql reset query cache;
sql_error select * from test.st1;
sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 2: database unauthorized and table read privilege
sql close
sql connect
sql grant read on test.st1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 2 then
return -1
endi
sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 3: database unauthorized and table read privilege with condition
sql close
sql connect
sql revoke read on test.st1 from wxy;
sql grant read on test.st1 with id = 1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 1 then
return -1
endi
sql_error insert into test.st1s1 values(now, 10);
sql_error insert into test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 4: database unauthorized and table write privilege
sql close
sql connect
sql revoke read on test.st1 with id = 1 from wxy;
sql grant write on test.st1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql_error select tbname, * from test.st1;
sql insert into test.st1s1 values(now, 10);
sql insert into test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 5: database unauthorized and table write privilege with condition
sql close
sql connect
sql revoke write on test.st1 from wxy;
sql grant write on test.st1 with id = 1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql_error select tbname, * from test.st1;
sql insert into test.st1s1 values(now, 10);
sql insert into test.st1s3 using test.st1 tags(1, 'dachang') values(now, 100);
sql_error insert into test.st1s2 values(now, 20);
sql_error insert into test.st1s4 using test.st1 tags(3, 'dachang') values(now, 300);
sql_error select * from test.st2;
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 6: database read privilege and table unauthorized
sql close
sql connect
sql revoke write on test.st1 with id = 1 from wxy;
sql grant read on test.* to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 6 then
return -1
endi
sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql select * from test.st2;
if $rows != 2 then
return -1
endi
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 7: database read privilege and table read privilege
sql close
sql connect
sql grant read on test.st1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 6 then
return -1
endi
sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql select * from test.st2;
if $rows != 2 then
return -1
endi
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 8: database read privilege and table read privilege with condition
sql close
sql connect
sql revoke read on test.st1 from wxy;
sql grant read on test.st1 with id = 1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 4 then
return -1
endi
sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql select * from test.st2;
if $rows != 2 then
return -1
endi
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 9: database read privilege and table write privilege
sql close
sql connect
sql revoke read on test.st1 with id = 1 from wxy;
sql grant write on test.st1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 6 then
return -1
endi
sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql select * from test.st2;
if $rows != 2 then
return -1
endi
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 10: database read privilege and table write privilege with condition
sql close
sql connect
sql revoke write on test.st1 from wxy;
sql grant write on test.st1 with id = 1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 8 then
return -1
endi
sql insert into test.st1s1 values(now, 10);
sql_error insert into test.st1s2 values(now, 20);
sql select * from test.st2;
if $rows != 2 then
return -1
endi
sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 11: database write privilege and table unauthorized
sql close
sql connect
sql revoke read on test.* from wxy;
sql revoke write on test.st1 with id = 1 from wxy;
sql grant write on test.* to wxy;
sql close
sql connect wxy
sql reset query cache;
sql_error select * from test.st1;
sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 12: database write privilege and table read privilege
sql close
sql connect
sql grant read on test.st1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 11 then
return -1
endi
sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 13: database write privilege and table read privilege with condition
sql close
sql connect
sql revoke read on test.st1 from wxy;
sql grant read on test.st1 with id = 1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql select * from test.st1;
if $rows != 8 then
return -1
endi
sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 14: database write privilege and table write privilege
sql close
sql connect
sql revoke read on test.st1 with id = 1 from wxy;
sql grant write on test.st1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql_error select * from test.st1;
sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
print =============== case 15: database write privilege and table write privilege with condition
sql close
sql connect
sql revoke write on test.st1 from wxy;
sql grant write on test.st1 with id = 1 to wxy;
sql close
sql connect wxy
sql reset query cache;
sql_error select * from test.st1;
sql insert into test.st1s1 values(now, 10);
sql_error insert into test.st1s2 values(now, 20);
sql_error select * from test.st2;
sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
system sh/exec.sh -n dnode1 -s stop -x SIGINT

1
tests/script/win-test-file
View File

@ -3,6 +3,7 @@
./test.sh -f tsim/user/privilege_db.sim
./test.sh -f tsim/user/privilege_sysinfo.sim
./test.sh -f tsim/user/privilege_topic.sim
./test.sh -f tsim/user/privilege_table.sim
./test.sh -f tsim/db/alter_option.sim
rem ./test.sh -f tsim/db/alter_replica_13.sim
./test.sh -f tsim/db/alter_replica_31.sim

Some files were not shown because too many files have changed in this diff Show More