xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #25317 from taosdata/enh/TD-28945-pk 

enh(s3/writing): enable writing to s3 always
This commit is contained in:
Hongze Cheng 2024-04-15 15:44:08 +08:00 committed by GitHub
parent 82ee169d61 4b3e3f9091
commit fb7db6a50c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
64 changed files with 8685 additions and 7073 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/common/cos.h
View File

@ -33,10 +33,12 @@ extern int32_t tsS3UploadDelaySec;
int32_t s3Init();
void s3CleanUp();
int32_t s3CheckCfg();
int32_t s3PutObjectFromFile(const char *file, const char *object);
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size);
void s3DeleteObjectsByPrefix(const char *prefix);
void s3DeleteObjects(const char *object_name[], int nobject);
int32_t s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name);
bool s3Get(const char *object_name, const char *path);
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
@ -45,6 +47,8 @@ void s3EvictCache(const char *path, long object_size);
long s3Size(const char *object_name);
int32_t s3GetObjectToFile(const char *object_name, char *fileName);
#define S3_DATA_CHUNK_PAGES (256 * 1024 * 1024)
#ifdef __cplusplus
}
#endif

14
include/common/tglobal.h
View File

@ -116,9 +116,9 @@ extern int32_t tsMonitorIntervalForBasic;
extern bool tsMonitorForceV2;
// audit
extern bool tsEnableAudit;
extern bool tsEnableAuditCreateTable;
extern int32_t tsAuditInterval;
extern bool tsEnableAudit;
extern bool tsEnableAuditCreateTable;
extern int32_t tsAuditInterval;
// telem
extern bool tsEnableTelem;
@ -126,9 +126,9 @@ extern int32_t tsTelemInterval;
extern char tsTelemServer[];
extern uint16_t tsTelemPort;
extern bool tsEnableCrashReport;
extern char * tsTelemUri;
extern char * tsClientCrashReportUri;
extern char * tsSvrCrashReportUri;
extern char *tsTelemUri;
extern char *tsClientCrashReportUri;
extern char *tsSvrCrashReportUri;
// query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
@ -215,6 +215,8 @@ extern int32_t tsTtlUnit;
extern int32_t tsTtlPushIntervalSec;
extern int32_t tsTtlBatchDropNum;
extern int32_t tsTrimVDbIntervalSec;
extern int32_t tsS3MigrateIntervalSec;
extern bool tsS3MigrateEnabled;
extern int32_t tsGrantHBInterval;
extern int32_t tsUptimeInterval;

50
include/common/tmsg.h
View File

@ -301,7 +301,8 @@ typedef enum ENodeType {
QUERY_NODE_GRANT_STMT,
QUERY_NODE_REVOKE_STMT,
QUERY_NODE_ALTER_CLUSTER_STMT,
// placeholder for [153, 180]
QUERY_NODE_S3MIGRATE_DATABASE_STMT,
// placeholder for [154, 180]
QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
QUERY_NODE_SHOW_CREATE_TABLE_STMT,
@ -586,7 +587,7 @@ typedef struct {
// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
// void tFreeSSubmitBlkRsp(void* param);
void tFreeSSubmitRsp(SSubmitRsp* pRsp);
void tFreeSSubmitRsp(SSubmitRsp* pRsp);
#define COL_SMA_ON ((int8_t)0x1)
#define COL_IDX_ON ((int8_t)0x2)
@ -1153,6 +1154,9 @@ typedef struct {
int32_t sstTrigger;
int16_t hashPrefix;
int16_t hashSuffix;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t tsdbPageSize;
int32_t sqlLen;
char* sql;
@ -1183,6 +1187,8 @@ typedef struct {
int32_t minRows;
int32_t walRetentionPeriod;
int32_t walRetentionSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t sqlLen;
char* sql;
int8_t withArbitrator;
@ -1263,6 +1269,20 @@ typedef struct {
int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
typedef struct {
char db[TSDB_DB_FNAME_LEN];
} SS3MigrateDbReq;
int32_t tSerializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq);
int32_t tDeserializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq);
typedef struct {
int32_t timestamp;
} SVS3MigrateDbReq;
int32_t tSerializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq);
int32_t tDeserializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq);
typedef struct {
int32_t timestampSec;
int32_t ttlDropMaxCount;
@ -1299,6 +1319,9 @@ typedef struct {
int8_t replications;
int8_t strict;
int8_t cacheLast;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t tsdbPageSize;
int32_t walRetentionPeriod;
int32_t walRollPeriod;
@ -1589,13 +1612,13 @@ int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
void tFreeSStatusReq(SStatusReq* pReq);
typedef struct {
int32_t contLen;
char* pCont;
int32_t contLen;
char* pCont;
} SStatisReq;
int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
void tFreeSStatisReq(SStatisReq *pReq);
void tFreeSStatisReq(SStatisReq* pReq);
typedef struct {
int32_t dnodeId;
@ -1700,7 +1723,10 @@ typedef struct {
int16_t hashPrefix;
int16_t hashSuffix;
int32_t tsdbPageSize;
int64_t reserved[8];
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int64_t reserved[6];
int8_t learnerReplica;
int8_t learnerSelfIndex;
SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA];
@ -1788,13 +1814,15 @@ typedef struct {
int8_t walLevel;
int8_t strict;
int8_t cacheLast;
int64_t reserved[8];
int64_t reserved[7];
// 1st modification
int16_t sttTrigger;
int32_t minRows;
// 2nd modification
int32_t walRetentionPeriod;
int32_t walRetentionSize;
int32_t s3KeepLocal;
int8_t s3Compact;
} SAlterVnodeConfigReq;
int32_t tSerializeSAlterVnodeConfigReq(void* buf, int32_t bufLen, SAlterVnodeConfigReq* pReq);
@ -1953,7 +1981,7 @@ typedef struct {
int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
void tFreeSShowReq(SShowReq* pReq);
void tFreeSShowReq(SShowReq* pReq);
typedef struct {
int64_t showId;
@ -2837,7 +2865,7 @@ typedef struct {
SVCreateTbReq* pReqs;
SArray* pArray;
};
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
} SVCreateTbBatchReq;
int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
@ -2932,7 +2960,7 @@ typedef struct {
int32_t newCommentLen;
char* newComment;
int64_t ctimeMs; // fill by vnode
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
} SVAlterTbReq;
int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@ -4051,7 +4079,7 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
#define TD_REQ_FROM_TAOX 0x8
#define SUBMIT_REQUEST_VERSION (1)
#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility
#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility
typedef struct {
int32_t flags;

3
include/common/tmsgdef.h
View File

@ -220,6 +220,8 @@
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
@ -272,6 +274,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_S3MIGRATE, "vnode-s3migrate", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)

586
include/common/ttokendef.h
View File

@ -84,304 +84,308 @@
#define TK_USE 66
#define TK_FLUSH 67
#define TK_TRIM 68
#define TK_COMPACT 69
#define TK_IF 70
#define TK_NOT 71
#define TK_EXISTS 72
#define TK_BUFFER 73
#define TK_CACHEMODEL 74
#define TK_CACHESIZE 75
#define TK_COMP 76
#define TK_DURATION 77
#define TK_NK_VARIABLE 78
#define TK_MAXROWS 79
#define TK_MINROWS 80
#define TK_KEEP 81
#define TK_PAGES 82
#define TK_PAGESIZE 83
#define TK_TSDB_PAGESIZE 84
#define TK_PRECISION 85
#define TK_REPLICA 86
#define TK_VGROUPS 87
#define TK_SINGLE_STABLE 88
#define TK_RETENTIONS 89
#define TK_SCHEMALESS 90
#define TK_WAL_LEVEL 91
#define TK_WAL_FSYNC_PERIOD 92
#define TK_WAL_RETENTION_PERIOD 93
#define TK_WAL_RETENTION_SIZE 94
#define TK_WAL_ROLL_PERIOD 95
#define TK_WAL_SEGMENT_SIZE 96
#define TK_STT_TRIGGER 97
#define TK_TABLE_PREFIX 98
#define TK_TABLE_SUFFIX 99
#define TK_KEEP_TIME_OFFSET 100
#define TK_NK_COLON 101
#define TK_BWLIMIT 102
#define TK_START 103
#define TK_TIMESTAMP 104
#define TK_END 105
#define TK_TABLE 106
#define TK_NK_LP 107
#define TK_NK_RP 108
#define TK_STABLE 109
#define TK_COLUMN 110
#define TK_MODIFY 111
#define TK_RENAME 112
#define TK_TAG 113
#define TK_SET 114
#define TK_NK_EQ 115
#define TK_USING 116
#define TK_TAGS 117
#define TK_PRIMARY 118
#define TK_KEY 119
#define TK_BOOL 120
#define TK_TINYINT 121
#define TK_SMALLINT 122
#define TK_INT 123
#define TK_INTEGER 124
#define TK_BIGINT 125
#define TK_FLOAT 126
#define TK_DOUBLE 127
#define TK_BINARY 128
#define TK_NCHAR 129
#define TK_UNSIGNED 130
#define TK_JSON 131
#define TK_VARCHAR 132
#define TK_MEDIUMBLOB 133
#define TK_BLOB 134
#define TK_VARBINARY 135
#define TK_GEOMETRY 136
#define TK_DECIMAL 137
#define TK_COMMENT 138
#define TK_MAX_DELAY 139
#define TK_WATERMARK 140
#define TK_ROLLUP 141
#define TK_TTL 142
#define TK_SMA 143
#define TK_DELETE_MARK 144
#define TK_FIRST 145
#define TK_LAST 146
#define TK_SHOW 147
#define TK_PRIVILEGES 148
#define TK_DATABASES 149
#define TK_TABLES 150
#define TK_STABLES 151
#define TK_MNODES 152
#define TK_QNODES 153
#define TK_ARBGROUPS 154
#define TK_FUNCTIONS 155
#define TK_INDEXES 156
#define TK_ACCOUNTS 157
#define TK_APPS 158
#define TK_CONNECTIONS 159
#define TK_LICENCES 160
#define TK_GRANTS 161
#define TK_FULL 162
#define TK_LOGS 163
#define TK_MACHINES 164
#define TK_QUERIES 165
#define TK_SCORES 166
#define TK_TOPICS 167
#define TK_VARIABLES 168
#define TK_BNODES 169
#define TK_SNODES 170
#define TK_TRANSACTIONS 171
#define TK_DISTRIBUTED 172
#define TK_CONSUMERS 173
#define TK_SUBSCRIPTIONS 174
#define TK_VNODES 175
#define TK_ALIVE 176
#define TK_VIEWS 177
#define TK_VIEW 178
#define TK_COMPACTS 179
#define TK_NORMAL 180
#define TK_CHILD 181
#define TK_LIKE 182
#define TK_TBNAME 183
#define TK_QTAGS 184
#define TK_AS 185
#define TK_SYSTEM 186
#define TK_INDEX 187
#define TK_FUNCTION 188
#define TK_INTERVAL 189
#define TK_COUNT 190
#define TK_LAST_ROW 191
#define TK_META 192
#define TK_ONLY 193
#define TK_TOPIC 194
#define TK_CONSUMER 195
#define TK_GROUP 196
#define TK_DESC 197
#define TK_DESCRIBE 198
#define TK_RESET 199
#define TK_QUERY 200
#define TK_CACHE 201
#define TK_EXPLAIN 202
#define TK_ANALYZE 203
#define TK_VERBOSE 204
#define TK_NK_BOOL 205
#define TK_RATIO 206
#define TK_NK_FLOAT 207
#define TK_OUTPUTTYPE 208
#define TK_AGGREGATE 209
#define TK_BUFSIZE 210
#define TK_LANGUAGE 211
#define TK_REPLACE 212
#define TK_STREAM 213
#define TK_INTO 214
#define TK_PAUSE 215
#define TK_RESUME 216
#define TK_TRIGGER 217
#define TK_AT_ONCE 218
#define TK_WINDOW_CLOSE 219
#define TK_IGNORE 220
#define TK_EXPIRED 221
#define TK_FILL_HISTORY 222
#define TK_UPDATE 223
#define TK_SUBTABLE 224
#define TK_UNTREATED 225
#define TK_KILL 226
#define TK_CONNECTION 227
#define TK_TRANSACTION 228
#define TK_BALANCE 229
#define TK_VGROUP 230
#define TK_LEADER 231
#define TK_MERGE 232
#define TK_REDISTRIBUTE 233
#define TK_SPLIT 234
#define TK_DELETE 235
#define TK_INSERT 236
#define TK_NK_BIN 237
#define TK_NK_HEX 238
#define TK_NULL 239
#define TK_NK_QUESTION 240
#define TK_NK_ALIAS 241
#define TK_NK_ARROW 242
#define TK_ROWTS 243
#define TK_QSTART 244
#define TK_QEND 245
#define TK_QDURATION 246
#define TK_WSTART 247
#define TK_WEND 248
#define TK_WDURATION 249
#define TK_IROWTS 250
#define TK_ISFILLED 251
#define TK_CAST 252
#define TK_NOW 253
#define TK_TODAY 254
#define TK_TIMEZONE 255
#define TK_CLIENT_VERSION 256
#define TK_SERVER_VERSION 257
#define TK_SERVER_STATUS 258
#define TK_CURRENT_USER 259
#define TK_CASE 260
#define TK_WHEN 261
#define TK_THEN 262
#define TK_ELSE 263
#define TK_BETWEEN 264
#define TK_IS 265
#define TK_NK_LT 266
#define TK_NK_GT 267
#define TK_NK_LE 268
#define TK_NK_GE 269
#define TK_NK_NE 270
#define TK_MATCH 271
#define TK_NMATCH 272
#define TK_CONTAINS 273
#define TK_IN 274
#define TK_JOIN 275
#define TK_INNER 276
#define TK_SELECT 277
#define TK_NK_HINT 278
#define TK_DISTINCT 279
#define TK_WHERE 280
#define TK_PARTITION 281
#define TK_BY 282
#define TK_SESSION 283
#define TK_STATE_WINDOW 284
#define TK_EVENT_WINDOW 285
#define TK_COUNT_WINDOW 286
#define TK_SLIDING 287
#define TK_FILL 288
#define TK_VALUE 289
#define TK_VALUE_F 290
#define TK_NONE 291
#define TK_PREV 292
#define TK_NULL_F 293
#define TK_LINEAR 294
#define TK_NEXT 295
#define TK_HAVING 296
#define TK_RANGE 297
#define TK_EVERY 298
#define TK_ORDER 299
#define TK_SLIMIT 300
#define TK_SOFFSET 301
#define TK_LIMIT 302
#define TK_OFFSET 303
#define TK_ASC 304
#define TK_NULLS 305
#define TK_ABORT 306
#define TK_AFTER 307
#define TK_ATTACH 308
#define TK_BEFORE 309
#define TK_BEGIN 310
#define TK_BITAND 311
#define TK_BITNOT 312
#define TK_BITOR 313
#define TK_BLOCKS 314
#define TK_CHANGE 315
#define TK_COMMA 316
#define TK_CONCAT 317
#define TK_CONFLICT 318
#define TK_COPY 319
#define TK_DEFERRED 320
#define TK_DELIMITERS 321
#define TK_DETACH 322
#define TK_DIVIDE 323
#define TK_DOT 324
#define TK_EACH 325
#define TK_FAIL 326
#define TK_FILE 327
#define TK_FOR 328
#define TK_GLOB 329
#define TK_ID 330
#define TK_IMMEDIATE 331
#define TK_IMPORT 332
#define TK_INITIALLY 333
#define TK_INSTEAD 334
#define TK_ISNULL 335
#define TK_MODULES 336
#define TK_NK_BITNOT 337
#define TK_NK_SEMI 338
#define TK_NOTNULL 339
#define TK_OF 340
#define TK_PLUS 341
#define TK_PRIVILEGE 342
#define TK_RAISE 343
#define TK_RESTRICT 344
#define TK_ROW 345
#define TK_SEMI 346
#define TK_STAR 347
#define TK_STATEMENT 348
#define TK_STRICT 349
#define TK_STRING 350
#define TK_TIMES 351
#define TK_VALUES 352
#define TK_VARIABLE 353
#define TK_WAL 354
#define TK_S3MIGRATE 69
#define TK_COMPACT 70
#define TK_IF 71
#define TK_NOT 72
#define TK_EXISTS 73
#define TK_BUFFER 74
#define TK_CACHEMODEL 75
#define TK_CACHESIZE 76
#define TK_COMP 77
#define TK_DURATION 78
#define TK_NK_VARIABLE 79
#define TK_MAXROWS 80
#define TK_MINROWS 81
#define TK_KEEP 82
#define TK_PAGES 83
#define TK_PAGESIZE 84
#define TK_TSDB_PAGESIZE 85
#define TK_PRECISION 86
#define TK_REPLICA 87
#define TK_VGROUPS 88
#define TK_SINGLE_STABLE 89
#define TK_RETENTIONS 90
#define TK_SCHEMALESS 91
#define TK_WAL_LEVEL 92
#define TK_WAL_FSYNC_PERIOD 93
#define TK_WAL_RETENTION_PERIOD 94
#define TK_WAL_RETENTION_SIZE 95
#define TK_WAL_ROLL_PERIOD 96
#define TK_WAL_SEGMENT_SIZE 97
#define TK_STT_TRIGGER 98
#define TK_TABLE_PREFIX 99
#define TK_TABLE_SUFFIX 100
#define TK_S3_CHUNKSIZE 101
#define TK_S3_KEEPLOCAL 102
#define TK_S3_COMPACT 103
#define TK_KEEP_TIME_OFFSET 104
#define TK_NK_COLON 105
#define TK_BWLIMIT 106
#define TK_START 107
#define TK_TIMESTAMP 108
#define TK_END 109
#define TK_TABLE 110
#define TK_NK_LP 111
#define TK_NK_RP 112
#define TK_STABLE 113
#define TK_COLUMN 114
#define TK_MODIFY 115
#define TK_RENAME 116
#define TK_TAG 117
#define TK_SET 118
#define TK_NK_EQ 119
#define TK_USING 120
#define TK_TAGS 121
#define TK_PRIMARY 122
#define TK_KEY 123
#define TK_BOOL 124
#define TK_TINYINT 125
#define TK_SMALLINT 126
#define TK_INT 127
#define TK_INTEGER 128
#define TK_BIGINT 129
#define TK_FLOAT 130
#define TK_DOUBLE 131
#define TK_BINARY 132
#define TK_NCHAR 133
#define TK_UNSIGNED 134
#define TK_JSON 135
#define TK_VARCHAR 136
#define TK_MEDIUMBLOB 137
#define TK_BLOB 138
#define TK_VARBINARY 139
#define TK_GEOMETRY 140
#define TK_DECIMAL 141
#define TK_COMMENT 142
#define TK_MAX_DELAY 143
#define TK_WATERMARK 144
#define TK_ROLLUP 145
#define TK_TTL 146
#define TK_SMA 147
#define TK_DELETE_MARK 148
#define TK_FIRST 149
#define TK_LAST 150
#define TK_SHOW 151
#define TK_PRIVILEGES 152
#define TK_DATABASES 153
#define TK_TABLES 154
#define TK_STABLES 155
#define TK_MNODES 156
#define TK_QNODES 157
#define TK_ARBGROUPS 158
#define TK_FUNCTIONS 159
#define TK_INDEXES 160
#define TK_ACCOUNTS 161
#define TK_APPS 162
#define TK_CONNECTIONS 163
#define TK_LICENCES 164
#define TK_GRANTS 165
#define TK_FULL 166
#define TK_LOGS 167
#define TK_MACHINES 168
#define TK_QUERIES 169
#define TK_SCORES 170
#define TK_TOPICS 171
#define TK_VARIABLES 172
#define TK_BNODES 173
#define TK_SNODES 174
#define TK_TRANSACTIONS 175
#define TK_DISTRIBUTED 176
#define TK_CONSUMERS 177
#define TK_SUBSCRIPTIONS 178
#define TK_VNODES 179
#define TK_ALIVE 180
#define TK_VIEWS 181
#define TK_VIEW 182
#define TK_COMPACTS 183
#define TK_NORMAL 184
#define TK_CHILD 185
#define TK_LIKE 186
#define TK_TBNAME 187
#define TK_QTAGS 188
#define TK_AS 189
#define TK_SYSTEM 190
#define TK_INDEX 191
#define TK_FUNCTION 192
#define TK_INTERVAL 193
#define TK_COUNT 194
#define TK_LAST_ROW 195
#define TK_META 196
#define TK_ONLY 197
#define TK_TOPIC 198
#define TK_CONSUMER 199
#define TK_GROUP 200
#define TK_DESC 201
#define TK_DESCRIBE 202
#define TK_RESET 203
#define TK_QUERY 204
#define TK_CACHE 205
#define TK_EXPLAIN 206
#define TK_ANALYZE 207
#define TK_VERBOSE 208
#define TK_NK_BOOL 209
#define TK_RATIO 210
#define TK_NK_FLOAT 211
#define TK_OUTPUTTYPE 212
#define TK_AGGREGATE 213
#define TK_BUFSIZE 214
#define TK_LANGUAGE 215
#define TK_REPLACE 216
#define TK_STREAM 217
#define TK_INTO 218
#define TK_PAUSE 219
#define TK_RESUME 220
#define TK_TRIGGER 221
#define TK_AT_ONCE 222
#define TK_WINDOW_CLOSE 223
#define TK_IGNORE 224
#define TK_EXPIRED 225
#define TK_FILL_HISTORY 226
#define TK_UPDATE 227
#define TK_SUBTABLE 228
#define TK_UNTREATED 229
#define TK_KILL 230
#define TK_CONNECTION 231
#define TK_TRANSACTION 232
#define TK_BALANCE 233
#define TK_VGROUP 234
#define TK_LEADER 235
#define TK_MERGE 236
#define TK_REDISTRIBUTE 237
#define TK_SPLIT 238
#define TK_DELETE 239
#define TK_INSERT 240
#define TK_NK_BIN 241
#define TK_NK_HEX 242
#define TK_NULL 243
#define TK_NK_QUESTION 244
#define TK_NK_ALIAS 245
#define TK_NK_ARROW 246
#define TK_ROWTS 247
#define TK_QSTART 248
#define TK_QEND 249
#define TK_QDURATION 250
#define TK_WSTART 251
#define TK_WEND 252
#define TK_WDURATION 253
#define TK_IROWTS 254
#define TK_ISFILLED 255
#define TK_CAST 256
#define TK_NOW 257
#define TK_TODAY 258
#define TK_TIMEZONE 259
#define TK_CLIENT_VERSION 260
#define TK_SERVER_VERSION 261
#define TK_SERVER_STATUS 262
#define TK_CURRENT_USER 263
#define TK_CASE 264
#define TK_WHEN 265
#define TK_THEN 266
#define TK_ELSE 267
#define TK_BETWEEN 268
#define TK_IS 269
#define TK_NK_LT 270
#define TK_NK_GT 271
#define TK_NK_LE 272
#define TK_NK_GE 273
#define TK_NK_NE 274
#define TK_MATCH 275
#define TK_NMATCH 276
#define TK_CONTAINS 277
#define TK_IN 278
#define TK_JOIN 279
#define TK_INNER 280
#define TK_SELECT 281
#define TK_NK_HINT 282
#define TK_DISTINCT 283
#define TK_WHERE 284
#define TK_PARTITION 285
#define TK_BY 286
#define TK_SESSION 287
#define TK_STATE_WINDOW 288
#define TK_EVENT_WINDOW 289
#define TK_COUNT_WINDOW 290
#define TK_SLIDING 291
#define TK_FILL 292
#define TK_VALUE 293
#define TK_VALUE_F 294
#define TK_NONE 295
#define TK_PREV 296
#define TK_NULL_F 297
#define TK_LINEAR 298
#define TK_NEXT 299
#define TK_HAVING 300
#define TK_RANGE 301
#define TK_EVERY 302
#define TK_ORDER 303
#define TK_SLIMIT 304
#define TK_SOFFSET 305
#define TK_LIMIT 306
#define TK_OFFSET 307
#define TK_ASC 308
#define TK_NULLS 309
#define TK_ABORT 310
#define TK_AFTER 311
#define TK_ATTACH 312
#define TK_BEFORE 313
#define TK_BEGIN 314
#define TK_BITAND 315
#define TK_BITNOT 316
#define TK_BITOR 317
#define TK_BLOCKS 318
#define TK_CHANGE 319
#define TK_COMMA 320
#define TK_CONCAT 321
#define TK_CONFLICT 322
#define TK_COPY 323
#define TK_DEFERRED 324
#define TK_DELIMITERS 325
#define TK_DETACH 326
#define TK_DIVIDE 327
#define TK_DOT 328
#define TK_EACH 329
#define TK_FAIL 330
#define TK_FILE 331
#define TK_FOR 332
#define TK_GLOB 333
#define TK_ID 334
#define TK_IMMEDIATE 335
#define TK_IMPORT 336
#define TK_INITIALLY 337
#define TK_INSTEAD 338
#define TK_ISNULL 339
#define TK_MODULES 340
#define TK_NK_BITNOT 341
#define TK_NK_SEMI 342
#define TK_NOTNULL 343
#define TK_OF 344
#define TK_PLUS 345
#define TK_PRIVILEGE 346
#define TK_RAISE 347
#define TK_RESTRICT 348
#define TK_ROW 349
#define TK_SEMI 350
#define TK_STAR 351
#define TK_STATEMENT 352
#define TK_STRICT 353
#define TK_STRING 354
#define TK_TIMES 355
#define TK_VALUES 356
#define TK_VARIABLE 357
#define TK_WAL 358
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601
#define TK_NK_ILLEGAL 602
// #define TK_NK_HEX 603 // hex number 0x123
#define TK_NK_OCT 604 // oct number
#define TK_NK_OCT 604 // oct number
// #define TK_NK_BIN 605 // bin format data 0b111
#define TK_BATCH_SCAN 606
#define TK_NO_BATCH_SCAN 607
#define TK_SORT_FOR_GROUP 608
#define TK_PARTITION_FIRST 609
#define TK_BATCH_SCAN 606
#define TK_NO_BATCH_SCAN 607
#define TK_SORT_FOR_GROUP 608
#define TK_PARTITION_FIRST 609
#define TK_PARA_TABLES_SORT 610
#define TK_SMALLDATA_TS_SORT 611

9
include/libs/nodes/cmdnodes.h
View File

@ -100,6 +100,10 @@ typedef struct SDatabaseOptions {
int32_t sstTrigger;
int32_t tablePrefix;
int32_t tableSuffix;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
SValueNode* s3KeepLocalStr;
int8_t s3Compact;
int8_t withArbitrator;
} SDatabaseOptions;
@ -138,6 +142,11 @@ typedef struct STrimDatabaseStmt {
int32_t maxSpeed;
} STrimDatabaseStmt;
typedef struct SS3MigrateDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SS3MigrateDatabaseStmt;
typedef struct SCompactDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];

7
include/util/tconfig.h
View File

@ -51,11 +51,7 @@ typedef enum {
CFG_DTYPE_TIMEZONE
} ECfgDataType;
typedef enum {
CFG_SCOPE_SERVER,
CFG_SCOPE_CLIENT,
CFG_SCOPE_BOTH
} ECfgScopeType;
typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType;
typedef enum {
CFG_DYN_NONE = 0,
@ -138,6 +134,7 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p
void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen);
void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump);
int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl);

42
include/util/tdef.h
View File

@ -197,8 +197,8 @@ typedef enum ELogicConditionType {
#define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788
// ACCOUNT is a 32 bit positive integer
// this is the length of its string representation, including the terminator zero
#define TSDB_ACCT_ID_LEN 11
#define TSDB_NODE_ID_LEN 11
#define TSDB_ACCT_ID_LEN 11
#define TSDB_NODE_ID_LEN 11
#define TSDB_VGROUP_ID_LEN 11
#define TSDB_MAX_COLUMNS 4096
@ -293,8 +293,8 @@ typedef enum ELogicConditionType {
#define TSDB_DNODE_CONFIG_LEN 128
#define TSDB_DNODE_VALUE_LEN 256
#define TSDB_CLUSTER_VALUE_LEN 1000
#define TSDB_GRANT_LOG_COL_LEN 15600
#define TSDB_CLUSTER_VALUE_LEN 1000
#define TSDB_GRANT_LOG_COL_LEN 15600
#define TSDB_ACTIVE_KEY_LEN 109
#define TSDB_CONN_ACTIVE_KEY_LEN 255
@ -416,6 +416,16 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_SUFFIX 0
#define TSDB_MIN_S3_CHUNK_SIZE (32 * 1024)
#define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024)
#define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024)
#define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute
#define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440)
#define TSDB_DEFAULT_S3_KEEP_LOCAL (3650 * 1440)
#define TSDB_MIN_S3_COMPACT 0
#define TSDB_MAX_S3_COMPACT 1
#define TSDB_DEFAULT_S3_COMPACT 0
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600
#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600
@ -530,12 +540,12 @@ enum {
SND_WORKER_TYPE__UNIQUE,
};
#define DEFAULT_HANDLE 0
#define MNODE_HANDLE 1
#define QNODE_HANDLE -1
#define SNODE_HANDLE -2
#define VNODE_HANDLE -3
#define CLIENT_HANDLE -5
#define DEFAULT_HANDLE 0
#define MNODE_HANDLE 1
#define QNODE_HANDLE -1
#define SNODE_HANDLE -2
#define VNODE_HANDLE -3
#define CLIENT_HANDLE -5
#define TSDB_CONFIG_OPTION_LEN 32
#define TSDB_CONFIG_VALUE_LEN 64
@ -555,13 +565,13 @@ enum {
// sort page size by default
#define DEFAULT_PAGESIZE 4096
#define VNODE_TIMEOUT_SEC 60
#define MNODE_TIMEOUT_SEC 60
#define VNODE_TIMEOUT_SEC 60
#define MNODE_TIMEOUT_SEC 60
#define MONITOR_TABLENAME_LEN 200
#define MONITOR_TAG_NAME_LEN 100
#define MONITOR_TAG_VALUE_LEN 300
#define MONITOR_METRIC_NAME_LEN 100
#define MONITOR_TABLENAME_LEN 200
#define MONITOR_TAG_NAME_LEN 100
#define MONITOR_TAG_VALUE_LEN 300
#define MONITOR_METRIC_NAME_LEN 100
#ifdef __cplusplus
}
#endif

5
source/common/CMakeLists.txt
View File

@ -16,14 +16,15 @@ ENDIF ()
IF (TD_STORAGE)
ADD_DEFINITIONS(-D_STORAGE)
TARGET_LINK_LIBRARIES(common PRIVATE storage)
ENDIF ()
IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
ENDIF()
ENDIF ()
ENDIF()
target_include_directories(
common

208
source/common/src/cos.c
View File

@ -53,6 +53,100 @@ int32_t s3Init() { return 0; /*s3Begin();*/ }
void s3CleanUp() { /*s3End();*/
}
static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() {
int32_t code = 0;
code = s3Begin();
if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n");
goto _exit;
}
// test put
char testdata[17] = "0123456789abcdef";
const char *objectname[] = {"s3test.txt"};
char path[PATH_MAX] = {0};
int ds_len = strlen(TD_DIRSEP);
int tmp_len = strlen(tsTempDir);
snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
snprintf(path + tmp_len, PATH_MAX, "%s", TD_DIRSEP);
snprintf(path + tmp_len + ds_len, PATH_MAX, "%s", objectname[0]);
} else {
snprintf(path + tmp_len, PATH_MAX, "%s", objectname[0]);
}
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (!fp) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to open test file: %s.\n", path);
// uError("ERROR: %s Failed to open %s", __func__, path);
goto _exit;
}
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to write test file: %s.\n", path);
goto _exit;
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to fsync test file: %s.\n", path);
goto _exit;
}
taosCloseFile(&fp);
fprintf(stderr, "\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata);
code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16);
if (code != 0) {
fprintf(stderr, "put object %s : failed.\n", objectname[0]);
goto _exit;
}
fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
// list buckets
fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName);
code = s3ListBucket(tsS3BucketName);
if (code != 0) {
fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
goto _exit;
}
fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
// test range get
uint8_t *pBlock = NULL;
int c_offset = 10;
int c_len = 6;
fprintf(stderr, "start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len);
code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
if (code != 0) {
fprintf(stderr, "get object %s : failed.\n", objectname[0]);
goto _exit;
}
char buf[7] = {0};
memcpy(buf, pBlock, c_len);
taosMemoryFree(pBlock);
fprintf(stderr, "object content: %s\n", buf);
fprintf(stderr, "get object %s: success.\n\n", objectname[0]);
// delete test object
fprintf(stderr, "start to delete object: %s.\n", objectname[0]);
code = s3DeleteObjects(objectname, 1);
if (code != 0) {
fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
goto _exit;
}
fprintf(stderr, "delete object %s: success.\n\n", objectname[0]);
s3End();
_exit:
return code;
}
static int should_retry() {
/*
if (retriesG--) {
@ -85,7 +179,7 @@ typedef struct {
} TS3GetData;
typedef struct {
char err_msg[128];
char err_msg[512];
S3Status status;
uint64_t content_length;
char *buf;
@ -137,6 +231,30 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
}
}
static SArray *getListByPrefix(const char *prefix);
static void s3FreeObjectKey(void *pItem);
static int32_t s3ListBucket(char const *bucketname) {
int32_t code = 0;
SArray *objectArray = getListByPrefix("s3");
if (objectArray == NULL) {
return -1;
}
const char **object_name = TARRAY_DATA(objectArray);
int size = TARRAY_SIZE(objectArray);
fprintf(stderr, "objects:\n");
for (int i = 0; i < size; ++i) {
fprintf(stderr, "%s\n", object_name[i]);
}
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return code;
}
typedef struct growbuffer {
// The total number of bytes, and the start byte
int size;
@ -471,10 +589,11 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c
} while (S3_status_is_retryable(data->status) && should_retry());
if (data->status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
// s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
s3PrintError(NULL, __LINE__, __func__, data->status, data->err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
} else if (data->contentLength) {
uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength);
uError("%s Failed to put remaining %llu bytes", __func__, (unsigned long long)data->contentLength);
code = TAOS_SYSTEM_ERROR(EIO);
}
@ -830,6 +949,66 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
return code;
}
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
int32_t code = 0;
int32_t lmtime = 0;
const char *filename = 0;
uint64_t contentLength = 0;
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
const char *contentDispositionFilename = 0, *contentEncoding = 0;
int64_t expires = -1;
S3CannedAcl cannedAcl = S3CannedAclPrivate;
int metaPropertiesCount = 0;
S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
char useServerSideEncryption = 0;
put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code;
}
contentLength = size;
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code;
}
if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) {
taosCloseFile(&data.infileFD);
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
contentLength;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG};
S3PutProperties putProperties = {contentType, md5,
cacheControl, contentDispositionFilename,
contentEncoding, expires,
cannedAcl, metaPropertiesCount,
metaProperties, useServerSideEncryption};
if (contentLength <= MULTIPART_CHUNK_SIZE) {
code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
} else {
code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
}
if (data.infileFD) {
taosCloseFile(&data.infileFD);
} else if (data.gb) {
growbuffer_destroy(data.gb);
}
return code;
}
typedef struct list_bucket_callback_data {
char err_msg[512];
S3Status status;
@ -888,7 +1067,7 @@ static SArray *getListByPrefix(const char *prefix) {
const char *marker = 0, *delimiter = 0;
int maxkeys = 0, allDetails = 0;
list_bucket_callback_data data;
list_bucket_callback_data data = {0};
data.objectArray = taosArrayInit(32, sizeof(void *));
if (!data.objectArray) {
uError("%s: %s", __func__, "out of memoty");
@ -918,14 +1097,17 @@ static SArray *getListByPrefix(const char *prefix) {
return data.objectArray;
}
} else {
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
uError("failed to list with prefix %s: %s", prefix, S3_get_status_name(data.status));
// s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
}
taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
return NULL;
}
void s3DeleteObjects(const char *object_name[], int nobject) {
int32_t s3DeleteObjects(const char *object_name[], int nobject) {
int32_t code = 0;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG};
S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
@ -938,8 +1120,11 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
code = -1;
}
}
return code;
}
void s3DeleteObjectsByPrefix(const char *prefix) {
@ -991,12 +1176,12 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
} while (S3_status_is_retryable(cbd.status) && should_retry());
if (cbd.status != S3StatusOK) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
}
if (check && cbd.buf_pos != size) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
}
@ -1233,7 +1418,7 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) {
cos_pool_destroy(p);
}
void s3DeleteObjects(const char *object_name[], int nobject) {
int32_t s3DeleteObjects(const char *object_name[], int nobject) {
cos_pool_t *p = NULL;
int is_cname = 0;
cos_string_t bucket;
@ -1267,6 +1452,8 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
} else {
cos_warn_log("delete objects failed\n");
}
return 0;
}
bool s3Exists(const char *object_name) {
@ -1535,8 +1722,9 @@ int32_t s3Init() { return 0; }
void s3CleanUp() {}
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; }
void s3DeleteObjectsByPrefix(const char *prefix) {}
void s3DeleteObjects(const char *object_name[], int nobject) {}
int32_t s3DeleteObjects(const char *object_name[], int nobject) { return 0; }
bool s3Exists(const char *object_name) { return false; }
bool s3Get(const char *object_name, const char *path) { return false; }
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {

5
source/common/src/systable.c
View File

@ -24,7 +24,7 @@
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
// clang-format off
static const SSysDbTableSchema dnodesSchema[] = {
@ -117,6 +117,9 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
};

34
source/common/src/tglobal.c
View File

@ -265,7 +265,9 @@ float tsSinkDataRate = 2.0;
int32_t tsStreamNodeCheckInterval = 16;
int32_t tsTtlUnit = 86400;
int32_t tsTtlPushIntervalSec = 10;
int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups
int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups
int32_t tsS3MigrateIntervalSec = 60 * 60; // interval of s3migrate db in all vgroups
bool tsS3MigrateEnabled = 1;
int32_t tsGrantHBInterval = 60;
int32_t tsUptimeInterval = 300; // seconds
char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits
@ -291,7 +293,7 @@ char tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
int32_t tsS3BlockSize = -1; // number of tsdb pages (4096)
int32_t tsS3BlockCacheSize = 16; // number of blocks
int32_t tsS3PageCacheSize = 4096; // number of pages
int32_t tsS3UploadDelaySec = 60 * 60 * 24;
int32_t tsS3UploadDelaySec = 60;
bool tsExperimental = true;
@ -351,7 +353,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
}
if (tsS3BucketName[0] != '<') {
#if defined(USE_COS) || defined(USE_S3)
if (tsDiskCfgNum > 1) tsS3Enabled = true;
#ifdef TD_ENTERPRISE
/*if (tsDiskCfgNum > 1) */ tsS3Enabled = true;
#endif
tsS3StreamEnabled = true;
#endif
}
@ -674,8 +678,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16,
(TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2),
CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
@ -700,7 +704,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0)
return -1;
if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -745,6 +750,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
@ -763,8 +772,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0)
@ -795,6 +803,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
/*
if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1;
if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) {
@ -804,10 +813,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
*/
if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0)
return -1;
@ -1247,8 +1257,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32;
tsMinDiskFreeSize = cfgGetItem(pCfg, "minDiskFreeSize")->i64;
tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32;
tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32;
// tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32;
// tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32;
tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
@ -1526,6 +1536,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"ttlBatchDropNum", &tsTtlBatchDropNum},
{"ttlFlushThreshold", &tsTtlFlushThreshold},
{"ttlPushInterval", &tsTtlPushIntervalSec},
{"s3MigrateIntervalSec", &tsS3MigrateIntervalSec},
{"s3MigrateEnabled", &tsS3MigrateEnabled},
//{"s3BlockSize", &tsS3BlockSize},
{"s3BlockCacheSize", &tsS3BlockCacheSize},
{"s3PageCacheSize", &tsS3PageCacheSize},

191
source/common/src/tmsg.c
View File

@ -1471,9 +1471,7 @@ int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
return 0;
}
void tFreeSStatisReq(SStatisReq *pReq) {
taosMemoryFreeClear(pReq->pCont);
}
void tFreeSStatisReq(SStatisReq *pReq) { taosMemoryFreeClear(pReq->pCont); }
// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
// SEncoder encoder = {0};
@ -1872,7 +1870,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
char *tb = taosHashIterate(pRsp->readTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1887,7 +1885,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1902,7 +1900,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1917,7 +1915,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->readViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1932,7 +1930,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1947,7 +1945,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen);
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1962,7 +1960,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
while (useDb != NULL) {
size_t keyLen = 0;
void * key = taosHashGetKey(useDb, &keyLen);
void *key = taosHashGetKey(useDb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -3022,6 +3020,9 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
}
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL();
@ -3094,6 +3095,15 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
@ -3139,6 +3149,10 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL();
if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1;
tEndEncode(&encoder);
@ -3189,6 +3203,13 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(&decoder)) {
@ -3885,6 +3906,57 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) {
return 0;
}
int32_t tSerializeSS3MigrateDbReq(void *buf, int32_t bufLen, SS3MigrateDbReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSS3MigrateDbReq(void *buf, int32_t bufLen, SS3MigrateDbReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSVS3MigrateDbReq(void *buf, int32_t bufLen, SVS3MigrateDbReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVS3MigrateDbReq(void *buf, int32_t bufLen, SVS3MigrateDbReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@ -3982,6 +4054,9 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) {
if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1;
if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1;
if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1;
if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1;
return 0;
}
@ -4059,6 +4134,15 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) {
if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1;
}
pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI32(decoder, &pRsp->s3ChunkSize) < 0) return -1;
if (tDecodeI32(decoder, &pRsp->s3KeepLocal) < 0) return -1;
if (tDecodeI8(decoder, &pRsp->s3Compact) < 0) return -1;
}
return 0;
}
@ -5082,7 +5166,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1;
if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1;
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) {
for (int32_t i = 0; i < 6; ++i) {
if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
}
if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1;
@ -5093,6 +5177,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
}
if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
tEndEncode(&encoder);
@ -5168,7 +5255,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1;
if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) {
for (int32_t i = 0; i < 6; ++i) {
if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
}
if (!tDecodeIsEnd(&decoder)) {
@ -5187,6 +5274,15 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -5442,7 +5538,7 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon
if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) {
for (int32_t i = 0; i < 7; ++i) {
if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
}
@ -5453,6 +5549,10 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon
if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -5478,7 +5578,7 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) {
for (int32_t i = 0; i < 7; ++i) {
if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
}
@ -5504,6 +5604,13 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -6557,9 +6664,7 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) {
return 0;
}
void tDestroySMqHbRsp(SMqHbRsp *pRsp) {
taosArrayDestroy(pRsp->topicPrivileges);
}
void tDestroySMqHbRsp(SMqHbRsp *pRsp) { taosArrayDestroy(pRsp->topicPrivileges); }
int32_t tSerializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) {
SEncoder encoder = {0};
@ -7345,27 +7450,27 @@ void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskSt
// return 0;
// }
//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
// SDecoder decoder = {0};
// int32_t num = 0;
// tDecoderInit(&decoder, buf, bufLen);
// int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
// SDecoder decoder = {0};
// int32_t num = 0;
// tDecoderInit(&decoder, buf, bufLen);
// if (tStartDecode(&decoder) < 0) return -1;
// if (tDecodeI32(&decoder, &num) < 0) return -1;
// if (num > 0) {
// pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
// if (NULL == pRsp->rspList) return -1;
// for (int32_t i = 0; i < num; ++i) {
// SVCreateTbRsp rsp = {0};
// if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
// if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
// }
// } else {
// pRsp->rspList = NULL;
// }
// tEndDecode(&decoder);
// if (tStartDecode(&decoder) < 0) return -1;
// if (tDecodeI32(&decoder, &num) < 0) return -1;
// if (num > 0) {
// pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
// if (NULL == pRsp->rspList) return -1;
// for (int32_t i = 0; i < num; ++i) {
// SVCreateTbRsp rsp = {0};
// if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
// if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
// }
// } else {
// pRsp->rspList = NULL;
// }
// tEndDecode(&decoder);
// tDecoderClear(&decoder);
// tDecoderClear(&decoder);
// return 0;
//}
@ -7642,8 +7747,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeI32(&encoder, taosArrayGetSize(pReq->pVgroupVerList)) < 0) return -1;
for(int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) {
SVgroupVer* p = taosArrayGet(pReq->pVgroupVerList, i);
for (int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) {
SVgroupVer *p = taosArrayGet(pReq->pVgroupVerList, i);
if (tEncodeI32(&encoder, p->vgId) < 0) return -1;
if (tEncodeI64(&encoder, p->ver) < 0) return -1;
}
@ -8907,7 +9012,7 @@ int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
void * data = taosArrayGetP(pRsp->blockData, i);
void *data = taosArrayGetP(pRsp->blockData, i);
if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
if (pRsp->withSchema) {
SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
@ -8946,7 +9051,7 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
}
for (int32_t i = 0; i < pRsp->blockNum; i++) {
void * data;
void *data;
uint64_t bLen;
if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
taosArrayPush(pRsp->blockData, &data);
@ -9006,7 +9111,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
if (pRsp->createTableNum) {
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * createTableReq = taosArrayGetP(pRsp->createTableReq, i);
void *createTableReq = taosArrayGetP(pRsp->createTableReq, i);
int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i);
if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1;
}
@ -9025,7 +9130,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion
pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * pCreate = NULL;
void *pCreate = NULL;
uint64_t len;
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
int32_t l = (int32_t)len;
@ -9337,7 +9442,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
taosArrayDestroy(pTbData->aCol);
} else {
int32_t nRow = TARRAY_SIZE(pTbData->aRowP);
SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
for (int32_t i = 0; i < nRow; ++i) {
tRowDestroy(rows[i]);

8
source/dnode/mgmt/CMakeLists.txt
View File

@ -14,6 +14,14 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc"
)
IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
ENDIF()
ENDIF()
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc)
SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc")

37
source/dnode/mgmt/exe/dmMain.c
View File

@ -27,15 +27,15 @@
#include "cus_name.h"
#else
#ifndef CUS_NAME
#define CUS_NAME "TDengine"
#define CUS_NAME "TDengine"
#endif
#ifndef CUS_PROMPT
#define CUS_PROMPT "taos"
#define CUS_PROMPT "taos"
#endif
#ifndef CUS_EMAIL
#define CUS_EMAIL "<support@taosdata.com>"
#define CUS_EMAIL "<support@taosdata.com>"
#endif
#endif
// clang-format off
@ -58,6 +58,7 @@ static struct {
bool dumpSdb;
bool generateGrant;
bool memDbg;
bool checkS3;
bool printAuth;
bool printVersion;
bool printHelp;
@ -169,7 +170,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
return -1;
}
} else if (strcmp(argv[i], "-a") == 0) {
if(i < argc - 1) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("apollo url overflow");
return -1;
@ -182,7 +183,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
} else if (strcmp(argv[i], "-s") == 0) {
global.dumpSdb = true;
} else if (strcmp(argv[i], "-E") == 0) {
if(i < argc - 1) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("env file path overflow");
return -1;
@ -207,6 +208,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
cmdEnvIndex++;
} else if (strcmp(argv[i], "-dm") == 0) {
global.memDbg = true;
} else if (strcmp(argv[i], "--checks3") == 0) {
global.checkS3 = true;
} else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 ||
strcmp(argv[i], "-?") == 0) {
global.printHelp = true;
@ -267,8 +270,21 @@ static void dmDumpCfg() {
cfgDumpCfg(pCfg, 0, true);
}
static int32_t dmCheckS3() {
int32_t code = 0;
SConfig *pCfg = taosGetCfg();
cfgDumpCfgS3(pCfg, 0, true);
#if defined(USE_S3)
extern int32_t s3CheckCfg();
code = s3CheckCfg();
#endif
return code;
}
static int32_t dmInitLog() {
return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs,
0);
}
static void taosCleanupArgs() {
@ -355,6 +371,15 @@ int mainWindows(int argc, char **argv) {
return -1;
}
if (global.checkS3) {
int32_t code = dmCheckS3();
taosCleanupCfg();
taosCloseLog();
taosCleanupArgs();
taosConvDestroy();
return code;
}
if (global.dumpConfig) {
dmDumpCfg();
taosCleanupCfg();

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -141,6 +141,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_S3MIGRATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -208,6 +209,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

40
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -53,7 +53,7 @@ void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj *pVnode = *ppVnode;
SVnodeObj *pVnode = *ppVnode;
if (!pVnode->failed) {
SVnodeLoadLite vload = {0};
if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) {
@ -159,6 +159,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->hashSuffix = pCreate->hashSuffix;
pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024;
pCfg->s3ChunkSize = pCreate->s3ChunkSize;
pCfg->s3KeepLocal = pCreate->s3KeepLocal;
pCfg->s3Compact = pCreate->s3Compact;
pCfg->standby = 0;
pCfg->syncCfg.replicaNum = 0;
pCfg->syncCfg.totalReplicaNum = 0;
@ -238,17 +242,18 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo(
"vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d "
"szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d"
", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d s3ChunkSize:%d s3KeepLocal:%d s3Compact:%d tsma:%d "
"precision:%d compression:%d minRows:%d maxRows:%d"
", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64
", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d "
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d",
req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024,
(uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize,
req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.keepTimeOffset, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod,
req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod,
req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica,
req.learnerSelfIndex, req.strict, req.changeVersion);
req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion);
for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
@ -347,7 +352,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
_OVER:
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(0, path, pMgmt->pTfs);
vnodeDestroy(0, path, pMgmt->pTfs, 0);
} else {
dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", req.vgId,
TMSG_INFO(pMsg->msgType));
@ -358,7 +363,7 @@ _OVER:
return code;
}
//alter replica doesn't use this, but restore dnode still use this
// alter replica doesn't use this, but restore dnode still use this
int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SAlterVnodeTypeReq req = {0};
if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
@ -400,8 +405,8 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name);
int32_t vgId = req.vgId;
dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d",
vgId, req.replica, req.selfIndex, req.strict, req.changeVersion);
dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d", vgId, req.replica,
req.selfIndex, req.strict, req.changeVersion);
for (int32_t i = 0; i < req.replica; ++i) {
SReplica *pReplica = &req.replicas[i];
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id);
@ -484,12 +489,12 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
if(req.learnerReplicas == 0){
if (req.learnerReplicas == 0) {
req.learnerSelfIndex = -1;
}
dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request",
req.vgId, TMSG_INFO(pMsg->msgType));
dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request", req.vgId,
TMSG_INFO(pMsg->msgType));
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
if (pVnode == NULL) {
@ -501,7 +506,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
ESyncRole role = vnodeGetRole(pVnode->pImpl);
dInfo("vgId:%d, checking node role:%d", req.vgId, role);
if(role == TAOS_SYNC_ROLE_VOTER){
if (role == TAOS_SYNC_ROLE_VOTER) {
dError("vgId:%d, failed to alter vnode type since node already is role:%d", req.vgId, role);
terrno = TSDB_CODE_VND_ALREADY_IS_VOTER;
vmReleaseVnode(pMgmt, pVnode);
@ -509,7 +514,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
dInfo("vgId:%d, checking node catch up", req.vgId);
if(vnodeIsCatchUp(pVnode->pImpl) != 1){
if (vnodeIsCatchUp(pVnode->pImpl) != 1) {
terrno = TSDB_CODE_VND_NOT_CATCH_UP;
vmReleaseVnode(pMgmt, pVnode);
return -1;
@ -519,8 +524,8 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
vmReleaseVnode(pMgmt, pVnode);
dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request",
req.vgId, TMSG_INFO(pMsg->msgType));
dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request", req.vgId,
TMSG_INFO(pMsg->msgType));
return 0;
}
@ -933,6 +938,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

5
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE
#include "vmInt.h"
#include "libs/function/tudf.h"
#include "tfs.h"
#include "vnd.h"
#include "libs/function/tudf.h"
int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
int32_t diskId = -1;
@ -234,6 +234,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
dInfo("vgId:%d, commit data finished", pVnode->vgId);
}
int32_t nodeId = vnodeNodeId(pVnode->pImpl);
vnodeClose(pVnode->pImpl);
pVnode->pImpl = NULL;
@ -250,7 +251,7 @@ _closed:
if (pVnode->dropped) {
dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs);
vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs, nodeId);
}
vmFreeVnodeObj(&pVnode);

10
source/dnode/mgmt/node_mgmt/CMakeLists.txt
View File

@ -4,15 +4,13 @@ target_link_libraries(
dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
)
IF (TD_STORAGE)
IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3)
add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
add_definitions(-DUSE_COS)
ENDIF()
ENDIF ()
ENDIF()
IF (DEFINED GRANT_CFG_INCLUDE_DIR)
add_definitions(-DGRANTS_CFG)

3
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -379,6 +379,9 @@ typedef struct {
int32_t walRollPeriod;
int64_t walRetentionSize;
int64_t walSegmentSize;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int8_t withArbitrator;
} SDbCfg;

140
source/dnode/mnode/impl/src/mndDb.c
View File

@ -36,7 +36,7 @@
#include "tjson.h"
#define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 41
#define DB_RESERVE_SIZE 32
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -49,6 +49,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq);
static int32_t mndProcessDropDbReq(SRpcMsg *pReq);
static int32_t mndProcessUseDbReq(SRpcMsg *pReq);
static int32_t mndProcessTrimDbReq(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq);
static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity);
static void mndCancelGetNextDb(SMnode *pMnode, void *pIter);
static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq);
@ -76,6 +77,7 @@ int32_t mndInitDb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq);
mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB, mndProcessTrimDbReq);
mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq);
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB, mndProcessS3MigrateDbReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb);
@ -140,6 +142,9 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER)
SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.keepTimeOffset, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3ChunkSize, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
@ -232,11 +237,26 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.keepTimeOffset, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3ChunkSize, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock);
if (pDb->cfg.s3ChunkSize == 0) {
pDb->cfg.s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
mInfo("db:%s, s3ChunkSize set from %d to default %d", pDb->name, pDb->cfg.s3ChunkSize, TSDB_DEFAULT_S3_CHUNK_SIZE);
}
if (pDb->cfg.s3KeepLocal == 0) {
pDb->cfg.s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
mInfo("db:%s, s3KeepLocal set from %d to default %d", pDb->name, pDb->cfg.s3KeepLocal, TSDB_DEFAULT_S3_KEEP_LOCAL);
}
if (pDb->cfg.tsdbPageSize != TSDB_MIN_TSDB_PAGESIZE) {
mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize,
TSDB_DEFAULT_TSDB_PAGESIZE);
@ -322,6 +342,9 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.minRows = pNew->cfg.minRows;
pOld->cfg.maxRows = pNew->cfg.maxRows;
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
pOld->cfg.s3ChunkSize = pNew->cfg.s3ChunkSize;
pOld->cfg.s3KeepLocal = pNew->cfg.s3KeepLocal;
pOld->cfg.s3Compact = pNew->cfg.s3Compact;
pOld->cfg.withArbitrator = pNew->cfg.withArbitrator;
pOld->compactStartTime = pNew->compactStartTime;
taosWUnLockLatch(&pOld->lock);
@ -422,6 +445,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
if (pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1;
if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1;
if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1;
terrno = 0;
return terrno;
}
@ -471,6 +498,9 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1;
}
if (pNewCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pNewCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1;
if (pNewCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pNewCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1;
if (pNewCfg->s3Compact < TSDB_MIN_S3_COMPACT || pNewCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1;
terrno = 0;
return terrno;
@ -507,6 +537,9 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
if (pCfg->s3ChunkSize <= 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
}
@ -679,6 +712,9 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
.sstTrigger = pCreate->sstTrigger,
.hashPrefix = pCreate->hashPrefix,
.hashSuffix = pCreate->hashSuffix,
.s3ChunkSize = pCreate->s3ChunkSize,
.s3KeepLocal = pCreate->s3KeepLocal,
.s3Compact = pCreate->s3Compact,
.tsdbPageSize = pCreate->tsdbPageSize,
.withArbitrator = pCreate->withArbitrator,
};
@ -945,6 +981,18 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
terrno = 0;
}
if (pAlter->s3KeepLocal > TSDB_MIN_S3_KEEP_LOCAL && pAlter->s3KeepLocal != pDb->cfg.s3KeepLocal) {
pDb->cfg.s3KeepLocal = pAlter->s3KeepLocal;
pDb->vgVersion++;
terrno = 0;
}
if (pAlter->s3Compact > TSDB_MIN_S3_COMPACT && pAlter->s3Compact != pDb->cfg.s3Compact) {
pDb->cfg.s3Compact = pAlter->s3Compact;
pDb->vgVersion++;
terrno = 0;
}
if (pAlter->withArbitrator >= TSDB_MIN_DB_WITH_ARBITRATOR && pAlter->withArbitrator != pDb->cfg.withArbitrator) {
pDb->cfg.withArbitrator = pAlter->withArbitrator;
pDb->vgVersion++;
@ -1151,6 +1199,9 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL);
cfgRsp->schemaless = pDb->cfg.schemaless;
cfgRsp->sstTrigger = pDb->cfg.sstTrigger;
cfgRsp->s3ChunkSize = pDb->cfg.s3ChunkSize;
cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal;
cfgRsp->s3Compact = pDb->cfg.s3Compact;
cfgRsp->withArbitrator = pDb->cfg.withArbitrator;
}
@ -1218,7 +1269,7 @@ static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
if (pIter == NULL) break;
if (pArbGroup->dbUid == pDb->uid) {
if (mndSetDropArbGroupPrepareLogs(pTrans,pArbGroup) != 0) {
if (mndSetDropArbGroupPrepareLogs(pTrans, pArbGroup) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pArbGroup);
return -1;
@ -1246,7 +1297,7 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
if (pIter == NULL) break;
if (pArbGroup->dbUid == pDb->uid) {
if (mndSetDropArbGroupCommitLogs(pTrans,pArbGroup) != 0) {
if (mndSetDropArbGroupCommitLogs(pTrans, pArbGroup) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pArbGroup);
return -1;
@ -1798,6 +1849,77 @@ _OVER:
return code;
}
static int32_t mndS3MigrateDb(SMnode *pMnode, SDbObj *pDb) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()};
int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq);
int32_t contLen = reqLen + sizeof(SMsgHead);
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid != pDb->uid) continue;
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), contLen, &s3migrateReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("vgId:%d, failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code);
} else {
mInfo("vgId:%d, send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp);
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SDbObj *pDb = NULL;
SS3MigrateDbReq s3migrateReq = {0};
if (tDeserializeSS3MigrateDbReq(pReq->pCont, pReq->contLen, &s3migrateReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("db:%s, start to s3migrate", s3migrateReq.db);
pDb = mndAcquireDb(pMnode, s3migrateReq.db);
if (pDb == NULL) {
goto _OVER;
}
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_TRIM_DB, pDb) != 0) {
goto _OVER;
}
code = mndS3MigrateDb(pMnode, pDb);
_OVER:
if (code != 0) {
mError("db:%s, failed to process s3migrate db req since %s", s3migrateReq.db, terrstr());
}
mndReleaseDb(pMnode, pDb);
return code;
}
const char *mndGetDbStr(const char *src) {
char *pos = strstr(src, TS_PATH_DELIMITER);
if (pos != NULL) ++pos;
@ -2096,6 +2218,18 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3ChunkSize, false);
char keeplocalVstr[128] = {0};
len = sprintf(&keeplocalVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.s3KeepLocal);
varDataSetLen(keeplocalVstr, len);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)keeplocalVstr, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3Compact, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false);
}

16
source/dnode/mnode/impl/src/mndMain.c
View File

@ -134,6 +134,14 @@ static void mndPullupTtl(SMnode *pMnode) {
}
static void mndPullupTrimDb(SMnode *pMnode) {
mTrace("pullup s3migrate");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_S3MIGRATE_DB_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
static void mndPullupS3MigrateDb(SMnode *pMnode) {
mTrace("pullup trim");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
@ -319,6 +327,7 @@ static int32_t minCronTime() {
int32_t min = INT32_MAX;
min = TMIN(min, tsTtlPushIntervalSec);
min = TMIN(min, tsTrimVDbIntervalSec);
min = TMIN(min, tsS3MigrateIntervalSec);
min = TMIN(min, tsTransPullupInterval);
min = TMIN(min, tsCompactPullupInterval);
min = TMIN(min, tsMqRebalanceInterval);
@ -342,6 +351,10 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
mndPullupTrimDb(pMnode);
}
if (tsS3MigrateEnabled && sec % tsS3MigrateIntervalSec == 0) {
mndPullupS3MigrateDb(pMnode);
}
if (sec % tsTransPullupInterval == 0) {
mndPullupTrans(pMnode);
}
@ -806,6 +819,7 @@ _OVER:
pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER ||
pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE ||
pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT ||
pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER ||
pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state));
@ -950,7 +964,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pObj->id == pMnode->selfDnodeId) {
pClusterInfo->first_ep_dnode_id = pObj->id;
tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
//pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
// pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode);
// pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f);
tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role));

66
source/dnode/mnode/impl/src/mndStb.c
View File

@ -42,6 +42,8 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew);
static int32_t mndProcessTtlTimer(SRpcMsg *pReq);
static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pReq);
static int32_t mndProcessCreateStbReq(SRpcMsg *pReq);
static int32_t mndProcessAlterStbReq(SRpcMsg *pReq);
static int32_t mndProcessDropStbReq(SRpcMsg *pReq);
@ -82,6 +84,8 @@ int32_t mndInitStb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq);
mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB_TIMER, mndProcessTrimDbTimer);
mndSetMsgHandle(pMnode, TDMT_VND_S3MIGRATE_RSP, mndProcessS3MigrateDbRsp);
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB_TIMER, mndProcessS3MigrateDbTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq);
// mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq);
@ -762,9 +766,9 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
pDst->createdTime = taosGetTimestampMs();
pDst->updateTime = pDst->createdTime;
pDst->uid =
(pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX)
? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
pDst->uid = (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX)
? pCreate->suid
: mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
pDst->dbUid = pDb->uid;
pDst->tagVer = 1;
pDst->colVer = 1;
@ -983,6 +987,43 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) {
return 0;
}
static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()};
int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq);
int32_t contLen = reqLen + sizeof(SMsgHead);
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("vgId:%d, timer failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code);
} else {
mInfo("vgId:%d, timer send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp);
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) {
for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
if (strcmp(pStb->pTags[tag].name, tagName) == 0) {
@ -1133,7 +1174,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
}
} else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) {
goto _OVER;
} else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) {
} else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) &&
(createReq.tagVer != 1 || createReq.colVer != 1)) {
mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name);
code = 0;
goto _OVER;
@ -1184,14 +1226,13 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
SName name = {0};
tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
if(createReq.sql == NULL && createReq.sqlLen == 0){
if (createReq.sql == NULL && createReq.sqlLen == 0) {
char detail[1000] = {0};
sprintf(detail, "dbname:%s, stable name:%s", name.dbname, name.tname);
auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, detail, strlen(detail));
}
else{
} else {
auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, createReq.sql, createReq.sqlLen);
}
_OVER:
@ -1573,7 +1614,7 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj
for (int32_t i = 0; i < pOld->numOfTags; ++i) {
nLen += (pOld->pTags[i].colId == colId) ? pField->bytes : pOld->pTags[i].bytes;
}
if (nLen > TSDB_MAX_TAGS_LEN) {
terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH;
return -1;
@ -1936,7 +1977,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
return 0;
}
static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bool* schema, bool* sma) {
static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion *pStbVer, bool *schema, bool *sma) {
char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName);
@ -1966,7 +2007,7 @@ static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bo
} else {
*schema = false;
}
if (pStbVer->smaVer && pStbVer->smaVer != pStb->smaVer) {
*sma = true;
} else {
@ -2537,6 +2578,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
static int32_t mndProcessDropTtltbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessTrimDbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
@ -2750,8 +2792,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t
pStbVersion->tversion = ntohl(pStbVersion->tversion);
pStbVersion->smaVer = ntohl(pStbVersion->smaVer);
bool schema = false;
bool sma = false;
bool schema = false;
bool sma = false;
int32_t code = mndValidateStbVersion(pMnode, pStbVersion, &schema, &sma);
if (TSDB_CODE_SUCCESS != code) {
STableMetaRsp metaRsp = {0};

13
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -32,10 +32,10 @@
#define VGROUP_VER_NUMBER 1
#define VGROUP_RESERVE_SIZE 64
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
@ -276,6 +276,9 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
createReq.daysToKeep2 = pDb->cfg.daysToKeep2;
createReq.keepTimeOffset = pDb->cfg.keepTimeOffset;
createReq.s3ChunkSize = pDb->cfg.s3ChunkSize;
createReq.s3KeepLocal = pDb->cfg.s3KeepLocal;
createReq.s3Compact = pDb->cfg.s3Compact;
createReq.minRows = pDb->cfg.minRows;
createReq.maxRows = pDb->cfg.maxRows;
createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
@ -399,6 +402,8 @@ static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pV
alterReq.minRows = pDb->cfg.minRows;
alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod;
alterReq.walRetentionSize = pDb->cfg.walRetentionSize;
alterReq.s3KeepLocal = pDb->cfg.s3KeepLocal;
alterReq.s3Compact = pDb->cfg.s3Compact;
mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId);
int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq);

17
source/dnode/vnode/inc/vnode.h
View File

@ -57,7 +57,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
int32_t diskPrimary, STfs *pTfs);
int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId,
int32_t diskPrimary, STfs *pTfs);
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs);
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId);
SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force);
void vnodePreClose(SVnode *pVnode);
void vnodePostClose(SVnode *pVnode);
@ -69,7 +69,7 @@ int32_t vnodeBegin(SVnode *pVnode);
int32_t vnodeStart(SVnode *pVnode);
void vnodeStop(SVnode *pVnode);
int64_t vnodeGetSyncHandle(SVnode *pVnode);
int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables);
int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList);
@ -137,8 +137,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen);
bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type);
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type);
bool metaTbInFilterCache(SMeta *pMeta, const void *key, int8_t type);
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void *key, int8_t type);
int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type);
int32_t metaInitTbFilterCache(SMeta *pMeta);
@ -175,8 +175,8 @@ void *tsdbGetIvtIdx2(SMeta *pMeta);
uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader);
void tsdbReaderSetCloseFlag(STsdbReader *pReader);
int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
void tsdbSetFilesetDelimited(STsdbReader* pReader);
void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param);
void tsdbSetFilesetDelimited(STsdbReader *pReader);
void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param);
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
@ -241,7 +241,7 @@ int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, i
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished);
int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished);
// sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
@ -319,6 +319,9 @@ struct SVnodeCfg {
int16_t hashPrefix;
int16_t hashSuffix;
int32_t tsdbPageSize;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
};
#define TABLE_ROLLUP_ON ((int8_t)0x1)

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -635,6 +635,7 @@ typedef struct {
STsdb *pTsdb;
const char *objName;
uint8_t s3File;
int32_t lcn;
int32_t fid;
int64_t cid;
int64_t blkno;

2
source/dnode/vnode/src/inc/vnd.h
View File

@ -133,6 +133,8 @@ int32_t vnodeAsyncCommit(SVnode* pVnode);
bool vnodeShouldRollback(SVnode* pVnode);
// vnodeSync.c
int64_t vnodeClusterId(SVnode* pVnode);
int32_t vnodeNodeId(SVnode* pVnode);
int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion);
int32_t vnodeSyncStart(SVnode* pVnode);
void vnodeSyncPreClose(SVnode* pVnode);

25
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -108,13 +108,13 @@ typedef struct SQueryNode SQueryNode;
#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count"
#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
#define VNODE_METRIC_TAG_NAME_USERNAME "username"
#define VNODE_METRIC_TAG_NAME_RESULT "result"
#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
#define VNODE_METRIC_TAG_NAME_USERNAME "username"
#define VNODE_METRIC_TAG_NAME_RESULT "result"
#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
//#define VNODE_METRIC_TAG_VALUE_INSERT "insert"
@ -239,6 +239,7 @@ int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t
int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey);
int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync);
int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp);
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);
@ -461,12 +462,12 @@ typedef struct SVCommitSched {
int64_t maxWaitMs;
} SVCommitSched;
typedef struct SVMonitorObj{
char strClusterId[TSDB_CLUSTER_ID_LEN];
char strDnodeId[TSDB_NODE_ID_LEN];
char strVgId[TSDB_VGROUP_ID_LEN];
taos_counter_t *insertCounter;
}SVMonitorObj;
typedef struct SVMonitorObj {
char strClusterId[TSDB_CLUSTER_ID_LEN];
char strDnodeId[TSDB_NODE_ID_LEN];
char strVgId[TSDB_VGROUP_ID_LEN];
taos_counter_t* insertCounter;
} SVMonitorObj;
struct SVnode {
char* path;

39
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -46,7 +46,6 @@ typedef struct {
STFileSet *fset;
TABLEID tbid[1];
bool hasTSData;
bool skipTsRow;
} ctx[1];
// reader
@ -128,21 +127,8 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
continue;
}
}
/*
extern int8_t tsS3Enabled;
int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs);
committer->ctx->skipTsRow = false;
if (tsS3Enabled && nlevel > 1 && committer->ctx->did.level == nlevel - 1) {
committer->ctx->skipTsRow = true;
}
*/
int64_t ts = TSDBROW_TS(&row->row);
if (committer->ctx->skipTsRow && ts <= committer->ctx->maxKey) {
ts = committer->ctx->maxKey + 1;
}
if (ts > committer->ctx->maxKey) {
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
@ -406,31 +392,6 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
// reset nextKey
committer->ctx->nextKey = TSKEY_MAX;
committer->ctx->skipTsRow = false;
extern int8_t tsS3Enabled;
extern int32_t tsS3UploadDelaySec;
long s3Size(const char *object_name);
int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1 && committer->ctx->fset) {
STFileObj *fobj = committer->ctx->fset->farr[TSDB_FTYPE_DATA];
if (fobj && fobj->f->did.level == nlevel - 1) {
// if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
if (taosCheckExistFile(fobj->fname)) {
int32_t mtime = 0;
taosStatFile(fobj->fname, NULL, &mtime, NULL);
if (mtime < committer->ctx->now - tsS3UploadDelaySec) {
committer->ctx->skipTsRow = true;
}
} else /*if (s3Size(object_name) > 0) */ {
committer->ctx->skipTsRow = true;
}
}
// new fset can be written with ts data
}
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);

11
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -29,15 +29,16 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon
reader[0]->config[0] = config[0];
int32_t lcn = config->file.lcn;
if (fname) {
if (fname) {
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd);
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn);
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, &config->file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -113,8 +114,8 @@ static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) {
static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; }
static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) {
int32_t code = 0;
int32_t lino = 0;
int32_t code = 0;
int32_t lino = 0;
ASSERT(writer->ctx->offset <= writer->file.size);
ASSERT(writer->config->fid == writer->file.fid);
@ -151,7 +152,7 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) {
}
tsdbTFileName(writer->config->tsdb, &writer->file, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:

95
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -103,7 +103,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig
if (fname) {
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
if (fname[i]) {
code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i]);
int32_t lcn = config->files[i].file.lcn;
code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -112,7 +113,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig
if (config->files[i].exist) {
char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, &config->files[i].file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i]);
int32_t lcn = config->files[i].file.lcn;
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -325,15 +327,15 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(br.offset == buffer0->size);
bool loadExtra = false;
int extraColIdx = -1;
for (int i = 0; i < ncid; i++) {
if (tBlockDataGetColData(bData, cids[i]) == NULL) {
loadExtra = true;
extraColIdx = i;
break;
}
}
if (!loadExtra) {
if (extraColIdx < 0) {
goto _exit;
}
@ -343,10 +345,78 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
buffer0, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// calc szHint
int64_t szHint = 0;
int extraCols = 1;
for (int i = extraColIdx + 1; i < ncid; ++i) {
if (tBlockDataGetColData(bData, cids[i]) == NULL) {
++extraCols;
break;
}
}
if (extraCols >= 2) {
br = BUFFER_READER_INITIALIZER(0, buffer0);
SBlockCol blockCol = {.cid = 0};
for (int32_t i = extraColIdx; i < ncid; ++i) {
int16_t extraColCid = cids[i];
while (extraColCid > blockCol.cid) {
if (br.offset >= buffer0->size) {
blockCol.cid = INT16_MAX;
break;
}
code = tGetBlockCol(&br, &blockCol);
TSDB_CHECK_CODE(code, lino, _exit);
}
if (extraColCid == blockCol.cid || blockCol.cid == INT16_MAX) {
extraColIdx = i;
break;
}
}
if (blockCol.cid > 0 && blockCol.cid < INT16_MAX /*&& blockCol->flag == HAS_VALUE*/) {
int64_t offset = blockCol.offset;
SBlockCol lastNonNoneBlockCol = {.cid = 0};
for (int32_t i = extraColIdx; i < ncid; ++i) {
int16_t extraColCid = cids[i];
while (extraColCid > blockCol.cid) {
if (br.offset >= buffer0->size) {
blockCol.cid = INT16_MAX;
break;
}
code = tGetBlockCol(&br, &blockCol);
TSDB_CHECK_CODE(code, lino, _exit);
}
if (extraColCid == blockCol.cid) {
lastNonNoneBlockCol = blockCol;
continue;
}
if (blockCol.cid == INT16_MAX) {
break;
}
}
if (lastNonNoneBlockCol.cid > 0) {
szHint = lastNonNoneBlockCol.offset + lastNonNoneBlockCol.szBitmap + lastNonNoneBlockCol.szOffset +
lastNonNoneBlockCol.szValue - offset;
}
}
}
// load each column
SBlockCol blockCol = {
.cid = 0,
};
bool firstRead = true;
br = BUFFER_READER_INITIALIZER(0, buffer0);
for (int32_t i = 0; i < ncid; i++) {
int16_t cid = cids[i];
@ -384,11 +454,13 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
} else if (cid == blockCol.cid) {
// load from file
tBufferClear(buffer1);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA],
record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0);
code = tsdbReadFileToBuffer(
reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, firstRead ? szHint : 0);
TSDB_CHECK_CODE(code, lino, _exit);
firstRead = false;
// decode the buffer
SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1);
code = tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist);
@ -657,6 +729,7 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) {
.fid = writer->config->fid,
.cid = writer->config->cid,
.size = 0,
.lcn = writer->config->lcn == -1 ? 0 : -1,
.minVer = VERSION_MAX,
.maxVer = VERSION_MIN,
};
@ -1640,8 +1713,9 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) {
flag |= (TD_FILE_CREATE | TD_FILE_TRUNC);
}
int32_t lcn = writer->files[ftype].lcn;
tsdbTFileName(writer->config->tsdb, &writer->files[ftype], fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype]);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn);
TSDB_CHECK_CODE(code, lino, _exit);
if (writer->files[ftype].size == 0) {
@ -1807,8 +1881,9 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) {
int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
int32_t lcn = writer->files[ftype].lcn;
tsdbTFileName(writer->config->tsdb, writer->files + ftype, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype]);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn);
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};

1
source/dnode/vnode/src/tsdb/tsdbDataFileRW.h
View File

@ -77,6 +77,7 @@ typedef struct SDataFileWriterConfig {
int64_t cid;
SDiskID did;
int64_t compactVersion;
int32_t lcn;
struct {
bool exist;
STFile file;

2
source/dnode/vnode/src/tsdb/tsdbDef.h
View File

@ -31,7 +31,7 @@ typedef struct SFDataPtr {
int64_t size;
} SFDataPtr;
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD);
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn);
extern void tsdbCloseFile(STsdbFD **ppFD);
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size);
extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint);

96
source/dnode/vnode/src/tsdb/tsdbFS2.c
View File

@ -20,7 +20,7 @@
#define BLOCK_COMMIT_FACTOR 3
extern void remove_file(const char *fname, bool last_level);
extern void remove_file(const char *fname);
#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT
#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1)
@ -355,17 +355,32 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) {
// check file existence
if (!taosCheckExistFile(fobj->fname)) {
if (tsS3Enabled) {
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
long s3_size = s3Size(object_name);
if (s3_size > 0) {
return 0;
bool found = false;
if (tsS3Enabled && fobj->f->lcn > 1) {
char fname1[TSDB_FILENAME_LEN];
tsdbTFileLastChunkName(fs->tsdb, fobj->f, fname1);
if (!taosCheckExistFile(fname1)) {
code = TSDB_CODE_FILE_CORRUPTED;
tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fname1);
return code;
}
found = true;
/*
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
long s3_size = s3Size(object_name);
if (s3_size > 0) {
return 0;
}
*/
}
code = TSDB_CODE_FILE_CORRUPTED;
tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fobj->fname);
return code;
if (!found) {
code = TSDB_CODE_FILE_CORRUPTED;
tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fobj->fname);
return code;
}
}
{ // TODO: check file size
@ -530,9 +545,9 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
if (taosIsDir(file->aname)) continue;
if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL &&
strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3)) {
int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs);
remove_file(file->aname, nlevel > 1 && file->did.level == nlevel - 1);
strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3) &&
strncmp(file->aname + strlen(file->aname) - 5, ".data", 5)) {
remove_file(file->aname);
}
}
@ -900,57 +915,28 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
continue;
}
bool skipMerge = false;
// bool skipMerge = false;
int32_t numFile = TARRAY2_SIZE(lvl->fobjArr);
if (numFile >= sttTrigger && (!fset->mergeScheduled)) {
// launch merge
{
extern int8_t tsS3Enabled;
extern int32_t tsS3UploadDelaySec;
long s3Size(const char *object_name);
int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1) {
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
if (fobj && fobj->f->did.level == nlevel - 1) {
// if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
code = tsdbTFileSetOpenChannel(fset);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosCheckExistFile(fobj->fname)) {
int32_t now = taosGetTimestampSec();
int32_t mtime = 0;
taosStatFile(fobj->fname, NULL, &mtime, NULL);
if (mtime < now - tsS3UploadDelaySec) {
skipMerge = true;
}
} else /* if (s3Size(object_name) > 0) */ {
skipMerge = true;
}
}
// new fset can be written with ts data
}
SMergeArg *arg = taosMemoryMalloc(sizeof(*arg));
if (arg == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (!skipMerge) {
code = tsdbTFileSetOpenChannel(fset);
TSDB_CHECK_CODE(code, lino, _exit);
arg->tsdb = fs->tsdb;
arg->fid = fset->fid;
SMergeArg *arg = taosMemoryMalloc(sizeof(*arg));
if (arg == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
arg->tsdb = fs->tsdb;
arg->fid = fset->fid;
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree,
arg, NULL);
TSDB_CHECK_CODE(code, lino, _exit);
fset->mergeScheduled = true;
}
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg,
NULL);
TSDB_CHECK_CODE(code, lino, _exit);
fset->mergeScheduled = true;
}
if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR && !skipMerge) {
if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) {
tsdbFSSetBlockCommit(fset, true);
} else {
tsdbFSSetBlockCommit(fset, false);

10
source/dnode/vnode/src/tsdb/tsdbFSet2.c
View File

@ -386,7 +386,11 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f
fobj2->f[0] = fobj1->f[0];
}
} else {
tsdbTFileObjRemove(fobj2);
if (fobj1->f->cid != fobj2->f->cid) {
tsdbTFileObjRemove(fobj2);
} else {
tsdbTFileObjRemoveUpdateLC(fobj2);
}
code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]);
if (code) return code;
}
@ -585,7 +589,7 @@ int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) {
return 0;
}
int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) {
int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) {
if (ppArr && ppArr[0]) {
TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear);
taosMemoryFree(ppArr[0]);
@ -663,4 +667,4 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) {
int32_t tsdbTFileSetOpenChannel(STFileSet *fset) {
if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0;
return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel);
}
}

1
source/dnode/vnode/src/tsdb/tsdbFSetRW.c
View File

@ -148,6 +148,7 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) {
.compactVersion = config->compactVersion,
.skmTb = writer[0]->skmTb,
.skmRow = writer[0]->skmRow,
.lcn = config->lcn,
.buffers = writer[0]->buffers,
};
for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {

1
source/dnode/vnode/src/tsdb/tsdbFSetRW.h
View File

@ -37,6 +37,7 @@ typedef struct {
int64_t cid;
SDiskID did;
int32_t level;
int32_t lcn;
struct {
bool exist;
STFile file;

145
source/dnode/vnode/src/tsdb/tsdbFile2.c
View File

@ -15,6 +15,7 @@
#include "tsdbFile2.h"
#include "cos.h"
#include "vnd.h"
// to_json
static int32_t head_to_json(const STFile *file, cJSON *json);
@ -42,24 +43,9 @@ static const struct {
[TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
};
void remove_file(const char *fname, bool last_level) {
int32_t code = taosRemoveFile(fname);
if (code) {
if (tsS3Enabled && last_level) {
const char *object_name = taosDirEntryBaseName((char *)fname);
long s3_size = tsS3Enabled ? s3Size(object_name) : 0;
if (!strncmp(fname + strlen(fname) - 5, ".data", 5) && s3_size > 0) {
s3DeleteObjects(&object_name, 1);
tsdbInfo("file:%s is removed from s3", fname);
} else {
tsdbError("file:%s remove failed", fname);
}
} else {
tsdbError("file:%s remove failed", fname);
}
} else {
tsdbInfo("file:%s is removed", fname);
}
void remove_file(const char *fname) {
taosRemoveFile(fname);
tsdbInfo("file:%s is removed", fname);
}
static int32_t tfile_to_json(const STFile *file, cJSON *json) {
@ -73,6 +59,11 @@ static int32_t tfile_to_json(const STFile *file, cJSON *json) {
return TSDB_CODE_OUT_OF_MEMORY;
}
/* lcn - last chunk number */
if (cJSON_AddNumberToObject(json, "lcn", file->lcn) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
/* fid */
if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -121,6 +112,14 @@ static int32_t tfile_from_json(const cJSON *json, STFile *file) {
return TSDB_CODE_FILE_CORRUPTED;
}
/* lcn */
item = cJSON_GetObjectItem(json, "lcn");
if (cJSON_IsNumber(item)) {
file->lcn = item->valuedouble;
} else {
// return TSDB_CODE_FILE_CORRUPTED;
}
/* fid */
item = cJSON_GetObjectItem(json, "fid");
if (cJSON_IsNumber(item)) {
@ -236,7 +235,8 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
fobj[0]->state = TSDB_FSTATE_LIVE;
fobj[0]->ref = 1;
tsdbTFileName(pTsdb, f, fobj[0]->fname);
fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
// fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode);
return 0;
}
@ -258,7 +258,7 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) {
tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) {
if (fobj->state == TSDB_FSTATE_DEAD) {
remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1);
remove_file(fobj->fname);
}
taosMemoryFree(fobj);
}
@ -266,6 +266,61 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) {
return 0;
}
static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {
if (fobj->f->type != TSDB_FTYPE_DATA) {
remove_file(fobj->fname);
return;
}
if (!remove_all) {
if (fobj->f->lcn < 1) {
remove_file(fobj->fname);
return;
} else {
// remove local last chunk file
char lc_path[TSDB_FILENAME_LEN];
tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);
char *dot = strrchr(lc_path, '.');
if (!dot) {
tsdbError("unexpected path: %s", lc_path);
return;
}
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);
remove_file(lc_path);
}
} else {
// delete by data file prefix
char lc_path[TSDB_FILENAME_LEN];
tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);
char *object_name = taosDirEntryBaseName(lc_path);
int32_t node_id = fobj->nlevel;
char object_name_prefix[TSDB_FILENAME_LEN];
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
char *dot = strrchr(object_name_prefix, '.');
if (!dot) {
tsdbError("unexpected path: %s", object_name_prefix);
return;
}
*(dot + 1) = 0;
s3DeleteObjectsByPrefix(object_name_prefix);
// remove local last chunk file
dot = strrchr(lc_path, '.');
if (!dot) {
tsdbError("unexpected path: %s", lc_path);
return;
}
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);
remove_file(lc_path);
}
}
int32_t tsdbTFileObjRemove(STFileObj *fobj) {
taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
@ -274,7 +329,21 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) {
taosThreadMutexUnlock(&fobj->mutex);
tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) {
remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1);
tsdbTFileObjRemoveLC(fobj, true);
taosMemoryFree(fobj);
}
return 0;
}
int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {
taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
fobj->state = TSDB_FSTATE_DEAD;
int32_t nRef = --fobj->ref;
taosThreadMutexUnlock(&fobj->mutex);
tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) {
tsdbTFileObjRemoveLC(fobj, false);
taosMemoryFree(fobj);
}
return 0;
@ -310,13 +379,45 @@ int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]) {
return 0;
}
int32_t tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]) {
SVnode *pVnode = pTsdb->pVnode;
STfs *pTfs = pVnode->pTfs;
if (pTfs) {
snprintf(fname, //
TSDB_FILENAME_LEN, //
"%s%s%s%sv%df%dver%" PRId64 ".%d.%s", //
tfsGetDiskPath(pTfs, f->did), //
TD_DIRSEP, //
pTsdb->path, //
TD_DIRSEP, //
TD_VID(pVnode), //
f->fid, //
f->cid, //
f->lcn, //
g_tfile_info[f->type].suffix);
} else {
snprintf(fname, //
TSDB_FILENAME_LEN, //
"%s%sv%df%dver%" PRId64 ".%d.%s", //
pTsdb->path, //
TD_DIRSEP, //
TD_VID(pVnode), //
f->fid, //
f->cid, //
f->lcn, //
g_tfile_info[f->type].suffix);
}
return 0;
}
bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) {
if (f1->type != f2->type) return false;
if (f1->did.level != f2->did.level) return false;
if (f1->did.id != f2->did.id) return false;
if (f1->fid != f2->fid) return false;
if (f1->cid != f2->cid) return false;
if (f1->s3flag != f2->s3flag) return false;
if (f1->lcn != f2->lcn) return false;
return true;
}

4
source/dnode/vnode/src/tsdb/tsdbFile2.h
View File

@ -45,6 +45,7 @@ enum {
int32_t tsdbTFileToJson(const STFile *f, cJSON *json);
int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f);
int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]);
int32_t tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]);
bool tsdbIsSameTFile(const STFile *f1, const STFile *f2);
bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2);
@ -53,12 +54,13 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj);
int32_t tsdbTFileObjRef(STFileObj *fobj);
int32_t tsdbTFileObjUnref(STFileObj *fobj);
int32_t tsdbTFileObjRemove(STFileObj *fobj);
int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj);
int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2);
struct STFile {
tsdb_ftype_t type;
SDiskID did; // disk id
int32_t s3flag;
int32_t lcn; // last chunk number
int32_t fid; // file id
int64_t cid; // commit id
int64_t size;

7
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -544,7 +544,7 @@ int32_t tsdbMerge(void *arg) {
TSDB_CHECK_CODE(code, lino, _exit);
if (merger->fset == NULL) return 0;
/*
bool skipMerge = false;
{
extern int8_t tsS3Enabled;
@ -565,7 +565,8 @@ int32_t tsdbMerge(void *arg) {
if (mtime < now - tsS3UploadDelaySec) {
skipMerge = true;
}
} else /* if (s3Size(object_name) > 0) */ {
} else // if (s3Size(object_name) > 0)
{
skipMerge = true;
}
}
@ -577,7 +578,7 @@ int32_t tsdbMerge(void *arg) {
code = 0;
goto _exit;
}
*/
// do merge
tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid);
code = tsdbDoMerge(merger);

219
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -15,16 +15,48 @@
#include "cos.h"
#include "tsdb.h"
#include "vnd.h"
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
int32_t code = 0;
const char *path = pFD->path;
int32_t szPage = pFD->szPage;
int32_t flag = pFD->flag;
int64_t lc_size = 0;
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
int errsv = errno;
if (tsS3Enabled && pFD->lcn > 1 && !strncmp(path + strlen(path) - 5, ".data", 5)) {
char lc_path[TSDB_FILENAME_LEN];
tstrncpy(lc_path, path, TSDB_FQDN_LEN);
char *dot = strrchr(lc_path, '.');
if (!dot) {
tsdbError("unexpected path: %s", lc_path);
code = TAOS_SYSTEM_ERROR(ENOENT);
goto _exit;
}
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn);
pFD->pFD = taosOpenFile(lc_path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
// taosMemoryFree(pFD);
goto _exit;
}
if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
}
} else {
tsdbInfo("no file: %s", path);
code = TAOS_SYSTEM_ERROR(errno);
// taosMemoryFree(pFD);
goto _exit;
}
/*
const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = 0;
if (tsS3Enabled) {
@ -43,7 +75,6 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
s3Get(object_name, path);
pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(ENOENT);
// taosMemoryFree(pFD);
@ -57,12 +88,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
pFD->objName = object_name;
// pFD->szFile = s3_size;
#endif
} else {
tsdbInfo("no file: %s", path);
code = TAOS_SYSTEM_ERROR(errsv);
// taosMemoryFree(pFD);
goto _exit;
}
*/
}
pFD->pBuf = taosMemoryCalloc(1, szPage);
@ -73,26 +99,33 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
goto _exit;
}
if (lc_size > 0) {
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
pFD->szFile = lc_size + chunksize * (pFD->lcn - 1);
}
// not check file size when reading data files.
if (flag != TD_FILE_READ && !pFD->s3File) {
if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
if (flag != TD_FILE_READ /* && !pFD->s3File*/) {
if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
// taosMemoryFree(pFD->pBuf);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
}
ASSERT(pFD->szFile % szPage == 0);
pFD->szFile = pFD->szFile / szPage;
}
ASSERT(pFD->szFile % szPage == 0);
pFD->szFile = pFD->szFile / szPage;
_exit:
return code;
}
// =============== PAGE-WISE FILE ===============
int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD) {
int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) {
int32_t code = 0;
STsdbFD *pFD = NULL;
int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
@ -111,6 +144,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF
pFD->flag = flag;
pFD->szPage = szPage;
pFD->pgno = 0;
pFD->lcn = lcn;
pFD->pTsdb = pTsdb;
*ppFD = pFD;
@ -123,9 +157,9 @@ void tsdbCloseFile(STsdbFD **ppFD) {
STsdbFD *pFD = *ppFD;
if (pFD) {
taosMemoryFree(pFD->pBuf);
if (!pFD->s3File) {
taosCloseFile(&pFD->pFD);
}
// if (!pFD->s3File) {
taosCloseFile(&pFD->pFD);
//}
taosMemoryFree(pFD);
*ppFD = NULL;
}
@ -141,12 +175,18 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
}
}
if (pFD->s3File) {
tsdbWarn("%s file: %s", __func__, pFD->path);
return code;
}
if (pFD->pgno > 0) {
int64_t n = taosLSeekFile(pFD->pFD, PAGE_OFFSET(pFD->pgno, pFD->szPage), SEEK_SET);
int64_t offset = PAGE_OFFSET(pFD->pgno, pFD->szPage);
if (pFD->lcn > 1) {
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t chunkoffset = chunksize * (pFD->lcn - 1);
offset -= chunkoffset;
}
ASSERT(offset >= 0);
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
@ -182,7 +222,15 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
}
int64_t offset = PAGE_OFFSET(pgno, pFD->szPage);
if (pFD->lcn > 1) {
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t chunkoffset = chunksize * (pFD->lcn - 1);
offset -= chunkoffset;
}
ASSERT(offset >= 0);
/*
if (pFD->s3File) {
LRUHandle *handle = NULL;
@ -203,24 +251,25 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
tsdbCacheRelease(pFD->pTsdb->bCache, handle);
} else {
// seek
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
// read
n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
} else if (n < pFD->szPage) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _exit;
}
*/
// seek
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
// read
n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
} else if (n < pFD->szPage) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _exit;
}
//}
// check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED;
@ -294,6 +343,74 @@ _exit:
return code;
}
static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
int32_t code = 0;
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t cOffset = offset % chunksize;
int64_t n = 0;
char *object_name = taosDirEntryBaseName(pFD->path);
char object_name_prefix[TSDB_FILENAME_LEN];
int32_t node_id = vnodeNodeId(pFD->pTsdb->pVnode);
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
char *dot = strrchr(object_name_prefix, '.');
if (!dot) {
tsdbError("unexpected path: %s", object_name_prefix);
code = TAOS_SYSTEM_ERROR(ENOENT);
goto _exit;
}
char *buf = taosMemoryCalloc(1, size);
for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) {
int64_t nRead = TMIN(chunksize - cOffset, size - n);
if (chunkno >= pFD->lcn) {
// read last chunk
int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET);
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(buf);
goto _exit;
}
ret = taosReadFile(pFD->pFD, buf + n, nRead);
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(buf);
goto _exit;
} else if (ret < nRead) {
code = TSDB_CODE_FILE_CORRUPTED;
taosMemoryFree(buf);
goto _exit;
}
} else {
uint8_t *pBlock = NULL;
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno);
code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(buf);
goto _exit;
}
memcpy(buf + n, pBlock, nRead);
taosMemoryFree(pBlock);
}
n += nRead;
cOffset = 0;
}
*ppBlock = buf;
_exit:
return code;
}
static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
int32_t code = 0;
int64_t n = 0;
@ -356,15 +473,22 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
}
int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
/*
code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
*/
code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
// 3, Store Pages in Cache
int nPage = pgnoEnd - pgno + 1;
for (int i = 0; i < nPage; ++i) {
tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage);
if (pFD->szFile != pgno) { // DONOT cache last volatile page
tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage);
}
if (szHint > 0 && n >= size) {
++pgno;
@ -404,7 +528,7 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size,
}
}
if (pFD->s3File && tsS3BlockSize < 0) {
if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
} else {
return tsdbReadFileImp(pFD, offset, pBuf, size);
@ -428,11 +552,12 @@ int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer
int32_t tsdbFsyncFile(STsdbFD *pFD) {
int32_t code = 0;
/*
if (pFD->s3File) {
tsdbWarn("%s file: %s", __func__, pFD->path);
return code;
}
*/
code = tsdbWriteFilePage(pFD);
if (code) goto _exit;
@ -464,23 +589,23 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
// head
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pHeadFD);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pHeadFD, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// data
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pDataFD);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pDataFD, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// sma
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pSmaFD);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pSmaFD, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// stt
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->aSttFD[iStt]);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->aSttFD[iStt], 0);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -658,7 +783,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
pDelFReader->fDel = *pFile;
tsdbDelFileName(pTsdb, pFile, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0);
if (code) {
taosMemoryFree(pDelFReader);
goto _exit;

569
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -38,24 +38,42 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
return TARRAY2_APPEND(rtner->fopArr, op);
}
static int32_t tsdbRemoveFileObjectS3(SRTNer *rtner, const STFileObj *fobj) {
int32_t code = 0, lino = 0;
static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
TdFilePtr fdFrom = NULL, fdTo = NULL;
char fname_from[TSDB_FILENAME_LEN];
char fname_to[TSDB_FILENAME_LEN];
STFileOp op = {
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
tsdbTFileLastChunkName(rtner->tsdb, from->f, fname_from);
tsdbTFileLastChunkName(rtner->tsdb, to, fname_to);
code = TARRAY2_APPEND(rtner->fopArr, op);
fdFrom = taosOpenFile(fname_from, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
s3DeleteObjects(&object_name, 1);
tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size);
fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1);
int64_t n = taosFSendFile(fdTo, fdFrom, 0, lc_size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
if (fdFrom) taosCloseFile(&fdFrom);
if (fdTo) taosCloseFile(&fdTo);
}
return code;
}
@ -97,38 +115,11 @@ _exit:
return code;
}
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
TdFilePtr fdFrom = NULL;
// TdFilePtr fdTo = NULL;
tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
char *object_name = taosDirEntryBaseName(fname);
code = s3PutObjectFromFile2(from->fname, object_name, 1);
TSDB_CHECK_CODE(code, lino, _exit);
taosCloseFile(&fdFrom);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
taosCloseFile(&fdFrom);
}
return code;
}
static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
int32_t lcn = fobj->f->lcn;
// remove old
op = (STFileOp){
@ -153,6 +144,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
.maxVer = fobj->f->maxVer,
.cid = fobj->f->cid,
.size = fobj->f->size,
.lcn = lcn,
.stt[0] =
{
.level = fobj->f->stt[0].level,
@ -164,59 +156,14 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
TSDB_CHECK_CODE(code, lino, _exit);
// do copy the file
code = tsdbDoCopyFile(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
if (lcn < 1) {
code = tsdbDoCopyFile(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDoCopyFileLC(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
}
return code;
}
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
// remove old
op = (STFileOp){
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = fobj->f->fid,
.nf =
{
.type = fobj->f->type,
.did = did[0],
.fid = fobj->f->fid,
.minVer = fobj->f->minVer,
.maxVer = fobj->f->maxVer,
.cid = fobj->f->cid,
.size = fobj->f->size,
.stt[0] =
{
.level = fobj->f->stt[0].level,
},
},
};
op.nf.s3flag = true;
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// do copy the file
code = tsdbCopyFileS3(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
@ -327,38 +274,14 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = fset->farr[ftype], 1); ++ftype) {
if (fobj == NULL) continue;
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
if (fobj->f->did.level == did.level) {
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1 &&
taosCheckExistFile(fobj->fname)) {
int32_t mtime = 0;
taosStatFile(fobj->fname, NULL, &mtime, NULL);
if (mtime < rtner->now - tsS3UploadDelaySec) {
tsdbInfo("file:%s size: %" PRId64 " do migrate s3", fobj->fname, fobj->f->size);
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
/*
code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did);
TSDB_CHECK_CODE(code, lino, _exit);
*/
continue;
}
/*
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) {
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
if (tsS3Enabled) {
int64_t fsize = 0;
if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(terrno);
tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode),
__func__, fobj->fname, tstrerror(code)); TSDB_CHECK_CODE(code, lino, _exit);
}
s3EvictCache(fobj->fname, fsize * 2);
}
*/
if (fobj->f->did.level > did.level) {
continue;
}
@ -367,7 +290,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
code = tsdbDoMigrateFileObj(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
//}
}
// stt
@ -471,3 +393,414 @@ int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
return code;
}
static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3KeepLocal, int64_t nowSec) {
int32_t localFid;
TSKEY key;
if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) {
nowSec = nowSec * 1000;
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) {
nowSec = nowSec * 1000000l;
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) {
nowSec = nowSec * 1000000000l;
} else {
ASSERT(0);
}
nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision];
key = nowSec - s3KeepLocal * tsTickPerMin[pKeepCfg->precision];
localFid = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision);
if (fid >= localFid) {
return 0;
} else {
return 1;
}
}
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
TdFilePtr fdFrom = NULL;
// TdFilePtr fdTo = NULL;
tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
char *object_name = taosDirEntryBaseName(fname);
code = s3PutObjectFromFile2(from->fname, object_name, 1);
TSDB_CHECK_CODE(code, lino, _exit);
taosCloseFile(&fdFrom);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
taosCloseFile(&fdFrom);
}
return code;
}
static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
TdFilePtr fdFrom = NULL, fdTo = NULL;
int32_t lcn = fobj->f->lcn + (size - 1) / chunksize;
// remove old
op = (STFileOp){
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = fobj->f->fid,
.nf =
{
.type = fobj->f->type,
.did = fobj->f->did,
.fid = fobj->f->fid,
.minVer = fobj->f->minVer,
.maxVer = fobj->f->maxVer,
.cid = fobj->f->cid,
.size = fobj->f->size,
.lcn = lcn,
.stt[0] =
{
.level = fobj->f->stt[0].level,
},
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
char fname[TSDB_FILENAME_LEN];
tsdbTFileName(rtner->tsdb, &op.nf, fname);
char *object_name = taosDirEntryBaseName(fname);
char object_name_prefix[TSDB_FILENAME_LEN];
int32_t node_id = vnodeNodeId(rtner->tsdb->pVnode);
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
char *dot = strrchr(object_name_prefix, '.');
if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1;
}
char *dot2 = strchr(object_name, '.');
if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1;
}
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn);
// do copy the file
for (int32_t cn = fobj->f->lcn; cn < lcn; ++cn) {
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
int64_t c_offset = chunksize * (cn - fobj->f->lcn);
code = s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize);
TSDB_CHECK_CODE(code, lino, _exit);
}
// copy last chunk
int64_t lc_offset = chunksize * (lcn - fobj->f->lcn);
int64_t lc_size = size - lc_offset;
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn);
fdFrom = taosOpenFile(fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size);
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
if (fdFrom) taosCloseFile(&fdFrom);
if (fdTo) taosCloseFile(&fdTo);
}
return code;
}
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
int32_t lcn = (size - 1) / chunksize + 1;
// remove old
op = (STFileOp){
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = fobj->f->fid,
.nf =
{
.type = fobj->f->type,
.did = fobj->f->did,
.fid = fobj->f->fid,
.minVer = fobj->f->minVer,
.maxVer = fobj->f->maxVer,
.cid = fobj->f->cid,
.size = fobj->f->size,
.lcn = lcn,
.stt[0] =
{
.level = fobj->f->stt[0].level,
},
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
char fname[TSDB_FILENAME_LEN];
tsdbTFileName(rtner->tsdb, &op.nf, fname);
char *object_name = taosDirEntryBaseName(fname);
char object_name_prefix[TSDB_FILENAME_LEN];
int32_t node_id = vnodeNodeId(rtner->tsdb->pVnode);
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
char *dot = strrchr(object_name_prefix, '.');
if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1;
}
// do copy the file
for (int32_t cn = 1; cn < lcn; ++cn) {
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
int64_t c_offset = chunksize * (cn - 1);
code = s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize);
TSDB_CHECK_CODE(code, lino, _exit);
}
// copy last chunk
TdFilePtr fdFrom = NULL, fdTo = NULL;
int64_t lc_offset = (int64_t)(lcn - 1) * chunksize;
int64_t lc_size = size - lc_offset;
dot = strchr(object_name, '.');
if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1;
}
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name), "%d.data", lcn);
fdFrom = taosOpenFile(fobj->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, fobj->f->size);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
}
return code;
}
static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) {
int32_t code = 0;
int32_t lino = 0;
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
if (!fobj) return code;
int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now);
if (expLevel < 0) return code; // expired
SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
int32_t s3KeepLocal = pCfg->s3KeepLocal;
int32_t s3ExpLevel = tsdbS3FidLevel(fset->fid, &rtner->tsdb->keepCfg, s3KeepLocal, rtner->now);
if (s3ExpLevel < 1) return code; // keep on local storage
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int32_t lcn = fobj->f->lcn;
if (lcn < 1 && taosCheckExistFile(fobj->fname)) {
int32_t mtime = 0;
int64_t size = 0;
taosStatFile(fobj->fname, &size, &mtime, NULL);
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
if (pCfg->s3Compact && lcn < 0) {
extern int32_t tsdbAsyncCompact(STsdb * tsdb, const STimeWindow *tw, bool sync);
STimeWindow win = {0};
tsdbFidKeyRange(fset->fid, rtner->tsdb->keepCfg.days, rtner->tsdb->keepCfg.precision, &win.skey, &win.ekey);
tsdbInfo("vgId:%d, compact begin lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn);
tsdbAsyncCompact(rtner->tsdb, &win, pCfg->sttTrigger == 1);
tsdbInfo("vgId:%d, compact end lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn);
return code;
}
code = tsdbMigrateDataFileS3(rtner, fobj, size, chunksize);
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
if (lcn <= 1) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return code;
}
char fname1[TSDB_FILENAME_LEN];
tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1);
if (taosCheckExistFile(fname1)) {
int32_t mtime = 0;
int64_t size = 0;
taosStatFile(fname1, &size, &mtime, NULL);
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
code = tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize);
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
tsdbError("vgId:%d, file: %s not found, %s at line %d", TD_VID(rtner->tsdb->pVnode), fname1, __func__, lino);
return code;
}
}
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbDoS3MigrateAsync(void *arg) {
int32_t code = 0;
int32_t lino = 0;
SRTNer rtner[1] = {0};
code = tsdbDoRetentionBegin(arg, rtner);
TSDB_CHECK_CODE(code, lino, _exit);
STFileSet *fset;
TARRAY2_FOREACH(rtner->fsetArr, fset) {
if (fset->fid != ((SRtnArg *)arg)->fid) continue;
code = tsdbDoS3MigrateOnFileSet(rtner, fset);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbDoRetentionEnd(rtner);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
if (TARRAY2_DATA(rtner->fopArr)) {
TARRAY2_DESTROY(rtner->fopArr, NULL);
}
TFileSetArray **fsetArr = &rtner->fsetArr;
if (fsetArr[0]) {
tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
}
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
int32_t code = 0;
if (!tsS3Enabled) {
return code;
}
taosThreadMutexLock(&tsdb->mutex);
if (tsdb->bgTaskDisabled) {
taosThreadMutexUnlock(&tsdb->mutex);
return 0;
}
STFileSet *fset;
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
code = tsdbTFileSetOpenChannel(fset);
if (code) {
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
if (arg == NULL) {
taosThreadMutexUnlock(&tsdb->mutex);
return TSDB_CODE_OUT_OF_MEMORY;
}
arg->tsdb = tsdb;
arg->now = now;
arg->fid = fset->fid;
if (sync) {
code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
tsdbFreeRtnArg, arg, NULL);
} else {
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
tsdbFreeRtnArg, arg, NULL);
}
if (code) {
tsdbFreeRtnArg(arg);
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
}
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}

6
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -49,12 +49,12 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
// open file
if (fname) {
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd);
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, config->file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -717,7 +717,7 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
char fname[TSDB_FILENAME_LEN];
tsdbTFileName(writer->config->tsdb, writer->file, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};

6
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -80,7 +80,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
char fname[TSDB_FILENAME_LEN];
tsdbTFileName(tsdb, &file, fname);
code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd);
code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// convert
@ -272,7 +272,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
code = tsdbTFileObjInit(tsdb, &file, &fobj);
TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd);
code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit1);
for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) {
@ -427,7 +427,7 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
}
char fname[TSDB_FILENAME_LEN] = {0};
code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd);
code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0);
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};

24
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -51,6 +51,9 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.hashEnd = 0,
.hashMethod = 0,
.sttTrigger = TSDB_DEFAULT_SST_TRIGGER,
.s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE,
.s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL,
.s3Compact = TSDB_DEFAULT_S3_COMPACT,
.tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE};
int vnodeCheckCfg(const SVnodeCfg *pCfg) {
@ -106,6 +109,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact) < 0) return -1;
if (pCfg->tsdbCfg.retentions[0].keep > 0) {
int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) {
@ -154,9 +160,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1;
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1;
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d",
pCfg->vgId, pCfg->syncCfg.replicaNum,
pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId,
pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) {
SJson *info = tjsonCreateObject();
SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i];
@ -317,6 +322,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024;
}
tjsonGetNumberValue(pJson, "s3ChunkSize", pCfg->s3ChunkSize, code);
if (code < 0) {
pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
}
tjsonGetNumberValue(pJson, "s3KeepLocal", pCfg->s3KeepLocal, code);
if (code < 0) {
pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
}
tjsonGetNumberValue(pJson, "s3Compact", pCfg->s3Compact, code);
if (code < 0) {
pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
}
return 0;
}

20
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -305,14 +305,14 @@ int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t s
return dstVgId;
}
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs) {
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) {
vInfo("path:%s is removed while destroy vnode", path);
tfsRmdir(pTfs, path);
int32_t nlevel = tfsGetLevel(pTfs);
if (vgId > 0 && nlevel > 1 && tsS3Enabled) {
// int32_t nlevel = tfsGetLevel(pTfs);
if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) {
char vnode_prefix[TSDB_FILENAME_LEN];
snprintf(vnode_prefix, TSDB_FILENAME_LEN, "v%df", vgId);
snprintf(vnode_prefix, TSDB_FILENAME_LEN, "%d/v%df", nodeId, vgId);
s3DeleteObjectsByPrefix(vnode_prefix);
}
}
@ -480,11 +480,11 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
vnodeRollback(pVnode);
}
snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId);
snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId);
snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%" PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId);
snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%" PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%" PRId32, pVnode->config.vgId);
if(tsEnableMonitor && pVnode->monitor.insertCounter == NULL){
if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) {
taos_counter_t *counter = NULL;
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
if(counter == NULL){
@ -499,11 +499,11 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
if(taos_collector_registry_register_metric(counter) == 1){
taos_counter_destroy(counter);
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter);
vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter);
}
}
pVnode->monitor.insertCounter = counter;
vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter);
vInfo("vgId:%d, succeed to set metric:%p", TD_VID(pVnode), counter);
}
return pVnode;

10
source/dnode/vnode/src/vnd/vnodeRetention.c
View File

@ -23,4 +23,12 @@ int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) {
if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now);
return code;
}
}
int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) {
int32_t code = TSDB_CODE_SUCCESS;
code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
return code;
}

38
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -38,6 +38,7 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pR
static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
SRpcMsg *pOriginalMsg);
static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
@ -565,6 +566,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
case TDMT_VND_TRIM:
if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_S3MIGRATE:
if (vnodeProcessS3MigrateReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_CREATE_SMA:
if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
@ -875,6 +879,26 @@ _exit:
return code;
}
extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now);
static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
SVS3MigrateDbReq s3migrateReq = {0};
// decode
if (tDeserializeSVS3MigrateDbReq(pReq, len, &s3migrateReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp);
code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp);
_exit:
return code;
}
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVDropTtlTableReq ttlReq = {0};
if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) {
@ -1893,11 +1917,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
}
vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
" cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d fsync:%d level:%d "
" cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d s3KeepLocal:%d "
"s3Compact:%d fsync:%d level:%d "
"walRetentionPeriod:%d walRetentionSize:%d",
TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024,
req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.keepTimeOffset, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize);
req.keepTimeOffset, req.s3KeepLocal, req.s3Compact, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod,
req.walRetentionSize);
if (pVnode->config.cacheLastSize != req.cacheLastSize) {
pVnode->config.cacheLastSize = req.cacheLastSize;
@ -1981,6 +2007,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
pVnode->config.tsdbCfg.minRows = req.minRows;
}
if (req.s3KeepLocal != -1 && req.s3KeepLocal != pVnode->config.s3KeepLocal) {
pVnode->config.s3KeepLocal = req.s3KeepLocal;
}
if (req.s3Compact != -1 && req.s3Compact != pVnode->config.s3Compact) {
pVnode->config.s3Compact = req.s3Compact;
}
if (walChanged) {
walAlter(pVnode->pWal, &pVnode->config.walCfg);
}
@ -2263,4 +2296,5 @@ _OVER:
int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
return 0;
}
int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync);
#endif

10
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -820,6 +820,16 @@ bool vnodeIsLeader(SVnode *pVnode) {
return true;
}
int64_t vnodeClusterId(SVnode *pVnode) {
SSyncCfg *syncCfg = &pVnode->config.syncCfg;
return syncCfg->nodeInfo[syncCfg->myIndex].clusterId;
}
int32_t vnodeNodeId(SVnode *pVnode) {
SSyncCfg *syncCfg = &pVnode->config.syncCfg;
return syncCfg->nodeInfo[syncCfg->myIndex].nodeId;
}
int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) {
int code = 0;
pSnap->lastApplyIndex = pVnode->state.committed;

4
source/libs/command/src/command.c
View File

@ -313,12 +313,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d",
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d",
dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize,
pCfg->keepTimeOffset);
pCfg->keepTimeOffset, pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact);
if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);

72
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -109,6 +109,8 @@ const char* nodesNodeName(ENodeType type) {
return "FlushDatabaseStmt";
case QUERY_NODE_TRIM_DATABASE_STMT:
return "TrimDatabaseStmt";
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
return "S3MigrateDatabaseStmt";
case QUERY_NODE_CREATE_TABLE_STMT:
return "CreateTableStmt";
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
@ -823,7 +825,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
static const char* jkProjectLogicPlanProjections = "Projections";
static const char* jkProjectLogicPlanIgnoreGroupId = "IgnoreGroupId";
static const char* jkProjectLogicPlanInputIgnoreGroup= "InputIgnoreGroup";
static const char* jkProjectLogicPlanInputIgnoreGroup = "InputIgnoreGroup";
static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj;
@ -1339,7 +1341,6 @@ static int32_t jsonToLogicDynQueryCtrlNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkSubplanIdQueryId = "QueryId";
static const char* jkSubplanIdGroupId = "GroupId";
static const char* jkSubplanIdSubplanId = "SubplanId";
@ -1846,8 +1847,6 @@ static int32_t jsonToFuncType(const SJson* pJson, void* pObj) {
return code;
}
static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj;
@ -1861,7 +1860,7 @@ static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkLastRowScanPhysiPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddTArray(pJson, jkLastRowScanPhysiPlanFuncTypes, funcTypeToJson, pNode->pFuncTypes);
}
@ -2255,7 +2254,6 @@ static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
SHashJoinPhysiNode* pNode = (SHashJoinPhysiNode*)pObj;
@ -2290,7 +2288,6 @@ static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkAggPhysiPlanExprs = "Exprs";
static const char* jkAggPhysiPlanGroupKeys = "GroupKeys";
static const char* jkAggPhysiPlanAggFuncs = "AggFuncs";
@ -2504,7 +2501,7 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) {
code = tjsonGetBoolValue(pJson, jkSortPhysiPlanCalcGroupIds, &pNode->calcGroupId);
}
if (TSDB_CODE_SUCCESS == code) {
code= tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol);
code = tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol);
}
return code;
@ -3187,7 +3184,6 @@ static const char* jkGroupCachePhysiPlanGroupByUid = "GroupByUid";
static const char* jkGroupCachePhysiPlanGlobalGroup = "GlobalGroup";
static const char* jkGroupCachePhysiPlanBatchFetch = "BatchFetch";
static int32_t physiGroupCacheNodeToJson(const void* pObj, SJson* pJson) {
const SGroupCachePhysiNode* pNode = (const SGroupCachePhysiNode*)pObj;
@ -3321,8 +3317,6 @@ static int32_t jsonToPhysiDynQueryCtrlNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkQueryNodeAddrId = "Id";
static const char* jkQueryNodeAddrInUse = "InUse";
static const char* jkQueryNodeAddrNumOfEps = "NumOfEps";
@ -4855,6 +4849,10 @@ static const char* jkDatabaseOptionsNumOfVgroups = "NumOfVgroups";
static const char* jkDatabaseOptionsSingleStable = "SingleStable";
static const char* jkDatabaseOptionsRetentions = "Retentions";
static const char* jkDatabaseOptionsSchemaless = "Schemaless";
static const char* jkDatabaseOptionsS3ChunkSize = "S3ChunkSize";
static const char* jkDatabaseOptionsS3KeepLocalNode = "S3KeepLocalNode";
static const char* jkDatabaseOptionsS3KeepLocal = "S3KeepLocal";
static const char* jkDatabaseOptionsS3Compact = "S3Compact";
static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
const SDatabaseOptions* pNode = (const SDatabaseOptions*)pObj;
@ -4914,6 +4912,18 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsSchemaless, pNode->schemaless);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3ChunkSize, pNode->s3ChunkSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkDatabaseOptionsS3KeepLocalNode, nodeToJson, pNode->s3KeepLocalStr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3KeepLocal, pNode->s3KeepLocal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3Compact, pNode->s3Compact);
}
return code;
}
@ -4976,6 +4986,18 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsSchemaless, &pNode->schemaless);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkDatabaseOptionsS3ChunkSize, &pNode->s3ChunkSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkDatabaseOptionsS3KeepLocalNode, (SNode**)&pNode->s3KeepLocalStr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkDatabaseOptionsS3KeepLocal, &pNode->s3KeepLocal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsS3Compact, &pNode->s3Compact);
}
return code;
}
@ -5564,6 +5586,24 @@ static int32_t jsonToTrimDatabaseStmt(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkS3MigrateDatabaseStmtDbName = "DbName";
static int32_t s3migrateDatabaseStmtToJson(const void* pObj, SJson* pJson) {
const SS3MigrateDatabaseStmt* pNode = (const SS3MigrateDatabaseStmt*)pObj;
int32_t code = tjsonAddStringToObject(pJson, jkS3MigrateDatabaseStmtDbName, pNode->dbName);
return code;
}
static int32_t jsonToS3MigrateDatabaseStmt(const SJson* pJson, void* pObj) {
SS3MigrateDatabaseStmt* pNode = (SS3MigrateDatabaseStmt*)pObj;
int32_t code = tjsonGetStringValue(pJson, jkS3MigrateDatabaseStmtDbName, pNode->dbName);
return code;
}
static const char* jkCreateTableStmtDbName = "DbName";
static const char* jkCreateTableStmtTableName = "TableName";
static const char* jkCreateTableStmtIgnoreExists = "IgnoreExists";
@ -6233,10 +6273,6 @@ static int32_t jsonToRestoreVnodeStmt(const SJson* pJson, void* pObj) {
return jsonToRestoreComponentNodeStmt(pJson, pObj);
}
static const char* jkCreateTopicStmtTopicName = "TopicName";
static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName";
static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists";
@ -6892,7 +6928,6 @@ static int32_t showCreateViewStmtToJson(const void* pObj, SJson* pJson) {
return code;
}
static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) {
SShowCreateViewStmt* pNode = (SShowCreateViewStmt*)pObj;
@ -6904,7 +6939,6 @@ static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkShowTableDistributedStmtDbName = "DbName";
static const char* jkShowTableDistributedStmtTableName = "TableName";
@ -7150,6 +7184,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return alterDatabaseStmtToJson(pObj, pJson);
case QUERY_NODE_TRIM_DATABASE_STMT:
return trimDatabaseStmtToJson(pObj, pJson);
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
return s3migrateDatabaseStmtToJson(pObj, pJson);
case QUERY_NODE_CREATE_TABLE_STMT:
return createTableStmtToJson(pObj, pJson);
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
@ -7489,6 +7525,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToAlterDatabaseStmt(pJson, pObj);
case QUERY_NODE_TRIM_DATABASE_STMT:
return jsonToTrimDatabaseStmt(pJson, pObj);
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
return jsonToS3MigrateDatabaseStmt(pJson, pObj);
case QUERY_NODE_CREATE_TABLE_STMT:
return jsonToCreateTableStmt(pJson, pObj);
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:

5
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -324,6 +324,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SFlushDatabaseStmt));
case QUERY_NODE_TRIM_DATABASE_STMT:
return makeNode(type, sizeof(STrimDatabaseStmt));
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
return makeNode(type, sizeof(SS3MigrateDatabaseStmt));
case QUERY_NODE_CREATE_TABLE_STMT:
return makeNode(type, sizeof(SCreateTableStmt));
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
@ -803,6 +805,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_DATABASE_OPTIONS: {
SDatabaseOptions* pOptions = (SDatabaseOptions*)pNode;
nodesDestroyNode((SNode*)pOptions->pDaysPerFile);
nodesDestroyNode((SNode*)pOptions->s3KeepLocalStr);
nodesDestroyList(pOptions->pKeep);
nodesDestroyList(pOptions->pRetentions);
break;
@ -940,6 +943,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field
case QUERY_NODE_TRIM_DATABASE_STMT: // no pointer field
break;
case QUERY_NODE_S3MIGRATE_DATABASE_STMT: // no pointer field
break;
case QUERY_NODE_CREATE_TABLE_STMT: {
SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode;
nodesDestroyList(pStmt->pCols);

71
source/libs/parser/inc/parAst.h
View File

@ -64,6 +64,9 @@ typedef enum EDatabaseOptionType {
DB_OPTION_STT_TRIGGER,
DB_OPTION_TABLE_PREFIX,
DB_OPTION_TABLE_SUFFIX,
DB_OPTION_S3_CHUNKSIZE,
DB_OPTION_S3_KEEPLOCAL,
DB_OPTION_S3_COMPACT,
DB_OPTION_KEEP_TIME_OFFSET
} EDatabaseOptionType;
@ -90,7 +93,7 @@ typedef struct STokenPair {
typedef struct SShowTablesOption {
EShowKind kind;
SToken dbName;
SToken dbName;
} SShowTablesOption;
extern SToken nil_token;
@ -111,37 +114,37 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pNode);
SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pLeft, SNode *pRight);
SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere);
SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList);
@ -154,7 +157,8 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange);
SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable,
SNodeList* pHint);
SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
@ -170,6 +174,7 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STo
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed);
SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createAlterTableOptions(SAstCreateContext* pCxt);

8
source/libs/parser/inc/sql.y
View File

@ -208,6 +208,7 @@ cmd ::= USE db_name(A).
cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); }
cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); }
cmd ::= S3MIGRATE DATABASE db_name(A). { pCxt->pRootNode = createS3MigrateDatabaseStmt(pCxt, &A); }
cmd ::= COMPACT DATABASE db_name(A) start_opt(B) end_opt(C). { pCxt->pRootNode = createCompactStmt(pCxt, &A, B, C); }
%type not_exists_opt { bool }
@ -260,6 +261,10 @@ db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C).
db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
db_options(A) ::= db_options(B) S3_CHUNKSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_CHUNKSIZE, &C); }
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); }
db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); }
alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
@ -291,6 +296,9 @@ alter_db_option(A) ::= WAL_RETENTION_SIZE NK_MINUS(B) NK_INTEGER(C).
t.n = (C.z + C.n) - B.z;
A.type = DB_OPTION_WAL_RETENTION_SIZE; A.val = t;
}
alter_db_option(A) ::= S3_KEEPLOCAL NK_INTEGER(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; }
alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; }
alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; }
alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; }
%type integer_list { SNodeList* }

38
source/libs/parser/src/parAstCreater.c
View File

@ -228,7 +228,6 @@ static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) {
return true;
}
static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) {
trimEscape(pStreamName);
if (pStreamName->n >= TSDB_STREAM_NAME_LEN) {
@ -918,7 +917,6 @@ SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewNam
return (SNode*)pView;
}
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) {
CHECK_PARSER_STATUS(pCxt);
SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
@ -1263,6 +1261,9 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
pOptions->tablePrefix = TSDB_DEFAULT_HASH_PREFIX;
pOptions->tableSuffix = TSDB_DEFAULT_HASH_SUFFIX;
pOptions->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pOptions->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pOptions->s3Compact = TSDB_DEFAULT_S3_COMPACT;
pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
return (SNode*)pOptions;
}
@ -1299,6 +1300,9 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->sstTrigger = -1;
pOptions->tablePrefix = -1;
pOptions->tableSuffix = -1;
pOptions->s3ChunkSize = -1;
pOptions->s3KeepLocal = -1;
pOptions->s3Compact = -1;
pOptions->withArbitrator = -1;
return (SNode*)pOptions;
}
@ -1415,6 +1419,21 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
nodesDestroyNode((SNode*)pNode);
break;
}
case DB_OPTION_S3_CHUNKSIZE:
pDbOptions->s3ChunkSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_S3_KEEPLOCAL: {
SToken* pToken = pVal;
if (TK_NK_INTEGER == pToken->type) {
pDbOptions->s3KeepLocal = taosStr2Int32(pToken->z, NULL, 10) * 1440;
} else {
pDbOptions->s3KeepLocalStr = (SValueNode*)createDurationValueNode(pCxt, pToken);
}
break;
}
case DB_OPTION_S3_COMPACT:
pDbOptions->s3Compact = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_KEEP_TIME_OFFSET: {
pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
@ -1501,6 +1520,17 @@ SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t
return (SNode*)pStmt;
}
SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false)) {
return NULL;
}
SS3MigrateDatabaseStmt* pStmt = (SS3MigrateDatabaseStmt*)nodesMakeNode(QUERY_NODE_S3MIGRATE_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
return (SNode*)pStmt;
}
SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false)) {
@ -1811,7 +1841,8 @@ SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pD
return (SNode*)pStmt;
}
SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, EOperatorType tableCondType) {
SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName,
EOperatorType tableCondType) {
CHECK_PARSER_STATUS(pCxt);
SNode* pDbName = NULL;
if (option.dbName.type == TK_NK_NIL) {
@ -1887,7 +1918,6 @@ SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode*
return (SNode*)pStmt;
}
SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable) {
CHECK_PARSER_STATUS(pCxt);
SShowTableDistributedStmt* pStmt = (SShowTableDistributedStmt*)nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT);

4
source/libs/parser/src/parTokenizer.c
View File

@ -305,6 +305,10 @@ static SKeyword keywordTable[] = {
{"_WSTART", TK_WSTART},
{"ALIVE", TK_ALIVE},
{"VARBINARY", TK_VARBINARY},
{"S3_CHUNKSIZE", TK_S3_CHUNKSIZE},
{"S3_KEEPLOCAL", TK_S3_KEEPLOCAL},
{"S3_COMPACT", TK_S3_COMPACT},
{"S3MIGRATE", TK_S3MIGRATE},
{"KEEP_TIME_OFFSET", TK_KEEP_TIME_OFFSET},
{"ARBGROUPS", TK_ARBGROUPS},
};

393
source/libs/parser/src/parTranslater.c
View File

@ -14,8 +14,8 @@
*/
#include "parTranslater.h"
#include "tdatablock.h"
#include "parInt.h"
#include "tdatablock.h"
#include "catalog.h"
#include "cmdnodes.h"
@ -381,7 +381,8 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pTable) {
}
#ifdef BUILD_NO_CALL
static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName,
STableMeta** pMeta) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
#endif
@ -406,7 +407,8 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac
}
#endif
int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) {
int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta,
bool couldBeView) {
int32_t code = TSDB_CODE_SUCCESS;
if (pParCxt->async) {
@ -1134,7 +1136,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
SExprNode* pExpr = (SExprNode*)pNode;
if (0 == strcmp((*pCol)->colName, pExpr->userAlias)) {
if (true == *pFound) {
if(nodesEqualNode(pFoundNode, pNode)) {
if (nodesEqualNode(pFoundNode, pNode)) {
continue;
}
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ORDERBY_AMBIGUOUS, (*pCol)->colName);
@ -1213,7 +1215,8 @@ static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) {
}
}
static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode, SListCell* pSelectListCell) {
static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode,
SListCell* pSelectListCell) {
SNodeList* pTbnameNodeList = nodesMakeList();
SFunctionNode* pFunc = (SFunctionNode*)pNode;
@ -1238,7 +1241,7 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt
nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList);
}
} else if (nodesIsTableStar(pPara)) {
char* pTableAlias = ((SColumnNode*)pPara)->tableAlias;
char* pTableAlias = ((SColumnNode*)pPara)->tableAlias;
STableNode* pTable = NULL;
int32_t code = findTable(pCxt, pTableAlias, &pTable);
if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE &&
@ -1259,12 +1262,12 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt
// after translate from
// before translate select list
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
SNode* pNode = NULL;
SNode* pNode = NULL;
SNodeList* pTbnameNodeList = nodesMakeList();
WHERE_EACH(pNode, pSelect->pProjectionList) {
if (nodesIsStar(pNode)) {
SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
size_t n = taosArrayGetSize(pTables);
size_t n = taosArrayGetSize(pTables);
for (int32_t i = 0; i < n; ++i) {
STableNode* pTable = taosArrayGetP(pTables, i);
if (nodeType(pTable) == QUERY_NODE_REAL_TABLE &&
@ -1278,7 +1281,7 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList);
}
} else if (nodesIsTableStar(pNode)) {
char* pTableAlias = ((SColumnNode*)pNode)->tableAlias;
char* pTableAlias = ((SColumnNode*)pNode)->tableAlias;
STableNode* pTable = NULL;
int32_t code = findTable(pCxt, pTableAlias, &pTable);
if (TSDB_CODE_SUCCESS == code &&
@ -1294,7 +1297,7 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
} else if (nodeType(pNode) == QUERY_NODE_FUNCTION) {
biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell);
}
WHERE_NEXT;
WHERE_NEXT;
}
return TSDB_CODE_SUCCESS;
@ -1302,11 +1305,11 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) {
SColumnNode* pCol = (SColumnNode*)(*ppNode);
if ((strcasecmp(pCol->colName, "tbname") == 0) &&
((SSelectStmt*)pCxt->pCurrStmt)->pFromTable &&
QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
if ((strcasecmp(pCol->colName, "tbname") == 0) && ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable &&
QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
SFunctionNode* tbnameFuncNode = NULL;
tbnameFuncNode = (SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0]!='\0') ? pCol->tableAlias : NULL);
tbnameFuncNode =
(SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0] != '\0') ? pCol->tableAlias : NULL);
tbnameFuncNode->node.resType = pCol->node.resType;
strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName);
strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias);
@ -1321,24 +1324,26 @@ bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) {
int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
if (pStmt->pTags) {
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pTags) {
SColumnDefNode* pTag = (SColumnDefNode*)pNode;
if (strcasecmp(pTag->colName, "tbname") == 0) {
int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, "tbname can not used for tags in BI mode");
return code;
}
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pTags) {
SColumnDefNode* pTag = (SColumnDefNode*)pNode;
if (strcasecmp(pTag->colName, "tbname") == 0) {
int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME,
"tbname can not used for tags in BI mode");
return code;
}
}
}
if (pStmt->pCols) {
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pCols) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if (strcasecmp(pCol->colName, "tbname") == 0) {
int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "tbname can not used for columns in BI mode");
return code;
}
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pCols) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if (strcasecmp(pCol->colName, "tbname") == 0) {
int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN,
"tbname can not used for columns in BI mode");
return code;
}
}
}
return TSDB_CODE_SUCCESS;
}
@ -1376,9 +1381,9 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
res = translateColumnWithoutPrefix(pCxt, pCol);
}
}
if(SQL_CLAUSE_ORDER_BY == pCxt->currClause && !(*pCol)->node.asParam
&& res != DEAL_RES_CONTINUE && res != DEAL_RES_END) {
res = translateColumnUseAlias(pCxt, pCol, &found);
if (SQL_CLAUSE_ORDER_BY == pCxt->currClause && !(*pCol)->node.asParam && res != DEAL_RES_CONTINUE &&
res != DEAL_RES_END) {
res = translateColumnUseAlias(pCxt, pCol, &found);
}
}
return res;
@ -1960,8 +1965,8 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode**
}
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SNode* pNode = NULL;
bool bFound = false;
SNode* pNode = NULL;
bool bFound = false;
FOREACH(pNode, pSelect->pProjectionList) {
if (nodeType(pNode) == QUERY_NODE_FUNCTION && strcasecmp(((SFunctionNode*)pNode)->functionName, "interp") == 0) {
bFound = true;
@ -2266,7 +2271,7 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode
return TSDB_CODE_OUT_OF_MEMORY;
}
SExprNode* pOldExpr = (SExprNode*)(*ppNode);
//rewrite a.tbname == tbname(a)
// rewrite a.tbname == tbname(a)
if (nodeType(*ppNode) == QUERY_NODE_FUNCTION && ((SFunctionNode*)(*ppNode))->funcType == FUNCTION_TYPE_TBNAME) {
SFunctionNode* pFunc = (SFunctionNode*)(*ppNode);
if (0 != LIST_LENGTH(pFunc->pParameterList)) {
@ -2327,7 +2332,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode**
static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) {
SFunctionNode* pFunc = (SFunctionNode*)(*ppNode);
if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) {
if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS;
}
if (0 == LIST_LENGTH(pFunc->pParameterList)) {
@ -2355,7 +2360,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp
static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) {
SFunctionNode* pFunc = (SFunctionNode*)(*ppNode);
int32_t code = translateAggFunc(pCxt, pFunc);
int32_t code = translateAggFunc(pCxt, pFunc);
if (TSDB_CODE_SUCCESS == code) {
bool bRewriteToColumn = false;
code = translateScanPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn);
@ -2843,18 +2848,17 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) {
}
static int32_t checkIsEmptyResult(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (pSelect->timeRange.skey > pSelect->timeRange.ekey
&& !pSelect->hasCountFunc) {
if (pSelect->timeRange.skey > pSelect->timeRange.ekey && !pSelect->hasCountFunc) {
pSelect->isEmptyResult = true;
}
return TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
static int32_t resetSelectFuncNumWithoutDup(SSelectStmt* pSelect) {
if (pSelect->selectFuncNum <= 1) return TSDB_CODE_SUCCESS;
pSelect->selectFuncNum = 0;
SNodeList* pNodeList = nodesMakeList();
int32_t code = nodesCollectSelectFuncs(pSelect, SQL_CLAUSE_FROM, NULL, fmIsSelectFunc, pNodeList);
int32_t code = nodesCollectSelectFuncs(pSelect, SQL_CLAUSE_FROM, NULL, fmIsSelectFunc, pNodeList);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pNodeList);
return code;
@ -3931,57 +3935,57 @@ static void convertVarDuration(SValueNode* pOffset, uint8_t precision) {
pOffset->unit = units[precision];
}
static const int64_t tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP;
static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) {
uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision;
static const int64_t tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP;
static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) {
uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision;
SValueNode* pInter = (SValueNode*)pInterval->pInterval;
bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit);
if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime,
getPrecisionStr(precision));
SValueNode* pInter = (SValueNode*)pInterval->pInterval;
bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit);
if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime,
getPrecisionStr(precision));
} else if (pInter->datum.i / getPrecisionMultiple(precision) > tsdbMaxKeepMS) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years");
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years");
}
if (NULL != pInterval->pOffset) {
SValueNode* pOffset = (SValueNode*)pInterval->pOffset;
if (pOffset->datum.i <= 0) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE);
if (NULL != pInterval->pOffset) {
SValueNode* pOffset = (SValueNode*)pInterval->pOffset;
if (pOffset->datum.i <= 0) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE);
}
if (pInter->unit == 'n' && pOffset->unit == 'y') {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT);
if (pInter->unit == 'n' && pOffset->unit == 'y') {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT);
}
bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter;
if ((fixed && pOffset->datum.i >= pInter->datum.i) ||
bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter;
if ((fixed && pOffset->datum.i >= pInter->datum.i) ||
(!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >=
getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG);
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG);
}
if (pOffset->unit == 'n' || pOffset->unit == 'y') {
convertVarDuration(pOffset, precision);
if (pOffset->unit == 'n' || pOffset->unit == 'y') {
convertVarDuration(pOffset, precision);
}
}
if (NULL != pInterval->pSliding) {
const static int32_t INTERVAL_SLIDING_FACTOR = 100;
if (NULL != pInterval->pSliding) {
const static int32_t INTERVAL_SLIDING_FACTOR = 100;
SValueNode* pSliding = (SValueNode*)pInterval->pSliding;
if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT);
SValueNode* pSliding = (SValueNode*)pInterval->pSliding;
if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT);
}
if ((pSliding->datum.i <
if ((pSliding->datum.i <
convertTimeFromPrecisionToUnit(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pSliding->unit)) ||
(pInter->datum.i / pSliding->datum.i > INTERVAL_SLIDING_FACTOR)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL);
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL);
}
if (pSliding->datum.i > pInter->datum.i) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG);
if (pSliding->datum.i > pInter->datum.i) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG);
}
}
return TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
static int32_t translateIntervalWindow(STranslateContext* pCxt, SSelectStmt* pSelect) {
@ -4122,11 +4126,11 @@ static EDealRes collectWindowsPseudocolumns(SNode* pNode, void* pContext) {
}
static int32_t checkWindowsConditonValid(SNode* pNode) {
int32_t code = TSDB_CODE_SUCCESS;
if(QUERY_NODE_EVENT_WINDOW != nodeType(pNode)) return code;
int32_t code = TSDB_CODE_SUCCESS;
if (QUERY_NODE_EVENT_WINDOW != nodeType(pNode)) return code;
SEventWindowNode* pEventWindowNode = (SEventWindowNode*)pNode;
SNodeList* pCols = nodesMakeList();
SNodeList* pCols = nodesMakeList();
if (NULL == pCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -4336,14 +4340,15 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec
typedef struct SEqCondTbNameTableInfo {
SRealTableNode* pRealTable;
SArray* aTbnames;
SArray* aTbnames;
} SEqCondTbNameTableInfo;
//[tableAlias.]tbname = tbNamVal
static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray** ppTabNames) {
static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias,
SArray** ppTabNames) {
if (pOperator->opType != OP_TYPE_EQUAL) return false;
SFunctionNode* pTbnameFunc = NULL;
SValueNode* pValueNode = NULL;
SValueNode* pValueNode = NULL;
if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION &&
((SFunctionNode*)(pOperator->pLeft))->funcType == FUNCTION_TYPE_TBNAME &&
nodeType(pOperator->pRight) == QUERY_NODE_VALUE) {
@ -4374,7 +4379,8 @@ static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOper
}
//[tableAlias.]tbname in (value1, value2, ...)
static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray** ppTbNames) {
static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias,
SArray** ppTbNames) {
if (pOperator->opType != OP_TYPE_IN) return false;
if (nodeType(pOperator->pLeft) != QUERY_NODE_FUNCTION ||
((SFunctionNode*)(pOperator->pLeft))->funcType != FUNCTION_TYPE_TBNAME ||
@ -4395,8 +4401,8 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper
}
*ppTbNames = taosArrayInit(1, sizeof(void*));
SNodeListNode* pValueListNode = (SNodeListNode*)pOperator->pRight;
SNodeList* pValueNodeList = pValueListNode->pNodeList;
SNode* pValNode = NULL;
SNodeList* pValueNodeList = pValueListNode->pNodeList;
SNode* pValNode = NULL;
FOREACH(pValNode, pValueNodeList) {
if (nodeType(pValNode) != QUERY_NODE_VALUE) {
return false;
@ -4404,13 +4410,12 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper
taosArrayPush(*ppTbNames, &((SValueNode*)pValNode)->literal);
}
return true;
}
static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo) {
int32_t code = TSDB_CODE_SUCCESS;
char* pTableAlias = NULL;
char* pTbNameVal = NULL;
char* pTableAlias = NULL;
char* pTbNameVal = NULL;
if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames) ||
isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames)) {
STableNode* pTable;
@ -4445,17 +4450,17 @@ static void findEqualCondTbnameInLogicCondAnd(STranslateContext* pCxt, SNode* pW
FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) {
if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) {
SEqCondTbNameTableInfo info = {0};
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info);
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info);
if (bIsEqTbnameCond) {
if (!isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) {
//TODO: intersect tbNames of same table? speed
// TODO: intersect tbNames of same table? speed
taosArrayPush(aTableTbnames, &info);
} else {
taosArrayDestroy(info.aTbnames);
}
}
}
//TODO: logic cond
// TODO: logic cond
}
}
@ -4480,10 +4485,10 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh
bool bAllTbName = true;
SNode* pTmpNode = NULL;
FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) {
//TODO: logic cond
// TODO: logic cond
if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) {
SEqCondTbNameTableInfo info = {0};
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info);
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info);
if (!bIsEqTbnameCond) {
bAllTbName = false;
break;
@ -4506,10 +4511,10 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh
}
static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) {
//TODO: optimize nested and/or condition. now only the fist level is processed.
// TODO: optimize nested and/or condition. now only the fist level is processed.
if (nodeType(pWhere) == QUERY_NODE_OPERATOR) {
SEqCondTbNameTableInfo info = {0};
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info);
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info);
if (bIsEqTbnameCond) {
taosArrayPush(aTableTbnames, &info);
}
@ -4523,7 +4528,8 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra
return TSDB_CODE_SUCCESS;
}
static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo, SVgroupsInfo* vgsInfo) {
static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo,
SVgroupsInfo* vgsInfo) {
int32_t nVgroups = 0;
int32_t nTbls = taosArrayGetSize(pInfo->aTbnames);
@ -4565,10 +4571,10 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt*
int32_t code = TSDB_CODE_SUCCESS;
for (int i = 0; i < taosArrayGetSize(aTables); ++i) {
SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i);
int32_t nTbls = taosArrayGetSize(pInfo->aTbnames);
int32_t nTbls = taosArrayGetSize(pInfo->aTbnames);
SVgroupsInfo* vgsInfo = taosMemoryMalloc(sizeof(SVgroupsInfo) + nTbls * sizeof(SVgroupInfo));
int32_t nVgroups = 0;
int32_t nVgroups = 0;
findVgroupsFromEqualTbname(pCxt, pInfo, vgsInfo);
if (vgsInfo->numOfVgroups != 0) {
taosMemoryFree(pInfo->pRealTable->pVgroupList);
@ -5161,7 +5167,6 @@ static int32_t translateInsertTable(STranslateContext* pCxt, SNode** pTable) {
if (TSDB_CODE_SUCCESS == code && TSDB_CHILD_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType &&
TSDB_NORMAL_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType) {
code = buildInvalidOperationMsg(&pCxt->msgBuf, "insert data into super table is not supported");
}
return code;
}
@ -5257,6 +5262,9 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
pReq->hashSuffix = pStmt->pOptions->tableSuffix;
pReq->tsdbPageSize = pStmt->pOptions->tsdbPageSize;
pReq->keepTimeOffset = pStmt->pOptions->keepTimeOffset;
pReq->s3ChunkSize = pStmt->pOptions->s3ChunkSize;
pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal;
pReq->s3Compact = pStmt->pOptions->s3Compact;
pReq->ignoreExist = pStmt->ignoreExists;
pReq->withArbitrator = pStmt->pOptions->withArbitrator;
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
@ -5282,6 +5290,22 @@ static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName,
return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal);
}
static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if (NULL != pOptions->s3KeepLocalStr) {
if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->s3KeepLocalStr)) {
return pCxt->errCode;
}
if (TIME_UNIT_MINUTE != pOptions->s3KeepLocalStr->unit && TIME_UNIT_HOUR != pOptions->s3KeepLocalStr->unit &&
TIME_UNIT_DAY != pOptions->s3KeepLocalStr->unit) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option s3_keeplocal unit: %c, only %c, %c, %c allowed",
pOptions->s3KeepLocalStr->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY);
}
pOptions->s3KeepLocal = getBigintFromValueNode(pOptions->s3KeepLocalStr);
}
return checkDbRangeOption(pCxt, "s3KeepLocal", pOptions->s3KeepLocal, TSDB_MIN_S3_KEEP_LOCAL, TSDB_MAX_S3_KEEP_LOCAL);
}
static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if (NULL != pOptions->pDaysPerFile) {
if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pDaysPerFile)) {
@ -5544,6 +5568,7 @@ static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbP
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
int32_t daysPerFile = pOptions->daysPerFile;
int32_t s3KeepLocal = pOptions->s3KeepLocal;
int64_t daysToKeep0 = pOptions->keep[0];
if (-1 == daysPerFile && -1 == daysToKeep0) {
return TSDB_CODE_SUCCESS;
@ -5560,6 +5585,10 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid duration value, should be keep2 >= keep1 >= keep0 >= 3 * duration");
}
if (s3KeepLocal > 0 && daysPerFile > s3KeepLocal / 3) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid parameters, should be s3_keeplocal >= 3 * duration");
}
if ((pOptions->replica == 2) ^ (pOptions->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
@ -5664,9 +5693,19 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
if (TSDB_CODE_SUCCESS == code) {
code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbS3KeepLocalOption(pCxt, pOptions);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkOptionsDependency(pCxt, pDbName, pOptions);
}
if (TSDB_CODE_SUCCESS == code) {
code =
checkDbRangeOption(pCxt, "s3_chunksize", pOptions->s3ChunkSize, TSDB_MIN_S3_CHUNK_SIZE, TSDB_MAX_S3_CHUNK_SIZE);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT);
}
return code;
}
@ -5679,18 +5718,18 @@ static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt*
}
#define FILL_CMD_SQL(sql, sqlLen, pCmdReq, CMD_TYPE, genericCmd) \
CMD_TYPE* pCmdReq = genericCmd; \
char* cmdSql = taosMemoryMalloc(sqlLen); \
if (cmdSql == NULL) { \
return TSDB_CODE_OUT_OF_MEMORY; \
} \
memcpy(cmdSql, sql, sqlLen); \
pCmdReq->sqlLen = sqlLen; \
pCmdReq->sql = cmdSql; \
CMD_TYPE* pCmdReq = genericCmd; \
char* cmdSql = taosMemoryMalloc(sqlLen); \
if (cmdSql == NULL) { \
return TSDB_CODE_OUT_OF_MEMORY; \
} \
memcpy(cmdSql, sql, sqlLen); \
pCmdReq->sqlLen = sqlLen; \
pCmdReq->sql = cmdSql;
static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq) {
const char* sql = pCxt->pParseCxt->pSql;
size_t sqlLen = pCxt->pParseCxt->sqlLen;
size_t sqlLen = pCxt->pParseCxt->sqlLen;
switch (msgType) {
case TDMT_MND_CREATE_DB: {
@ -5800,7 +5839,6 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq)
default: {
break;
}
}
return TSDB_CODE_SUCCESS;
@ -5873,6 +5911,8 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
pReq->minRows = pStmt->pOptions->minRowsPerBlock;
pReq->walRetentionPeriod = pStmt->pOptions->walRetentionPeriod;
pReq->walRetentionSize = pStmt->pOptions->walRetentionSize;
pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal;
pReq->s3Compact = pStmt->pOptions->s3Compact;
pReq->withArbitrator = pStmt->pOptions->withArbitrator;
return;
}
@ -5899,6 +5939,14 @@ static int32_t translateTrimDatabase(STranslateContext* pCxt, STrimDatabaseStmt*
return buildCmdMsg(pCxt, TDMT_MND_TRIM_DB, (FSerializeFunc)tSerializeSTrimDbReq, &req);
}
static int32_t translateS3MigrateDatabase(STranslateContext* pCxt, SS3MigrateDatabaseStmt* pStmt) {
SS3MigrateDbReq req = {0};
SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, req.db);
return buildCmdMsg(pCxt, TDMT_MND_S3MIGRATE_DB, (FSerializeFunc)tSerializeSS3MigrateDbReq, &req);
}
static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calBytes) {
*pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
SNode* pNode;
@ -5950,7 +5998,7 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
}
static bool validRollupFunc(const char* pFunc) {
static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"};
static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"};
static const int32_t numOfRollupFuncs = (sizeof(rollupFuncs) / sizeof(char*));
for (int i = 0; i < numOfRollupFuncs; ++i) {
if (0 == strcmp(rollupFuncs[i], pFunc)) {
@ -6258,7 +6306,6 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions
return code;
}
static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) {
if (NULL != strchr(pStmt->tableName, '.')) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME,
@ -7357,8 +7404,8 @@ static int16_t getCreateComponentNodeMsgType(ENodeType type) {
static int32_t translateCreateComponentNode(STranslateContext* pCxt, SCreateComponentNodeStmt* pStmt) {
SMCreateQnodeReq createReq = {.dnodeId = pStmt->dnodeId};
int32_t code = buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)),
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq);
int32_t code = buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)),
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq);
tFreeSMCreateQnodeReq(&createReq);
return code;
}
@ -7381,8 +7428,8 @@ static int16_t getDropComponentNodeMsgType(ENodeType type) {
static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponentNodeStmt* pStmt) {
SDDropQnodeReq dropReq = {.dnodeId = pStmt->dnodeId};
int32_t code = buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)),
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq);
int32_t code = buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)),
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq);
tFreeSDDropQnodeReq(&dropReq);
return code;
}
@ -7507,7 +7554,8 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt
.mgmtEps = pParCxt->mgmtEpSet};
SName name;
STableMeta* pMeta = NULL;
int32_t code = getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false);
int32_t code =
getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false);
if (code) {
taosMemoryFree(pMeta);
return code;
@ -7615,7 +7663,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt)
#ifdef TD_ENTERPRISE
if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
int32_t origCode = code;
SName name;
SName name;
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name);
SViewMeta* pMeta = NULL;
code = getViewMetaFromMetaCache(pCxt, &name, &pMeta);
@ -7623,9 +7671,10 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt)
code = origCode;
} else {
SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA};
char dbFName[TSDB_DB_FNAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&name, dbFName);
code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false, pMeta->user, &res);
code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false,
pMeta->user, &res);
if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(&name, pCxt->pTargetTables);
}
@ -7638,7 +7687,8 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt)
viewMeta.numOfCols = res.schemaRes.numOfCols;
viewMeta.pSchema = res.schemaRes.pSchema;
code = buildTableMetaFromViewMeta(&pStmt->pMeta, &viewMeta);
parserDebug("rebuild view meta, view:%s.%s, numOfCols:%d, code:0x%x", dbFName, pStmt->tableName, viewMeta.numOfCols, code);
parserDebug("rebuild view meta, view:%s.%s, numOfCols:%d, code:0x%x", dbFName, pStmt->tableName,
viewMeta.numOfCols, code);
}
taosMemoryFree(res.schemaRes.pSchema);
}
@ -7728,12 +7778,12 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt
#ifdef TD_ENTERPRISE
SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pStmt->pQuery)->pFromTable;
SName name;
STableMeta* pMeta = NULL;
int8_t tableType = 0;
int32_t code = getTargetMeta(
pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name),
&pMeta, true);
SName name;
STableMeta* pMeta = NULL;
int8_t tableType = 0;
int32_t code =
getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name),
&pMeta, true);
if (NULL != pMeta) {
tableType = pMeta->tableType;
taosMemoryFree(pMeta);
@ -8055,17 +8105,17 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
&& TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType
&& !hasTbnameFunction(pSelect->pPartitionByList) ) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Count window for stream on super table must patitioned by table name");
"Count window for stream on super table must patitioned by table name");
}
int64_t watermark = 0;
if (pStmt->pOptions->pWatermark) {
translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark);
watermark =((SValueNode*)pStmt->pOptions->pWatermark)->datum.i;
watermark = ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i;
}
if (watermark <= 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Watermark of Count window must exceed 0.");
"Watermark of Count window must exceed 0.");
}
if (pStmt->pOptions->ignoreExpired != 1) {
@ -8498,7 +8548,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta
}
snprintf(pFunc1->node.aliasName, sizeof(pFunc1->node.aliasName), "%s.%p", pFunc1->functionName, pFunc1);
code = nodesListStrictAppend(pProjectionList, (SNode*) pFunc1);
code = nodesListStrictAppend(pProjectionList, (SNode*)pFunc1);
if (code) {
nodesDestroyList(pProjectionList);
return code;
@ -8511,7 +8561,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta
}
snprintf(pFunc2->node.aliasName, sizeof(pFunc2->node.aliasName), "%s.%p", pFunc2->functionName, pFunc2);
code = nodesListStrictAppend(pProjectionList, (SNode*) pFunc2);
code = nodesListStrictAppend(pProjectionList, (SNode*)pFunc2);
if (code) {
nodesDestroyList(pProjectionList);
return code;
@ -8777,7 +8827,7 @@ static int32_t createStreamReqVersionInfo(SSDataBlock* pBlock, SArray** pArray,
taosArrayPush(*pArray, &v);
}
} else {
int32_t precision = (pInterval->interval > 0)? pInterval->precision:TSDB_TIME_PRECISION_MILLI;
int32_t precision = (pInterval->interval > 0) ? pInterval->precision : TSDB_TIME_PRECISION_MILLI;
*lastTs = taosGetTimestamp(precision);
}
@ -8801,7 +8851,8 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDa
if (TSDB_CODE_SUCCESS == code) {
if (interval.interval > 0) {
pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, interval.precision);
pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit,
interval.precision);
} else {
pStmt->pReq->lastTs = lastTs + 1; // start key of the next time window
}
@ -8856,26 +8907,27 @@ static int32_t validateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStm
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type");
}
/*
STableMeta* pMetaCache = NULL;
int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->viewName, &pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
taosMemoryFreeClear(pMetaCache);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with table");
}
*/
/*
STableMeta* pMetaCache = NULL;
int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->viewName, &pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
taosMemoryFreeClear(pMetaCache);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with
table");
}
*/
return TSDB_CODE_SUCCESS;
}
static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStmt) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
return TSDB_CODE_OPS_NOT_SUPPORT;
#endif
SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA};
SName name;
char dbFName[TSDB_DB_FNAME_LEN];
SName name;
char dbFName[TSDB_DB_FNAME_LEN];
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name);
tNameGetFullDbName(&name, dbFName);
@ -8892,7 +8944,8 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt
pStmt->createReq.pSchema = res.schemaRes.pSchema;
strncpy(pStmt->createReq.name, pStmt->viewName, sizeof(pStmt->createReq.name) - 1);
tstrncpy(pStmt->createReq.dbFName, dbFName, sizeof(pStmt->createReq.dbFName));
snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName, pStmt->viewName);
snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName,
pStmt->viewName);
TSWAP(pStmt->createReq.querySql, pStmt->pQuerySql);
pStmt->createReq.orReplace = pStmt->orReplace;
pStmt->createReq.sql = tstrdup(pCxt->pParseCxt->pSql);
@ -8908,14 +8961,13 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt
return code;
}
static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#endif
SCMDropViewReq dropReq = {0};
SName name;
SCMDropViewReq dropReq = {0};
SName name;
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, dropReq.dbFName);
strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1);
@ -8935,7 +8987,6 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt)
return buildCmdMsg(pCxt, TDMT_MND_DROP_VIEW, (FSerializeFunc)tSerializeSCMDropViewReq, &dropReq);
}
static int32_t readFromFile(char* pName, int32_t* len, char** buf) {
int64_t filesize = 0;
if (taosStatFile(pName, &filesize, NULL, NULL) < 0) {
@ -9036,7 +9087,7 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt,
if (TSDB_CODE_SUCCESS == code) {
SName name;
code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name),
&(pTable->pMeta), false);
&(pTable->pMeta), false);
if (code) {
nodesDestroyNode((SNode*)pTable);
return code;
@ -9070,15 +9121,16 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt,
}
static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
int32_t code = 0;
int32_t code = 0;
SAlterUserReq req = {0};
req.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
req.privileges = pStmt->privileges;
#ifdef TD_ENTERPRISE
if (0 != pStmt->tabName[0]) {
SName name;
SName name;
STableMeta* pTableMeta = NULL;
code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true);
code =
getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true);
if (TSDB_CODE_SUCCESS != code) {
if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
@ -9104,16 +9156,17 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
}
static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
int32_t code = 0;
int32_t code = 0;
SAlterUserReq req = {0};
req.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
req.privileges = pStmt->privileges;
#ifdef TD_ENTERPRISE
if (0 != pStmt->tabName[0]) {
SName name;
SName name;
STableMeta* pTableMeta = NULL;
code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true);
code =
getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true);
if (TSDB_CODE_SUCCESS != code) {
if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
@ -9143,7 +9196,8 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
SBalanceVgroupLeaderReq req = {0};
req.vgId = pStmt->vgId;
int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
int32_t code =
buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
tFreeSBalanceVgroupLeaderReq(&req);
return code;
}
@ -9267,6 +9321,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_TRIM_DATABASE_STMT:
code = translateTrimDatabase(pCxt, (STrimDatabaseStmt*)pNode);
break;
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
code = translateS3MigrateDatabase(pCxt, (SS3MigrateDatabaseStmt*)pNode);
break;
case QUERY_NODE_CREATE_TABLE_STMT:
code = translateCreateSuperTable(pCxt, (SCreateTableStmt*)pNode);
break;
@ -9576,7 +9633,6 @@ static int32_t extractShowCreateViewResultSchema(int32_t* numOfCols, SSchema** p
return TSDB_CODE_SUCCESS;
}
static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) {
*numOfCols = 3;
*pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema));
@ -9767,17 +9823,16 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN
static const char* getTbNameColName(ENodeType type) {
const char* colName;
switch (type)
{
case QUERY_NODE_SHOW_VIEWS_STMT:
colName = "view_name";
break;
case QUERY_NODE_SHOW_STABLES_STMT:
colName = "stable_name";
break;
default:
colName = "table_name";
break;
switch (type) {
case QUERY_NODE_SHOW_VIEWS_STMT:
colName = "view_name";
break;
case QUERY_NODE_SHOW_STABLES_STMT:
colName = "stable_name";
break;
default:
colName = "table_name";
break;
}
return colName;
}
@ -9811,14 +9866,14 @@ static int32_t insertCondIntoSelectStmt(SSelectStmt* pSelect, SNode* pCond) {
SNodeList* pLogicCondList2 = NULL;
if (nodeType(pSelect->pWhere) == QUERY_NODE_LOGIC_CONDITION &&
((SLogicConditionNode*)pSelect->pWhere)->condType == LOGIC_COND_TYPE_AND) {
pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList;
pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList;
} else {
nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere);
}
if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION &&
((SLogicConditionNode*)pCond)->condType == LOGIC_COND_TYPE_AND) {
pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList;
pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList;
} else {
nodesListMakeAppend(&pLogicCondList2, pCond);
}
@ -11117,8 +11172,8 @@ static int32_t rewriteFlushDatabase(STranslateContext* pCxt, SQuery* pQuery) {
static int32_t rewriteShowCompacts(STranslateContext* pCxt, SQuery* pQuery) {
SShowCompactsStmt* pShow = (SShowCompactsStmt*)(pQuery->pRoot);
SSelectStmt* pStmt = NULL;
int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACTS_STMT, &pStmt);
SSelectStmt* pStmt = NULL;
int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACTS_STMT, &pStmt);
if (TSDB_CODE_SUCCESS == code) {
pCxt->showRewrite = true;
pQuery->showRewrite = true;
@ -11130,8 +11185,8 @@ static int32_t rewriteShowCompacts(STranslateContext* pCxt, SQuery* pQuery) {
static int32_t rewriteShowCompactDetailsStmt(STranslateContext* pCxt, SQuery* pQuery) {
SShowCompactDetailsStmt* pShow = (SShowCompactDetailsStmt*)(pQuery->pRoot);
SSelectStmt* pStmt = NULL;
int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACT_DETAILS_STMT, &pStmt);
SSelectStmt* pStmt = NULL;
int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACT_DETAILS_STMT, &pStmt);
if (TSDB_CODE_SUCCESS == code) {
if (NULL != pShow->pCompactId) {
code = createOperatorNode(OP_TYPE_EQUAL, "compact_id", pShow->pCompactId, &pStmt->pWhere);

12133
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

110
source/util/src/tconfig.c
View File

@ -21,8 +21,8 @@
#include "tgrant.h"
#include "tjson.h"
#include "tlog.h"
#include "tutil.h"
#include "tunit.h"
#include "tutil.h"
#define CFG_NAME_PRINT_LEN 24
#define CFG_SRC_PRINT_LEN 12
@ -315,19 +315,19 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value,
static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) {
SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag");
if (resetArray) {
// reset
if (pDebugFlagItem == NULL) return -1;
// reset
if (pDebugFlagItem == NULL) return -1;
// logflag names that should 'not' be set by 'debugFlag'
// logflag names that should 'not' be set by 'debugFlag'
if (pDebugFlagItem->array == NULL) {
pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar));
if (pDebugFlagItem->array == NULL) {
pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar));
if (pDebugFlagItem->array == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
taosArrayClear(pDebugFlagItem->array);
return 0;
}
taosArrayClear(pDebugFlagItem->array);
return 0;
}
// update
@ -406,8 +406,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
case CFG_DTYPE_BOOL: {
int32_t ival = (int32_t)atoi(pVal);
if (ival != 0 && ival != 1) {
uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name,
cfgDtypeStr(pItem->dtype), ival);
uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival);
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
@ -691,6 +690,89 @@ void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p
*pLen = len;
}
void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) {
if (dump) {
printf(" s3 config");
printf("\n");
printf("=================================================================");
printf("\n");
} else {
uInfo(" s3 config");
uInfo("=================================================================");
}
char src[CFG_SRC_PRINT_LEN + 1] = {0};
char name[CFG_NAME_PRINT_LEN + 1] = {0};
int32_t size = taosArrayGetSize(pCfg->array);
for (int32_t i = 0; i < size; ++i) {
SConfigItem *pItem = taosArrayGet(pCfg->array, i);
if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue;
if (dump && strcmp(pItem->name, "scriptDir") == 0) continue;
if (dump && strncmp(pItem->name, "s3", 2) != 0) continue;
tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN);
for (int32_t j = 0; j < CFG_SRC_PRINT_LEN; ++j) {
if (src[j] == 0) src[j] = ' ';
}
tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN);
for (int32_t j = 0; j < CFG_NAME_PRINT_LEN; ++j) {
if (name[j] == 0) name[j] = ' ';
}
switch (pItem->dtype) {
case CFG_DTYPE_BOOL:
if (dump) {
printf("%s %s %u\n", src, name, pItem->bval);
} else {
uInfo("%s %s %u", src, name, pItem->bval);
}
break;
case CFG_DTYPE_INT32:
if (dump) {
printf("%s %s %d\n", src, name, pItem->i32);
} else {
uInfo("%s %s %d", src, name, pItem->i32);
}
break;
case CFG_DTYPE_INT64:
if (dump) {
printf("%s %s %" PRId64 "\n", src, name, pItem->i64);
} else {
uInfo("%s %s %" PRId64, src, name, pItem->i64);
}
break;
case CFG_DTYPE_DOUBLE:
case CFG_DTYPE_FLOAT:
if (dump) {
printf("%s %s %.2f\n", src, name, pItem->fval);
} else {
uInfo("%s %s %.2f", src, name, pItem->fval);
}
break;
case CFG_DTYPE_STRING:
case CFG_DTYPE_DIR:
case CFG_DTYPE_LOCALE:
case CFG_DTYPE_CHARSET:
case CFG_DTYPE_TIMEZONE:
case CFG_DTYPE_NONE:
if (dump) {
printf("%s %s %s\n", src, name, pItem->str);
} else {
uInfo("%s %s %s", src, name, pItem->str);
}
break;
}
}
if (dump) {
printf("=================================================================\n");
} else {
uInfo("=================================================================");
}
}
void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
if (dump) {
printf(" global config");
@ -738,7 +820,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
break;
case CFG_DTYPE_INT64:
if (dump) {
printf("%s %s %" PRId64"\n", src, name, pItem->i64);
printf("%s %s %" PRId64 "\n", src, name, pItem->i64);
} else {
uInfo("%s %s %" PRId64, src, name, pItem->i64);
}

12
tests/develop-test/2-query/show_create_db.py
View File

@ -42,17 +42,17 @@ class TDTestCase:
tdSql.query('show create database scd;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd4')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
self.restartTaosd(1, dbname='scd')
@ -60,17 +60,17 @@ class TDTestCase:
tdSql.query('show create database scd;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd4')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.execute('drop database scd')

40
tests/script/local.supp
View File

@ -1655,3 +1655,43 @@
fun:start_thread
fun:clone
}
{
<insert_a_suppression_name_here>
Memcheck:Param
write(buf)
fun:__libc_write
fun:write
fun:uv__try_write
fun:uv__write
fun:uv_write2
fun:uvStartSendRespImpl
fun:uvStartSendResp
fun:uvHandleResp
fun:uvPrepareCb
fun:uv__run_prepare
fun:uv_run
fun:transWorkerThread
fun:start_thread
fun:clone
}
{
<memalign suppression>
Memcheck:Param
write(buf)
fun:__libc_write
fun:write
fun:uv__try_write
fun:uv__write
fun:uv_write2
fun:uvStartSendRespImpl
fun:uvStartSendResp
fun:uvHandleResp
fun:uvWorkerAsyncCb
fun:uv__async_io
fun:uv__io_poll
fun:uv_run
fun:transWorkerThread
fun:start_thread
fun:clone
}

2
tests/script/tsim/db/alter_option.sim
View File

@ -46,7 +46,7 @@ print ============= create database
# | REPLICA value [1 | 3]
# | WAL_LEVEL value [1 | 2]
sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 WAL_FSYNC_PERIOD 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL_LEVEL 2 VGROUPS 6 SINGLE_STABLE 1
sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 WAL_FSYNC_PERIOD 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 S3_KEEPLOCAL 720 PRECISION 'ns' REPLICA 3 WAL_LEVEL 2 VGROUPS 6 SINGLE_STABLE 1
sql select * from information_schema.ins_databases
print rows: $rows
print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09

2
tests/system-test/2-query/distribute_agg_apercentile.py
View File

@ -18,7 +18,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5")
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 s3_keeplocal 3000 vgroups 5")
tdSql.execute(f" use {dbname} ")
tdSql.execute(
f'''create table {dbname}.stb1

5
tools/shell/src/shellAuto.c
View File

@ -214,6 +214,7 @@ SWords shellCommands[] = {
{"insert into <tb_name> using <stb_name> <anyword> values(", 0, 0, NULL},
{"insert into <tb_name> file ", 0, 0, NULL},
{"trim database <db_name>", 0, 0, NULL},
{"s3migrate database <db_name>", 0, 0, NULL},
{"use <db_name>", 0, 0, NULL},
{"quit", 0, 0, NULL}};
@ -283,6 +284,9 @@ char* db_options[] = {"keep ",
"wal_level ",
"vgroups ",
"single_stable ",
"s3_chunksize ",
"s3_keeplocal ",
"s3_compact ",
"wal_retention_period ",
"wal_roll_period ",
"wal_retention_size ",
@ -290,6 +294,7 @@ char* db_options[] = {"keep ",
char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "stt_trigger ",
"wal_retention_period ", "wal_retention_size ", "cachesize ",
"s3_keeplocal ", "s3_compact ",
"wal_fsync_period ", "buffer ", "pages " ,"wal_level "};
char* data_types[] = {"timestamp", "int",