xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: solve conflicts

This commit is contained in:
kailixu 2024-04-15 16:12:15 +08:00
parent a6c36a6cf3 fb7db6a50c
commit 123090e18a
64 changed files with 7466 additions and 5835 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/common/cos.h
View File

@ -33,10 +33,12 @@ extern int32_t tsS3UploadDelaySec;
int32_t s3Init(); int32_t s3Init();
void s3CleanUp(); void s3CleanUp();
int32_t s3CheckCfg();
int32_t s3PutObjectFromFile(const char *file, const char *object); int32_t s3PutObjectFromFile(const char *file, const char *object);
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp); int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size);
void s3DeleteObjectsByPrefix(const char *prefix); void s3DeleteObjectsByPrefix(const char *prefix);
void s3DeleteObjects(const char *object_name[], int nobject); int32_t s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name); bool s3Exists(const char *object_name);
bool s3Get(const char *object_name, const char *path); bool s3Get(const char *object_name, const char *path);
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock); int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
@ -45,6 +47,8 @@ void s3EvictCache(const char *path, long object_size);
long s3Size(const char *object_name); long s3Size(const char *object_name);
int32_t s3GetObjectToFile(const char *object_name, char *fileName); int32_t s3GetObjectToFile(const char *object_name, char *fileName);
#define S3_DATA_CHUNK_PAGES (256 * 1024 * 1024)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

14
include/common/tglobal.h
View File

@ -136,9 +136,9 @@ extern int32_t tsMonitorIntervalForBasic;
extern bool tsMonitorForceV2; extern bool tsMonitorForceV2;
// audit // audit
extern bool tsEnableAudit; extern bool tsEnableAudit;
extern bool tsEnableAuditCreateTable; extern bool tsEnableAuditCreateTable;
extern int32_t tsAuditInterval; extern int32_t tsAuditInterval;
// telem // telem
extern bool tsEnableTelem; extern bool tsEnableTelem;
@ -146,9 +146,9 @@ extern int32_t tsTelemInterval;
extern char tsTelemServer[]; extern char tsTelemServer[];
extern uint16_t tsTelemPort; extern uint16_t tsTelemPort;
extern bool tsEnableCrashReport; extern bool tsEnableCrashReport;
extern char * tsTelemUri; extern char *tsTelemUri;
extern char * tsClientCrashReportUri; extern char *tsClientCrashReportUri;
extern char * tsSvrCrashReportUri; extern char *tsSvrCrashReportUri;
// query buffer management // query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
@ -235,6 +235,8 @@ extern int32_t tsTtlUnit;
extern int32_t tsTtlPushIntervalSec; extern int32_t tsTtlPushIntervalSec;
extern int32_t tsTtlBatchDropNum; extern int32_t tsTtlBatchDropNum;
extern int32_t tsTrimVDbIntervalSec; extern int32_t tsTrimVDbIntervalSec;
extern int32_t tsS3MigrateIntervalSec;
extern bool tsS3MigrateEnabled;
extern int32_t tsGrantHBInterval; extern int32_t tsGrantHBInterval;
extern int32_t tsUptimeInterval; extern int32_t tsUptimeInterval;

50
include/common/tmsg.h
View File

@ -302,7 +302,8 @@ typedef enum ENodeType {
QUERY_NODE_GRANT_STMT, QUERY_NODE_GRANT_STMT,
QUERY_NODE_REVOKE_STMT, QUERY_NODE_REVOKE_STMT,
QUERY_NODE_ALTER_CLUSTER_STMT, QUERY_NODE_ALTER_CLUSTER_STMT,
// placeholder for [153, 180] QUERY_NODE_S3MIGRATE_DATABASE_STMT,
// placeholder for [154, 180]
QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181, QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
QUERY_NODE_SHOW_CREATE_DATABASE_STMT, QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
QUERY_NODE_SHOW_CREATE_TABLE_STMT, QUERY_NODE_SHOW_CREATE_TABLE_STMT,
@ -588,7 +589,7 @@ typedef struct {
// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); // int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); // int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
// void tFreeSSubmitBlkRsp(void* param); // void tFreeSSubmitBlkRsp(void* param);
void tFreeSSubmitRsp(SSubmitRsp* pRsp); void tFreeSSubmitRsp(SSubmitRsp* pRsp);
#define COL_SMA_ON ((int8_t)0x1) #define COL_SMA_ON ((int8_t)0x1)
#define COL_IDX_ON ((int8_t)0x2) #define COL_IDX_ON ((int8_t)0x2)
@ -1155,6 +1156,9 @@ typedef struct {
int32_t sstTrigger; int32_t sstTrigger;
int16_t hashPrefix; int16_t hashPrefix;
int16_t hashSuffix; int16_t hashSuffix;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t tsdbPageSize; int32_t tsdbPageSize;
int32_t sqlLen; int32_t sqlLen;
char* sql; char* sql;
@ -1186,6 +1190,8 @@ typedef struct {
int32_t minRows; int32_t minRows;
int32_t walRetentionPeriod; int32_t walRetentionPeriod;
int32_t walRetentionSize; int32_t walRetentionSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t sqlLen; int32_t sqlLen;
char* sql; char* sql;
int8_t withArbitrator; int8_t withArbitrator;
@ -1266,6 +1272,20 @@ typedef struct {
int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
typedef struct {
char db[TSDB_DB_FNAME_LEN];
} SS3MigrateDbReq;
int32_t tSerializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq);
int32_t tDeserializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq);
typedef struct {
int32_t timestamp;
} SVS3MigrateDbReq;
int32_t tSerializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq);
int32_t tDeserializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq);
typedef struct { typedef struct {
int32_t timestampSec; int32_t timestampSec;
int32_t ttlDropMaxCount; int32_t ttlDropMaxCount;
@ -1303,6 +1323,9 @@ typedef struct {
int8_t strict; int8_t strict;
int8_t cacheLast; int8_t cacheLast;
int8_t encryptAlgorithm; int8_t encryptAlgorithm;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t tsdbPageSize; int32_t tsdbPageSize;
int32_t walRetentionPeriod; int32_t walRetentionPeriod;
int32_t walRollPeriod; int32_t walRollPeriod;
@ -1595,13 +1618,13 @@ int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
void tFreeSStatusReq(SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq);
typedef struct { typedef struct {
int32_t contLen; int32_t contLen;
char* pCont; char* pCont;
} SStatisReq; } SStatisReq;
int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
void tFreeSStatisReq(SStatisReq *pReq); void tFreeSStatisReq(SStatisReq* pReq);
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
@ -1706,7 +1729,10 @@ typedef struct {
int16_t hashPrefix; int16_t hashPrefix;
int16_t hashSuffix; int16_t hashSuffix;
int32_t tsdbPageSize; int32_t tsdbPageSize;
int64_t reserved[8]; int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int64_t reserved[6];
int8_t learnerReplica; int8_t learnerReplica;
int8_t learnerSelfIndex; int8_t learnerSelfIndex;
SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA];
@ -1795,13 +1821,15 @@ typedef struct {
int8_t walLevel; int8_t walLevel;
int8_t strict; int8_t strict;
int8_t cacheLast; int8_t cacheLast;
int64_t reserved[8]; int64_t reserved[7];
// 1st modification // 1st modification
int16_t sttTrigger; int16_t sttTrigger;
int32_t minRows; int32_t minRows;
// 2nd modification // 2nd modification
int32_t walRetentionPeriod; int32_t walRetentionPeriod;
int32_t walRetentionSize; int32_t walRetentionSize;
int32_t s3KeepLocal;
int8_t s3Compact;
} SAlterVnodeConfigReq; } SAlterVnodeConfigReq;
int32_t tSerializeSAlterVnodeConfigReq(void* buf, int32_t bufLen, SAlterVnodeConfigReq* pReq); int32_t tSerializeSAlterVnodeConfigReq(void* buf, int32_t bufLen, SAlterVnodeConfigReq* pReq);
@ -1960,7 +1988,7 @@ typedef struct {
int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); // int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
void tFreeSShowReq(SShowReq* pReq); void tFreeSShowReq(SShowReq* pReq);
typedef struct { typedef struct {
int64_t showId; int64_t showId;
@ -2844,7 +2872,7 @@ typedef struct {
SVCreateTbReq* pReqs; SVCreateTbReq* pReqs;
SArray* pArray; SArray* pArray;
}; };
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
} SVCreateTbBatchReq; } SVCreateTbBatchReq;
int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq); int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
@ -2939,7 +2967,7 @@ typedef struct {
int32_t newCommentLen; int32_t newCommentLen;
char* newComment; char* newComment;
int64_t ctimeMs; // fill by vnode int64_t ctimeMs; // fill by vnode
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
} SVAlterTbReq; } SVAlterTbReq;
int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@ -4058,7 +4086,7 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
#define TD_REQ_FROM_TAOX 0x8 #define TD_REQ_FROM_TAOX 0x8
#define SUBMIT_REQUEST_VERSION (1) #define SUBMIT_REQUEST_VERSION (1)
#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility #define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility
typedef struct { typedef struct {
int32_t flags; int32_t flags;

3
include/common/tmsgdef.h
View File

@ -222,6 +222,8 @@
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
@ -274,6 +276,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_S3MIGRATE, "vnode-s3migrate", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)

590
include/common/ttokendef.h
View File

@ -85,305 +85,309 @@
#define TK_USE 67 #define TK_USE 67
#define TK_FLUSH 68 #define TK_FLUSH 68
#define TK_TRIM 69 #define TK_TRIM 69
#define TK_COMPACT 70 #define TK_S3MIGRATE 70
#define TK_IF 71 #define TK_COMPACT 71
#define TK_NOT 72 #define TK_IF 72
#define TK_EXISTS 73 #define TK_NOT 73
#define TK_BUFFER 74 #define TK_EXISTS 74
#define TK_CACHEMODEL 75 #define TK_BUFFER 75
#define TK_CACHESIZE 76 #define TK_CACHEMODEL 76
#define TK_COMP 77 #define TK_CACHESIZE 77
#define TK_DURATION 78 #define TK_COMP 78
#define TK_NK_VARIABLE 79 #define TK_DURATION 79
#define TK_MAXROWS 80 #define TK_NK_VARIABLE 80
#define TK_MINROWS 81 #define TK_MAXROWS 81
#define TK_KEEP 82 #define TK_MINROWS 82
#define TK_PAGES 83 #define TK_KEEP 83
#define TK_PAGESIZE 84 #define TK_PAGES 84
#define TK_TSDB_PAGESIZE 85 #define TK_PAGESIZE 85
#define TK_PRECISION 86 #define TK_TSDB_PAGESIZE 86
#define TK_REPLICA 87 #define TK_PRECISION 87
#define TK_VGROUPS 88 #define TK_REPLICA 88
#define TK_SINGLE_STABLE 89 #define TK_VGROUPS 89
#define TK_RETENTIONS 90 #define TK_SINGLE_STABLE 90
#define TK_SCHEMALESS 91 #define TK_RETENTIONS 91
#define TK_WAL_LEVEL 92 #define TK_SCHEMALESS 92
#define TK_WAL_FSYNC_PERIOD 93 #define TK_WAL_LEVEL 93
#define TK_WAL_RETENTION_PERIOD 94 #define TK_WAL_FSYNC_PERIOD 94
#define TK_WAL_RETENTION_SIZE 95 #define TK_WAL_RETENTION_PERIOD 95
#define TK_WAL_ROLL_PERIOD 96 #define TK_WAL_RETENTION_SIZE 96
#define TK_WAL_SEGMENT_SIZE 97 #define TK_WAL_ROLL_PERIOD 97
#define TK_STT_TRIGGER 98 #define TK_WAL_SEGMENT_SIZE 98
#define TK_TABLE_PREFIX 99 #define TK_STT_TRIGGER 99
#define TK_TABLE_SUFFIX 100 #define TK_TABLE_PREFIX 100
#define TK_KEEP_TIME_OFFSET 101 #define TK_TABLE_SUFFIX 101
#define TK_ENCRYPT_ALGORITHM 102 #define TK_S3_CHUNKSIZE 102
#define TK_NK_COLON 103 #define TK_S3_KEEPLOCAL 103
#define TK_BWLIMIT 104 #define TK_S3_COMPACT 104
#define TK_START 105 #define TK_KEEP_TIME_OFFSET 105
#define TK_TIMESTAMP 106 #define TK_ENCRYPT_ALGORITHM 106
#define TK_END 107 #define TK_NK_COLON 107
#define TK_TABLE 108 #define TK_BWLIMIT 108
#define TK_NK_LP 109 #define TK_START 109
#define TK_NK_RP 110 #define TK_TIMESTAMP 110
#define TK_STABLE 111 #define TK_END 111
#define TK_COLUMN 112 #define TK_TABLE 112
#define TK_MODIFY 113 #define TK_NK_LP 113
#define TK_RENAME 114 #define TK_NK_RP 114
#define TK_TAG 115 #define TK_STABLE 115
#define TK_SET 116 #define TK_COLUMN 116
#define TK_NK_EQ 117 #define TK_MODIFY 117
#define TK_USING 118 #define TK_RENAME 118
#define TK_TAGS 119 #define TK_TAG 119
#define TK_PRIMARY 120 #define TK_SET 120
#define TK_KEY 121 #define TK_NK_EQ 121
#define TK_BOOL 122 #define TK_USING 122
#define TK_TINYINT 123 #define TK_TAGS 123
#define TK_SMALLINT 124 #define TK_PRIMARY 124
#define TK_INT 125 #define TK_KEY 125
#define TK_INTEGER 126 #define TK_BOOL 126
#define TK_BIGINT 127 #define TK_TINYINT 127
#define TK_FLOAT 128 #define TK_SMALLINT 128
#define TK_DOUBLE 129 #define TK_INT 129
#define TK_BINARY 130 #define TK_INTEGER 130
#define TK_NCHAR 131 #define TK_BIGINT 131
#define TK_UNSIGNED 132 #define TK_FLOAT 132
#define TK_JSON 133 #define TK_DOUBLE 133
#define TK_VARCHAR 134 #define TK_BINARY 134
#define TK_MEDIUMBLOB 135 #define TK_NCHAR 135
#define TK_BLOB 136 #define TK_UNSIGNED 136
#define TK_VARBINARY 137 #define TK_JSON 137
#define TK_GEOMETRY 138 #define TK_VARCHAR 138
#define TK_DECIMAL 139 #define TK_MEDIUMBLOB 139
#define TK_COMMENT 140 #define TK_BLOB 140
#define TK_MAX_DELAY 141 #define TK_VARBINARY 141
#define TK_WATERMARK 142 #define TK_GEOMETRY 142
#define TK_ROLLUP 143 #define TK_DECIMAL 143
#define TK_TTL 144 #define TK_COMMENT 144
#define TK_SMA 145 #define TK_MAX_DELAY 145
#define TK_DELETE_MARK 146 #define TK_WATERMARK 146
#define TK_FIRST 147 #define TK_ROLLUP 147
#define TK_LAST 148 #define TK_TTL 148
#define TK_SHOW 149 #define TK_SMA 149
#define TK_PRIVILEGES 150 #define TK_DELETE_MARK 150
#define TK_DATABASES 151 #define TK_FIRST 151
#define TK_TABLES 152 #define TK_LAST 152
#define TK_STABLES 153 #define TK_SHOW 153
#define TK_MNODES 154 #define TK_PRIVILEGES 154
#define TK_QNODES 155 #define TK_DATABASES 155
#define TK_ARBGROUPS 156 #define TK_TABLES 156
#define TK_FUNCTIONS 157 #define TK_STABLES 157
#define TK_INDEXES 158 #define TK_MNODES 158
#define TK_ACCOUNTS 159 #define TK_QNODES 159
#define TK_APPS 160 #define TK_ARBGROUPS 160
#define TK_CONNECTIONS 161 #define TK_FUNCTIONS 161
#define TK_LICENCES 162 #define TK_INDEXES 162
#define TK_GRANTS 163 #define TK_ACCOUNTS 163
#define TK_FULL 164 #define TK_APPS 164
#define TK_LOGS 165 #define TK_CONNECTIONS 165
#define TK_MACHINES 166 #define TK_LICENCES 166
#define TK_ENCRYPTIONS 167 #define TK_GRANTS 167
#define TK_QUERIES 168 #define TK_FULL 168
#define TK_SCORES 169 #define TK_LOGS 169
#define TK_TOPICS 170 #define TK_MACHINES 170
#define TK_VARIABLES 171 #define TK_ENCRYPTIONS 171
#define TK_BNODES 172 #define TK_QUERIES 172
#define TK_SNODES 173 #define TK_SCORES 173
#define TK_TRANSACTIONS 174 #define TK_TOPICS 174
#define TK_DISTRIBUTED 175 #define TK_VARIABLES 175
#define TK_CONSUMERS 176 #define TK_BNODES 176
#define TK_SUBSCRIPTIONS 177 #define TK_SNODES 177
#define TK_VNODES 178 #define TK_TRANSACTIONS 178
#define TK_ALIVE 179 #define TK_DISTRIBUTED 179
#define TK_VIEWS 180 #define TK_CONSUMERS 180
#define TK_VIEW 181 #define TK_SUBSCRIPTIONS 181
#define TK_COMPACTS 182 #define TK_VNODES 182
#define TK_NORMAL 183 #define TK_ALIVE 183
#define TK_CHILD 184 #define TK_VIEWS 184
#define TK_LIKE 185 #define TK_VIEW 185
#define TK_TBNAME 186 #define TK_COMPACTS 186
#define TK_QTAGS 187 #define TK_NORMAL 187
#define TK_AS 188 #define TK_CHILD 188
#define TK_SYSTEM 189 #define TK_LIKE 189
#define TK_INDEX 190 #define TK_TBNAME 190
#define TK_FUNCTION 191 #define TK_QTAGS 191
#define TK_INTERVAL 192 #define TK_AS 192
#define TK_COUNT 193 #define TK_SYSTEM 193
#define TK_LAST_ROW 194 #define TK_INDEX 194
#define TK_META 195 #define TK_FUNCTION 195
#define TK_ONLY 196 #define TK_INTERVAL 196
#define TK_TOPIC 197 #define TK_COUNT 197
#define TK_CONSUMER 198 #define TK_LAST_ROW 198
#define TK_GROUP 199 #define TK_META 199
#define TK_DESC 200 #define TK_ONLY 200
#define TK_DESCRIBE 201 #define TK_TOPIC 201
#define TK_RESET 202 #define TK_CONSUMER 202
#define TK_QUERY 203 #define TK_GROUP 203
#define TK_CACHE 204 #define TK_DESC 204
#define TK_EXPLAIN 205 #define TK_DESCRIBE 205
#define TK_ANALYZE 206 #define TK_RESET 206
#define TK_VERBOSE 207 #define TK_QUERY 207
#define TK_NK_BOOL 208 #define TK_CACHE 208
#define TK_RATIO 209 #define TK_EXPLAIN 209
#define TK_NK_FLOAT 210 #define TK_ANALYZE 210
#define TK_OUTPUTTYPE 211 #define TK_VERBOSE 211
#define TK_AGGREGATE 212 #define TK_NK_BOOL 212
#define TK_BUFSIZE 213 #define TK_RATIO 213
#define TK_LANGUAGE 214 #define TK_NK_FLOAT 214
#define TK_REPLACE 215 #define TK_OUTPUTTYPE 215
#define TK_STREAM 216 #define TK_AGGREGATE 216
#define TK_INTO 217 #define TK_BUFSIZE 217
#define TK_PAUSE 218 #define TK_LANGUAGE 218
#define TK_RESUME 219 #define TK_REPLACE 219
#define TK_TRIGGER 220 #define TK_STREAM 220
#define TK_AT_ONCE 221 #define TK_INTO 221
#define TK_WINDOW_CLOSE 222 #define TK_PAUSE 222
#define TK_IGNORE 223 #define TK_RESUME 223
#define TK_EXPIRED 224 #define TK_TRIGGER 224
#define TK_FILL_HISTORY 225 #define TK_AT_ONCE 225
#define TK_UPDATE 226 #define TK_WINDOW_CLOSE 226
#define TK_SUBTABLE 227 #define TK_IGNORE 227
#define TK_UNTREATED 228 #define TK_EXPIRED 228
#define TK_KILL 229 #define TK_FILL_HISTORY 229
#define TK_CONNECTION 230 #define TK_UPDATE 230
#define TK_TRANSACTION 231 #define TK_SUBTABLE 231
#define TK_BALANCE 232 #define TK_UNTREATED 232
#define TK_VGROUP 233 #define TK_KILL 233
#define TK_LEADER 234 #define TK_CONNECTION 234
#define TK_MERGE 235 #define TK_TRANSACTION 235
#define TK_REDISTRIBUTE 236 #define TK_BALANCE 236
#define TK_SPLIT 237 #define TK_VGROUP 237
#define TK_DELETE 238 #define TK_LEADER 238
#define TK_INSERT 239 #define TK_MERGE 239
#define TK_NK_BIN 240 #define TK_REDISTRIBUTE 240
#define TK_NK_HEX 241 #define TK_SPLIT 241
#define TK_NULL 242 #define TK_DELETE 242
#define TK_NK_QUESTION 243 #define TK_INSERT 243
#define TK_NK_ALIAS 244 #define TK_NK_BIN 244
#define TK_NK_ARROW 245 #define TK_NK_HEX 245
#define TK_ROWTS 246 #define TK_NULL 246
#define TK_QSTART 247 #define TK_NK_QUESTION 247
#define TK_QEND 248 #define TK_NK_ALIAS 248
#define TK_QDURATION 249 #define TK_NK_ARROW 249
#define TK_WSTART 250 #define TK_ROWTS 250
#define TK_WEND 251 #define TK_QSTART 251
#define TK_WDURATION 252 #define TK_QEND 252
#define TK_IROWTS 253 #define TK_QDURATION 253
#define TK_ISFILLED 254 #define TK_WSTART 254
#define TK_CAST 255 #define TK_WEND 255
#define TK_NOW 256 #define TK_WDURATION 256
#define TK_TODAY 257 #define TK_IROWTS 257
#define TK_TIMEZONE 258 #define TK_ISFILLED 258
#define TK_CLIENT_VERSION 259 #define TK_CAST 259
#define TK_SERVER_VERSION 260 #define TK_NOW 260
#define TK_SERVER_STATUS 261 #define TK_TODAY 261
#define TK_CURRENT_USER 262 #define TK_TIMEZONE 262
#define TK_CASE 263 #define TK_CLIENT_VERSION 263
#define TK_WHEN 264 #define TK_SERVER_VERSION 264
#define TK_THEN 265 #define TK_SERVER_STATUS 265
#define TK_ELSE 266 #define TK_CURRENT_USER 266
#define TK_BETWEEN 267 #define TK_CASE 267
#define TK_IS 268 #define TK_WHEN 268
#define TK_NK_LT 269 #define TK_THEN 269
#define TK_NK_GT 270 #define TK_ELSE 270
#define TK_NK_LE 271 #define TK_BETWEEN 271
#define TK_NK_GE 272 #define TK_IS 272
#define TK_NK_NE 273 #define TK_NK_LT 273
#define TK_MATCH 274 #define TK_NK_GT 274
#define TK_NMATCH 275 #define TK_NK_LE 275
#define TK_CONTAINS 276 #define TK_NK_GE 276
#define TK_IN 277 #define TK_NK_NE 277
#define TK_JOIN 278 #define TK_MATCH 278
#define TK_INNER 279 #define TK_NMATCH 279
#define TK_SELECT 280 #define TK_CONTAINS 280
#define TK_NK_HINT 281 #define TK_IN 281
#define TK_DISTINCT 282 #define TK_JOIN 282
#define TK_WHERE 283 #define TK_INNER 283
#define TK_PARTITION 284 #define TK_SELECT 284
#define TK_BY 285 #define TK_NK_HINT 285
#define TK_SESSION 286 #define TK_DISTINCT 286
#define TK_STATE_WINDOW 287 #define TK_WHERE 287
#define TK_EVENT_WINDOW 288 #define TK_PARTITION 288
#define TK_COUNT_WINDOW 289 #define TK_BY 289
#define TK_SLIDING 290 #define TK_SESSION 290
#define TK_FILL 291 #define TK_STATE_WINDOW 291
#define TK_VALUE 292 #define TK_EVENT_WINDOW 292
#define TK_VALUE_F 293 #define TK_COUNT_WINDOW 293
#define TK_NONE 294 #define TK_SLIDING 294
#define TK_PREV 295 #define TK_FILL 295
#define TK_NULL_F 296 #define TK_VALUE 296
#define TK_LINEAR 297 #define TK_VALUE_F 297
#define TK_NEXT 298 #define TK_NONE 298
#define TK_HAVING 299 #define TK_PREV 299
#define TK_RANGE 300 #define TK_NULL_F 300
#define TK_EVERY 301 #define TK_LINEAR 301
#define TK_ORDER 302 #define TK_NEXT 302
#define TK_SLIMIT 303 #define TK_HAVING 303
#define TK_SOFFSET 304 #define TK_RANGE 304
#define TK_LIMIT 305 #define TK_EVERY 305
#define TK_OFFSET 306 #define TK_ORDER 306
#define TK_ASC 307 #define TK_SLIMIT 307
#define TK_NULLS 308 #define TK_SOFFSET 308
#define TK_ABORT 309 #define TK_LIMIT 309
#define TK_AFTER 310 #define TK_OFFSET 310
#define TK_ATTACH 311 #define TK_ASC 311
#define TK_BEFORE 312 #define TK_NULLS 312
#define TK_BEGIN 313 #define TK_ABORT 313
#define TK_BITAND 314 #define TK_AFTER 314
#define TK_BITNOT 315 #define TK_ATTACH 315
#define TK_BITOR 316 #define TK_BEFORE 316
#define TK_BLOCKS 317 #define TK_BEGIN 317
#define TK_CHANGE 318 #define TK_BITAND 318
#define TK_COMMA 319 #define TK_BITNOT 319
#define TK_CONCAT 320 #define TK_BITOR 320
#define TK_CONFLICT 321 #define TK_BLOCKS 321
#define TK_COPY 322 #define TK_CHANGE 322
#define TK_DEFERRED 323 #define TK_COMMA 323
#define TK_DELIMITERS 324 #define TK_CONCAT 324
#define TK_DETACH 325 #define TK_CONFLICT 325
#define TK_DIVIDE 326 #define TK_COPY 326
#define TK_DOT 327 #define TK_DEFERRED 327
#define TK_EACH 328 #define TK_DELIMITERS 328
#define TK_FAIL 329 #define TK_DETACH 329
#define TK_FILE 330 #define TK_DIVIDE 330
#define TK_FOR 331 #define TK_DOT 331
#define TK_GLOB 332 #define TK_EACH 332
#define TK_ID 333 #define TK_FAIL 333
#define TK_IMMEDIATE 334 #define TK_FILE 334
#define TK_IMPORT 335 #define TK_FOR 335
#define TK_INITIALLY 336 #define TK_GLOB 336
#define TK_INSTEAD 337 #define TK_ID 337
#define TK_ISNULL 338 #define TK_IMMEDIATE 338
#define TK_MODULES 339 #define TK_IMPORT 339
#define TK_NK_BITNOT 340 #define TK_INITIALLY 340
#define TK_NK_SEMI 341 #define TK_INSTEAD 341
#define TK_NOTNULL 342 #define TK_ISNULL 342
#define TK_OF 343 #define TK_MODULES 343
#define TK_PLUS 344 #define TK_NK_BITNOT 344
#define TK_PRIVILEGE 345 #define TK_NK_SEMI 345
#define TK_RAISE 346 #define TK_NOTNULL 346
#define TK_RESTRICT 347 #define TK_OF 347
#define TK_ROW 348 #define TK_PLUS 348
#define TK_SEMI 349 #define TK_PRIVILEGE 349
#define TK_STAR 350 #define TK_RAISE 350
#define TK_STATEMENT 351 #define TK_RESTRICT 351
#define TK_STRICT 352 #define TK_ROW 352
#define TK_STRING 353 #define TK_SEMI 353
#define TK_TIMES 354 #define TK_STAR 354
#define TK_VALUES 355 #define TK_STATEMENT 355
#define TK_VARIABLE 356 #define TK_STRICT 356
#define TK_WAL 357 #define TK_STRING 357
#define TK_TIMES 358
#define TK_VALUES 359
#define TK_VARIABLE 360
#define TK_WAL 361
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601
#define TK_NK_ILLEGAL 602 #define TK_NK_ILLEGAL 602
// #define TK_NK_HEX 603 // hex number 0x123 // #define TK_NK_HEX 603 // hex number 0x123
#define TK_NK_OCT 604 // oct number #define TK_NK_OCT 604 // oct number
// #define TK_NK_BIN 605 // bin format data 0b111 // #define TK_NK_BIN 605 // bin format data 0b111
#define TK_BATCH_SCAN 606 #define TK_BATCH_SCAN 606
#define TK_NO_BATCH_SCAN 607 #define TK_NO_BATCH_SCAN 607
#define TK_SORT_FOR_GROUP 608 #define TK_SORT_FOR_GROUP 608
#define TK_PARTITION_FIRST 609 #define TK_PARTITION_FIRST 609
#define TK_PARA_TABLES_SORT 610 #define TK_PARA_TABLES_SORT 610
#define TK_SMALLDATA_TS_SORT 611 #define TK_SMALLDATA_TS_SORT 611

9
include/libs/nodes/cmdnodes.h
View File

@ -102,6 +102,10 @@ typedef struct SDatabaseOptions {
int32_t sstTrigger; int32_t sstTrigger;
int32_t tablePrefix; int32_t tablePrefix;
int32_t tableSuffix; int32_t tableSuffix;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
SValueNode* s3KeepLocalStr;
int8_t s3Compact;
int8_t withArbitrator; int8_t withArbitrator;
} SDatabaseOptions; } SDatabaseOptions;
@ -140,6 +144,11 @@ typedef struct STrimDatabaseStmt {
int32_t maxSpeed; int32_t maxSpeed;
} STrimDatabaseStmt; } STrimDatabaseStmt;
typedef struct SS3MigrateDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SS3MigrateDatabaseStmt;
typedef struct SCompactDatabaseStmt { typedef struct SCompactDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];

7
include/util/tconfig.h
View File

@ -51,11 +51,7 @@ typedef enum {
CFG_DTYPE_TIMEZONE CFG_DTYPE_TIMEZONE
} ECfgDataType; } ECfgDataType;
typedef enum { typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType;
CFG_SCOPE_SERVER,
CFG_SCOPE_CLIENT,
CFG_SCOPE_BOTH
} ECfgScopeType;
typedef enum { typedef enum {
CFG_DYN_NONE = 0, CFG_DYN_NONE = 0,
@ -138,6 +134,7 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p
void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen);
void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump);
int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl); int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl);

42
include/util/tdef.h
View File

@ -201,8 +201,8 @@ typedef enum ELogicConditionType {
#define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788 #define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788
// ACCOUNT is a 32 bit positive integer // ACCOUNT is a 32 bit positive integer
// this is the length of its string representation, including the terminator zero // this is the length of its string representation, including the terminator zero
#define TSDB_ACCT_ID_LEN 11 #define TSDB_ACCT_ID_LEN 11
#define TSDB_NODE_ID_LEN 11 #define TSDB_NODE_ID_LEN 11
#define TSDB_VGROUP_ID_LEN 11 #define TSDB_VGROUP_ID_LEN 11
#define TSDB_MAX_COLUMNS 4096 #define TSDB_MAX_COLUMNS 4096
@ -297,8 +297,8 @@ typedef enum ELogicConditionType {
#define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_DNODE_CONFIG_LEN 128
#define TSDB_DNODE_VALUE_LEN 256 #define TSDB_DNODE_VALUE_LEN 256
#define TSDB_CLUSTER_VALUE_LEN 1000 #define TSDB_CLUSTER_VALUE_LEN 1000
#define TSDB_GRANT_LOG_COL_LEN 15600 #define TSDB_GRANT_LOG_COL_LEN 15600
#define TSDB_ACTIVE_KEY_LEN 109 #define TSDB_ACTIVE_KEY_LEN 109
#define TSDB_CONN_ACTIVE_KEY_LEN 255 #define TSDB_CONN_ACTIVE_KEY_LEN 255
@ -428,6 +428,16 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DEFAULT_HASH_SUFFIX 0
#define TSDB_MIN_S3_CHUNK_SIZE (32 * 1024)
#define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024)
#define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024)
#define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute
#define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440)
#define TSDB_DEFAULT_S3_KEEP_LOCAL (3650 * 1440)
#define TSDB_MIN_S3_COMPACT 0
#define TSDB_MAX_S3_COMPACT 1
#define TSDB_DEFAULT_S3_COMPACT 0
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600 #define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600
#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600 #define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600
@ -543,12 +553,12 @@ enum {
SND_WORKER_TYPE__UNIQUE, SND_WORKER_TYPE__UNIQUE,
}; };
#define DEFAULT_HANDLE 0 #define DEFAULT_HANDLE 0
#define MNODE_HANDLE 1 #define MNODE_HANDLE 1
#define QNODE_HANDLE -1 #define QNODE_HANDLE -1
#define SNODE_HANDLE -2 #define SNODE_HANDLE -2
#define VNODE_HANDLE -3 #define VNODE_HANDLE -3
#define CLIENT_HANDLE -5 #define CLIENT_HANDLE -5
#define TSDB_CONFIG_OPTION_LEN 32 #define TSDB_CONFIG_OPTION_LEN 32
#define TSDB_CONFIG_VALUE_LEN 64 #define TSDB_CONFIG_VALUE_LEN 64
@ -568,13 +578,13 @@ enum {
// sort page size by default // sort page size by default
#define DEFAULT_PAGESIZE 4096 #define DEFAULT_PAGESIZE 4096
#define VNODE_TIMEOUT_SEC 60 #define VNODE_TIMEOUT_SEC 60
#define MNODE_TIMEOUT_SEC 60 #define MNODE_TIMEOUT_SEC 60
#define MONITOR_TABLENAME_LEN 200 #define MONITOR_TABLENAME_LEN 200
#define MONITOR_TAG_NAME_LEN 100 #define MONITOR_TAG_NAME_LEN 100
#define MONITOR_TAG_VALUE_LEN 300 #define MONITOR_TAG_VALUE_LEN 300
#define MONITOR_METRIC_NAME_LEN 100 #define MONITOR_METRIC_NAME_LEN 100
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

5
source/common/CMakeLists.txt
View File

@ -16,14 +16,15 @@ ENDIF ()
IF (TD_STORAGE) IF (TD_STORAGE)
ADD_DEFINITIONS(-D_STORAGE) ADD_DEFINITIONS(-D_STORAGE)
TARGET_LINK_LIBRARIES(common PRIVATE storage) TARGET_LINK_LIBRARIES(common PRIVATE storage)
ENDIF ()
IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3}) IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3) add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS}) ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS) add_definitions(-DUSE_COS)
ENDIF() ENDIF()
ENDIF()
ENDIF ()
target_include_directories( target_include_directories(
common common

208
source/common/src/cos.c
View File

@ -53,6 +53,100 @@ int32_t s3Init() { return 0; /*s3Begin();*/ }
void s3CleanUp() { /*s3End();*/ void s3CleanUp() { /*s3End();*/
} }
static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() {
int32_t code = 0;
code = s3Begin();
if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n");
goto _exit;
}
// test put
char testdata[17] = "0123456789abcdef";
const char *objectname[] = {"s3test.txt"};
char path[PATH_MAX] = {0};
int ds_len = strlen(TD_DIRSEP);
int tmp_len = strlen(tsTempDir);
snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
snprintf(path + tmp_len, PATH_MAX, "%s", TD_DIRSEP);
snprintf(path + tmp_len + ds_len, PATH_MAX, "%s", objectname[0]);
} else {
snprintf(path + tmp_len, PATH_MAX, "%s", objectname[0]);
}
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (!fp) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to open test file: %s.\n", path);
// uError("ERROR: %s Failed to open %s", __func__, path);
goto _exit;
}
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to write test file: %s.\n", path);
goto _exit;
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to fsync test file: %s.\n", path);
goto _exit;
}
taosCloseFile(&fp);
fprintf(stderr, "\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata);
code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16);
if (code != 0) {
fprintf(stderr, "put object %s : failed.\n", objectname[0]);
goto _exit;
}
fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
// list buckets
fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName);
code = s3ListBucket(tsS3BucketName);
if (code != 0) {
fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
goto _exit;
}
fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
// test range get
uint8_t *pBlock = NULL;
int c_offset = 10;
int c_len = 6;
fprintf(stderr, "start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len);
code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
if (code != 0) {
fprintf(stderr, "get object %s : failed.\n", objectname[0]);
goto _exit;
}
char buf[7] = {0};
memcpy(buf, pBlock, c_len);
taosMemoryFree(pBlock);
fprintf(stderr, "object content: %s\n", buf);
fprintf(stderr, "get object %s: success.\n\n", objectname[0]);
// delete test object
fprintf(stderr, "start to delete object: %s.\n", objectname[0]);
code = s3DeleteObjects(objectname, 1);
if (code != 0) {
fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
goto _exit;
}
fprintf(stderr, "delete object %s: success.\n\n", objectname[0]);
s3End();
_exit:
return code;
}
static int should_retry() { static int should_retry() {
/* /*
if (retriesG--) { if (retriesG--) {
@ -85,7 +179,7 @@ typedef struct {
} TS3GetData; } TS3GetData;
typedef struct { typedef struct {
char err_msg[128]; char err_msg[512];
S3Status status; S3Status status;
uint64_t content_length; uint64_t content_length;
char *buf; char *buf;
@ -137,6 +231,30 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
} }
} }
static SArray *getListByPrefix(const char *prefix);
static void s3FreeObjectKey(void *pItem);
static int32_t s3ListBucket(char const *bucketname) {
int32_t code = 0;
SArray *objectArray = getListByPrefix("s3");
if (objectArray == NULL) {
return -1;
}
const char **object_name = TARRAY_DATA(objectArray);
int size = TARRAY_SIZE(objectArray);
fprintf(stderr, "objects:\n");
for (int i = 0; i < size; ++i) {
fprintf(stderr, "%s\n", object_name[i]);
}
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return code;
}
typedef struct growbuffer { typedef struct growbuffer {
// The total number of bytes, and the start byte // The total number of bytes, and the start byte
int size; int size;
@ -471,10 +589,11 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c
} while (S3_status_is_retryable(data->status) && should_retry()); } while (S3_status_is_retryable(data->status) && should_retry());
if (data->status != S3StatusOK) { if (data->status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg); // s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
s3PrintError(NULL, __LINE__, __func__, data->status, data->err_msg);
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
} else if (data->contentLength) { } else if (data->contentLength) {
uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength); uError("%s Failed to put remaining %llu bytes", __func__, (unsigned long long)data->contentLength);
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
} }
@ -830,6 +949,66 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
return code; return code;
} }
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
int32_t code = 0;
int32_t lmtime = 0;
const char *filename = 0;
uint64_t contentLength = 0;
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
const char *contentDispositionFilename = 0, *contentEncoding = 0;
int64_t expires = -1;
S3CannedAcl cannedAcl = S3CannedAclPrivate;
int metaPropertiesCount = 0;
S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
char useServerSideEncryption = 0;
put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code;
}
contentLength = size;
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code;
}
if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) {
taosCloseFile(&data.infileFD);
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
contentLength;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG};
S3PutProperties putProperties = {contentType, md5,
cacheControl, contentDispositionFilename,
contentEncoding, expires,
cannedAcl, metaPropertiesCount,
metaProperties, useServerSideEncryption};
if (contentLength <= MULTIPART_CHUNK_SIZE) {
code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
} else {
code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
}
if (data.infileFD) {
taosCloseFile(&data.infileFD);
} else if (data.gb) {
growbuffer_destroy(data.gb);
}
return code;
}
typedef struct list_bucket_callback_data { typedef struct list_bucket_callback_data {
char err_msg[512]; char err_msg[512];
S3Status status; S3Status status;
@ -888,7 +1067,7 @@ static SArray *getListByPrefix(const char *prefix) {
const char *marker = 0, *delimiter = 0; const char *marker = 0, *delimiter = 0;
int maxkeys = 0, allDetails = 0; int maxkeys = 0, allDetails = 0;
list_bucket_callback_data data; list_bucket_callback_data data = {0};
data.objectArray = taosArrayInit(32, sizeof(void *)); data.objectArray = taosArrayInit(32, sizeof(void *));
if (!data.objectArray) { if (!data.objectArray) {
uError("%s: %s", __func__, "out of memoty"); uError("%s: %s", __func__, "out of memoty");
@ -918,14 +1097,17 @@ static SArray *getListByPrefix(const char *prefix) {
return data.objectArray; return data.objectArray;
} }
} else { } else {
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); uError("failed to list with prefix %s: %s", prefix, S3_get_status_name(data.status));
// s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
} }
taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
return NULL; return NULL;
} }
void s3DeleteObjects(const char *object_name[], int nobject) { int32_t s3DeleteObjects(const char *object_name[], int nobject) {
int32_t code = 0;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG}; 0, awsRegionG};
S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
@ -938,8 +1120,11 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
code = -1;
} }
} }
return code;
} }
void s3DeleteObjectsByPrefix(const char *prefix) { void s3DeleteObjectsByPrefix(const char *prefix) {
@ -991,12 +1176,12 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
} while (S3_status_is_retryable(cbd.status) && should_retry()); } while (S3_status_is_retryable(cbd.status) && should_retry());
if (cbd.status != S3StatusOK) { if (cbd.status != S3StatusOK) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO); return TAOS_SYSTEM_ERROR(EIO);
} }
if (check && cbd.buf_pos != size) { if (check && cbd.buf_pos != size) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO); return TAOS_SYSTEM_ERROR(EIO);
} }
@ -1233,7 +1418,7 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) {
cos_pool_destroy(p); cos_pool_destroy(p);
} }
void s3DeleteObjects(const char *object_name[], int nobject) { int32_t s3DeleteObjects(const char *object_name[], int nobject) {
cos_pool_t *p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_string_t bucket; cos_string_t bucket;
@ -1267,6 +1452,8 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
} else { } else {
cos_warn_log("delete objects failed\n"); cos_warn_log("delete objects failed\n");
} }
return 0;
} }
bool s3Exists(const char *object_name) { bool s3Exists(const char *object_name) {
@ -1535,8 +1722,9 @@ int32_t s3Init() { return 0; }
void s3CleanUp() {} void s3CleanUp() {}
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; } int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; }
void s3DeleteObjectsByPrefix(const char *prefix) {} void s3DeleteObjectsByPrefix(const char *prefix) {}
void s3DeleteObjects(const char *object_name[], int nobject) {} int32_t s3DeleteObjects(const char *object_name[], int nobject) { return 0; }
bool s3Exists(const char *object_name) { return false; } bool s3Exists(const char *object_name) { return false; }
bool s3Get(const char *object_name, const char *path) { return false; } bool s3Get(const char *object_name, const char *path) { return false; }
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {

5
source/common/src/systable.c
View File

@ -24,7 +24,7 @@
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
// clang-format off // clang-format off
static const SSysDbTableSchema dnodesSchema[] = { static const SSysDbTableSchema dnodesSchema[] = {
@ -117,6 +117,9 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "encrypt_algorithm", .bytes = TSDB_ENCRYPT_ALGO_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "encrypt_algorithm", .bytes = TSDB_ENCRYPT_ALGO_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
}; };

34
source/common/src/tglobal.c
View File

@ -275,7 +275,9 @@ float tsSinkDataRate = 2.0;
int32_t tsStreamNodeCheckInterval = 16; int32_t tsStreamNodeCheckInterval = 16;
int32_t tsTtlUnit = 86400; int32_t tsTtlUnit = 86400;
int32_t tsTtlPushIntervalSec = 10; int32_t tsTtlPushIntervalSec = 10;
int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups
int32_t tsS3MigrateIntervalSec = 60 * 60; // interval of s3migrate db in all vgroups
bool tsS3MigrateEnabled = 1;
int32_t tsGrantHBInterval = 60; int32_t tsGrantHBInterval = 60;
int32_t tsUptimeInterval = 300; // seconds int32_t tsUptimeInterval = 300; // seconds
char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits
@ -301,7 +303,7 @@ char tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) int32_t tsS3BlockSize = -1; // number of tsdb pages (4096)
int32_t tsS3BlockCacheSize = 16; // number of blocks int32_t tsS3BlockCacheSize = 16; // number of blocks
int32_t tsS3PageCacheSize = 4096; // number of pages int32_t tsS3PageCacheSize = 4096; // number of pages
int32_t tsS3UploadDelaySec = 60 * 60 * 24; int32_t tsS3UploadDelaySec = 60;
bool tsExperimental = true; bool tsExperimental = true;
@ -361,7 +363,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
} }
if (tsS3BucketName[0] != '<') { if (tsS3BucketName[0] != '<') {
#if defined(USE_COS) || defined(USE_S3) #if defined(USE_COS) || defined(USE_S3)
if (tsDiskCfgNum > 1) tsS3Enabled = true; #ifdef TD_ENTERPRISE
/*if (tsDiskCfgNum > 1) */ tsS3Enabled = true;
#endif
tsS3StreamEnabled = true; tsS3StreamEnabled = true;
#endif #endif
} }
@ -688,8 +692,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0) CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2),
(TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
@ -713,7 +717,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -758,6 +763,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1; return -1;
@ -776,8 +785,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
return -1;
if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0) CFG_DYN_ENT_SERVER) != 0)
@ -808,6 +816,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
/*
if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1; return -1;
if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) { if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) {
@ -817,10 +826,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
*/
if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0) CFG_DYN_ENT_SERVER) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0) CFG_DYN_ENT_SERVER) != 0)
return -1; return -1;
@ -1262,8 +1272,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32; tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32;
tsMinDiskFreeSize = cfgGetItem(pCfg, "minDiskFreeSize")->i64; tsMinDiskFreeSize = cfgGetItem(pCfg, "minDiskFreeSize")->i64;
tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32; // tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32;
tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32; // tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32;
tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32; tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32; tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
@ -1541,6 +1551,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"ttlBatchDropNum", &tsTtlBatchDropNum}, {"ttlBatchDropNum", &tsTtlBatchDropNum},
{"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlFlushThreshold", &tsTtlFlushThreshold},
{"ttlPushInterval", &tsTtlPushIntervalSec}, {"ttlPushInterval", &tsTtlPushIntervalSec},
{"s3MigrateIntervalSec", &tsS3MigrateIntervalSec},
{"s3MigrateEnabled", &tsS3MigrateEnabled},
//{"s3BlockSize", &tsS3BlockSize}, //{"s3BlockSize", &tsS3BlockSize},
{"s3BlockCacheSize", &tsS3BlockCacheSize}, {"s3BlockCacheSize", &tsS3BlockCacheSize},
{"s3PageCacheSize", &tsS3PageCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize},

188
source/common/src/tmsg.c
View File

@ -1475,9 +1475,7 @@ int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
return 0; return 0;
} }
void tFreeSStatisReq(SStatisReq *pReq) { void tFreeSStatisReq(SStatisReq *pReq) { taosMemoryFreeClear(pReq->pCont); }
taosMemoryFreeClear(pReq->pCont);
}
// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { // int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
// SEncoder encoder = {0}; // SEncoder encoder = {0};
@ -1876,7 +1874,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
char *tb = taosHashIterate(pRsp->readTbs, NULL); char *tb = taosHashIterate(pRsp->readTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1891,7 +1889,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeTbs, NULL); tb = taosHashIterate(pRsp->writeTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1906,7 +1904,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterTbs, NULL); tb = taosHashIterate(pRsp->alterTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1921,7 +1919,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->readViews, NULL); tb = taosHashIterate(pRsp->readViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1936,7 +1934,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeViews, NULL); tb = taosHashIterate(pRsp->writeViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1951,7 +1949,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->alterViews, NULL); tb = taosHashIterate(pRsp->alterViews, NULL);
while (tb != NULL) { while (tb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(tb, &keyLen); void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -1966,7 +1964,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
while (useDb != NULL) { while (useDb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
void * key = taosHashGetKey(useDb, &keyLen); void *key = taosHashGetKey(useDb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1;
@ -3026,6 +3024,9 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
} }
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL(); ENCODESQL();
@ -3099,6 +3100,15 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
} }
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
DECODESQL(); DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
@ -3146,6 +3156,10 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL(); ENCODESQL();
if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1; if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
@ -3196,6 +3210,13 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
} }
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
DECODESQL(); DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(&decoder)) { if (!tDecodeIsEnd(&decoder)) {
@ -3892,6 +3913,57 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) {
return 0; return 0;
} }
int32_t tSerializeSS3MigrateDbReq(void *buf, int32_t bufLen, SS3MigrateDbReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSS3MigrateDbReq(void *buf, int32_t bufLen, SS3MigrateDbReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSVS3MigrateDbReq(void *buf, int32_t bufLen, SVS3MigrateDbReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSVS3MigrateDbReq(void *buf, int32_t bufLen, SVS3MigrateDbReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq *pReq) { int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq *pReq) {
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);
@ -3990,6 +4062,9 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) {
if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1; if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1;
if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1; if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1;
if (tEncodeI8(encoder, pRsp->encryptAlgorithm) < 0) return -1; if (tEncodeI8(encoder, pRsp->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1;
if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1;
return 0; return 0;
} }
@ -4064,9 +4139,15 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) {
} }
pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(decoder)) { if (!tDecodeIsEnd(decoder)) {
if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1; if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1;
if (tDecodeI8(decoder, &pRsp->encryptAlgorithm) < 0) return -1; if (tDecodeI8(decoder, &pRsp->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(decoder, &pRsp->s3ChunkSize) < 0) return -1;
if (tDecodeI32(decoder, &pRsp->s3KeepLocal) < 0) return -1;
if (tDecodeI8(decoder, &pRsp->s3Compact) < 0) return -1;
} }
return 0; return 0;
@ -5092,7 +5173,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1; if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1;
if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1; if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1;
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) { for (int32_t i = 0; i < 6; ++i) {
if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
} }
if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1;
@ -5104,6 +5185,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI8(&encoder, pReq->encryptAlgorithm) < 0) return -1; if (tEncodeI8(&encoder, pReq->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
@ -5179,7 +5263,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1; if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1;
if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1; if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) { for (int32_t i = 0; i < 6; ++i) {
if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
} }
if (!tDecodeIsEnd(&decoder)) { if (!tDecodeIsEnd(&decoder)) {
@ -5201,6 +5285,15 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1; if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1;
} }
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
return 0; return 0;
@ -5456,7 +5549,7 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon
if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) { for (int32_t i = 0; i < 7; ++i) {
if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
} }
@ -5467,6 +5560,10 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon
if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1;
if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -5492,7 +5589,7 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) { for (int32_t i = 0; i < 7; ++i) {
if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
} }
@ -5518,6 +5615,13 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
} }
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
return 0; return 0;
@ -6571,9 +6675,7 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) {
return 0; return 0;
} }
void tDestroySMqHbRsp(SMqHbRsp *pRsp) { void tDestroySMqHbRsp(SMqHbRsp *pRsp) { taosArrayDestroy(pRsp->topicPrivileges); }
taosArrayDestroy(pRsp->topicPrivileges);
}
int32_t tSerializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) { int32_t tSerializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) {
SEncoder encoder = {0}; SEncoder encoder = {0};
@ -7359,27 +7461,27 @@ void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskSt
// return 0; // return 0;
// } // }
//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { // int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
// SDecoder decoder = {0}; // SDecoder decoder = {0};
// int32_t num = 0; // int32_t num = 0;
// tDecoderInit(&decoder, buf, bufLen); // tDecoderInit(&decoder, buf, bufLen);
// if (tStartDecode(&decoder) < 0) return -1; // if (tStartDecode(&decoder) < 0) return -1;
// if (tDecodeI32(&decoder, &num) < 0) return -1; // if (tDecodeI32(&decoder, &num) < 0) return -1;
// if (num > 0) { // if (num > 0) {
// pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp)); // pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
// if (NULL == pRsp->rspList) return -1; // if (NULL == pRsp->rspList) return -1;
// for (int32_t i = 0; i < num; ++i) { // for (int32_t i = 0; i < num; ++i) {
// SVCreateTbRsp rsp = {0}; // SVCreateTbRsp rsp = {0};
// if (tDecodeI32(&decoder, &rsp.code) < 0) return -1; // if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
// if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1; // if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
// } // }
// } else { // } else {
// pRsp->rspList = NULL; // pRsp->rspList = NULL;
// } // }
// tEndDecode(&decoder); // tEndDecode(&decoder);
// tDecoderClear(&decoder); // tDecoderClear(&decoder);
// return 0; // return 0;
//} //}
@ -7656,8 +7758,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeI32(&encoder, taosArrayGetSize(pReq->pVgroupVerList)) < 0) return -1; if (tEncodeI32(&encoder, taosArrayGetSize(pReq->pVgroupVerList)) < 0) return -1;
for(int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) {
SVgroupVer* p = taosArrayGet(pReq->pVgroupVerList, i); SVgroupVer *p = taosArrayGet(pReq->pVgroupVerList, i);
if (tEncodeI32(&encoder, p->vgId) < 0) return -1; if (tEncodeI32(&encoder, p->vgId) < 0) return -1;
if (tEncodeI64(&encoder, p->ver) < 0) return -1; if (tEncodeI64(&encoder, p->ver) < 0) return -1;
} }
@ -8921,7 +9023,7 @@ int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
for (int32_t i = 0; i < pRsp->blockNum; i++) { for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
void * data = taosArrayGetP(pRsp->blockData, i); void *data = taosArrayGetP(pRsp->blockData, i);
if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
if (pRsp->withSchema) { if (pRsp->withSchema) {
SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
@ -8960,7 +9062,7 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
} }
for (int32_t i = 0; i < pRsp->blockNum; i++) { for (int32_t i = 0; i < pRsp->blockNum; i++) {
void * data; void *data;
uint64_t bLen; uint64_t bLen;
if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
taosArrayPush(pRsp->blockData, &data); taosArrayPush(pRsp->blockData, &data);
@ -9020,7 +9122,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
if (pRsp->createTableNum) { if (pRsp->createTableNum) {
for (int32_t i = 0; i < pRsp->createTableNum; i++) { for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * createTableReq = taosArrayGetP(pRsp->createTableReq, i); void *createTableReq = taosArrayGetP(pRsp->createTableReq, i);
int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i); int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i);
if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1; if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1;
} }
@ -9039,7 +9141,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion
pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *)); pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
for (int32_t i = 0; i < pRsp->createTableNum; i++) { for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void * pCreate = NULL; void *pCreate = NULL;
uint64_t len; uint64_t len;
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
int32_t l = (int32_t)len; int32_t l = (int32_t)len;
@ -9351,7 +9453,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
taosArrayDestroy(pTbData->aCol); taosArrayDestroy(pTbData->aCol);
} else { } else {
int32_t nRow = TARRAY_SIZE(pTbData->aRowP); int32_t nRow = TARRAY_SIZE(pTbData->aRowP);
SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP); SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
for (int32_t i = 0; i < nRow; ++i) { for (int32_t i = 0; i < nRow; ++i) {
tRowDestroy(rows[i]); tRowDestroy(rows[i]);

8
source/dnode/mgmt/CMakeLists.txt
View File

@ -14,6 +14,14 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc"
) )
IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
ENDIF()
ENDIF()
IF (TD_LINUX_64 AND JEMALLOC_ENABLED) IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc)
SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc")

37
source/dnode/mgmt/exe/dmMain.c
View File

@ -28,15 +28,15 @@
#include "cus_name.h" #include "cus_name.h"
#else #else
#ifndef CUS_NAME #ifndef CUS_NAME
#define CUS_NAME "TDengine" #define CUS_NAME "TDengine"
#endif #endif
#ifndef CUS_PROMPT #ifndef CUS_PROMPT
#define CUS_PROMPT "taos" #define CUS_PROMPT "taos"
#endif #endif
#ifndef CUS_EMAIL #ifndef CUS_EMAIL
#define CUS_EMAIL "<support@taosdata.com>" #define CUS_EMAIL "<support@taosdata.com>"
#endif #endif
#endif #endif
// clang-format off // clang-format off
@ -61,6 +61,7 @@ static struct {
bool dumpSdb; bool dumpSdb;
bool generateGrant; bool generateGrant;
bool memDbg; bool memDbg;
bool checkS3;
bool printAuth; bool printAuth;
bool printVersion; bool printVersion;
bool printHelp; bool printHelp;
@ -174,7 +175,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
return -1; return -1;
} }
} else if (strcmp(argv[i], "-a") == 0) { } else if (strcmp(argv[i], "-a") == 0) {
if(i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("apollo url overflow"); printf("apollo url overflow");
return -1; return -1;
@ -187,7 +188,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
} else if (strcmp(argv[i], "-s") == 0) { } else if (strcmp(argv[i], "-s") == 0) {
global.dumpSdb = true; global.dumpSdb = true;
} else if (strcmp(argv[i], "-E") == 0) { } else if (strcmp(argv[i], "-E") == 0) {
if(i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("env file path overflow"); printf("env file path overflow");
return -1; return -1;
@ -229,6 +230,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
cmdEnvIndex++; cmdEnvIndex++;
} else if (strcmp(argv[i], "-dm") == 0) { } else if (strcmp(argv[i], "-dm") == 0) {
global.memDbg = true; global.memDbg = true;
} else if (strcmp(argv[i], "--checks3") == 0) {
global.checkS3 = true;
} else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 ||
strcmp(argv[i], "-?") == 0) { strcmp(argv[i], "-?") == 0) {
global.printHelp = true; global.printHelp = true;
@ -290,8 +293,21 @@ static void dmDumpCfg() {
cfgDumpCfg(pCfg, 0, true); cfgDumpCfg(pCfg, 0, true);
} }
static int32_t dmCheckS3() {
int32_t code = 0;
SConfig *pCfg = taosGetCfg();
cfgDumpCfgS3(pCfg, 0, true);
#if defined(USE_S3)
extern int32_t s3CheckCfg();
code = s3CheckCfg();
#endif
return code;
}
static int32_t dmInitLog() { static int32_t dmInitLog() {
return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs,
0);
} }
static void taosCleanupArgs() { static void taosCleanupArgs() {
@ -396,6 +412,15 @@ int mainWindows(int argc, char **argv) {
return -1; return -1;
} }
if (global.checkS3) {
int32_t code = dmCheckS3();
taosCleanupCfg();
taosCloseLog();
taosCleanupArgs();
taosConvDestroy();
return code;
}
if (global.dumpConfig) { if (global.dumpConfig) {
dmDumpCfg(); dmDumpCfg();
taosCleanupCfg(); taosCleanupCfg();

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -142,6 +142,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_S3MIGRATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -210,6 +211,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

39
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -53,7 +53,7 @@ void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
SVnodeObj **ppVnode = pIter; SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue; if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj *pVnode = *ppVnode; SVnodeObj *pVnode = *ppVnode;
if (!pVnode->failed) { if (!pVnode->failed) {
SVnodeLoadLite vload = {0}; SVnodeLoadLite vload = {0};
if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) { if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) {
@ -184,6 +184,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->hashSuffix = pCreate->hashSuffix; pCfg->hashSuffix = pCreate->hashSuffix;
pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024;
pCfg->s3ChunkSize = pCreate->s3ChunkSize;
pCfg->s3KeepLocal = pCreate->s3KeepLocal;
pCfg->s3Compact = pCreate->s3Compact;
pCfg->standby = 0; pCfg->standby = 0;
pCfg->syncCfg.replicaNum = 0; pCfg->syncCfg.replicaNum = 0;
pCfg->syncCfg.totalReplicaNum = 0; pCfg->syncCfg.totalReplicaNum = 0;
@ -263,17 +267,25 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo( dInfo(
"vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d " "vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d "
"szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64 "szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d" ", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d s3ChunkSize:%d s3KeepLocal:%d s3Compact:%d tsma:%d "
"precision:%d compression:%d minRows:%d maxRows:%d"
", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64 ", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64
", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d " ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d "
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d encryptAlgorithm:%d", "learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d encryptAlgorithm:%d",
req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024,
(uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize,
req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
<<<<<<< HEAD
req.keepTimeOffset, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod, req.keepTimeOffset, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod,
req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod,
req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica,
req.learnerSelfIndex, req.strict, req.changeVersion, req.encryptAlgorithm); req.learnerSelfIndex, req.strict, req.changeVersion, req.encryptAlgorithm);
=======
req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion);
>>>>>>> 3.0
for (int32_t i = 0; i < req.replica; ++i) { for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port, dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
@ -380,7 +392,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
vnodeClose(pImpl); vnodeClose(pImpl);
vnodeDestroy(0, path, pMgmt->pTfs); vnodeDestroy(0, path, pMgmt->pTfs, 0);
} else { } else {
dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", req.vgId, dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", req.vgId,
TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
@ -391,7 +403,7 @@ _OVER:
return code; return code;
} }
//alter replica doesn't use this, but restore dnode still use this // alter replica doesn't use this, but restore dnode still use this
int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SAlterVnodeTypeReq req = {0}; SAlterVnodeTypeReq req = {0};
if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) { if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
@ -433,8 +445,8 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name); dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name);
int32_t vgId = req.vgId; int32_t vgId = req.vgId;
dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d", dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d", vgId, req.replica,
vgId, req.replica, req.selfIndex, req.strict, req.changeVersion); req.selfIndex, req.strict, req.changeVersion);
for (int32_t i = 0; i < req.replica; ++i) { for (int32_t i = 0; i < req.replica; ++i) {
SReplica *pReplica = &req.replicas[i]; SReplica *pReplica = &req.replicas[i];
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id);
@ -517,12 +529,12 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1; return -1;
} }
if(req.learnerReplicas == 0){ if (req.learnerReplicas == 0) {
req.learnerSelfIndex = -1; req.learnerSelfIndex = -1;
} }
dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request", dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request", req.vgId,
req.vgId, TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
@ -534,7 +546,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
ESyncRole role = vnodeGetRole(pVnode->pImpl); ESyncRole role = vnodeGetRole(pVnode->pImpl);
dInfo("vgId:%d, checking node role:%d", req.vgId, role); dInfo("vgId:%d, checking node role:%d", req.vgId, role);
if(role == TAOS_SYNC_ROLE_VOTER){ if (role == TAOS_SYNC_ROLE_VOTER) {
dError("vgId:%d, failed to alter vnode type since node already is role:%d", req.vgId, role); dError("vgId:%d, failed to alter vnode type since node already is role:%d", req.vgId, role);
terrno = TSDB_CODE_VND_ALREADY_IS_VOTER; terrno = TSDB_CODE_VND_ALREADY_IS_VOTER;
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
@ -542,7 +554,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
dInfo("vgId:%d, checking node catch up", req.vgId); dInfo("vgId:%d, checking node catch up", req.vgId);
if(vnodeIsCatchUp(pVnode->pImpl) != 1){ if (vnodeIsCatchUp(pVnode->pImpl) != 1) {
terrno = TSDB_CODE_VND_NOT_CATCH_UP; terrno = TSDB_CODE_VND_NOT_CATCH_UP;
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
return -1; return -1;
@ -552,8 +564,8 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request", dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request", req.vgId,
req.vgId, TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
return 0; return 0;
} }
@ -966,6 +978,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

5
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "vmInt.h" #include "vmInt.h"
#include "libs/function/tudf.h"
#include "tfs.h" #include "tfs.h"
#include "vnd.h" #include "vnd.h"
#include "libs/function/tudf.h"
int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
int32_t diskId = -1; int32_t diskId = -1;
@ -234,6 +234,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
dInfo("vgId:%d, commit data finished", pVnode->vgId); dInfo("vgId:%d, commit data finished", pVnode->vgId);
} }
int32_t nodeId = vnodeNodeId(pVnode->pImpl);
vnodeClose(pVnode->pImpl); vnodeClose(pVnode->pImpl);
pVnode->pImpl = NULL; pVnode->pImpl = NULL;
@ -250,7 +251,7 @@ _closed:
if (pVnode->dropped) { if (pVnode->dropped) {
dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped); dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId); snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs); vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs, nodeId);
} }
vmFreeVnodeObj(&pVnode); vmFreeVnodeObj(&pVnode);

10
source/dnode/mgmt/node_mgmt/CMakeLists.txt
View File

@ -4,15 +4,13 @@ target_link_libraries(
dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
) )
IF (TD_STORAGE) IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3}) IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3) add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS}) ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS) add_definitions(-DUSE_COS)
ENDIF() ENDIF()
ENDIF()
ENDIF ()
IF (DEFINED GRANT_CFG_INCLUDE_DIR) IF (DEFINED GRANT_CFG_INCLUDE_DIR)
add_definitions(-DGRANTS_CFG) add_definitions(-DGRANTS_CFG)

3
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -382,6 +382,9 @@ typedef struct {
int32_t walRollPeriod; int32_t walRollPeriod;
int64_t walRetentionSize; int64_t walRetentionSize;
int64_t walSegmentSize; int64_t walSegmentSize;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int8_t withArbitrator; int8_t withArbitrator;
int8_t encryptAlgorithm; int8_t encryptAlgorithm;
} SDbCfg; } SDbCfg;

140
source/dnode/mnode/impl/src/mndDb.c
View File

@ -36,7 +36,7 @@
#include "tjson.h" #include "tjson.h"
#define DB_VER_NUMBER 1 #define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 40 #define DB_RESERVE_SIZE 31
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -49,6 +49,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq);
static int32_t mndProcessDropDbReq(SRpcMsg *pReq); static int32_t mndProcessDropDbReq(SRpcMsg *pReq);
static int32_t mndProcessUseDbReq(SRpcMsg *pReq); static int32_t mndProcessUseDbReq(SRpcMsg *pReq);
static int32_t mndProcessTrimDbReq(SRpcMsg *pReq); static int32_t mndProcessTrimDbReq(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq);
static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity);
static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter);
static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq); static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq);
@ -76,6 +77,7 @@ int32_t mndInitDb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq);
mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB, mndProcessTrimDbReq); mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB, mndProcessTrimDbReq);
mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq);
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB, mndProcessS3MigrateDbReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb);
@ -140,6 +142,9 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER)
SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.keepTimeOffset, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.keepTimeOffset, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3ChunkSize, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.encryptAlgorithm, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.encryptAlgorithm, _OVER)
@ -233,12 +238,27 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.keepTimeOffset, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.keepTimeOffset, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3ChunkSize, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.encryptAlgorithm, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.encryptAlgorithm, _OVER)
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock); taosInitRWLatch(&pDb->lock);
if (pDb->cfg.s3ChunkSize == 0) {
pDb->cfg.s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
mInfo("db:%s, s3ChunkSize set from %d to default %d", pDb->name, pDb->cfg.s3ChunkSize, TSDB_DEFAULT_S3_CHUNK_SIZE);
}
if (pDb->cfg.s3KeepLocal == 0) {
pDb->cfg.s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
mInfo("db:%s, s3KeepLocal set from %d to default %d", pDb->name, pDb->cfg.s3KeepLocal, TSDB_DEFAULT_S3_KEEP_LOCAL);
}
if (pDb->cfg.tsdbPageSize != TSDB_MIN_TSDB_PAGESIZE) { if (pDb->cfg.tsdbPageSize != TSDB_MIN_TSDB_PAGESIZE) {
mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize, mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize,
TSDB_DEFAULT_TSDB_PAGESIZE); TSDB_DEFAULT_TSDB_PAGESIZE);
@ -324,6 +344,9 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.minRows = pNew->cfg.minRows; pOld->cfg.minRows = pNew->cfg.minRows;
pOld->cfg.maxRows = pNew->cfg.maxRows; pOld->cfg.maxRows = pNew->cfg.maxRows;
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
pOld->cfg.s3ChunkSize = pNew->cfg.s3ChunkSize;
pOld->cfg.s3KeepLocal = pNew->cfg.s3KeepLocal;
pOld->cfg.s3Compact = pNew->cfg.s3Compact;
pOld->cfg.withArbitrator = pNew->cfg.withArbitrator; pOld->cfg.withArbitrator = pNew->cfg.withArbitrator;
pOld->compactStartTime = pNew->compactStartTime; pOld->compactStartTime = pNew->compactStartTime;
taosWUnLockLatch(&pOld->lock); taosWUnLockLatch(&pOld->lock);
@ -427,6 +450,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1; if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
if (pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1;
if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1;
if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1;
terrno = 0; terrno = 0;
return terrno; return terrno;
} }
@ -476,6 +503,9 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1; return -1;
} }
if (pNewCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pNewCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1;
if (pNewCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pNewCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1;
if (pNewCfg->s3Compact < TSDB_MIN_S3_COMPACT || pNewCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1;
terrno = 0; terrno = 0;
return terrno; return terrno;
@ -512,6 +542,9 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
if (pCfg->s3ChunkSize <= 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
} }
@ -684,6 +717,9 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
.sstTrigger = pCreate->sstTrigger, .sstTrigger = pCreate->sstTrigger,
.hashPrefix = pCreate->hashPrefix, .hashPrefix = pCreate->hashPrefix,
.hashSuffix = pCreate->hashSuffix, .hashSuffix = pCreate->hashSuffix,
.s3ChunkSize = pCreate->s3ChunkSize,
.s3KeepLocal = pCreate->s3KeepLocal,
.s3Compact = pCreate->s3Compact,
.tsdbPageSize = pCreate->tsdbPageSize, .tsdbPageSize = pCreate->tsdbPageSize,
.withArbitrator = pCreate->withArbitrator, .withArbitrator = pCreate->withArbitrator,
.encryptAlgorithm = pCreate->encryptAlgorithm, .encryptAlgorithm = pCreate->encryptAlgorithm,
@ -996,6 +1032,18 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
terrno = 0; terrno = 0;
} }
if (pAlter->s3KeepLocal > TSDB_MIN_S3_KEEP_LOCAL && pAlter->s3KeepLocal != pDb->cfg.s3KeepLocal) {
pDb->cfg.s3KeepLocal = pAlter->s3KeepLocal;
pDb->vgVersion++;
terrno = 0;
}
if (pAlter->s3Compact > TSDB_MIN_S3_COMPACT && pAlter->s3Compact != pDb->cfg.s3Compact) {
pDb->cfg.s3Compact = pAlter->s3Compact;
pDb->vgVersion++;
terrno = 0;
}
if (pAlter->withArbitrator >= TSDB_MIN_DB_WITH_ARBITRATOR && pAlter->withArbitrator != pDb->cfg.withArbitrator) { if (pAlter->withArbitrator >= TSDB_MIN_DB_WITH_ARBITRATOR && pAlter->withArbitrator != pDb->cfg.withArbitrator) {
pDb->cfg.withArbitrator = pAlter->withArbitrator; pDb->cfg.withArbitrator = pAlter->withArbitrator;
pDb->vgVersion++; pDb->vgVersion++;
@ -1202,6 +1250,9 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL);
cfgRsp->schemaless = pDb->cfg.schemaless; cfgRsp->schemaless = pDb->cfg.schemaless;
cfgRsp->sstTrigger = pDb->cfg.sstTrigger; cfgRsp->sstTrigger = pDb->cfg.sstTrigger;
cfgRsp->s3ChunkSize = pDb->cfg.s3ChunkSize;
cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal;
cfgRsp->s3Compact = pDb->cfg.s3Compact;
cfgRsp->withArbitrator = pDb->cfg.withArbitrator; cfgRsp->withArbitrator = pDb->cfg.withArbitrator;
cfgRsp->encryptAlgorithm = pDb->cfg.encryptAlgorithm; cfgRsp->encryptAlgorithm = pDb->cfg.encryptAlgorithm;
} }
@ -1270,7 +1321,7 @@ static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
if (pIter == NULL) break; if (pIter == NULL) break;
if (pArbGroup->dbUid == pDb->uid) { if (pArbGroup->dbUid == pDb->uid) {
if (mndSetDropArbGroupPrepareLogs(pTrans,pArbGroup) != 0) { if (mndSetDropArbGroupPrepareLogs(pTrans, pArbGroup) != 0) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pArbGroup); sdbRelease(pSdb, pArbGroup);
return -1; return -1;
@ -1298,7 +1349,7 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
if (pIter == NULL) break; if (pIter == NULL) break;
if (pArbGroup->dbUid == pDb->uid) { if (pArbGroup->dbUid == pDb->uid) {
if (mndSetDropArbGroupCommitLogs(pTrans,pArbGroup) != 0) { if (mndSetDropArbGroupCommitLogs(pTrans, pArbGroup) != 0) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pArbGroup); sdbRelease(pSdb, pArbGroup);
return -1; return -1;
@ -1850,6 +1901,77 @@ _OVER:
return code; return code;
} }
static int32_t mndS3MigrateDb(SMnode *pMnode, SDbObj *pDb) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()};
int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq);
int32_t contLen = reqLen + sizeof(SMsgHead);
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid != pDb->uid) continue;
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), contLen, &s3migrateReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("vgId:%d, failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code);
} else {
mInfo("vgId:%d, send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp);
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SDbObj *pDb = NULL;
SS3MigrateDbReq s3migrateReq = {0};
if (tDeserializeSS3MigrateDbReq(pReq->pCont, pReq->contLen, &s3migrateReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("db:%s, start to s3migrate", s3migrateReq.db);
pDb = mndAcquireDb(pMnode, s3migrateReq.db);
if (pDb == NULL) {
goto _OVER;
}
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_TRIM_DB, pDb) != 0) {
goto _OVER;
}
code = mndS3MigrateDb(pMnode, pDb);
_OVER:
if (code != 0) {
mError("db:%s, failed to process s3migrate db req since %s", s3migrateReq.db, terrstr());
}
mndReleaseDb(pMnode, pDb);
return code;
}
const char *mndGetDbStr(const char *src) { const char *mndGetDbStr(const char *src) {
char *pos = strstr(src, TS_PATH_DELIMITER); char *pos = strstr(src, TS_PATH_DELIMITER);
if (pos != NULL) ++pos; if (pos != NULL) ++pos;
@ -2160,6 +2282,18 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3ChunkSize, false);
char keeplocalVstr[128] = {0};
len = sprintf(&keeplocalVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.s3KeepLocal);
varDataSetLen(keeplocalVstr, len);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)keeplocalVstr, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3Compact, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false);

16
source/dnode/mnode/impl/src/mndMain.c
View File

@ -134,6 +134,14 @@ static void mndPullupTtl(SMnode *pMnode) {
} }
static void mndPullupTrimDb(SMnode *pMnode) { static void mndPullupTrimDb(SMnode *pMnode) {
mTrace("pullup s3migrate");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_S3MIGRATE_DB_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
static void mndPullupS3MigrateDb(SMnode *pMnode) {
mTrace("pullup trim"); mTrace("pullup trim");
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
@ -319,6 +327,7 @@ static int32_t minCronTime() {
int32_t min = INT32_MAX; int32_t min = INT32_MAX;
min = TMIN(min, tsTtlPushIntervalSec); min = TMIN(min, tsTtlPushIntervalSec);
min = TMIN(min, tsTrimVDbIntervalSec); min = TMIN(min, tsTrimVDbIntervalSec);
min = TMIN(min, tsS3MigrateIntervalSec);
min = TMIN(min, tsTransPullupInterval); min = TMIN(min, tsTransPullupInterval);
min = TMIN(min, tsCompactPullupInterval); min = TMIN(min, tsCompactPullupInterval);
min = TMIN(min, tsMqRebalanceInterval); min = TMIN(min, tsMqRebalanceInterval);
@ -342,6 +351,10 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
mndPullupTrimDb(pMnode); mndPullupTrimDb(pMnode);
} }
if (tsS3MigrateEnabled && sec % tsS3MigrateIntervalSec == 0) {
mndPullupS3MigrateDb(pMnode);
}
if (sec % tsTransPullupInterval == 0) { if (sec % tsTransPullupInterval == 0) {
mndPullupTrans(pMnode); mndPullupTrans(pMnode);
} }
@ -821,6 +834,7 @@ _OVER:
pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER || pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER ||
pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE || pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE ||
pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT ||
pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER ||
pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) { pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state)); pMnode->stopped, state.restored, syncStr(state.state));
@ -965,7 +979,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pObj->id == pMnode->selfDnodeId) { if (pObj->id == pMnode->selfDnodeId) {
pClusterInfo->first_ep_dnode_id = pObj->id; pClusterInfo->first_ep_dnode_id = pObj->id;
tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
//pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; // pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode);
// pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f);
tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role));

66
source/dnode/mnode/impl/src/mndStb.c
View File

@ -42,6 +42,8 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew);
static int32_t mndProcessTtlTimer(SRpcMsg *pReq); static int32_t mndProcessTtlTimer(SRpcMsg *pReq);
static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq); static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pReq);
static int32_t mndProcessCreateStbReq(SRpcMsg *pReq); static int32_t mndProcessCreateStbReq(SRpcMsg *pReq);
static int32_t mndProcessAlterStbReq(SRpcMsg *pReq); static int32_t mndProcessAlterStbReq(SRpcMsg *pReq);
static int32_t mndProcessDropStbReq(SRpcMsg *pReq); static int32_t mndProcessDropStbReq(SRpcMsg *pReq);
@ -82,6 +84,8 @@ int32_t mndInitStb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq);
mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer); mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB_TIMER, mndProcessTrimDbTimer); mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB_TIMER, mndProcessTrimDbTimer);
mndSetMsgHandle(pMnode, TDMT_VND_S3MIGRATE_RSP, mndProcessS3MigrateDbRsp);
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB_TIMER, mndProcessS3MigrateDbTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq);
// mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); // mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq);
@ -762,9 +766,9 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
pDst->createdTime = taosGetTimestampMs(); pDst->createdTime = taosGetTimestampMs();
pDst->updateTime = pDst->createdTime; pDst->updateTime = pDst->createdTime;
pDst->uid = pDst->uid = (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX)
(pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX) ? pCreate->suid
? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
pDst->dbUid = pDb->uid; pDst->dbUid = pDb->uid;
pDst->tagVer = 1; pDst->tagVer = 1;
pDst->colVer = 1; pDst->colVer = 1;
@ -983,6 +987,43 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) {
return 0; return 0;
} }
static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()};
int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq);
int32_t contLen = reqLen + sizeof(SMsgHead);
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("vgId:%d, timer failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code);
} else {
mInfo("vgId:%d, timer send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp);
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) { static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) {
for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
if (strcmp(pStb->pTags[tag].name, tagName) == 0) { if (strcmp(pStb->pTags[tag].name, tagName) == 0) {
@ -1133,7 +1174,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
} }
} else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) {
goto _OVER; goto _OVER;
} else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) { } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) &&
(createReq.tagVer != 1 || createReq.colVer != 1)) {
mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name); mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name);
code = 0; code = 0;
goto _OVER; goto _OVER;
@ -1184,14 +1226,13 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
SName name = {0}; SName name = {0};
tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
if(createReq.sql == NULL && createReq.sqlLen == 0){ if (createReq.sql == NULL && createReq.sqlLen == 0) {
char detail[1000] = {0}; char detail[1000] = {0};
sprintf(detail, "dbname:%s, stable name:%s", name.dbname, name.tname); sprintf(detail, "dbname:%s, stable name:%s", name.dbname, name.tname);
auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, detail, strlen(detail)); auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, detail, strlen(detail));
} } else {
else{
auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, createReq.sql, createReq.sqlLen); auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, createReq.sql, createReq.sqlLen);
} }
_OVER: _OVER:
@ -1573,7 +1614,7 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj
for (int32_t i = 0; i < pOld->numOfTags; ++i) { for (int32_t i = 0; i < pOld->numOfTags; ++i) {
nLen += (pOld->pTags[i].colId == colId) ? pField->bytes : pOld->pTags[i].bytes; nLen += (pOld->pTags[i].colId == colId) ? pField->bytes : pOld->pTags[i].bytes;
} }
if (nLen > TSDB_MAX_TAGS_LEN) { if (nLen > TSDB_MAX_TAGS_LEN) {
terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH; terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH;
return -1; return -1;
@ -1936,7 +1977,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
return 0; return 0;
} }
static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bool* schema, bool* sma) { static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion *pStbVer, bool *schema, bool *sma) {
char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName); snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName);
@ -1966,7 +2007,7 @@ static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bo
} else { } else {
*schema = false; *schema = false;
} }
if (pStbVer->smaVer && pStbVer->smaVer != pStb->smaVer) { if (pStbVer->smaVer && pStbVer->smaVer != pStb->smaVer) {
*sma = true; *sma = true;
} else { } else {
@ -2537,6 +2578,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
static int32_t mndProcessDropTtltbRsp(SRpcMsg *pRsp) { return 0; } static int32_t mndProcessDropTtltbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessTrimDbRsp(SRpcMsg *pRsp) { return 0; } static int32_t mndProcessTrimDbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
@ -2750,8 +2792,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t
pStbVersion->tversion = ntohl(pStbVersion->tversion); pStbVersion->tversion = ntohl(pStbVersion->tversion);
pStbVersion->smaVer = ntohl(pStbVersion->smaVer); pStbVersion->smaVer = ntohl(pStbVersion->smaVer);
bool schema = false; bool schema = false;
bool sma = false; bool sma = false;
int32_t code = mndValidateStbVersion(pMnode, pStbVersion, &schema, &sma); int32_t code = mndValidateStbVersion(pMnode, pStbVersion, &schema, &sma);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
STableMetaRsp metaRsp = {0}; STableMetaRsp metaRsp = {0};

13
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -32,10 +32,10 @@
#define VGROUP_VER_NUMBER 1 #define VGROUP_VER_NUMBER 1
#define VGROUP_RESERVE_SIZE 64 #define VGROUP_RESERVE_SIZE 64
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
@ -276,6 +276,9 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
createReq.daysToKeep1 = pDb->cfg.daysToKeep1; createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
createReq.daysToKeep2 = pDb->cfg.daysToKeep2; createReq.daysToKeep2 = pDb->cfg.daysToKeep2;
createReq.keepTimeOffset = pDb->cfg.keepTimeOffset; createReq.keepTimeOffset = pDb->cfg.keepTimeOffset;
createReq.s3ChunkSize = pDb->cfg.s3ChunkSize;
createReq.s3KeepLocal = pDb->cfg.s3KeepLocal;
createReq.s3Compact = pDb->cfg.s3Compact;
createReq.minRows = pDb->cfg.minRows; createReq.minRows = pDb->cfg.minRows;
createReq.maxRows = pDb->cfg.maxRows; createReq.maxRows = pDb->cfg.maxRows;
createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod; createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
@ -400,6 +403,8 @@ static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pV
alterReq.minRows = pDb->cfg.minRows; alterReq.minRows = pDb->cfg.minRows;
alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod; alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod;
alterReq.walRetentionSize = pDb->cfg.walRetentionSize; alterReq.walRetentionSize = pDb->cfg.walRetentionSize;
alterReq.s3KeepLocal = pDb->cfg.s3KeepLocal;
alterReq.s3Compact = pDb->cfg.s3Compact;
mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId); mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId);
int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq); int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq);

17
source/dnode/vnode/inc/vnode.h
View File

@ -57,7 +57,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
int32_t diskPrimary, STfs *pTfs); int32_t diskPrimary, STfs *pTfs);
int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId,
int32_t diskPrimary, STfs *pTfs); int32_t diskPrimary, STfs *pTfs);
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs); void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId);
SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force); SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force);
void vnodePreClose(SVnode *pVnode); void vnodePreClose(SVnode *pVnode);
void vnodePostClose(SVnode *pVnode); void vnodePostClose(SVnode *pVnode);
@ -69,7 +69,7 @@ int32_t vnodeBegin(SVnode *pVnode);
int32_t vnodeStart(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode);
void vnodeStop(SVnode *pVnode); void vnodeStop(SVnode *pVnode);
int64_t vnodeGetSyncHandle(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode);
int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables); void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables);
int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList); int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList);
@ -137,8 +137,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen); int32_t payloadLen);
bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type); bool metaTbInFilterCache(SMeta *pMeta, const void *key, int8_t type);
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type); int32_t metaPutTbToFilterCache(SMeta *pMeta, const void *key, int8_t type);
int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type); int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type);
int32_t metaInitTbFilterCache(SMeta *pMeta); int32_t metaInitTbFilterCache(SMeta *pMeta);
@ -175,8 +175,8 @@ void *tsdbGetIvtIdx2(SMeta *pMeta);
uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader);
void tsdbReaderSetCloseFlag(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader);
int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
void tsdbSetFilesetDelimited(STsdbReader* pReader); void tsdbSetFilesetDelimited(STsdbReader *pReader);
void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param);
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
@ -241,7 +241,7 @@ int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, i
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished); int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished);
// sma // sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
@ -323,6 +323,9 @@ struct SVnodeCfg {
int32_t tsdbPageSize; int32_t tsdbPageSize;
int32_t tdbEncryptAlgorithm; int32_t tdbEncryptAlgorithm;
char tdbEncryptKey[ENCRYPT_KEY_LEN]; char tdbEncryptKey[ENCRYPT_KEY_LEN];
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
}; };
#define TABLE_ROLLUP_ON ((int8_t)0x1) #define TABLE_ROLLUP_ON ((int8_t)0x1)

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -635,6 +635,7 @@ typedef struct {
STsdb *pTsdb; STsdb *pTsdb;
const char *objName; const char *objName;
uint8_t s3File; uint8_t s3File;
int32_t lcn;
int32_t fid; int32_t fid;
int64_t cid; int64_t cid;
int64_t blkno; int64_t blkno;

2
source/dnode/vnode/src/inc/vnd.h
View File

@ -133,6 +133,8 @@ int32_t vnodeAsyncCommit(SVnode* pVnode);
bool vnodeShouldRollback(SVnode* pVnode); bool vnodeShouldRollback(SVnode* pVnode);
// vnodeSync.c // vnodeSync.c
int64_t vnodeClusterId(SVnode* pVnode);
int32_t vnodeNodeId(SVnode* pVnode);
int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion); int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion);
int32_t vnodeSyncStart(SVnode* pVnode); int32_t vnodeSyncStart(SVnode* pVnode);
void vnodeSyncPreClose(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode);

25
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -108,13 +108,13 @@ typedef struct SQueryNode SQueryNode;
#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count" #define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count"
#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" #define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" #define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" #define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" #define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
#define VNODE_METRIC_TAG_NAME_USERNAME "username" #define VNODE_METRIC_TAG_NAME_USERNAME "username"
#define VNODE_METRIC_TAG_NAME_RESULT "result" #define VNODE_METRIC_TAG_NAME_RESULT "result"
#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" #define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
//#define VNODE_METRIC_TAG_VALUE_INSERT "insert" //#define VNODE_METRIC_TAG_VALUE_INSERT "insert"
@ -239,6 +239,7 @@ int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t
int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey); int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey);
int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync); int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync);
int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp);
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);
@ -461,12 +462,12 @@ typedef struct SVCommitSched {
int64_t maxWaitMs; int64_t maxWaitMs;
} SVCommitSched; } SVCommitSched;
typedef struct SVMonitorObj{ typedef struct SVMonitorObj {
char strClusterId[TSDB_CLUSTER_ID_LEN]; char strClusterId[TSDB_CLUSTER_ID_LEN];
char strDnodeId[TSDB_NODE_ID_LEN]; char strDnodeId[TSDB_NODE_ID_LEN];
char strVgId[TSDB_VGROUP_ID_LEN]; char strVgId[TSDB_VGROUP_ID_LEN];
taos_counter_t *insertCounter; taos_counter_t* insertCounter;
}SVMonitorObj; } SVMonitorObj;
struct SVnode { struct SVnode {
char* path; char* path;

39
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -46,7 +46,6 @@ typedef struct {
STFileSet *fset; STFileSet *fset;
TABLEID tbid[1]; TABLEID tbid[1];
bool hasTSData; bool hasTSData;
bool skipTsRow;
} ctx[1]; } ctx[1];
// reader // reader
@ -128,21 +127,8 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
continue; continue;
} }
} }
/*
extern int8_t tsS3Enabled;
int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs);
committer->ctx->skipTsRow = false;
if (tsS3Enabled && nlevel > 1 && committer->ctx->did.level == nlevel - 1) {
committer->ctx->skipTsRow = true;
}
*/
int64_t ts = TSDBROW_TS(&row->row); int64_t ts = TSDBROW_TS(&row->row);
if (committer->ctx->skipTsRow && ts <= committer->ctx->maxKey) {
ts = committer->ctx->maxKey + 1;
}
if (ts > committer->ctx->maxKey) { if (ts > committer->ctx->maxKey) {
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
@ -406,31 +392,6 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
// reset nextKey // reset nextKey
committer->ctx->nextKey = TSKEY_MAX; committer->ctx->nextKey = TSKEY_MAX;
committer->ctx->skipTsRow = false;
extern int8_t tsS3Enabled;
extern int32_t tsS3UploadDelaySec;
long s3Size(const char *object_name);
int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1 && committer->ctx->fset) {
STFileObj *fobj = committer->ctx->fset->farr[TSDB_FTYPE_DATA];
if (fobj && fobj->f->did.level == nlevel - 1) {
// if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
if (taosCheckExistFile(fobj->fname)) {
int32_t mtime = 0;
taosStatFile(fobj->fname, NULL, &mtime, NULL);
if (mtime < committer->ctx->now - tsS3UploadDelaySec) {
committer->ctx->skipTsRow = true;
}
} else /*if (s3Size(object_name) > 0) */ {
committer->ctx->skipTsRow = true;
}
}
// new fset can be written with ts data
}
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);

11
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -29,15 +29,16 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon
reader[0]->config[0] = config[0]; reader[0]->config[0] = config[0];
int32_t lcn = config->file.lcn;
if (fname) { if (fname) {
if (fname) { if (fname) {
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd); code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
} else { } else {
char fname1[TSDB_FILENAME_LEN]; char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, &config->file, fname1); tsdbTFileName(config->tsdb, &config->file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd); code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -115,8 +116,8 @@ static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) {
static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; } static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; }
static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) { static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
ASSERT(writer->ctx->offset <= writer->file.size); ASSERT(writer->ctx->offset <= writer->file.size);
ASSERT(writer->config->fid == writer->file.fid); ASSERT(writer->config->fid == writer->file.fid);
@ -156,7 +157,7 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) {
} }
tsdbTFileName(writer->config->tsdb, &writer->file, fname); tsdbTFileName(writer->config->tsdb, &writer->file, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd); code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:

95
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -107,7 +107,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig
if (fname) { if (fname) {
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
if (fname[i]) { if (fname[i]) {
code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i]); int32_t lcn = config->files[i].file.lcn;
code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
@ -116,7 +117,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig
if (config->files[i].exist) { if (config->files[i].exist) {
char fname1[TSDB_FILENAME_LEN]; char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, &config->files[i].file, fname1); tsdbTFileName(config->tsdb, &config->files[i].file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i]); int32_t lcn = config->files[i].file.lcn;
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
@ -340,15 +342,15 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(br.offset == buffer0->size); ASSERT(br.offset == buffer0->size);
bool loadExtra = false; int extraColIdx = -1;
for (int i = 0; i < ncid; i++) { for (int i = 0; i < ncid; i++) {
if (tBlockDataGetColData(bData, cids[i]) == NULL) { if (tBlockDataGetColData(bData, cids[i]) == NULL) {
loadExtra = true; extraColIdx = i;
break; break;
} }
} }
if (!loadExtra) { if (extraColIdx < 0) {
goto _exit; goto _exit;
} }
@ -358,10 +360,78 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
buffer0, 0, encryptAlgorithm, encryptKey); buffer0, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// calc szHint
int64_t szHint = 0;
int extraCols = 1;
for (int i = extraColIdx + 1; i < ncid; ++i) {
if (tBlockDataGetColData(bData, cids[i]) == NULL) {
++extraCols;
break;
}
}
if (extraCols >= 2) {
br = BUFFER_READER_INITIALIZER(0, buffer0);
SBlockCol blockCol = {.cid = 0};
for (int32_t i = extraColIdx; i < ncid; ++i) {
int16_t extraColCid = cids[i];
while (extraColCid > blockCol.cid) {
if (br.offset >= buffer0->size) {
blockCol.cid = INT16_MAX;
break;
}
code = tGetBlockCol(&br, &blockCol);
TSDB_CHECK_CODE(code, lino, _exit);
}
if (extraColCid == blockCol.cid || blockCol.cid == INT16_MAX) {
extraColIdx = i;
break;
}
}
if (blockCol.cid > 0 && blockCol.cid < INT16_MAX /*&& blockCol->flag == HAS_VALUE*/) {
int64_t offset = blockCol.offset;
SBlockCol lastNonNoneBlockCol = {.cid = 0};
for (int32_t i = extraColIdx; i < ncid; ++i) {
int16_t extraColCid = cids[i];
while (extraColCid > blockCol.cid) {
if (br.offset >= buffer0->size) {
blockCol.cid = INT16_MAX;
break;
}
code = tGetBlockCol(&br, &blockCol);
TSDB_CHECK_CODE(code, lino, _exit);
}
if (extraColCid == blockCol.cid) {
lastNonNoneBlockCol = blockCol;
continue;
}
if (blockCol.cid == INT16_MAX) {
break;
}
}
if (lastNonNoneBlockCol.cid > 0) {
szHint = lastNonNoneBlockCol.offset + lastNonNoneBlockCol.szBitmap + lastNonNoneBlockCol.szOffset +
lastNonNoneBlockCol.szValue - offset;
}
}
}
// load each column // load each column
SBlockCol blockCol = { SBlockCol blockCol = {
.cid = 0, .cid = 0,
}; };
bool firstRead = true;
br = BUFFER_READER_INITIALIZER(0, buffer0); br = BUFFER_READER_INITIALIZER(0, buffer0);
for (int32_t i = 0; i < ncid; i++) { for (int32_t i = 0; i < ncid; i++) {
int16_t cid = cids[i]; int16_t cid = cids[i];
@ -401,12 +471,20 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load from file // load from file
tBufferClear(buffer1); tBufferClear(buffer1);
<<<<<<< HEAD
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA],
record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset, record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey);
=======
code = tsdbReadFileToBuffer(
reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, firstRead ? szHint : 0);
>>>>>>> 3.0
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
firstRead = false;
// decode the buffer // decode the buffer
SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1); SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1);
code = tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist); code = tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist);
@ -683,6 +761,7 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) {
.fid = writer->config->fid, .fid = writer->config->fid,
.cid = writer->config->cid, .cid = writer->config->cid,
.size = 0, .size = 0,
.lcn = writer->config->lcn == -1 ? 0 : -1,
.minVer = VERSION_MAX, .minVer = VERSION_MAX,
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
@ -1698,8 +1777,9 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) {
flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); flag |= (TD_FILE_CREATE | TD_FILE_TRUNC);
} }
int32_t lcn = writer->files[ftype].lcn;
tsdbTFileName(writer->config->tsdb, &writer->files[ftype], fname); tsdbTFileName(writer->config->tsdb, &writer->files[ftype], fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype]); code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
if (writer->files[ftype].size == 0) { if (writer->files[ftype].size == 0) {
@ -1868,8 +1948,9 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) {
int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
int32_t lcn = writer->files[ftype].lcn;
tsdbTFileName(writer->config->tsdb, writer->files + ftype, fname); tsdbTFileName(writer->config->tsdb, writer->files + ftype, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype]); code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};

1
source/dnode/vnode/src/tsdb/tsdbDataFileRW.h
View File

@ -77,6 +77,7 @@ typedef struct SDataFileWriterConfig {
int64_t cid; int64_t cid;
SDiskID did; SDiskID did;
int64_t compactVersion; int64_t compactVersion;
int32_t lcn;
struct { struct {
bool exist; bool exist;
STFile file; STFile file;

2
source/dnode/vnode/src/tsdb/tsdbDef.h
View File

@ -31,7 +31,7 @@ typedef struct SFDataPtr {
int64_t size; int64_t size;
} SFDataPtr; } SFDataPtr;
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD); extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn);
extern void tsdbCloseFile(STsdbFD **ppFD); extern void tsdbCloseFile(STsdbFD **ppFD);
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size,

96
source/dnode/vnode/src/tsdb/tsdbFS2.c
View File

@ -20,7 +20,7 @@
#define BLOCK_COMMIT_FACTOR 3 #define BLOCK_COMMIT_FACTOR 3
extern void remove_file(const char *fname, bool last_level); extern void remove_file(const char *fname);
#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT #define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT
#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1) #define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1)
@ -355,17 +355,32 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) {
// check file existence // check file existence
if (!taosCheckExistFile(fobj->fname)) { if (!taosCheckExistFile(fobj->fname)) {
if (tsS3Enabled) { bool found = false;
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
long s3_size = s3Size(object_name); if (tsS3Enabled && fobj->f->lcn > 1) {
if (s3_size > 0) { char fname1[TSDB_FILENAME_LEN];
return 0; tsdbTFileLastChunkName(fs->tsdb, fobj->f, fname1);
if (!taosCheckExistFile(fname1)) {
code = TSDB_CODE_FILE_CORRUPTED;
tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fname1);
return code;
} }
found = true;
/*
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
long s3_size = s3Size(object_name);
if (s3_size > 0) {
return 0;
}
*/
} }
code = TSDB_CODE_FILE_CORRUPTED; if (!found) {
tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fobj->fname); code = TSDB_CODE_FILE_CORRUPTED;
return code; tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fobj->fname);
return code;
}
} }
{ // TODO: check file size { // TODO: check file size
@ -530,9 +545,9 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
if (taosIsDir(file->aname)) continue; if (taosIsDir(file->aname)) continue;
if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL && if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL &&
strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3)) { strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3) &&
int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs); strncmp(file->aname + strlen(file->aname) - 5, ".data", 5)) {
remove_file(file->aname, nlevel > 1 && file->did.level == nlevel - 1); remove_file(file->aname);
} }
} }
@ -900,57 +915,28 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
continue; continue;
} }
bool skipMerge = false; // bool skipMerge = false;
int32_t numFile = TARRAY2_SIZE(lvl->fobjArr); int32_t numFile = TARRAY2_SIZE(lvl->fobjArr);
if (numFile >= sttTrigger && (!fset->mergeScheduled)) { if (numFile >= sttTrigger && (!fset->mergeScheduled)) {
// launch merge code = tsdbTFileSetOpenChannel(fset);
{ TSDB_CHECK_CODE(code, lino, _exit);
extern int8_t tsS3Enabled;
extern int32_t tsS3UploadDelaySec;
long s3Size(const char *object_name);
int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1) {
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
if (fobj && fobj->f->did.level == nlevel - 1) {
// if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
if (taosCheckExistFile(fobj->fname)) { SMergeArg *arg = taosMemoryMalloc(sizeof(*arg));
int32_t now = taosGetTimestampSec(); if (arg == NULL) {
int32_t mtime = 0; code = TSDB_CODE_OUT_OF_MEMORY;
taosStatFile(fobj->fname, NULL, &mtime, NULL); TSDB_CHECK_CODE(code, lino, _exit);
if (mtime < now - tsS3UploadDelaySec) {
skipMerge = true;
}
} else /* if (s3Size(object_name) > 0) */ {
skipMerge = true;
}
}
// new fset can be written with ts data
}
} }
if (!skipMerge) { arg->tsdb = fs->tsdb;
code = tsdbTFileSetOpenChannel(fset); arg->fid = fset->fid;
TSDB_CHECK_CODE(code, lino, _exit);
SMergeArg *arg = taosMemoryMalloc(sizeof(*arg)); code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg,
if (arg == NULL) { NULL);
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); fset->mergeScheduled = true;
}
arg->tsdb = fs->tsdb;
arg->fid = fset->fid;
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree,
arg, NULL);
TSDB_CHECK_CODE(code, lino, _exit);
fset->mergeScheduled = true;
}
} }
if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR && !skipMerge) { if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) {
tsdbFSSetBlockCommit(fset, true); tsdbFSSetBlockCommit(fset, true);
} else { } else {
tsdbFSSetBlockCommit(fset, false); tsdbFSSetBlockCommit(fset, false);

10
source/dnode/vnode/src/tsdb/tsdbFSet2.c
View File

@ -386,7 +386,11 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f
fobj2->f[0] = fobj1->f[0]; fobj2->f[0] = fobj1->f[0];
} }
} else { } else {
tsdbTFileObjRemove(fobj2); if (fobj1->f->cid != fobj2->f->cid) {
tsdbTFileObjRemove(fobj2);
} else {
tsdbTFileObjRemoveUpdateLC(fobj2);
}
code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]);
if (code) return code; if (code) return code;
} }
@ -585,7 +589,7 @@ int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) {
return 0; return 0;
} }
int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) { int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) {
if (ppArr && ppArr[0]) { if (ppArr && ppArr[0]) {
TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear); TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear);
taosMemoryFree(ppArr[0]); taosMemoryFree(ppArr[0]);
@ -663,4 +667,4 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) {
int32_t tsdbTFileSetOpenChannel(STFileSet *fset) { int32_t tsdbTFileSetOpenChannel(STFileSet *fset) {
if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0; if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0;
return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel); return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel);
} }

1
source/dnode/vnode/src/tsdb/tsdbFSetRW.c
View File

@ -148,6 +148,7 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) {
.compactVersion = config->compactVersion, .compactVersion = config->compactVersion,
.skmTb = writer[0]->skmTb, .skmTb = writer[0]->skmTb,
.skmRow = writer[0]->skmRow, .skmRow = writer[0]->skmRow,
.lcn = config->lcn,
.buffers = writer[0]->buffers, .buffers = writer[0]->buffers,
}; };
for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {

1
source/dnode/vnode/src/tsdb/tsdbFSetRW.h
View File

@ -37,6 +37,7 @@ typedef struct {
int64_t cid; int64_t cid;
SDiskID did; SDiskID did;
int32_t level; int32_t level;
int32_t lcn;
struct { struct {
bool exist; bool exist;
STFile file; STFile file;

145
source/dnode/vnode/src/tsdb/tsdbFile2.c
View File

@ -15,6 +15,7 @@
#include "tsdbFile2.h" #include "tsdbFile2.h"
#include "cos.h" #include "cos.h"
#include "vnd.h"
// to_json // to_json
static int32_t head_to_json(const STFile *file, cJSON *json); static int32_t head_to_json(const STFile *file, cJSON *json);
@ -42,24 +43,9 @@ static const struct {
[TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json}, [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
}; };
void remove_file(const char *fname, bool last_level) { void remove_file(const char *fname) {
int32_t code = taosRemoveFile(fname); taosRemoveFile(fname);
if (code) { tsdbInfo("file:%s is removed", fname);
if (tsS3Enabled && last_level) {
const char *object_name = taosDirEntryBaseName((char *)fname);
long s3_size = tsS3Enabled ? s3Size(object_name) : 0;
if (!strncmp(fname + strlen(fname) - 5, ".data", 5) && s3_size > 0) {
s3DeleteObjects(&object_name, 1);
tsdbInfo("file:%s is removed from s3", fname);
} else {
tsdbError("file:%s remove failed", fname);
}
} else {
tsdbError("file:%s remove failed", fname);
}
} else {
tsdbInfo("file:%s is removed", fname);
}
} }
static int32_t tfile_to_json(const STFile *file, cJSON *json) { static int32_t tfile_to_json(const STFile *file, cJSON *json) {
@ -73,6 +59,11 @@ static int32_t tfile_to_json(const STFile *file, cJSON *json) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
/* lcn - last chunk number */
if (cJSON_AddNumberToObject(json, "lcn", file->lcn) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
/* fid */ /* fid */
if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) { if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -121,6 +112,14 @@ static int32_t tfile_from_json(const cJSON *json, STFile *file) {
return TSDB_CODE_FILE_CORRUPTED; return TSDB_CODE_FILE_CORRUPTED;
} }
/* lcn */
item = cJSON_GetObjectItem(json, "lcn");
if (cJSON_IsNumber(item)) {
file->lcn = item->valuedouble;
} else {
// return TSDB_CODE_FILE_CORRUPTED;
}
/* fid */ /* fid */
item = cJSON_GetObjectItem(json, "fid"); item = cJSON_GetObjectItem(json, "fid");
if (cJSON_IsNumber(item)) { if (cJSON_IsNumber(item)) {
@ -236,7 +235,8 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
fobj[0]->state = TSDB_FSTATE_LIVE; fobj[0]->state = TSDB_FSTATE_LIVE;
fobj[0]->ref = 1; fobj[0]->ref = 1;
tsdbTFileName(pTsdb, f, fobj[0]->fname); tsdbTFileName(pTsdb, f, fobj[0]->fname);
fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs); // fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode);
return 0; return 0;
} }
@ -258,7 +258,7 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) {
tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) { if (nRef == 0) {
if (fobj->state == TSDB_FSTATE_DEAD) { if (fobj->state == TSDB_FSTATE_DEAD) {
remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1); remove_file(fobj->fname);
} }
taosMemoryFree(fobj); taosMemoryFree(fobj);
} }
@ -266,6 +266,61 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) {
return 0; return 0;
} }
static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {
if (fobj->f->type != TSDB_FTYPE_DATA) {
remove_file(fobj->fname);
return;
}
if (!remove_all) {
if (fobj->f->lcn < 1) {
remove_file(fobj->fname);
return;
} else {
// remove local last chunk file
char lc_path[TSDB_FILENAME_LEN];
tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);
char *dot = strrchr(lc_path, '.');
if (!dot) {
tsdbError("unexpected path: %s", lc_path);
return;
}
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);
remove_file(lc_path);
}
} else {
// delete by data file prefix
char lc_path[TSDB_FILENAME_LEN];
tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);
char *object_name = taosDirEntryBaseName(lc_path);
int32_t node_id = fobj->nlevel;
char object_name_prefix[TSDB_FILENAME_LEN];
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
char *dot = strrchr(object_name_prefix, '.');
if (!dot) {
tsdbError("unexpected path: %s", object_name_prefix);
return;
}
*(dot + 1) = 0;
s3DeleteObjectsByPrefix(object_name_prefix);
// remove local last chunk file
dot = strrchr(lc_path, '.');
if (!dot) {
tsdbError("unexpected path: %s", lc_path);
return;
}
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);
remove_file(lc_path);
}
}
int32_t tsdbTFileObjRemove(STFileObj *fobj) { int32_t tsdbTFileObjRemove(STFileObj *fobj) {
taosThreadMutexLock(&fobj->mutex); taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0); ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
@ -274,7 +329,21 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) {
taosThreadMutexUnlock(&fobj->mutex); taosThreadMutexUnlock(&fobj->mutex);
tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) { if (nRef == 0) {
remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1); tsdbTFileObjRemoveLC(fobj, true);
taosMemoryFree(fobj);
}
return 0;
}
int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {
taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
fobj->state = TSDB_FSTATE_DEAD;
int32_t nRef = --fobj->ref;
taosThreadMutexUnlock(&fobj->mutex);
tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) {
tsdbTFileObjRemoveLC(fobj, false);
taosMemoryFree(fobj); taosMemoryFree(fobj);
} }
return 0; return 0;
@ -310,13 +379,45 @@ int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]) {
return 0; return 0;
} }
int32_t tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]) {
SVnode *pVnode = pTsdb->pVnode;
STfs *pTfs = pVnode->pTfs;
if (pTfs) {
snprintf(fname, //
TSDB_FILENAME_LEN, //
"%s%s%s%sv%df%dver%" PRId64 ".%d.%s", //
tfsGetDiskPath(pTfs, f->did), //
TD_DIRSEP, //
pTsdb->path, //
TD_DIRSEP, //
TD_VID(pVnode), //
f->fid, //
f->cid, //
f->lcn, //
g_tfile_info[f->type].suffix);
} else {
snprintf(fname, //
TSDB_FILENAME_LEN, //
"%s%sv%df%dver%" PRId64 ".%d.%s", //
pTsdb->path, //
TD_DIRSEP, //
TD_VID(pVnode), //
f->fid, //
f->cid, //
f->lcn, //
g_tfile_info[f->type].suffix);
}
return 0;
}
bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) { bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) {
if (f1->type != f2->type) return false; if (f1->type != f2->type) return false;
if (f1->did.level != f2->did.level) return false; if (f1->did.level != f2->did.level) return false;
if (f1->did.id != f2->did.id) return false; if (f1->did.id != f2->did.id) return false;
if (f1->fid != f2->fid) return false; if (f1->fid != f2->fid) return false;
if (f1->cid != f2->cid) return false; if (f1->cid != f2->cid) return false;
if (f1->s3flag != f2->s3flag) return false; if (f1->lcn != f2->lcn) return false;
return true; return true;
} }

4
source/dnode/vnode/src/tsdb/tsdbFile2.h
View File

@ -45,6 +45,7 @@ enum {
int32_t tsdbTFileToJson(const STFile *f, cJSON *json); int32_t tsdbTFileToJson(const STFile *f, cJSON *json);
int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f); int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f);
int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]); int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]);
int32_t tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]);
bool tsdbIsSameTFile(const STFile *f1, const STFile *f2); bool tsdbIsSameTFile(const STFile *f1, const STFile *f2);
bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2); bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2);
@ -53,12 +54,13 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj);
int32_t tsdbTFileObjRef(STFileObj *fobj); int32_t tsdbTFileObjRef(STFileObj *fobj);
int32_t tsdbTFileObjUnref(STFileObj *fobj); int32_t tsdbTFileObjUnref(STFileObj *fobj);
int32_t tsdbTFileObjRemove(STFileObj *fobj); int32_t tsdbTFileObjRemove(STFileObj *fobj);
int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj);
int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2); int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2);
struct STFile { struct STFile {
tsdb_ftype_t type; tsdb_ftype_t type;
SDiskID did; // disk id SDiskID did; // disk id
int32_t s3flag; int32_t lcn; // last chunk number
int32_t fid; // file id int32_t fid; // file id
int64_t cid; // commit id int64_t cid; // commit id
int64_t size; int64_t size;

7
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -544,7 +544,7 @@ int32_t tsdbMerge(void *arg) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
if (merger->fset == NULL) return 0; if (merger->fset == NULL) return 0;
/*
bool skipMerge = false; bool skipMerge = false;
{ {
extern int8_t tsS3Enabled; extern int8_t tsS3Enabled;
@ -565,7 +565,8 @@ int32_t tsdbMerge(void *arg) {
if (mtime < now - tsS3UploadDelaySec) { if (mtime < now - tsS3UploadDelaySec) {
skipMerge = true; skipMerge = true;
} }
} else /* if (s3Size(object_name) > 0) */ { } else // if (s3Size(object_name) > 0)
{
skipMerge = true; skipMerge = true;
} }
} }
@ -577,7 +578,7 @@ int32_t tsdbMerge(void *arg) {
code = 0; code = 0;
goto _exit; goto _exit;
} }
*/
// do merge // do merge
tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid); tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid);
code = tsdbDoMerge(merger); code = tsdbDoMerge(merger);

218
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -15,17 +15,52 @@
#include "cos.h" #include "cos.h"
#include "tsdb.h" #include "tsdb.h"
<<<<<<< HEAD
#include "crypt.h" #include "crypt.h"
=======
#include "vnd.h"
>>>>>>> 3.0
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
int32_t code = 0; int32_t code = 0;
const char *path = pFD->path; const char *path = pFD->path;
int32_t szPage = pFD->szPage; int32_t szPage = pFD->szPage;
int32_t flag = pFD->flag; int32_t flag = pFD->flag;
int64_t lc_size = 0;
pFD->pFD = taosOpenFile(path, flag); pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) { if (pFD->pFD == NULL) {
int errsv = errno; if (tsS3Enabled && pFD->lcn > 1 && !strncmp(path + strlen(path) - 5, ".data", 5)) {
char lc_path[TSDB_FILENAME_LEN];
tstrncpy(lc_path, path, TSDB_FQDN_LEN);
char *dot = strrchr(lc_path, '.');
if (!dot) {
tsdbError("unexpected path: %s", lc_path);
code = TAOS_SYSTEM_ERROR(ENOENT);
goto _exit;
}
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn);
pFD->pFD = taosOpenFile(lc_path, flag);
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
// taosMemoryFree(pFD);
goto _exit;
}
if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
}
} else {
tsdbInfo("no file: %s", path);
code = TAOS_SYSTEM_ERROR(errno);
// taosMemoryFree(pFD);
goto _exit;
}
/*
const char *object_name = taosDirEntryBaseName((char *)path); const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = 0; long s3_size = 0;
if (tsS3Enabled) { if (tsS3Enabled) {
@ -44,7 +79,6 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
s3Get(object_name, path); s3Get(object_name, path);
pFD->pFD = taosOpenFile(path, flag); pFD->pFD = taosOpenFile(path, flag);
if (pFD->pFD == NULL) { if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(ENOENT); code = TAOS_SYSTEM_ERROR(ENOENT);
// taosMemoryFree(pFD); // taosMemoryFree(pFD);
@ -58,12 +92,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
pFD->objName = object_name; pFD->objName = object_name;
// pFD->szFile = s3_size; // pFD->szFile = s3_size;
#endif #endif
} else { */
tsdbInfo("no file: %s", path);
code = TAOS_SYSTEM_ERROR(errsv);
// taosMemoryFree(pFD);
goto _exit;
}
} }
pFD->pBuf = taosMemoryCalloc(1, szPage); pFD->pBuf = taosMemoryCalloc(1, szPage);
@ -74,26 +103,33 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
goto _exit; goto _exit;
} }
if (lc_size > 0) {
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
pFD->szFile = lc_size + chunksize * (pFD->lcn - 1);
}
// not check file size when reading data files. // not check file size when reading data files.
if (flag != TD_FILE_READ && !pFD->s3File) { if (flag != TD_FILE_READ /* && !pFD->s3File*/) {
if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) { if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
// taosMemoryFree(pFD->pBuf); // taosMemoryFree(pFD->pBuf);
// taosCloseFile(&pFD->pFD); // taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD); // taosMemoryFree(pFD);
goto _exit; goto _exit;
} }
ASSERT(pFD->szFile % szPage == 0);
pFD->szFile = pFD->szFile / szPage;
} }
ASSERT(pFD->szFile % szPage == 0);
pFD->szFile = pFD->szFile / szPage;
_exit: _exit:
return code; return code;
} }
// =============== PAGE-WISE FILE =============== // =============== PAGE-WISE FILE ===============
int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD) { int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) {
int32_t code = 0; int32_t code = 0;
STsdbFD *pFD = NULL; STsdbFD *pFD = NULL;
int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
@ -112,6 +148,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF
pFD->flag = flag; pFD->flag = flag;
pFD->szPage = szPage; pFD->szPage = szPage;
pFD->pgno = 0; pFD->pgno = 0;
pFD->lcn = lcn;
pFD->pTsdb = pTsdb; pFD->pTsdb = pTsdb;
*ppFD = pFD; *ppFD = pFD;
@ -124,9 +161,9 @@ void tsdbCloseFile(STsdbFD **ppFD) {
STsdbFD *pFD = *ppFD; STsdbFD *pFD = *ppFD;
if (pFD) { if (pFD) {
taosMemoryFree(pFD->pBuf); taosMemoryFree(pFD->pBuf);
if (!pFD->s3File) { // if (!pFD->s3File) {
taosCloseFile(&pFD->pFD); taosCloseFile(&pFD->pFD);
} //}
taosMemoryFree(pFD); taosMemoryFree(pFD);
*ppFD = NULL; *ppFD = NULL;
} }
@ -142,12 +179,18 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* e
} }
} }
if (pFD->s3File) {
tsdbWarn("%s file: %s", __func__, pFD->path);
return code;
}
if (pFD->pgno > 0) { if (pFD->pgno > 0) {
int64_t n = taosLSeekFile(pFD->pFD, PAGE_OFFSET(pFD->pgno, pFD->szPage), SEEK_SET); int64_t offset = PAGE_OFFSET(pFD->pgno, pFD->szPage);
if (pFD->lcn > 1) {
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t chunkoffset = chunksize * (pFD->lcn - 1);
offset -= chunkoffset;
}
ASSERT(offset >= 0);
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _exit; goto _exit;
@ -205,7 +248,15 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
} }
int64_t offset = PAGE_OFFSET(pgno, pFD->szPage); int64_t offset = PAGE_OFFSET(pgno, pFD->szPage);
if (pFD->lcn > 1) {
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t chunkoffset = chunksize * (pFD->lcn - 1);
offset -= chunkoffset;
}
ASSERT(offset >= 0);
/*
if (pFD->s3File) { if (pFD->s3File) {
LRUHandle *handle = NULL; LRUHandle *handle = NULL;
@ -226,13 +277,15 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
tsdbCacheRelease(pFD->pTsdb->bCache, handle); tsdbCacheRelease(pFD->pTsdb->bCache, handle);
} else { } else {
// seek */
int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); // seek
if (n < 0) { int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
code = TAOS_SYSTEM_ERROR(errno); if (n < 0) {
goto _exit; code = TAOS_SYSTEM_ERROR(errno);
} goto _exit;
}
<<<<<<< HEAD
// read // read
n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) { if (n < 0) {
@ -266,7 +319,18 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor
} }
//tsdbDebug("CBC_Decrypt count:%d %s", count, __FUNCTION__); //tsdbDebug("CBC_Decrypt count:%d %s", count, __FUNCTION__);
} }
=======
// read
n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
} else if (n < pFD->szPage) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _exit;
>>>>>>> 3.0
} }
//}
// check // check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
@ -343,6 +407,74 @@ _exit:
return code; return code;
} }
static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
int32_t code = 0;
SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t cOffset = offset % chunksize;
int64_t n = 0;
char *object_name = taosDirEntryBaseName(pFD->path);
char object_name_prefix[TSDB_FILENAME_LEN];
int32_t node_id = vnodeNodeId(pFD->pTsdb->pVnode);
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
char *dot = strrchr(object_name_prefix, '.');
if (!dot) {
tsdbError("unexpected path: %s", object_name_prefix);
code = TAOS_SYSTEM_ERROR(ENOENT);
goto _exit;
}
char *buf = taosMemoryCalloc(1, size);
for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) {
int64_t nRead = TMIN(chunksize - cOffset, size - n);
if (chunkno >= pFD->lcn) {
// read last chunk
int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET);
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(buf);
goto _exit;
}
ret = taosReadFile(pFD->pFD, buf + n, nRead);
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(buf);
goto _exit;
} else if (ret < nRead) {
code = TSDB_CODE_FILE_CORRUPTED;
taosMemoryFree(buf);
goto _exit;
}
} else {
uint8_t *pBlock = NULL;
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno);
code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(buf);
goto _exit;
}
memcpy(buf + n, pBlock, nRead);
taosMemoryFree(pBlock);
}
n += nRead;
cOffset = 0;
}
*ppBlock = buf;
_exit:
return code;
}
static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) { static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
int32_t code = 0; int32_t code = 0;
int64_t n = 0; int64_t n = 0;
@ -405,15 +537,22 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
} }
int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage; int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
/*
code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock); code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _exit; goto _exit;
} }
*/
code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
// 3, Store Pages in Cache // 3, Store Pages in Cache
int nPage = pgnoEnd - pgno + 1; int nPage = pgnoEnd - pgno + 1;
for (int i = 0; i < nPage; ++i) { for (int i = 0; i < nPage; ++i) {
tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage); if (pFD->szFile != pgno) { // DONOT cache last volatile page
tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage);
}
if (szHint > 0 && n >= size) { if (szHint > 0 && n >= size) {
++pgno; ++pgno;
@ -454,7 +593,7 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size,
} }
} }
if (pFD->s3File && tsS3BlockSize < 0) { if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint); return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
} else { } else {
return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey); return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey);
@ -480,12 +619,17 @@ int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer
int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) { int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0; int32_t code = 0;
/*
if (pFD->s3File) { if (pFD->s3File) {
tsdbWarn("%s file: %s", __func__, pFD->path); tsdbWarn("%s file: %s", __func__, pFD->path);
return code; return code;
} }
<<<<<<< HEAD
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
=======
*/
code = tsdbWriteFilePage(pFD);
>>>>>>> 3.0
if (code) goto _exit; if (code) goto _exit;
if (taosFsyncFile(pFD->pFD) < 0) { if (taosFsyncFile(pFD->pFD) < 0) {
@ -516,23 +660,23 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
// head // head
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pHeadFD); code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pHeadFD, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// data // data
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pDataFD); code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pDataFD, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// sma // sma
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pSmaFD); code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pSmaFD, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// stt // stt
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->aSttFD[iStt]); code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->aSttFD[iStt], 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -716,7 +860,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
pDelFReader->fDel = *pFile; pDelFReader->fDel = *pFile;
tsdbDelFileName(pTsdb, pFile, fname); tsdbDelFileName(pTsdb, pFile, fname);
code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH); code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0);
if (code) { if (code) {
taosMemoryFree(pDelFReader); taosMemoryFree(pDelFReader);
goto _exit; goto _exit;

569
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -38,24 +38,42 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
return TARRAY2_APPEND(rtner->fopArr, op); return TARRAY2_APPEND(rtner->fopArr, op);
} }
static int32_t tsdbRemoveFileObjectS3(SRTNer *rtner, const STFileObj *fobj) { static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0, lino = 0; int32_t code = 0;
int32_t lino = 0;
TdFilePtr fdFrom = NULL, fdTo = NULL;
char fname_from[TSDB_FILENAME_LEN];
char fname_to[TSDB_FILENAME_LEN];
STFileOp op = { tsdbTFileLastChunkName(rtner->tsdb, from->f, fname_from);
.optype = TSDB_FOP_REMOVE, tsdbTFileLastChunkName(rtner->tsdb, to, fname_to);
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op); fdFrom = taosOpenFile(fname_from, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
const char *object_name = taosDirEntryBaseName((char *)fobj->fname); tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size);
s3DeleteObjects(&object_name, 1);
fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1);
int64_t n = taosFSendFile(fdTo, fdFrom, 0, lc_size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
if (fdFrom) taosCloseFile(&fdFrom);
if (fdTo) taosCloseFile(&fdTo);
} }
return code; return code;
} }
@ -97,38 +115,11 @@ _exit:
return code; return code;
} }
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
TdFilePtr fdFrom = NULL;
// TdFilePtr fdTo = NULL;
tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
char *object_name = taosDirEntryBaseName(fname);
code = s3PutObjectFromFile2(from->fname, object_name, 1);
TSDB_CHECK_CODE(code, lino, _exit);
taosCloseFile(&fdFrom);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
taosCloseFile(&fdFrom);
}
return code;
}
static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) { static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
STFileOp op = {0}; STFileOp op = {0};
int32_t lcn = fobj->f->lcn;
// remove old // remove old
op = (STFileOp){ op = (STFileOp){
@ -153,6 +144,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
.maxVer = fobj->f->maxVer, .maxVer = fobj->f->maxVer,
.cid = fobj->f->cid, .cid = fobj->f->cid,
.size = fobj->f->size, .size = fobj->f->size,
.lcn = lcn,
.stt[0] = .stt[0] =
{ {
.level = fobj->f->stt[0].level, .level = fobj->f->stt[0].level,
@ -164,59 +156,14 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// do copy the file // do copy the file
code = tsdbDoCopyFile(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: if (lcn < 1) {
if (code) { code = tsdbDoCopyFile(rtner, fobj, &op.nf);
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDoCopyFileLC(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
} }
return code;
}
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
// remove old
op = (STFileOp){
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = fobj->f->fid,
.nf =
{
.type = fobj->f->type,
.did = did[0],
.fid = fobj->f->fid,
.minVer = fobj->f->minVer,
.maxVer = fobj->f->maxVer,
.cid = fobj->f->cid,
.size = fobj->f->size,
.stt[0] =
{
.level = fobj->f->stt[0].level,
},
},
};
op.nf.s3flag = true;
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// do copy the file
code = tsdbCopyFileS3(rtner, fobj, &op.nf);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
@ -327,38 +274,14 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = fset->farr[ftype], 1); ++ftype) { for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = fset->farr[ftype], 1); ++ftype) {
if (fobj == NULL) continue; if (fobj == NULL) continue;
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
if (fobj->f->did.level == did.level) { if (fobj->f->did.level == did.level) {
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1 && /*
taosCheckExistFile(fobj->fname)) { code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did);
int32_t mtime = 0; TSDB_CHECK_CODE(code, lino, _exit);
taosStatFile(fobj->fname, NULL, &mtime, NULL); */
if (mtime < rtner->now - tsS3UploadDelaySec) {
tsdbInfo("file:%s size: %" PRId64 " do migrate s3", fobj->fname, fobj->f->size);
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
continue; continue;
} }
/*
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) {
code = tsdbMigrateDataFileS3(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
if (tsS3Enabled) {
int64_t fsize = 0;
if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(terrno);
tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode),
__func__, fobj->fname, tstrerror(code)); TSDB_CHECK_CODE(code, lino, _exit);
}
s3EvictCache(fobj->fname, fsize * 2);
}
*/
if (fobj->f->did.level > did.level) { if (fobj->f->did.level > did.level) {
continue; continue;
} }
@ -367,7 +290,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
code = tsdbDoMigrateFileObj(rtner, fobj, &did); code = tsdbDoMigrateFileObj(rtner, fobj, &did);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
//}
} }
// stt // stt
@ -471,3 +393,414 @@ int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
return code; return code;
} }
static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3KeepLocal, int64_t nowSec) {
int32_t localFid;
TSKEY key;
if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) {
nowSec = nowSec * 1000;
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) {
nowSec = nowSec * 1000000l;
} else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) {
nowSec = nowSec * 1000000000l;
} else {
ASSERT(0);
}
nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision];
key = nowSec - s3KeepLocal * tsTickPerMin[pKeepCfg->precision];
localFid = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision);
if (fid >= localFid) {
return 0;
} else {
return 1;
}
}
static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0;
int32_t lino = 0;
char fname[TSDB_FILENAME_LEN];
TdFilePtr fdFrom = NULL;
// TdFilePtr fdTo = NULL;
tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
char *object_name = taosDirEntryBaseName(fname);
code = s3PutObjectFromFile2(from->fname, object_name, 1);
TSDB_CHECK_CODE(code, lino, _exit);
taosCloseFile(&fdFrom);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
taosCloseFile(&fdFrom);
}
return code;
}
static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
TdFilePtr fdFrom = NULL, fdTo = NULL;
int32_t lcn = fobj->f->lcn + (size - 1) / chunksize;
// remove old
op = (STFileOp){
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = fobj->f->fid,
.nf =
{
.type = fobj->f->type,
.did = fobj->f->did,
.fid = fobj->f->fid,
.minVer = fobj->f->minVer,
.maxVer = fobj->f->maxVer,
.cid = fobj->f->cid,
.size = fobj->f->size,
.lcn = lcn,
.stt[0] =
{
.level = fobj->f->stt[0].level,
},
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
char fname[TSDB_FILENAME_LEN];
tsdbTFileName(rtner->tsdb, &op.nf, fname);
char *object_name = taosDirEntryBaseName(fname);
char object_name_prefix[TSDB_FILENAME_LEN];
int32_t node_id = vnodeNodeId(rtner->tsdb->pVnode);
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
char *dot = strrchr(object_name_prefix, '.');
if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1;
}
char *dot2 = strchr(object_name, '.');
if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1;
}
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn);
// do copy the file
for (int32_t cn = fobj->f->lcn; cn < lcn; ++cn) {
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
int64_t c_offset = chunksize * (cn - fobj->f->lcn);
code = s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize);
TSDB_CHECK_CODE(code, lino, _exit);
}
// copy last chunk
int64_t lc_offset = chunksize * (lcn - fobj->f->lcn);
int64_t lc_size = size - lc_offset;
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn);
fdFrom = taosOpenFile(fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size);
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
if (fdFrom) taosCloseFile(&fdFrom);
if (fdTo) taosCloseFile(&fdTo);
}
return code;
}
static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) {
int32_t code = 0;
int32_t lino = 0;
STFileOp op = {0};
int32_t lcn = (size - 1) / chunksize + 1;
// remove old
op = (STFileOp){
.optype = TSDB_FOP_REMOVE,
.fid = fobj->f->fid,
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
op = (STFileOp){
.optype = TSDB_FOP_CREATE,
.fid = fobj->f->fid,
.nf =
{
.type = fobj->f->type,
.did = fobj->f->did,
.fid = fobj->f->fid,
.minVer = fobj->f->minVer,
.maxVer = fobj->f->maxVer,
.cid = fobj->f->cid,
.size = fobj->f->size,
.lcn = lcn,
.stt[0] =
{
.level = fobj->f->stt[0].level,
},
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
char fname[TSDB_FILENAME_LEN];
tsdbTFileName(rtner->tsdb, &op.nf, fname);
char *object_name = taosDirEntryBaseName(fname);
char object_name_prefix[TSDB_FILENAME_LEN];
int32_t node_id = vnodeNodeId(rtner->tsdb->pVnode);
snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);
char *dot = strrchr(object_name_prefix, '.');
if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1;
}
// do copy the file
for (int32_t cn = 1; cn < lcn; ++cn) {
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
int64_t c_offset = chunksize * (cn - 1);
code = s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize);
TSDB_CHECK_CODE(code, lino, _exit);
}
// copy last chunk
TdFilePtr fdFrom = NULL, fdTo = NULL;
int64_t lc_offset = (int64_t)(lcn - 1) * chunksize;
int64_t lc_size = size - lc_offset;
dot = strchr(object_name, '.');
if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1;
}
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name), "%d.data", lcn);
fdFrom = taosOpenFile(fobj->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, fobj->f->size);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
}
return code;
}
static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) {
int32_t code = 0;
int32_t lino = 0;
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
if (!fobj) return code;
int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now);
if (expLevel < 0) return code; // expired
SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
int32_t s3KeepLocal = pCfg->s3KeepLocal;
int32_t s3ExpLevel = tsdbS3FidLevel(fset->fid, &rtner->tsdb->keepCfg, s3KeepLocal, rtner->now);
if (s3ExpLevel < 1) return code; // keep on local storage
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int32_t lcn = fobj->f->lcn;
if (lcn < 1 && taosCheckExistFile(fobj->fname)) {
int32_t mtime = 0;
int64_t size = 0;
taosStatFile(fobj->fname, &size, &mtime, NULL);
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
if (pCfg->s3Compact && lcn < 0) {
extern int32_t tsdbAsyncCompact(STsdb * tsdb, const STimeWindow *tw, bool sync);
STimeWindow win = {0};
tsdbFidKeyRange(fset->fid, rtner->tsdb->keepCfg.days, rtner->tsdb->keepCfg.precision, &win.skey, &win.ekey);
tsdbInfo("vgId:%d, compact begin lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn);
tsdbAsyncCompact(rtner->tsdb, &win, pCfg->sttTrigger == 1);
tsdbInfo("vgId:%d, compact end lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn);
return code;
}
code = tsdbMigrateDataFileS3(rtner, fobj, size, chunksize);
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
if (lcn <= 1) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return code;
}
char fname1[TSDB_FILENAME_LEN];
tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1);
if (taosCheckExistFile(fname1)) {
int32_t mtime = 0;
int64_t size = 0;
taosStatFile(fname1, &size, &mtime, NULL);
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
code = tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize);
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
tsdbError("vgId:%d, file: %s not found, %s at line %d", TD_VID(rtner->tsdb->pVnode), fname1, __func__, lino);
return code;
}
}
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbDoS3MigrateAsync(void *arg) {
int32_t code = 0;
int32_t lino = 0;
SRTNer rtner[1] = {0};
code = tsdbDoRetentionBegin(arg, rtner);
TSDB_CHECK_CODE(code, lino, _exit);
STFileSet *fset;
TARRAY2_FOREACH(rtner->fsetArr, fset) {
if (fset->fid != ((SRtnArg *)arg)->fid) continue;
code = tsdbDoS3MigrateOnFileSet(rtner, fset);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbDoRetentionEnd(rtner);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
if (TARRAY2_DATA(rtner->fopArr)) {
TARRAY2_DESTROY(rtner->fopArr, NULL);
}
TFileSetArray **fsetArr = &rtner->fsetArr;
if (fsetArr[0]) {
tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
}
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
int32_t code = 0;
if (!tsS3Enabled) {
return code;
}
taosThreadMutexLock(&tsdb->mutex);
if (tsdb->bgTaskDisabled) {
taosThreadMutexUnlock(&tsdb->mutex);
return 0;
}
STFileSet *fset;
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
code = tsdbTFileSetOpenChannel(fset);
if (code) {
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
if (arg == NULL) {
taosThreadMutexUnlock(&tsdb->mutex);
return TSDB_CODE_OUT_OF_MEMORY;
}
arg->tsdb = tsdb;
arg->now = now;
arg->fid = fset->fid;
if (sync) {
code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
tsdbFreeRtnArg, arg, NULL);
} else {
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
tsdbFreeRtnArg, arg, NULL);
}
if (code) {
tsdbFreeRtnArg(arg);
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
}
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}

6
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -49,12 +49,12 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
// open file // open file
if (fname) { if (fname) {
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd); code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
char fname1[TSDB_FILENAME_LEN]; char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, config->file, fname1); tsdbTFileName(config->tsdb, config->file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd); code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -772,7 +772,7 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
tsdbTFileName(writer->config->tsdb, writer->file, fname); tsdbTFileName(writer->config->tsdb, writer->file, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd); code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};

6
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -84,7 +84,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
tsdbTFileName(tsdb, &file, fname); tsdbTFileName(tsdb, &file, fname);
code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd); code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// convert // convert
@ -281,7 +281,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
code = tsdbTFileObjInit(tsdb, &file, &fobj); code = tsdbTFileObjInit(tsdb, &file, &fobj);
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd); code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0);
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) { for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) {
@ -437,7 +437,7 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
} }
char fname[TSDB_FILENAME_LEN] = {0}; char fname[TSDB_FILENAME_LEN] = {0};
code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd); code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};

24
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -52,6 +52,9 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.hashEnd = 0, .hashEnd = 0,
.hashMethod = 0, .hashMethod = 0,
.sttTrigger = TSDB_DEFAULT_SST_TRIGGER, .sttTrigger = TSDB_DEFAULT_SST_TRIGGER,
.s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE,
.s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL,
.s3Compact = TSDB_DEFAULT_S3_COMPACT,
.tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE}; .tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE};
int vnodeCheckCfg(const SVnodeCfg *pCfg) { int vnodeCheckCfg(const SVnodeCfg *pCfg) {
@ -107,6 +110,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact) < 0) return -1;
if (pCfg->tsdbCfg.retentions[0].keep > 0) { if (pCfg->tsdbCfg.retentions[0].keep > 0) {
int32_t nRetention = 1; int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) { if (pCfg->tsdbCfg.retentions[1].freq > 0) {
@ -158,9 +164,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
SJson *nodeInfo = tjsonCreateArray(); SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1; if (nodeInfo == NULL) return -1;
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1;
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId,
pCfg->vgId, pCfg->syncCfg.replicaNum, pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) { for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) {
SJson *info = tjsonCreateObject(); SJson *info = tjsonCreateObject();
SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i];
@ -360,6 +365,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024; pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024;
} }
tjsonGetNumberValue(pJson, "s3ChunkSize", pCfg->s3ChunkSize, code);
if (code < 0) {
pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
}
tjsonGetNumberValue(pJson, "s3KeepLocal", pCfg->s3KeepLocal, code);
if (code < 0) {
pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
}
tjsonGetNumberValue(pJson, "s3Compact", pCfg->s3Compact, code);
if (code < 0) {
pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
}
return 0; return 0;
} }

20
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -305,14 +305,14 @@ int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t s
return dstVgId; return dstVgId;
} }
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs) { void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) {
vInfo("path:%s is removed while destroy vnode", path); vInfo("path:%s is removed while destroy vnode", path);
tfsRmdir(pTfs, path); tfsRmdir(pTfs, path);
int32_t nlevel = tfsGetLevel(pTfs); // int32_t nlevel = tfsGetLevel(pTfs);
if (vgId > 0 && nlevel > 1 && tsS3Enabled) { if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) {
char vnode_prefix[TSDB_FILENAME_LEN]; char vnode_prefix[TSDB_FILENAME_LEN];
snprintf(vnode_prefix, TSDB_FILENAME_LEN, "v%df", vgId); snprintf(vnode_prefix, TSDB_FILENAME_LEN, "%d/v%df", nodeId, vgId);
s3DeleteObjectsByPrefix(vnode_prefix); s3DeleteObjectsByPrefix(vnode_prefix);
} }
} }
@ -480,11 +480,11 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
vnodeRollback(pVnode); vnodeRollback(pVnode);
} }
snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId); snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%" PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId);
snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId); snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%" PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId); snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%" PRId32, pVnode->config.vgId);
if(tsEnableMonitor && pVnode->monitor.insertCounter == NULL){ if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) {
taos_counter_t *counter = NULL; taos_counter_t *counter = NULL;
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
if(counter == NULL){ if(counter == NULL){
@ -499,11 +499,11 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
if(taos_collector_registry_register_metric(counter) == 1){ if(taos_collector_registry_register_metric(counter) == 1){
taos_counter_destroy(counter); taos_counter_destroy(counter);
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter); vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter);
} }
} }
pVnode->monitor.insertCounter = counter; pVnode->monitor.insertCounter = counter;
vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter); vInfo("vgId:%d, succeed to set metric:%p", TD_VID(pVnode), counter);
} }
return pVnode; return pVnode;

10
source/dnode/vnode/src/vnd/vnodeRetention.c
View File

@ -23,4 +23,12 @@ int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) {
if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now); if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now);
return code; return code;
} }
int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) {
int32_t code = TSDB_CODE_SUCCESS;
code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
return code;
}

38
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -38,6 +38,7 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pR
static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
SRpcMsg *pOriginalMsg); SRpcMsg *pOriginalMsg);
static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
@ -565,6 +566,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
case TDMT_VND_TRIM: case TDMT_VND_TRIM:
if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break; break;
case TDMT_VND_S3MIGRATE:
if (vnodeProcessS3MigrateReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_CREATE_SMA: case TDMT_VND_CREATE_SMA:
if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break; break;
@ -875,6 +879,26 @@ _exit:
return code; return code;
} }
extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now);
static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
SVS3MigrateDbReq s3migrateReq = {0};
// decode
if (tDeserializeSVS3MigrateDbReq(pReq, len, &s3migrateReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp);
code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp);
_exit:
return code;
}
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVDropTtlTableReq ttlReq = {0}; SVDropTtlTableReq ttlReq = {0};
if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) { if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) {
@ -1893,11 +1917,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
} }
vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64
" cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d fsync:%d level:%d " " cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d s3KeepLocal:%d "
"s3Compact:%d fsync:%d level:%d "
"walRetentionPeriod:%d walRetentionSize:%d", "walRetentionPeriod:%d walRetentionSize:%d",
TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024,
req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.keepTimeOffset, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize); req.keepTimeOffset, req.s3KeepLocal, req.s3Compact, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod,
req.walRetentionSize);
if (pVnode->config.cacheLastSize != req.cacheLastSize) { if (pVnode->config.cacheLastSize != req.cacheLastSize) {
pVnode->config.cacheLastSize = req.cacheLastSize; pVnode->config.cacheLastSize = req.cacheLastSize;
@ -1981,6 +2007,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe
pVnode->config.tsdbCfg.minRows = req.minRows; pVnode->config.tsdbCfg.minRows = req.minRows;
} }
if (req.s3KeepLocal != -1 && req.s3KeepLocal != pVnode->config.s3KeepLocal) {
pVnode->config.s3KeepLocal = req.s3KeepLocal;
}
if (req.s3Compact != -1 && req.s3Compact != pVnode->config.s3Compact) {
pVnode->config.s3Compact = req.s3Compact;
}
if (walChanged) { if (walChanged) {
walAlter(pVnode->pWal, &pVnode->config.walCfg); walAlter(pVnode->pWal, &pVnode->config.walCfg);
} }
@ -2263,4 +2296,5 @@ _OVER:
int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
return 0; return 0;
} }
int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync);
#endif #endif

10
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -820,6 +820,16 @@ bool vnodeIsLeader(SVnode *pVnode) {
return true; return true;
} }
int64_t vnodeClusterId(SVnode *pVnode) {
SSyncCfg *syncCfg = &pVnode->config.syncCfg;
return syncCfg->nodeInfo[syncCfg->myIndex].clusterId;
}
int32_t vnodeNodeId(SVnode *pVnode) {
SSyncCfg *syncCfg = &pVnode->config.syncCfg;
return syncCfg->nodeInfo[syncCfg->myIndex].nodeId;
}
int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) {
int code = 0; int code = 0;
pSnap->lastApplyIndex = pVnode->state.committed; pSnap->lastApplyIndex = pVnode->state.committed;

4
source/libs/command/src/command.c
View File

@ -325,12 +325,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s'", "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d",
dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize, 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize,
pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm)); pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact);
if (retentions) { if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);

72
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -109,6 +109,8 @@ const char* nodesNodeName(ENodeType type) {
return "FlushDatabaseStmt"; return "FlushDatabaseStmt";
case QUERY_NODE_TRIM_DATABASE_STMT: case QUERY_NODE_TRIM_DATABASE_STMT:
return "TrimDatabaseStmt"; return "TrimDatabaseStmt";
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
return "S3MigrateDatabaseStmt";
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
return "CreateTableStmt"; return "CreateTableStmt";
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
@ -825,7 +827,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
static const char* jkProjectLogicPlanProjections = "Projections"; static const char* jkProjectLogicPlanProjections = "Projections";
static const char* jkProjectLogicPlanIgnoreGroupId = "IgnoreGroupId"; static const char* jkProjectLogicPlanIgnoreGroupId = "IgnoreGroupId";
static const char* jkProjectLogicPlanInputIgnoreGroup= "InputIgnoreGroup"; static const char* jkProjectLogicPlanInputIgnoreGroup = "InputIgnoreGroup";
static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj; const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj;
@ -1341,7 +1343,6 @@ static int32_t jsonToLogicDynQueryCtrlNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkSubplanIdQueryId = "QueryId"; static const char* jkSubplanIdQueryId = "QueryId";
static const char* jkSubplanIdGroupId = "GroupId"; static const char* jkSubplanIdGroupId = "GroupId";
static const char* jkSubplanIdSubplanId = "SubplanId"; static const char* jkSubplanIdSubplanId = "SubplanId";
@ -1848,8 +1849,6 @@ static int32_t jsonToFuncType(const SJson* pJson, void* pObj) {
return code; return code;
} }
static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) { static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj; const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj;
@ -1863,7 +1862,7 @@ static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkLastRowScanPhysiPlanTargets, pNode->pTargets); code = nodeListToJson(pJson, jkLastRowScanPhysiPlanTargets, pNode->pTargets);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddTArray(pJson, jkLastRowScanPhysiPlanFuncTypes, funcTypeToJson, pNode->pFuncTypes); code = tjsonAddTArray(pJson, jkLastRowScanPhysiPlanFuncTypes, funcTypeToJson, pNode->pFuncTypes);
} }
@ -2257,7 +2256,6 @@ static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) { static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
SHashJoinPhysiNode* pNode = (SHashJoinPhysiNode*)pObj; SHashJoinPhysiNode* pNode = (SHashJoinPhysiNode*)pObj;
@ -2292,7 +2290,6 @@ static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkAggPhysiPlanExprs = "Exprs"; static const char* jkAggPhysiPlanExprs = "Exprs";
static const char* jkAggPhysiPlanGroupKeys = "GroupKeys"; static const char* jkAggPhysiPlanGroupKeys = "GroupKeys";
static const char* jkAggPhysiPlanAggFuncs = "AggFuncs"; static const char* jkAggPhysiPlanAggFuncs = "AggFuncs";
@ -2506,7 +2503,7 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) {
code = tjsonGetBoolValue(pJson, jkSortPhysiPlanCalcGroupIds, &pNode->calcGroupId); code = tjsonGetBoolValue(pJson, jkSortPhysiPlanCalcGroupIds, &pNode->calcGroupId);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code= tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol); code = tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol);
} }
return code; return code;
@ -3189,7 +3186,6 @@ static const char* jkGroupCachePhysiPlanGroupByUid = "GroupByUid";
static const char* jkGroupCachePhysiPlanGlobalGroup = "GlobalGroup"; static const char* jkGroupCachePhysiPlanGlobalGroup = "GlobalGroup";
static const char* jkGroupCachePhysiPlanBatchFetch = "BatchFetch"; static const char* jkGroupCachePhysiPlanBatchFetch = "BatchFetch";
static int32_t physiGroupCacheNodeToJson(const void* pObj, SJson* pJson) { static int32_t physiGroupCacheNodeToJson(const void* pObj, SJson* pJson) {
const SGroupCachePhysiNode* pNode = (const SGroupCachePhysiNode*)pObj; const SGroupCachePhysiNode* pNode = (const SGroupCachePhysiNode*)pObj;
@ -3323,8 +3319,6 @@ static int32_t jsonToPhysiDynQueryCtrlNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkQueryNodeAddrId = "Id"; static const char* jkQueryNodeAddrId = "Id";
static const char* jkQueryNodeAddrInUse = "InUse"; static const char* jkQueryNodeAddrInUse = "InUse";
static const char* jkQueryNodeAddrNumOfEps = "NumOfEps"; static const char* jkQueryNodeAddrNumOfEps = "NumOfEps";
@ -4857,6 +4851,10 @@ static const char* jkDatabaseOptionsNumOfVgroups = "NumOfVgroups";
static const char* jkDatabaseOptionsSingleStable = "SingleStable"; static const char* jkDatabaseOptionsSingleStable = "SingleStable";
static const char* jkDatabaseOptionsRetentions = "Retentions"; static const char* jkDatabaseOptionsRetentions = "Retentions";
static const char* jkDatabaseOptionsSchemaless = "Schemaless"; static const char* jkDatabaseOptionsSchemaless = "Schemaless";
static const char* jkDatabaseOptionsS3ChunkSize = "S3ChunkSize";
static const char* jkDatabaseOptionsS3KeepLocalNode = "S3KeepLocalNode";
static const char* jkDatabaseOptionsS3KeepLocal = "S3KeepLocal";
static const char* jkDatabaseOptionsS3Compact = "S3Compact";
static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
const SDatabaseOptions* pNode = (const SDatabaseOptions*)pObj; const SDatabaseOptions* pNode = (const SDatabaseOptions*)pObj;
@ -4916,6 +4914,18 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsSchemaless, pNode->schemaless); code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsSchemaless, pNode->schemaless);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3ChunkSize, pNode->s3ChunkSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkDatabaseOptionsS3KeepLocalNode, nodeToJson, pNode->s3KeepLocalStr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3KeepLocal, pNode->s3KeepLocal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3Compact, pNode->s3Compact);
}
return code; return code;
} }
@ -4978,6 +4988,18 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsSchemaless, &pNode->schemaless); code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsSchemaless, &pNode->schemaless);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkDatabaseOptionsS3ChunkSize, &pNode->s3ChunkSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkDatabaseOptionsS3KeepLocalNode, (SNode**)&pNode->s3KeepLocalStr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkDatabaseOptionsS3KeepLocal, &pNode->s3KeepLocal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsS3Compact, &pNode->s3Compact);
}
return code; return code;
} }
@ -5566,6 +5588,24 @@ static int32_t jsonToTrimDatabaseStmt(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkS3MigrateDatabaseStmtDbName = "DbName";
static int32_t s3migrateDatabaseStmtToJson(const void* pObj, SJson* pJson) {
const SS3MigrateDatabaseStmt* pNode = (const SS3MigrateDatabaseStmt*)pObj;
int32_t code = tjsonAddStringToObject(pJson, jkS3MigrateDatabaseStmtDbName, pNode->dbName);
return code;
}
static int32_t jsonToS3MigrateDatabaseStmt(const SJson* pJson, void* pObj) {
SS3MigrateDatabaseStmt* pNode = (SS3MigrateDatabaseStmt*)pObj;
int32_t code = tjsonGetStringValue(pJson, jkS3MigrateDatabaseStmtDbName, pNode->dbName);
return code;
}
static const char* jkCreateTableStmtDbName = "DbName"; static const char* jkCreateTableStmtDbName = "DbName";
static const char* jkCreateTableStmtTableName = "TableName"; static const char* jkCreateTableStmtTableName = "TableName";
static const char* jkCreateTableStmtIgnoreExists = "IgnoreExists"; static const char* jkCreateTableStmtIgnoreExists = "IgnoreExists";
@ -6235,10 +6275,6 @@ static int32_t jsonToRestoreVnodeStmt(const SJson* pJson, void* pObj) {
return jsonToRestoreComponentNodeStmt(pJson, pObj); return jsonToRestoreComponentNodeStmt(pJson, pObj);
} }
static const char* jkCreateTopicStmtTopicName = "TopicName"; static const char* jkCreateTopicStmtTopicName = "TopicName";
static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName";
static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists";
@ -6898,7 +6934,6 @@ static int32_t showCreateViewStmtToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) { static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) {
SShowCreateViewStmt* pNode = (SShowCreateViewStmt*)pObj; SShowCreateViewStmt* pNode = (SShowCreateViewStmt*)pObj;
@ -6910,7 +6945,6 @@ static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkShowTableDistributedStmtDbName = "DbName"; static const char* jkShowTableDistributedStmtDbName = "DbName";
static const char* jkShowTableDistributedStmtTableName = "TableName"; static const char* jkShowTableDistributedStmtTableName = "TableName";
@ -7156,6 +7190,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return alterDatabaseStmtToJson(pObj, pJson); return alterDatabaseStmtToJson(pObj, pJson);
case QUERY_NODE_TRIM_DATABASE_STMT: case QUERY_NODE_TRIM_DATABASE_STMT:
return trimDatabaseStmtToJson(pObj, pJson); return trimDatabaseStmtToJson(pObj, pJson);
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
return s3migrateDatabaseStmtToJson(pObj, pJson);
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
return createTableStmtToJson(pObj, pJson); return createTableStmtToJson(pObj, pJson);
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
@ -7497,6 +7533,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToAlterDatabaseStmt(pJson, pObj); return jsonToAlterDatabaseStmt(pJson, pObj);
case QUERY_NODE_TRIM_DATABASE_STMT: case QUERY_NODE_TRIM_DATABASE_STMT:
return jsonToTrimDatabaseStmt(pJson, pObj); return jsonToTrimDatabaseStmt(pJson, pObj);
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
return jsonToS3MigrateDatabaseStmt(pJson, pObj);
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
return jsonToCreateTableStmt(pJson, pObj); return jsonToCreateTableStmt(pJson, pObj);
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:

5
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -324,6 +324,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SFlushDatabaseStmt)); return makeNode(type, sizeof(SFlushDatabaseStmt));
case QUERY_NODE_TRIM_DATABASE_STMT: case QUERY_NODE_TRIM_DATABASE_STMT:
return makeNode(type, sizeof(STrimDatabaseStmt)); return makeNode(type, sizeof(STrimDatabaseStmt));
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
return makeNode(type, sizeof(SS3MigrateDatabaseStmt));
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
return makeNode(type, sizeof(SCreateTableStmt)); return makeNode(type, sizeof(SCreateTableStmt));
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
@ -804,6 +806,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_DATABASE_OPTIONS: { case QUERY_NODE_DATABASE_OPTIONS: {
SDatabaseOptions* pOptions = (SDatabaseOptions*)pNode; SDatabaseOptions* pOptions = (SDatabaseOptions*)pNode;
nodesDestroyNode((SNode*)pOptions->pDaysPerFile); nodesDestroyNode((SNode*)pOptions->pDaysPerFile);
nodesDestroyNode((SNode*)pOptions->s3KeepLocalStr);
nodesDestroyList(pOptions->pKeep); nodesDestroyList(pOptions->pKeep);
nodesDestroyList(pOptions->pRetentions); nodesDestroyList(pOptions->pRetentions);
break; break;
@ -941,6 +944,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field
case QUERY_NODE_TRIM_DATABASE_STMT: // no pointer field case QUERY_NODE_TRIM_DATABASE_STMT: // no pointer field
break; break;
case QUERY_NODE_S3MIGRATE_DATABASE_STMT: // no pointer field
break;
case QUERY_NODE_CREATE_TABLE_STMT: { case QUERY_NODE_CREATE_TABLE_STMT: {
SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode; SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode;
nodesDestroyList(pStmt->pCols); nodesDestroyList(pStmt->pCols);

71
source/libs/parser/inc/parAst.h
View File

@ -64,6 +64,9 @@ typedef enum EDatabaseOptionType {
DB_OPTION_STT_TRIGGER, DB_OPTION_STT_TRIGGER,
DB_OPTION_TABLE_PREFIX, DB_OPTION_TABLE_PREFIX,
DB_OPTION_TABLE_SUFFIX, DB_OPTION_TABLE_SUFFIX,
DB_OPTION_S3_CHUNKSIZE,
DB_OPTION_S3_KEEPLOCAL,
DB_OPTION_S3_COMPACT,
DB_OPTION_KEEP_TIME_OFFSET, DB_OPTION_KEEP_TIME_OFFSET,
DB_OPTION_ENCRYPT_ALGORITHM, DB_OPTION_ENCRYPT_ALGORITHM,
} EDatabaseOptionType; } EDatabaseOptionType;
@ -91,7 +94,7 @@ typedef struct STokenPair {
typedef struct SShowTablesOption { typedef struct SShowTablesOption {
EShowKind kind; EShowKind kind;
SToken dbName; SToken dbName;
} SShowTablesOption; } SShowTablesOption;
extern SToken nil_token; extern SToken nil_token;
@ -112,37 +115,37 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pNode); SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pNode);
SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pLeft, SNode *pRight); SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pLeft, SNode *pRight);
SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral); SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral); SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt); SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias); SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt); SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2); SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias); SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName); SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond); SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken); SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill); SNode* pFill);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd); SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint); SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere);
SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList);
@ -155,7 +158,8 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange); SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange);
SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery); SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill); SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint); SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable,
SNodeList* pHint);
SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags); SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
@ -171,6 +175,7 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STo
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions); SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed); SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed);
SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd); SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createAlterTableOptions(SAstCreateContext* pCxt); SNode* createAlterTableOptions(SAstCreateContext* pCxt);

8
source/libs/parser/inc/sql.y
View File

@ -211,6 +211,7 @@ cmd ::= USE db_name(A).
cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); } cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); }
cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); } cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); }
cmd ::= S3MIGRATE DATABASE db_name(A). { pCxt->pRootNode = createS3MigrateDatabaseStmt(pCxt, &A); }
cmd ::= COMPACT DATABASE db_name(A) start_opt(B) end_opt(C). { pCxt->pRootNode = createCompactStmt(pCxt, &A, B, C); } cmd ::= COMPACT DATABASE db_name(A) start_opt(B) end_opt(C). { pCxt->pRootNode = createCompactStmt(pCxt, &A, B, C); }
%type not_exists_opt { bool } %type not_exists_opt { bool }
@ -263,6 +264,10 @@ db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C).
db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); } db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); } db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); } db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
db_options(A) ::= db_options(B) S3_CHUNKSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_CHUNKSIZE, &C); }
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); }
db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); } db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); }
db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_ENCRYPT_ALGORITHM, &C); } db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_ENCRYPT_ALGORITHM, &C); }
@ -295,6 +300,9 @@ alter_db_option(A) ::= WAL_RETENTION_SIZE NK_MINUS(B) NK_INTEGER(C).
t.n = (C.z + C.n) - B.z; t.n = (C.z + C.n) - B.z;
A.type = DB_OPTION_WAL_RETENTION_SIZE; A.val = t; A.type = DB_OPTION_WAL_RETENTION_SIZE; A.val = t;
} }
alter_db_option(A) ::= S3_KEEPLOCAL NK_INTEGER(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; }
alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; }
alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; }
alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; } alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; }
alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; } alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; }

38
source/libs/parser/src/parAstCreater.c
View File

@ -228,7 +228,6 @@ static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) {
return true; return true;
} }
static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) { static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) {
trimEscape(pStreamName); trimEscape(pStreamName);
if (pStreamName->n >= TSDB_STREAM_NAME_LEN) { if (pStreamName->n >= TSDB_STREAM_NAME_LEN) {
@ -918,7 +917,6 @@ SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewNam
return (SNode*)pView; return (SNode*)pView;
} }
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
@ -1263,6 +1261,9 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; pOptions->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
pOptions->tablePrefix = TSDB_DEFAULT_HASH_PREFIX; pOptions->tablePrefix = TSDB_DEFAULT_HASH_PREFIX;
pOptions->tableSuffix = TSDB_DEFAULT_HASH_SUFFIX; pOptions->tableSuffix = TSDB_DEFAULT_HASH_SUFFIX;
pOptions->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pOptions->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pOptions->s3Compact = TSDB_DEFAULT_S3_COMPACT;
pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
pOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; pOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
return (SNode*)pOptions; return (SNode*)pOptions;
@ -1300,6 +1301,9 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->sstTrigger = -1; pOptions->sstTrigger = -1;
pOptions->tablePrefix = -1; pOptions->tablePrefix = -1;
pOptions->tableSuffix = -1; pOptions->tableSuffix = -1;
pOptions->s3ChunkSize = -1;
pOptions->s3KeepLocal = -1;
pOptions->s3Compact = -1;
pOptions->withArbitrator = -1; pOptions->withArbitrator = -1;
pOptions->encryptAlgorithm = -1; pOptions->encryptAlgorithm = -1;
return (SNode*)pOptions; return (SNode*)pOptions;
@ -1417,6 +1421,21 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
nodesDestroyNode((SNode*)pNode); nodesDestroyNode((SNode*)pNode);
break; break;
} }
case DB_OPTION_S3_CHUNKSIZE:
pDbOptions->s3ChunkSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_S3_KEEPLOCAL: {
SToken* pToken = pVal;
if (TK_NK_INTEGER == pToken->type) {
pDbOptions->s3KeepLocal = taosStr2Int32(pToken->z, NULL, 10) * 1440;
} else {
pDbOptions->s3KeepLocalStr = (SValueNode*)createDurationValueNode(pCxt, pToken);
}
break;
}
case DB_OPTION_S3_COMPACT:
pDbOptions->s3Compact = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_KEEP_TIME_OFFSET: { case DB_OPTION_KEEP_TIME_OFFSET: {
pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break; break;
@ -1507,6 +1526,17 @@ SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false)) {
return NULL;
}
SS3MigrateDatabaseStmt* pStmt = (SS3MigrateDatabaseStmt*)nodesMakeNode(QUERY_NODE_S3MIGRATE_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
return (SNode*)pStmt;
}
SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd) { SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, false)) { if (!checkDbName(pCxt, pDbName, false)) {
@ -1817,7 +1847,8 @@ SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pD
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, EOperatorType tableCondType) { SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName,
EOperatorType tableCondType) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SNode* pDbName = NULL; SNode* pDbName = NULL;
if (option.dbName.type == TK_NK_NIL) { if (option.dbName.type == TK_NK_NIL) {
@ -1893,7 +1924,6 @@ SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode*
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable) { SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SShowTableDistributedStmt* pStmt = (SShowTableDistributedStmt*)nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT); SShowTableDistributedStmt* pStmt = (SShowTableDistributedStmt*)nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT);

4
source/libs/parser/src/parTokenizer.c
View File

@ -308,6 +308,10 @@ static SKeyword keywordTable[] = {
{"_WSTART", TK_WSTART}, {"_WSTART", TK_WSTART},
{"ALIVE", TK_ALIVE}, {"ALIVE", TK_ALIVE},
{"VARBINARY", TK_VARBINARY}, {"VARBINARY", TK_VARBINARY},
{"S3_CHUNKSIZE", TK_S3_CHUNKSIZE},
{"S3_KEEPLOCAL", TK_S3_KEEPLOCAL},
{"S3_COMPACT", TK_S3_COMPACT},
{"S3MIGRATE", TK_S3MIGRATE},
{"KEEP_TIME_OFFSET", TK_KEEP_TIME_OFFSET}, {"KEEP_TIME_OFFSET", TK_KEEP_TIME_OFFSET},
{"ARBGROUPS", TK_ARBGROUPS}, {"ARBGROUPS", TK_ARBGROUPS},
}; };

393
source/libs/parser/src/parTranslater.c
View File

@ -14,8 +14,8 @@
*/ */
#include "parTranslater.h" #include "parTranslater.h"
#include "tdatablock.h"
#include "parInt.h" #include "parInt.h"
#include "tdatablock.h"
#include "catalog.h" #include "catalog.h"
#include "cmdnodes.h" #include "cmdnodes.h"
@ -387,7 +387,8 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pTable) {
} }
#ifdef BUILD_NO_CALL #ifdef BUILD_NO_CALL
static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName,
STableMeta** pMeta) {
#ifndef TD_ENTERPRISE #ifndef TD_ENTERPRISE
return TSDB_CODE_PAR_TABLE_NOT_EXIST; return TSDB_CODE_PAR_TABLE_NOT_EXIST;
#endif #endif
@ -412,7 +413,8 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac
} }
#endif #endif
int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta,
bool couldBeView) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (pParCxt->async) { if (pParCxt->async) {
@ -1140,7 +1142,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
SExprNode* pExpr = (SExprNode*)pNode; SExprNode* pExpr = (SExprNode*)pNode;
if (0 == strcmp((*pCol)->colName, pExpr->userAlias)) { if (0 == strcmp((*pCol)->colName, pExpr->userAlias)) {
if (true == *pFound) { if (true == *pFound) {
if(nodesEqualNode(pFoundNode, pNode)) { if (nodesEqualNode(pFoundNode, pNode)) {
continue; continue;
} }
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ORDERBY_AMBIGUOUS, (*pCol)->colName); pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ORDERBY_AMBIGUOUS, (*pCol)->colName);
@ -1219,7 +1221,8 @@ static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) {
} }
} }
static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode, SListCell* pSelectListCell) { static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode,
SListCell* pSelectListCell) {
SNodeList* pTbnameNodeList = nodesMakeList(); SNodeList* pTbnameNodeList = nodesMakeList();
SFunctionNode* pFunc = (SFunctionNode*)pNode; SFunctionNode* pFunc = (SFunctionNode*)pNode;
@ -1244,7 +1247,7 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt
nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList); nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList);
} }
} else if (nodesIsTableStar(pPara)) { } else if (nodesIsTableStar(pPara)) {
char* pTableAlias = ((SColumnNode*)pPara)->tableAlias; char* pTableAlias = ((SColumnNode*)pPara)->tableAlias;
STableNode* pTable = NULL; STableNode* pTable = NULL;
int32_t code = findTable(pCxt, pTableAlias, &pTable); int32_t code = findTable(pCxt, pTableAlias, &pTable);
if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE && if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE &&
@ -1265,12 +1268,12 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt
// after translate from // after translate from
// before translate select list // before translate select list
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
SNode* pNode = NULL; SNode* pNode = NULL;
SNodeList* pTbnameNodeList = nodesMakeList(); SNodeList* pTbnameNodeList = nodesMakeList();
WHERE_EACH(pNode, pSelect->pProjectionList) { WHERE_EACH(pNode, pSelect->pProjectionList) {
if (nodesIsStar(pNode)) { if (nodesIsStar(pNode)) {
SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
size_t n = taosArrayGetSize(pTables); size_t n = taosArrayGetSize(pTables);
for (int32_t i = 0; i < n; ++i) { for (int32_t i = 0; i < n; ++i) {
STableNode* pTable = taosArrayGetP(pTables, i); STableNode* pTable = taosArrayGetP(pTables, i);
if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && if (nodeType(pTable) == QUERY_NODE_REAL_TABLE &&
@ -1284,7 +1287,7 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList); nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList);
} }
} else if (nodesIsTableStar(pNode)) { } else if (nodesIsTableStar(pNode)) {
char* pTableAlias = ((SColumnNode*)pNode)->tableAlias; char* pTableAlias = ((SColumnNode*)pNode)->tableAlias;
STableNode* pTable = NULL; STableNode* pTable = NULL;
int32_t code = findTable(pCxt, pTableAlias, &pTable); int32_t code = findTable(pCxt, pTableAlias, &pTable);
if (TSDB_CODE_SUCCESS == code && if (TSDB_CODE_SUCCESS == code &&
@ -1300,7 +1303,7 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
} else if (nodeType(pNode) == QUERY_NODE_FUNCTION) { } else if (nodeType(pNode) == QUERY_NODE_FUNCTION) {
biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell); biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell);
} }
WHERE_NEXT; WHERE_NEXT;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1308,11 +1311,11 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) {
SColumnNode* pCol = (SColumnNode*)(*ppNode); SColumnNode* pCol = (SColumnNode*)(*ppNode);
if ((strcasecmp(pCol->colName, "tbname") == 0) && if ((strcasecmp(pCol->colName, "tbname") == 0) && ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable &&
((SSelectStmt*)pCxt->pCurrStmt)->pFromTable && QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
SFunctionNode* tbnameFuncNode = NULL; SFunctionNode* tbnameFuncNode = NULL;
tbnameFuncNode = (SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0]!='\0') ? pCol->tableAlias : NULL); tbnameFuncNode =
(SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0] != '\0') ? pCol->tableAlias : NULL);
tbnameFuncNode->node.resType = pCol->node.resType; tbnameFuncNode->node.resType = pCol->node.resType;
strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName); strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName);
strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias); strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias);
@ -1327,24 +1330,26 @@ bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) {
int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
if (pStmt->pTags) { if (pStmt->pTags) {
SNode* pNode = NULL; SNode* pNode = NULL;
FOREACH(pNode, pStmt->pTags) { FOREACH(pNode, pStmt->pTags) {
SColumnDefNode* pTag = (SColumnDefNode*)pNode; SColumnDefNode* pTag = (SColumnDefNode*)pNode;
if (strcasecmp(pTag->colName, "tbname") == 0) { if (strcasecmp(pTag->colName, "tbname") == 0) {
int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, "tbname can not used for tags in BI mode"); int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME,
return code; "tbname can not used for tags in BI mode");
} return code;
} }
}
} }
if (pStmt->pCols) { if (pStmt->pCols) {
SNode* pNode = NULL; SNode* pNode = NULL;
FOREACH(pNode, pStmt->pCols) { FOREACH(pNode, pStmt->pCols) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode; SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if (strcasecmp(pCol->colName, "tbname") == 0) { if (strcasecmp(pCol->colName, "tbname") == 0) {
int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "tbname can not used for columns in BI mode"); int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN,
return code; "tbname can not used for columns in BI mode");
} return code;
} }
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1382,9 +1387,9 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
res = translateColumnWithoutPrefix(pCxt, pCol); res = translateColumnWithoutPrefix(pCxt, pCol);
} }
} }
if(SQL_CLAUSE_ORDER_BY == pCxt->currClause && !(*pCol)->node.asParam if (SQL_CLAUSE_ORDER_BY == pCxt->currClause && !(*pCol)->node.asParam && res != DEAL_RES_CONTINUE &&
&& res != DEAL_RES_CONTINUE && res != DEAL_RES_END) { res != DEAL_RES_END) {
res = translateColumnUseAlias(pCxt, pCol, &found); res = translateColumnUseAlias(pCxt, pCol, &found);
} }
} }
return res; return res;
@ -1966,8 +1971,8 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode**
} }
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SNode* pNode = NULL; SNode* pNode = NULL;
bool bFound = false; bool bFound = false;
FOREACH(pNode, pSelect->pProjectionList) { FOREACH(pNode, pSelect->pProjectionList) {
if (nodeType(pNode) == QUERY_NODE_FUNCTION && strcasecmp(((SFunctionNode*)pNode)->functionName, "interp") == 0) { if (nodeType(pNode) == QUERY_NODE_FUNCTION && strcasecmp(((SFunctionNode*)pNode)->functionName, "interp") == 0) {
bFound = true; bFound = true;
@ -2272,7 +2277,7 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
SExprNode* pOldExpr = (SExprNode*)(*ppNode); SExprNode* pOldExpr = (SExprNode*)(*ppNode);
//rewrite a.tbname == tbname(a) // rewrite a.tbname == tbname(a)
if (nodeType(*ppNode) == QUERY_NODE_FUNCTION && ((SFunctionNode*)(*ppNode))->funcType == FUNCTION_TYPE_TBNAME) { if (nodeType(*ppNode) == QUERY_NODE_FUNCTION && ((SFunctionNode*)(*ppNode))->funcType == FUNCTION_TYPE_TBNAME) {
SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); SFunctionNode* pFunc = (SFunctionNode*)(*ppNode);
if (0 != LIST_LENGTH(pFunc->pParameterList)) { if (0 != LIST_LENGTH(pFunc->pParameterList)) {
@ -2333,7 +2338,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode**
static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) {
SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); SFunctionNode* pFunc = (SFunctionNode*)(*ppNode);
if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (0 == LIST_LENGTH(pFunc->pParameterList)) { if (0 == LIST_LENGTH(pFunc->pParameterList)) {
@ -2361,7 +2366,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp
static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) { static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) {
SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); SFunctionNode* pFunc = (SFunctionNode*)(*ppNode);
int32_t code = translateAggFunc(pCxt, pFunc); int32_t code = translateAggFunc(pCxt, pFunc);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
bool bRewriteToColumn = false; bool bRewriteToColumn = false;
code = translateScanPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn); code = translateScanPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn);
@ -2849,18 +2854,17 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) {
} }
static int32_t checkIsEmptyResult(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t checkIsEmptyResult(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (pSelect->timeRange.skey > pSelect->timeRange.ekey if (pSelect->timeRange.skey > pSelect->timeRange.ekey && !pSelect->hasCountFunc) {
&& !pSelect->hasCountFunc) {
pSelect->isEmptyResult = true; pSelect->isEmptyResult = true;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t resetSelectFuncNumWithoutDup(SSelectStmt* pSelect) { static int32_t resetSelectFuncNumWithoutDup(SSelectStmt* pSelect) {
if (pSelect->selectFuncNum <= 1) return TSDB_CODE_SUCCESS; if (pSelect->selectFuncNum <= 1) return TSDB_CODE_SUCCESS;
pSelect->selectFuncNum = 0; pSelect->selectFuncNum = 0;
SNodeList* pNodeList = nodesMakeList(); SNodeList* pNodeList = nodesMakeList();
int32_t code = nodesCollectSelectFuncs(pSelect, SQL_CLAUSE_FROM, NULL, fmIsSelectFunc, pNodeList); int32_t code = nodesCollectSelectFuncs(pSelect, SQL_CLAUSE_FROM, NULL, fmIsSelectFunc, pNodeList);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pNodeList); nodesDestroyList(pNodeList);
return code; return code;
@ -3937,57 +3941,57 @@ static void convertVarDuration(SValueNode* pOffset, uint8_t precision) {
pOffset->unit = units[precision]; pOffset->unit = units[precision];
} }
static const int64_t tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP; static const int64_t tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP;
static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) {
uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision;
SValueNode* pInter = (SValueNode*)pInterval->pInterval; SValueNode* pInter = (SValueNode*)pInterval->pInterval;
bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit);
if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime,
getPrecisionStr(precision)); getPrecisionStr(precision));
} else if (pInter->datum.i / getPrecisionMultiple(precision) > tsdbMaxKeepMS) { } else if (pInter->datum.i / getPrecisionMultiple(precision) > tsdbMaxKeepMS) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years"); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years");
} }
if (NULL != pInterval->pOffset) { if (NULL != pInterval->pOffset) {
SValueNode* pOffset = (SValueNode*)pInterval->pOffset; SValueNode* pOffset = (SValueNode*)pInterval->pOffset;
if (pOffset->datum.i <= 0) { if (pOffset->datum.i <= 0) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE);
} }
if (pInter->unit == 'n' && pOffset->unit == 'y') { if (pInter->unit == 'n' && pOffset->unit == 'y') {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT);
} }
bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter;
if ((fixed && pOffset->datum.i >= pInter->datum.i) || if ((fixed && pOffset->datum.i >= pInter->datum.i) ||
(!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >=
getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG);
} }
if (pOffset->unit == 'n' || pOffset->unit == 'y') { if (pOffset->unit == 'n' || pOffset->unit == 'y') {
convertVarDuration(pOffset, precision); convertVarDuration(pOffset, precision);
} }
} }
if (NULL != pInterval->pSliding) { if (NULL != pInterval->pSliding) {
const static int32_t INTERVAL_SLIDING_FACTOR = 100; const static int32_t INTERVAL_SLIDING_FACTOR = 100;
SValueNode* pSliding = (SValueNode*)pInterval->pSliding; SValueNode* pSliding = (SValueNode*)pInterval->pSliding;
if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT);
} }
if ((pSliding->datum.i < if ((pSliding->datum.i <
convertTimeFromPrecisionToUnit(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pSliding->unit)) || convertTimeFromPrecisionToUnit(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pSliding->unit)) ||
(pInter->datum.i / pSliding->datum.i > INTERVAL_SLIDING_FACTOR)) { (pInter->datum.i / pSliding->datum.i > INTERVAL_SLIDING_FACTOR)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL);
} }
if (pSliding->datum.i > pInter->datum.i) { if (pSliding->datum.i > pInter->datum.i) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG);
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateIntervalWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translateIntervalWindow(STranslateContext* pCxt, SSelectStmt* pSelect) {
@ -4128,11 +4132,11 @@ static EDealRes collectWindowsPseudocolumns(SNode* pNode, void* pContext) {
} }
static int32_t checkWindowsConditonValid(SNode* pNode) { static int32_t checkWindowsConditonValid(SNode* pNode) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if(QUERY_NODE_EVENT_WINDOW != nodeType(pNode)) return code; if (QUERY_NODE_EVENT_WINDOW != nodeType(pNode)) return code;
SEventWindowNode* pEventWindowNode = (SEventWindowNode*)pNode; SEventWindowNode* pEventWindowNode = (SEventWindowNode*)pNode;
SNodeList* pCols = nodesMakeList(); SNodeList* pCols = nodesMakeList();
if (NULL == pCols) { if (NULL == pCols) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -4342,14 +4346,15 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec
typedef struct SEqCondTbNameTableInfo { typedef struct SEqCondTbNameTableInfo {
SRealTableNode* pRealTable; SRealTableNode* pRealTable;
SArray* aTbnames; SArray* aTbnames;
} SEqCondTbNameTableInfo; } SEqCondTbNameTableInfo;
//[tableAlias.]tbname = tbNamVal //[tableAlias.]tbname = tbNamVal
static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray** ppTabNames) { static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias,
SArray** ppTabNames) {
if (pOperator->opType != OP_TYPE_EQUAL) return false; if (pOperator->opType != OP_TYPE_EQUAL) return false;
SFunctionNode* pTbnameFunc = NULL; SFunctionNode* pTbnameFunc = NULL;
SValueNode* pValueNode = NULL; SValueNode* pValueNode = NULL;
if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION && if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION &&
((SFunctionNode*)(pOperator->pLeft))->funcType == FUNCTION_TYPE_TBNAME && ((SFunctionNode*)(pOperator->pLeft))->funcType == FUNCTION_TYPE_TBNAME &&
nodeType(pOperator->pRight) == QUERY_NODE_VALUE) { nodeType(pOperator->pRight) == QUERY_NODE_VALUE) {
@ -4380,7 +4385,8 @@ static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOper
} }
//[tableAlias.]tbname in (value1, value2, ...) //[tableAlias.]tbname in (value1, value2, ...)
static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray** ppTbNames) { static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias,
SArray** ppTbNames) {
if (pOperator->opType != OP_TYPE_IN) return false; if (pOperator->opType != OP_TYPE_IN) return false;
if (nodeType(pOperator->pLeft) != QUERY_NODE_FUNCTION || if (nodeType(pOperator->pLeft) != QUERY_NODE_FUNCTION ||
((SFunctionNode*)(pOperator->pLeft))->funcType != FUNCTION_TYPE_TBNAME || ((SFunctionNode*)(pOperator->pLeft))->funcType != FUNCTION_TYPE_TBNAME ||
@ -4401,8 +4407,8 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper
} }
*ppTbNames = taosArrayInit(1, sizeof(void*)); *ppTbNames = taosArrayInit(1, sizeof(void*));
SNodeListNode* pValueListNode = (SNodeListNode*)pOperator->pRight; SNodeListNode* pValueListNode = (SNodeListNode*)pOperator->pRight;
SNodeList* pValueNodeList = pValueListNode->pNodeList; SNodeList* pValueNodeList = pValueListNode->pNodeList;
SNode* pValNode = NULL; SNode* pValNode = NULL;
FOREACH(pValNode, pValueNodeList) { FOREACH(pValNode, pValueNodeList) {
if (nodeType(pValNode) != QUERY_NODE_VALUE) { if (nodeType(pValNode) != QUERY_NODE_VALUE) {
return false; return false;
@ -4410,13 +4416,12 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper
taosArrayPush(*ppTbNames, &((SValueNode*)pValNode)->literal); taosArrayPush(*ppTbNames, &((SValueNode*)pValNode)->literal);
} }
return true; return true;
} }
static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo) { static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
char* pTableAlias = NULL; char* pTableAlias = NULL;
char* pTbNameVal = NULL; char* pTbNameVal = NULL;
if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames) || if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames) ||
isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames)) { isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames)) {
STableNode* pTable; STableNode* pTable;
@ -4451,17 +4456,17 @@ static void findEqualCondTbnameInLogicCondAnd(STranslateContext* pCxt, SNode* pW
FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) {
if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) {
SEqCondTbNameTableInfo info = {0}; SEqCondTbNameTableInfo info = {0};
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info);
if (bIsEqTbnameCond) { if (bIsEqTbnameCond) {
if (!isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { if (!isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) {
//TODO: intersect tbNames of same table? speed // TODO: intersect tbNames of same table? speed
taosArrayPush(aTableTbnames, &info); taosArrayPush(aTableTbnames, &info);
} else { } else {
taosArrayDestroy(info.aTbnames); taosArrayDestroy(info.aTbnames);
} }
} }
} }
//TODO: logic cond // TODO: logic cond
} }
} }
@ -4486,10 +4491,10 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh
bool bAllTbName = true; bool bAllTbName = true;
SNode* pTmpNode = NULL; SNode* pTmpNode = NULL;
FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) {
//TODO: logic cond // TODO: logic cond
if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) {
SEqCondTbNameTableInfo info = {0}; SEqCondTbNameTableInfo info = {0};
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info);
if (!bIsEqTbnameCond) { if (!bIsEqTbnameCond) {
bAllTbName = false; bAllTbName = false;
break; break;
@ -4512,10 +4517,10 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh
} }
static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) {
//TODO: optimize nested and/or condition. now only the fist level is processed. // TODO: optimize nested and/or condition. now only the fist level is processed.
if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { if (nodeType(pWhere) == QUERY_NODE_OPERATOR) {
SEqCondTbNameTableInfo info = {0}; SEqCondTbNameTableInfo info = {0};
bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info); bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info);
if (bIsEqTbnameCond) { if (bIsEqTbnameCond) {
taosArrayPush(aTableTbnames, &info); taosArrayPush(aTableTbnames, &info);
} }
@ -4529,7 +4534,8 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo, SVgroupsInfo* vgsInfo) { static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo,
SVgroupsInfo* vgsInfo) {
int32_t nVgroups = 0; int32_t nVgroups = 0;
int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); int32_t nTbls = taosArrayGetSize(pInfo->aTbnames);
@ -4571,10 +4577,10 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt*
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
for (int i = 0; i < taosArrayGetSize(aTables); ++i) { for (int i = 0; i < taosArrayGetSize(aTables); ++i) {
SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i); SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i);
int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); int32_t nTbls = taosArrayGetSize(pInfo->aTbnames);
SVgroupsInfo* vgsInfo = taosMemoryMalloc(sizeof(SVgroupsInfo) + nTbls * sizeof(SVgroupInfo)); SVgroupsInfo* vgsInfo = taosMemoryMalloc(sizeof(SVgroupsInfo) + nTbls * sizeof(SVgroupInfo));
int32_t nVgroups = 0; int32_t nVgroups = 0;
findVgroupsFromEqualTbname(pCxt, pInfo, vgsInfo); findVgroupsFromEqualTbname(pCxt, pInfo, vgsInfo);
if (vgsInfo->numOfVgroups != 0) { if (vgsInfo->numOfVgroups != 0) {
taosMemoryFree(pInfo->pRealTable->pVgroupList); taosMemoryFree(pInfo->pRealTable->pVgroupList);
@ -5167,7 +5173,6 @@ static int32_t translateInsertTable(STranslateContext* pCxt, SNode** pTable) {
if (TSDB_CODE_SUCCESS == code && TSDB_CHILD_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType && if (TSDB_CODE_SUCCESS == code && TSDB_CHILD_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType &&
TSDB_NORMAL_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType) { TSDB_NORMAL_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType) {
code = buildInvalidOperationMsg(&pCxt->msgBuf, "insert data into super table is not supported"); code = buildInvalidOperationMsg(&pCxt->msgBuf, "insert data into super table is not supported");
} }
return code; return code;
} }
@ -5263,6 +5268,9 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
pReq->hashSuffix = pStmt->pOptions->tableSuffix; pReq->hashSuffix = pStmt->pOptions->tableSuffix;
pReq->tsdbPageSize = pStmt->pOptions->tsdbPageSize; pReq->tsdbPageSize = pStmt->pOptions->tsdbPageSize;
pReq->keepTimeOffset = pStmt->pOptions->keepTimeOffset; pReq->keepTimeOffset = pStmt->pOptions->keepTimeOffset;
pReq->s3ChunkSize = pStmt->pOptions->s3ChunkSize;
pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal;
pReq->s3Compact = pStmt->pOptions->s3Compact;
pReq->ignoreExist = pStmt->ignoreExists; pReq->ignoreExist = pStmt->ignoreExists;
pReq->withArbitrator = pStmt->pOptions->withArbitrator; pReq->withArbitrator = pStmt->pOptions->withArbitrator;
pReq->encryptAlgorithm = pStmt->pOptions->encryptAlgorithm; pReq->encryptAlgorithm = pStmt->pOptions->encryptAlgorithm;
@ -5289,6 +5297,22 @@ static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName,
return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal); return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal);
} }
static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if (NULL != pOptions->s3KeepLocalStr) {
if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->s3KeepLocalStr)) {
return pCxt->errCode;
}
if (TIME_UNIT_MINUTE != pOptions->s3KeepLocalStr->unit && TIME_UNIT_HOUR != pOptions->s3KeepLocalStr->unit &&
TIME_UNIT_DAY != pOptions->s3KeepLocalStr->unit) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option s3_keeplocal unit: %c, only %c, %c, %c allowed",
pOptions->s3KeepLocalStr->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY);
}
pOptions->s3KeepLocal = getBigintFromValueNode(pOptions->s3KeepLocalStr);
}
return checkDbRangeOption(pCxt, "s3KeepLocal", pOptions->s3KeepLocal, TSDB_MIN_S3_KEEP_LOCAL, TSDB_MAX_S3_KEEP_LOCAL);
}
static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if (NULL != pOptions->pDaysPerFile) { if (NULL != pOptions->pDaysPerFile) {
if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pDaysPerFile)) { if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pDaysPerFile)) {
@ -5566,6 +5590,7 @@ static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbP
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
int32_t daysPerFile = pOptions->daysPerFile; int32_t daysPerFile = pOptions->daysPerFile;
int32_t s3KeepLocal = pOptions->s3KeepLocal;
int64_t daysToKeep0 = pOptions->keep[0]; int64_t daysToKeep0 = pOptions->keep[0];
if (-1 == daysPerFile && -1 == daysToKeep0) { if (-1 == daysPerFile && -1 == daysToKeep0) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -5582,6 +5607,10 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid duration value, should be keep2 >= keep1 >= keep0 >= 3 * duration"); "Invalid duration value, should be keep2 >= keep1 >= keep0 >= 3 * duration");
} }
if (s3KeepLocal > 0 && daysPerFile > s3KeepLocal / 3) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid parameters, should be s3_keeplocal >= 3 * duration");
}
if ((pOptions->replica == 2) ^ (pOptions->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) { if ((pOptions->replica == 2) ^ (pOptions->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
@ -5693,9 +5722,19 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix); code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
} }
if (TSDB_CODE_SUCCESS == code) {
code = checkDbS3KeepLocalOption(pCxt, pOptions);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkOptionsDependency(pCxt, pDbName, pOptions); code = checkOptionsDependency(pCxt, pDbName, pOptions);
} }
if (TSDB_CODE_SUCCESS == code) {
code =
checkDbRangeOption(pCxt, "s3_chunksize", pOptions->s3ChunkSize, TSDB_MIN_S3_CHUNK_SIZE, TSDB_MAX_S3_CHUNK_SIZE);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT);
}
return code; return code;
} }
@ -5708,18 +5747,18 @@ static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt*
} }
#define FILL_CMD_SQL(sql, sqlLen, pCmdReq, CMD_TYPE, genericCmd) \ #define FILL_CMD_SQL(sql, sqlLen, pCmdReq, CMD_TYPE, genericCmd) \
CMD_TYPE* pCmdReq = genericCmd; \ CMD_TYPE* pCmdReq = genericCmd; \
char* cmdSql = taosMemoryMalloc(sqlLen); \ char* cmdSql = taosMemoryMalloc(sqlLen); \
if (cmdSql == NULL) { \ if (cmdSql == NULL) { \
return TSDB_CODE_OUT_OF_MEMORY; \ return TSDB_CODE_OUT_OF_MEMORY; \
} \ } \
memcpy(cmdSql, sql, sqlLen); \ memcpy(cmdSql, sql, sqlLen); \
pCmdReq->sqlLen = sqlLen; \ pCmdReq->sqlLen = sqlLen; \
pCmdReq->sql = cmdSql; \ pCmdReq->sql = cmdSql;
static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq) { static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq) {
const char* sql = pCxt->pParseCxt->pSql; const char* sql = pCxt->pParseCxt->pSql;
size_t sqlLen = pCxt->pParseCxt->sqlLen; size_t sqlLen = pCxt->pParseCxt->sqlLen;
switch (msgType) { switch (msgType) {
case TDMT_MND_CREATE_DB: { case TDMT_MND_CREATE_DB: {
@ -5834,7 +5873,6 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq)
default: { default: {
break; break;
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -5907,6 +5945,8 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
pReq->minRows = pStmt->pOptions->minRowsPerBlock; pReq->minRows = pStmt->pOptions->minRowsPerBlock;
pReq->walRetentionPeriod = pStmt->pOptions->walRetentionPeriod; pReq->walRetentionPeriod = pStmt->pOptions->walRetentionPeriod;
pReq->walRetentionSize = pStmt->pOptions->walRetentionSize; pReq->walRetentionSize = pStmt->pOptions->walRetentionSize;
pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal;
pReq->s3Compact = pStmt->pOptions->s3Compact;
pReq->withArbitrator = pStmt->pOptions->withArbitrator; pReq->withArbitrator = pStmt->pOptions->withArbitrator;
return; return;
} }
@ -5938,6 +5978,14 @@ static int32_t translateTrimDatabase(STranslateContext* pCxt, STrimDatabaseStmt*
return buildCmdMsg(pCxt, TDMT_MND_TRIM_DB, (FSerializeFunc)tSerializeSTrimDbReq, &req); return buildCmdMsg(pCxt, TDMT_MND_TRIM_DB, (FSerializeFunc)tSerializeSTrimDbReq, &req);
} }
static int32_t translateS3MigrateDatabase(STranslateContext* pCxt, SS3MigrateDatabaseStmt* pStmt) {
SS3MigrateDbReq req = {0};
SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, req.db);
return buildCmdMsg(pCxt, TDMT_MND_S3MIGRATE_DB, (FSerializeFunc)tSerializeSS3MigrateDbReq, &req);
}
static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calBytes) { static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calBytes) {
*pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
SNode* pNode; SNode* pNode;
@ -5989,7 +6037,7 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
} }
static bool validRollupFunc(const char* pFunc) { static bool validRollupFunc(const char* pFunc) {
static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"}; static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"};
static const int32_t numOfRollupFuncs = (sizeof(rollupFuncs) / sizeof(char*)); static const int32_t numOfRollupFuncs = (sizeof(rollupFuncs) / sizeof(char*));
for (int i = 0; i < numOfRollupFuncs; ++i) { for (int i = 0; i < numOfRollupFuncs; ++i) {
if (0 == strcmp(rollupFuncs[i], pFunc)) { if (0 == strcmp(rollupFuncs[i], pFunc)) {
@ -6297,7 +6345,6 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions
return code; return code;
} }
static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) {
if (NULL != strchr(pStmt->tableName, '.')) { if (NULL != strchr(pStmt->tableName, '.')) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME,
@ -7409,8 +7456,8 @@ static int16_t getCreateComponentNodeMsgType(ENodeType type) {
static int32_t translateCreateComponentNode(STranslateContext* pCxt, SCreateComponentNodeStmt* pStmt) { static int32_t translateCreateComponentNode(STranslateContext* pCxt, SCreateComponentNodeStmt* pStmt) {
SMCreateQnodeReq createReq = {.dnodeId = pStmt->dnodeId}; SMCreateQnodeReq createReq = {.dnodeId = pStmt->dnodeId};
int32_t code = buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)), int32_t code = buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)),
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq); (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq);
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return code; return code;
} }
@ -7433,8 +7480,8 @@ static int16_t getDropComponentNodeMsgType(ENodeType type) {
static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponentNodeStmt* pStmt) { static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponentNodeStmt* pStmt) {
SDDropQnodeReq dropReq = {.dnodeId = pStmt->dnodeId}; SDDropQnodeReq dropReq = {.dnodeId = pStmt->dnodeId};
int32_t code = buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)), int32_t code = buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)),
(FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq); (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq);
tFreeSDDropQnodeReq(&dropReq); tFreeSDDropQnodeReq(&dropReq);
return code; return code;
} }
@ -7559,7 +7606,8 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt
.mgmtEps = pParCxt->mgmtEpSet}; .mgmtEps = pParCxt->mgmtEpSet};
SName name; SName name;
STableMeta* pMeta = NULL; STableMeta* pMeta = NULL;
int32_t code = getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false); int32_t code =
getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false);
if (code) { if (code) {
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
return code; return code;
@ -7667,7 +7715,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt)
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
int32_t origCode = code; int32_t origCode = code;
SName name; SName name;
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name);
SViewMeta* pMeta = NULL; SViewMeta* pMeta = NULL;
code = getViewMetaFromMetaCache(pCxt, &name, &pMeta); code = getViewMetaFromMetaCache(pCxt, &name, &pMeta);
@ -7675,9 +7723,10 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt)
code = origCode; code = origCode;
} else { } else {
SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA};
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&name, dbFName); tNameGetFullDbName(&name, dbFName);
code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false, pMeta->user, &res); code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false,
pMeta->user, &res);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(&name, pCxt->pTargetTables); code = collectUseTable(&name, pCxt->pTargetTables);
} }
@ -7690,7 +7739,8 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt)
viewMeta.numOfCols = res.schemaRes.numOfCols; viewMeta.numOfCols = res.schemaRes.numOfCols;
viewMeta.pSchema = res.schemaRes.pSchema; viewMeta.pSchema = res.schemaRes.pSchema;
code = buildTableMetaFromViewMeta(&pStmt->pMeta, &viewMeta); code = buildTableMetaFromViewMeta(&pStmt->pMeta, &viewMeta);
parserDebug("rebuild view meta, view:%s.%s, numOfCols:%d, code:0x%x", dbFName, pStmt->tableName, viewMeta.numOfCols, code); parserDebug("rebuild view meta, view:%s.%s, numOfCols:%d, code:0x%x", dbFName, pStmt->tableName,
viewMeta.numOfCols, code);
} }
taosMemoryFree(res.schemaRes.pSchema); taosMemoryFree(res.schemaRes.pSchema);
} }
@ -7780,12 +7830,12 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pStmt->pQuery)->pFromTable; SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pStmt->pQuery)->pFromTable;
SName name; SName name;
STableMeta* pMeta = NULL; STableMeta* pMeta = NULL;
int8_t tableType = 0; int8_t tableType = 0;
int32_t code = getTargetMeta( int32_t code =
pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name),
&pMeta, true); &pMeta, true);
if (NULL != pMeta) { if (NULL != pMeta) {
tableType = pMeta->tableType; tableType = pMeta->tableType;
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
@ -8107,17 +8157,17 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
&& TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType
&& !hasTbnameFunction(pSelect->pPartitionByList) ) { && !hasTbnameFunction(pSelect->pPartitionByList) ) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Count window for stream on super table must patitioned by table name"); "Count window for stream on super table must patitioned by table name");
} }
int64_t watermark = 0; int64_t watermark = 0;
if (pStmt->pOptions->pWatermark) { if (pStmt->pOptions->pWatermark) {
translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark); translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark);
watermark =((SValueNode*)pStmt->pOptions->pWatermark)->datum.i; watermark = ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i;
} }
if (watermark <= 0) { if (watermark <= 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"Watermark of Count window must exceed 0."); "Watermark of Count window must exceed 0.");
} }
if (pStmt->pOptions->ignoreExpired != 1) { if (pStmt->pOptions->ignoreExpired != 1) {
@ -8550,7 +8600,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta
} }
snprintf(pFunc1->node.aliasName, sizeof(pFunc1->node.aliasName), "%s.%p", pFunc1->functionName, pFunc1); snprintf(pFunc1->node.aliasName, sizeof(pFunc1->node.aliasName), "%s.%p", pFunc1->functionName, pFunc1);
code = nodesListStrictAppend(pProjectionList, (SNode*) pFunc1); code = nodesListStrictAppend(pProjectionList, (SNode*)pFunc1);
if (code) { if (code) {
nodesDestroyList(pProjectionList); nodesDestroyList(pProjectionList);
return code; return code;
@ -8563,7 +8613,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta
} }
snprintf(pFunc2->node.aliasName, sizeof(pFunc2->node.aliasName), "%s.%p", pFunc2->functionName, pFunc2); snprintf(pFunc2->node.aliasName, sizeof(pFunc2->node.aliasName), "%s.%p", pFunc2->functionName, pFunc2);
code = nodesListStrictAppend(pProjectionList, (SNode*) pFunc2); code = nodesListStrictAppend(pProjectionList, (SNode*)pFunc2);
if (code) { if (code) {
nodesDestroyList(pProjectionList); nodesDestroyList(pProjectionList);
return code; return code;
@ -8829,7 +8879,7 @@ static int32_t createStreamReqVersionInfo(SSDataBlock* pBlock, SArray** pArray,
taosArrayPush(*pArray, &v); taosArrayPush(*pArray, &v);
} }
} else { } else {
int32_t precision = (pInterval->interval > 0)? pInterval->precision:TSDB_TIME_PRECISION_MILLI; int32_t precision = (pInterval->interval > 0) ? pInterval->precision : TSDB_TIME_PRECISION_MILLI;
*lastTs = taosGetTimestamp(precision); *lastTs = taosGetTimestamp(precision);
} }
@ -8853,7 +8903,8 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDa
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (interval.interval > 0) { if (interval.interval > 0) {
pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, interval.precision); pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit,
interval.precision);
} else { } else {
pStmt->pReq->lastTs = lastTs + 1; // start key of the next time window pStmt->pReq->lastTs = lastTs + 1; // start key of the next time window
} }
@ -8908,26 +8959,27 @@ static int32_t validateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStm
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type"); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type");
} }
/* /*
STableMeta* pMetaCache = NULL; STableMeta* pMetaCache = NULL;
int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->viewName, &pMetaCache); int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->viewName, &pMetaCache);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
taosMemoryFreeClear(pMetaCache); taosMemoryFreeClear(pMetaCache);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with table"); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with
} table");
*/ }
*/
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStmt) { static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStmt) {
#ifndef TD_ENTERPRISE #ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT; return TSDB_CODE_OPS_NOT_SUPPORT;
#endif #endif
SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA};
SName name; SName name;
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name);
tNameGetFullDbName(&name, dbFName); tNameGetFullDbName(&name, dbFName);
@ -8944,7 +8996,8 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt
pStmt->createReq.pSchema = res.schemaRes.pSchema; pStmt->createReq.pSchema = res.schemaRes.pSchema;
strncpy(pStmt->createReq.name, pStmt->viewName, sizeof(pStmt->createReq.name) - 1); strncpy(pStmt->createReq.name, pStmt->viewName, sizeof(pStmt->createReq.name) - 1);
tstrncpy(pStmt->createReq.dbFName, dbFName, sizeof(pStmt->createReq.dbFName)); tstrncpy(pStmt->createReq.dbFName, dbFName, sizeof(pStmt->createReq.dbFName));
snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName, pStmt->viewName); snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName,
pStmt->viewName);
TSWAP(pStmt->createReq.querySql, pStmt->pQuerySql); TSWAP(pStmt->createReq.querySql, pStmt->pQuerySql);
pStmt->createReq.orReplace = pStmt->orReplace; pStmt->createReq.orReplace = pStmt->orReplace;
pStmt->createReq.sql = tstrdup(pCxt->pParseCxt->pSql); pStmt->createReq.sql = tstrdup(pCxt->pParseCxt->pSql);
@ -8960,14 +9013,13 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt
return code; return code;
} }
static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) { static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) {
#ifndef TD_ENTERPRISE #ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT; return TSDB_CODE_OPS_NOT_SUPPORT;
#endif #endif
SCMDropViewReq dropReq = {0}; SCMDropViewReq dropReq = {0};
SName name; SName name;
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, dropReq.dbFName); tNameGetFullDbName(&name, dropReq.dbFName);
strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1); strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1);
@ -8987,7 +9039,6 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt)
return buildCmdMsg(pCxt, TDMT_MND_DROP_VIEW, (FSerializeFunc)tSerializeSCMDropViewReq, &dropReq); return buildCmdMsg(pCxt, TDMT_MND_DROP_VIEW, (FSerializeFunc)tSerializeSCMDropViewReq, &dropReq);
} }
static int32_t readFromFile(char* pName, int32_t* len, char** buf) { static int32_t readFromFile(char* pName, int32_t* len, char** buf) {
int64_t filesize = 0; int64_t filesize = 0;
if (taosStatFile(pName, &filesize, NULL, NULL) < 0) { if (taosStatFile(pName, &filesize, NULL, NULL) < 0) {
@ -9088,7 +9139,7 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt,
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SName name; SName name;
code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name),
&(pTable->pMeta), false); &(pTable->pMeta), false);
if (code) { if (code) {
nodesDestroyNode((SNode*)pTable); nodesDestroyNode((SNode*)pTable);
return code; return code;
@ -9122,15 +9173,16 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt,
} }
static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
int32_t code = 0; int32_t code = 0;
SAlterUserReq req = {0}; SAlterUserReq req = {0};
req.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES; req.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
req.privileges = pStmt->privileges; req.privileges = pStmt->privileges;
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
if (0 != pStmt->tabName[0]) { if (0 != pStmt->tabName[0]) {
SName name; SName name;
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); code =
getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
@ -9156,16 +9208,17 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
} }
static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
int32_t code = 0; int32_t code = 0;
SAlterUserReq req = {0}; SAlterUserReq req = {0};
req.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES; req.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
req.privileges = pStmt->privileges; req.privileges = pStmt->privileges;
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
if (0 != pStmt->tabName[0]) { if (0 != pStmt->tabName[0]) {
SName name; SName name;
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); code =
getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
@ -9195,7 +9248,8 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) { static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
SBalanceVgroupLeaderReq req = {0}; SBalanceVgroupLeaderReq req = {0};
req.vgId = pStmt->vgId; req.vgId = pStmt->vgId;
int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req); int32_t code =
buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
tFreeSBalanceVgroupLeaderReq(&req); tFreeSBalanceVgroupLeaderReq(&req);
return code; return code;
} }
@ -9319,6 +9373,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_TRIM_DATABASE_STMT: case QUERY_NODE_TRIM_DATABASE_STMT:
code = translateTrimDatabase(pCxt, (STrimDatabaseStmt*)pNode); code = translateTrimDatabase(pCxt, (STrimDatabaseStmt*)pNode);
break; break;
case QUERY_NODE_S3MIGRATE_DATABASE_STMT:
code = translateS3MigrateDatabase(pCxt, (SS3MigrateDatabaseStmt*)pNode);
break;
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
code = translateCreateSuperTable(pCxt, (SCreateTableStmt*)pNode); code = translateCreateSuperTable(pCxt, (SCreateTableStmt*)pNode);
break; break;
@ -9628,7 +9685,6 @@ static int32_t extractShowCreateViewResultSchema(int32_t* numOfCols, SSchema** p
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) {
*numOfCols = 3; *numOfCols = 3;
*pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema));
@ -9819,17 +9875,16 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN
static const char* getTbNameColName(ENodeType type) { static const char* getTbNameColName(ENodeType type) {
const char* colName; const char* colName;
switch (type) switch (type) {
{ case QUERY_NODE_SHOW_VIEWS_STMT:
case QUERY_NODE_SHOW_VIEWS_STMT: colName = "view_name";
colName = "view_name"; break;
break; case QUERY_NODE_SHOW_STABLES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT: colName = "stable_name";
colName = "stable_name"; break;
break; default:
default: colName = "table_name";
colName = "table_name"; break;
break;
} }
return colName; return colName;
} }
@ -9863,14 +9918,14 @@ static int32_t insertCondIntoSelectStmt(SSelectStmt* pSelect, SNode* pCond) {
SNodeList* pLogicCondList2 = NULL; SNodeList* pLogicCondList2 = NULL;
if (nodeType(pSelect->pWhere) == QUERY_NODE_LOGIC_CONDITION && if (nodeType(pSelect->pWhere) == QUERY_NODE_LOGIC_CONDITION &&
((SLogicConditionNode*)pSelect->pWhere)->condType == LOGIC_COND_TYPE_AND) { ((SLogicConditionNode*)pSelect->pWhere)->condType == LOGIC_COND_TYPE_AND) {
pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList; pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList;
} else { } else {
nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere); nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere);
} }
if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION && if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION &&
((SLogicConditionNode*)pCond)->condType == LOGIC_COND_TYPE_AND) { ((SLogicConditionNode*)pCond)->condType == LOGIC_COND_TYPE_AND) {
pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList; pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList;
} else { } else {
nodesListMakeAppend(&pLogicCondList2, pCond); nodesListMakeAppend(&pLogicCondList2, pCond);
} }
@ -11169,8 +11224,8 @@ static int32_t rewriteFlushDatabase(STranslateContext* pCxt, SQuery* pQuery) {
static int32_t rewriteShowCompacts(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteShowCompacts(STranslateContext* pCxt, SQuery* pQuery) {
SShowCompactsStmt* pShow = (SShowCompactsStmt*)(pQuery->pRoot); SShowCompactsStmt* pShow = (SShowCompactsStmt*)(pQuery->pRoot);
SSelectStmt* pStmt = NULL; SSelectStmt* pStmt = NULL;
int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACTS_STMT, &pStmt); int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACTS_STMT, &pStmt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pCxt->showRewrite = true; pCxt->showRewrite = true;
pQuery->showRewrite = true; pQuery->showRewrite = true;
@ -11182,8 +11237,8 @@ static int32_t rewriteShowCompacts(STranslateContext* pCxt, SQuery* pQuery) {
static int32_t rewriteShowCompactDetailsStmt(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteShowCompactDetailsStmt(STranslateContext* pCxt, SQuery* pQuery) {
SShowCompactDetailsStmt* pShow = (SShowCompactDetailsStmt*)(pQuery->pRoot); SShowCompactDetailsStmt* pShow = (SShowCompactDetailsStmt*)(pQuery->pRoot);
SSelectStmt* pStmt = NULL; SSelectStmt* pStmt = NULL;
int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACT_DETAILS_STMT, &pStmt); int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACT_DETAILS_STMT, &pStmt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (NULL != pShow->pCompactId) { if (NULL != pShow->pCompactId) {
code = createOperatorNode(OP_TYPE_EQUAL, "compact_id", pShow->pCompactId, &pStmt->pWhere); code = createOperatorNode(OP_TYPE_EQUAL, "compact_id", pShow->pCompactId, &pStmt->pWhere);

9676
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

110
source/util/src/tconfig.c
View File

@ -21,8 +21,8 @@
#include "tgrant.h" #include "tgrant.h"
#include "tjson.h" #include "tjson.h"
#include "tlog.h" #include "tlog.h"
#include "tutil.h"
#include "tunit.h" #include "tunit.h"
#include "tutil.h"
#define CFG_NAME_PRINT_LEN 24 #define CFG_NAME_PRINT_LEN 24
#define CFG_SRC_PRINT_LEN 12 #define CFG_SRC_PRINT_LEN 12
@ -315,19 +315,19 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value,
static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) { static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) {
SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag"); SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag");
if (resetArray) { if (resetArray) {
// reset // reset
if (pDebugFlagItem == NULL) return -1; if (pDebugFlagItem == NULL) return -1;
// logflag names that should 'not' be set by 'debugFlag' // logflag names that should 'not' be set by 'debugFlag'
if (pDebugFlagItem->array == NULL) {
pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar));
if (pDebugFlagItem->array == NULL) { if (pDebugFlagItem->array == NULL) {
pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar)); terrno = TSDB_CODE_OUT_OF_MEMORY;
if (pDebugFlagItem->array == NULL) { return -1;
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} }
taosArrayClear(pDebugFlagItem->array); }
return 0; taosArrayClear(pDebugFlagItem->array);
return 0;
} }
// update // update
@ -406,8 +406,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
case CFG_DTYPE_BOOL: { case CFG_DTYPE_BOOL: {
int32_t ival = (int32_t)atoi(pVal); int32_t ival = (int32_t)atoi(pVal);
if (ival != 0 && ival != 1) { if (ival != 0 && ival != 1) {
uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival);
cfgDtypeStr(pItem->dtype), ival);
terrno = TSDB_CODE_OUT_OF_RANGE; terrno = TSDB_CODE_OUT_OF_RANGE;
return -1; return -1;
} }
@ -691,6 +690,89 @@ void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p
*pLen = len; *pLen = len;
} }
void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) {
if (dump) {
printf(" s3 config");
printf("\n");
printf("=================================================================");
printf("\n");
} else {
uInfo(" s3 config");
uInfo("=================================================================");
}
char src[CFG_SRC_PRINT_LEN + 1] = {0};
char name[CFG_NAME_PRINT_LEN + 1] = {0};
int32_t size = taosArrayGetSize(pCfg->array);
for (int32_t i = 0; i < size; ++i) {
SConfigItem *pItem = taosArrayGet(pCfg->array, i);
if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue;
if (dump && strcmp(pItem->name, "scriptDir") == 0) continue;
if (dump && strncmp(pItem->name, "s3", 2) != 0) continue;
tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN);
for (int32_t j = 0; j < CFG_SRC_PRINT_LEN; ++j) {
if (src[j] == 0) src[j] = ' ';
}
tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN);
for (int32_t j = 0; j < CFG_NAME_PRINT_LEN; ++j) {
if (name[j] == 0) name[j] = ' ';
}
switch (pItem->dtype) {
case CFG_DTYPE_BOOL:
if (dump) {
printf("%s %s %u\n", src, name, pItem->bval);
} else {
uInfo("%s %s %u", src, name, pItem->bval);
}
break;
case CFG_DTYPE_INT32:
if (dump) {
printf("%s %s %d\n", src, name, pItem->i32);
} else {
uInfo("%s %s %d", src, name, pItem->i32);
}
break;
case CFG_DTYPE_INT64:
if (dump) {
printf("%s %s %" PRId64 "\n", src, name, pItem->i64);
} else {
uInfo("%s %s %" PRId64, src, name, pItem->i64);
}
break;
case CFG_DTYPE_DOUBLE:
case CFG_DTYPE_FLOAT:
if (dump) {
printf("%s %s %.2f\n", src, name, pItem->fval);
} else {
uInfo("%s %s %.2f", src, name, pItem->fval);
}
break;
case CFG_DTYPE_STRING:
case CFG_DTYPE_DIR:
case CFG_DTYPE_LOCALE:
case CFG_DTYPE_CHARSET:
case CFG_DTYPE_TIMEZONE:
case CFG_DTYPE_NONE:
if (dump) {
printf("%s %s %s\n", src, name, pItem->str);
} else {
uInfo("%s %s %s", src, name, pItem->str);
}
break;
}
}
if (dump) {
printf("=================================================================\n");
} else {
uInfo("=================================================================");
}
}
void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
if (dump) { if (dump) {
printf(" global config"); printf(" global config");
@ -738,7 +820,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
break; break;
case CFG_DTYPE_INT64: case CFG_DTYPE_INT64:
if (dump) { if (dump) {
printf("%s %s %" PRId64"\n", src, name, pItem->i64); printf("%s %s %" PRId64 "\n", src, name, pItem->i64);
} else { } else {
uInfo("%s %s %" PRId64, src, name, pItem->i64); uInfo("%s %s %" PRId64, src, name, pItem->i64);
} }

13
tests/develop-test/2-query/show_create_db.py
View File

@ -42,17 +42,17 @@ class TDTestCase:
tdSql.query('show create database scd;') tdSql.query('show create database scd;')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd') tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none'") tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd2;') tdSql.query('show create database scd2;')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2') tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none'") tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd4') tdSql.query('show create database scd4')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4') tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none'") tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
self.restartTaosd(1, dbname='scd') self.restartTaosd(1, dbname='scd')
@ -60,17 +60,16 @@ class TDTestCase:
tdSql.query('show create database scd;') tdSql.query('show create database scd;')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd') tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none'") tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd2;') tdSql.query('show create database scd2;')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2') tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none'") tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd4') tdSql.query('show create database scd4')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4') tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none'") tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.execute('drop database scd') tdSql.execute('drop database scd')

40
tests/script/local.supp
View File

@ -1655,3 +1655,43 @@
fun:start_thread fun:start_thread
fun:clone fun:clone
} }
{
<insert_a_suppression_name_here>
Memcheck:Param
write(buf)
fun:__libc_write
fun:write
fun:uv__try_write
fun:uv__write
fun:uv_write2
fun:uvStartSendRespImpl
fun:uvStartSendResp
fun:uvHandleResp
fun:uvPrepareCb
fun:uv__run_prepare
fun:uv_run
fun:transWorkerThread
fun:start_thread
fun:clone
}
{
<memalign suppression>
Memcheck:Param
write(buf)
fun:__libc_write
fun:write
fun:uv__try_write
fun:uv__write
fun:uv_write2
fun:uvStartSendRespImpl
fun:uvStartSendResp
fun:uvHandleResp
fun:uvWorkerAsyncCb
fun:uv__async_io
fun:uv__io_poll
fun:uv_run
fun:transWorkerThread
fun:start_thread
fun:clone
}

2
tests/script/tsim/db/alter_option.sim
View File

@ -46,7 +46,7 @@ print ============= create database
# | REPLICA value [1 | 3] # | REPLICA value [1 | 3]
# | WAL_LEVEL value [1 | 2] # | WAL_LEVEL value [1 | 2]
sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 WAL_FSYNC_PERIOD 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL_LEVEL 2 VGROUPS 6 SINGLE_STABLE 1 sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 WAL_FSYNC_PERIOD 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 S3_KEEPLOCAL 720 PRECISION 'ns' REPLICA 3 WAL_LEVEL 2 VGROUPS 6 SINGLE_STABLE 1
sql select * from information_schema.ins_databases sql select * from information_schema.ins_databases
print rows: $rows print rows: $rows
print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09

2
tests/system-test/2-query/distribute_agg_apercentile.py
View File

@ -18,7 +18,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 s3_keeplocal 3000 vgroups 5")
tdSql.execute(f" use {dbname} ") tdSql.execute(f" use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

5
tools/shell/src/shellAuto.c
View File

@ -214,6 +214,7 @@ SWords shellCommands[] = {
{"insert into <tb_name> using <stb_name> <anyword> values(", 0, 0, NULL}, {"insert into <tb_name> using <stb_name> <anyword> values(", 0, 0, NULL},
{"insert into <tb_name> file ", 0, 0, NULL}, {"insert into <tb_name> file ", 0, 0, NULL},
{"trim database <db_name>", 0, 0, NULL}, {"trim database <db_name>", 0, 0, NULL},
{"s3migrate database <db_name>", 0, 0, NULL},
{"use <db_name>", 0, 0, NULL}, {"use <db_name>", 0, 0, NULL},
{"quit", 0, 0, NULL}}; {"quit", 0, 0, NULL}};
@ -283,6 +284,9 @@ char* db_options[] = {"keep ",
"wal_level ", "wal_level ",
"vgroups ", "vgroups ",
"single_stable ", "single_stable ",
"s3_chunksize ",
"s3_keeplocal ",
"s3_compact ",
"wal_retention_period ", "wal_retention_period ",
"wal_roll_period ", "wal_roll_period ",
"wal_retention_size ", "wal_retention_size ",
@ -290,6 +294,7 @@ char* db_options[] = {"keep ",
char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "stt_trigger ", char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "stt_trigger ",
"wal_retention_period ", "wal_retention_size ", "cachesize ", "wal_retention_period ", "wal_retention_size ", "cachesize ",
"s3_keeplocal ", "s3_compact ",
"wal_fsync_period ", "buffer ", "pages " ,"wal_level "}; "wal_fsync_period ", "buffer ", "pages " ,"wal_level "};
char* data_types[] = {"timestamp", "int", char* data_types[] = {"timestamp", "int",