diff --git a/docs/en/14-reference/03-taos-sql/22-meta.md b/docs/en/14-reference/03-taos-sql/22-meta.md index 0d4dd8fe2e..3a17353b02 100644 --- a/docs/en/14-reference/03-taos-sql/22-meta.md +++ b/docs/en/14-reference/03-taos-sql/22-meta.md @@ -342,3 +342,18 @@ Note: Users with SYSINFO property set to 0 cannot view this table. | 10 | raw_data | BIGINT | Estimated size of raw data, in KB | note: + +## INS_FILESETS + +Provides information about file sets. + +| # | **Column** | **Data Type** | **Description** | ** | +| --- | :-----------: | ------------- | ---------------------------------------------------- | +| 1 | db_name | VARCHAR(65) | Database name | +| 2 | vgroup_id | INT | Vgroup ID | +| 3 | fileset_id | INT | File set ID | +| 4 | start_time | TIMESTAMP | Start time of the time range covered by the file set | +| 5 | end_time | TIMESTAMP | End time of the time range covered by the file set | +| 6 | total_size | BIGINT | Total size of the file set | +| 7 | last_compact | TIMESTAMP | Time of the last compaction | +| 8 | shold_compact | bool | Whether the file set should be compacted | diff --git a/docs/zh/08-operation/04-maintenance.md b/docs/zh/08-operation/04-maintenance.md index 88122fed69..9ef165179d 100644 --- a/docs/zh/08-operation/04-maintenance.md +++ b/docs/zh/08-operation/04-maintenance.md @@ -18,6 +18,7 @@ TDengine 面向多种写入场景,而很多写入场景下,TDengine 的存 ```SQL COMPACT DATABASE db_name [start with 'XXXX'] [end with 'YYYY']; +COMPACT [db_name.]VGROUPS IN (vgroup_id1, vgroup_id2, ...) [start with 'XXXX'] [end with 'YYYY']; SHOW COMPACTS [compact_id]; KILL COMPACT compact_id; ``` @@ -25,6 +26,7 @@ KILL COMPACT compact_id; ### 效果 - 扫描并压缩指定的 DB 中所有 VGROUP 中 VNODE 的所有数据文件 +- 扫描并压缩 DB 中指定的 VGROUP 列表中 VNODE 的所有数据文件, 若 db_name 为空,则默认为当前数据库 - COMPCAT 会删除被删除数据以及被删除的表的数据 - COMPACT 会合并多个 STT 文件 - 可通过 start with 关键字指定 COMPACT 数据的起始时间 diff --git a/docs/zh/14-reference/03-taos-sql/22-meta.md b/docs/zh/14-reference/03-taos-sql/22-meta.md index 5708d814a7..b269453ed2 100644 --- a/docs/zh/14-reference/03-taos-sql/22-meta.md +++ b/docs/zh/14-reference/03-taos-sql/22-meta.md @@ -343,3 +343,17 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库,提供对数 | 10 | raw_data | BIGINT | 预估的原始数据的大小, 单位为KB | +## INS_FILESETS + +提供当前数据存储的文件组的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | --------------------------------------- | +| 1 | db_name | VARCHAR(65) | 数据库名 | +| 2 | vgroup_id | INT | vgroup id | +| 3 | fileset_id | INT | 文件组 id | +| 4 | start_time | TIMESTAMP | 文件组的覆盖数据的开始时间 | +| 5 | end_time | TIMESTAMP | 文件组的覆盖数据的结束时间 | +| 6 | total_size | BIGINT | 文件组的总大小 | +| 7 | last_compact | TIMESTAMP | 最后一次压缩的时间 | +| 8 | shold_compact | bool | 是否需要压缩,true:需要,false:不需要 | diff --git a/include/common/streamMsg.h b/include/common/streamMsg.h index 3db92ba58d..d410bd17e0 100644 --- a/include/common/streamMsg.h +++ b/include/common/streamMsg.h @@ -188,6 +188,7 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg); typedef struct { SMsgHead head; int32_t msgId; + SEpSet mndEpset; } SMStreamHbRspMsg; int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp); diff --git a/include/common/systable.h b/include/common/systable.h index 71d29dd318..bd8ba76f4f 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -62,6 +62,7 @@ extern "C" { #define TSDB_INS_TABLE_ENCRYPTIONS "ins_encryptions" #define TSDB_INS_TABLE_TSMAS "ins_tsmas" #define TSDB_INS_DISK_USAGE "ins_disk_usage" +#define TSDB_INS_TABLE_FILESETS "ins_filesets" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFS_TABLE_SMAS "perf_smas" diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 638e5e0278..90465f611e 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -112,6 +112,9 @@ extern int32_t tsNumOfSnodeWriteThreads; extern int64_t tsQueueMemoryAllowed; extern int32_t tsRetentionSpeedLimitMB; +extern const char *tsAlterCompactTaskKeywords; +extern int32_t tsNumOfCompactThreads; + // sync raft extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b26d98f4b4..0b6a8b3f1b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -162,6 +162,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_ANODE, TSDB_MGMT_TABLE_ANODE_FULL, TSDB_MGMT_TABLE_USAGE, + TSDB_MGMT_TABLE_FILESETS, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -310,6 +311,7 @@ typedef enum ENodeType { QUERY_NODE_DESCRIBE_STMT, QUERY_NODE_RESET_QUERY_CACHE_STMT, QUERY_NODE_COMPACT_DATABASE_STMT, + QUERY_NODE_COMPACT_VGROUPS_STMT, QUERY_NODE_CREATE_FUNCTION_STMT, QUERY_NODE_DROP_FUNCTION_STMT, QUERY_NODE_CREATE_STREAM_STMT, @@ -402,6 +404,7 @@ typedef enum ENodeType { QUERY_NODE_CREATE_TSMA_STMT, QUERY_NODE_SHOW_CREATE_TSMA_STMT, QUERY_NODE_DROP_TSMA_STMT, + QUERY_NODE_SHOW_FILESETS_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, @@ -1347,6 +1350,11 @@ typedef struct { int8_t withArbitrator; int8_t encryptAlgorithm; char dnodeListStr[TSDB_DNODE_LIST_LEN]; + // 1. add auto-compact parameters + int32_t compactInterval; // minutes + int32_t compactStartTime; // minutes + int32_t compactEndTime; // minutes + int8_t compactTimeOffset; // hour } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); @@ -1378,6 +1386,11 @@ typedef struct { int32_t sqlLen; char* sql; int8_t withArbitrator; + // 1. add auto-compact parameters + int32_t compactInterval; + int32_t compactStartTime; + int32_t compactEndTime; + int8_t compactTimeOffset; } SAlterDbReq; int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); @@ -1510,6 +1523,10 @@ typedef struct { int32_t s3ChunkSize; int32_t s3KeepLocal; int8_t s3Compact; + int8_t compactTimeOffset; + int32_t compactInterval; + int32_t compactStartTime; + int32_t compactEndTime; int32_t tsdbPageSize; int32_t walRetentionPeriod; int32_t walRollPeriod; @@ -1617,6 +1634,7 @@ typedef struct { STimeWindow timeRange; int32_t sqlLen; char* sql; + SArray* vgroupIds; } SCompactDbReq; int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); @@ -2024,6 +2042,8 @@ typedef struct { int32_t dnodeId; int32_t numberFileset; int32_t finished; + int32_t progress; + int64_t remainingTime; } SQueryCompactProgressRsp; int32_t tSerializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq); diff --git a/include/libs/crypt/crypt.h b/include/libs/crypt/crypt.h index c294877a57..5f981b7ac8 100644 --- a/include/libs/crypt/crypt.h +++ b/include/libs/crypt/crypt.h @@ -21,19 +21,19 @@ extern "C" { #endif -typedef struct SCryptOpts{ - int32_t len; - char* source; - char* result; - int32_t unitLen; - unsigned char key[17]; -}SCryptOpts; +typedef struct SCryptOpts { + int32_t len; + char* source; + char* result; + int32_t unitLen; + char key[17]; +} SCryptOpts; -int32_t CBC_Decrypt(SCryptOpts *opts); -int32_t CBC_Encrypt(SCryptOpts *opts); +int32_t CBC_Decrypt(SCryptOpts* opts); +int32_t CBC_Encrypt(SCryptOpts* opts); #ifdef __cplusplus } #endif -#endif // _CRYPT_H_ \ No newline at end of file +#endif // _CRYPT_H_ \ No newline at end of file diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 52e740b3df..6721d72996 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -171,6 +171,8 @@ typedef union { typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param); +struct SFileSetReader; + typedef struct TsdReader { int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables); @@ -191,6 +193,13 @@ typedef struct TsdReader { void (*tsdSetFilesetDelimited)(void* pReader); void (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param); + + // for fileset query + int32_t (*fileSetReaderOpen)(void *pVnode, struct SFileSetReader **ppReader); + int32_t (*fileSetReadNext)(struct SFileSetReader *); + int32_t (*fileSetGetEntryField)(struct SFileSetReader *, const char *, void *); + void (*fileSetReaderClose)(struct SFileSetReader **); + } TsdReader; typedef struct SStoreCacheReader { @@ -431,7 +440,7 @@ typedef struct SStateStore { int32_t (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark, const char* id, int64_t ckId, int8_t type, struct SStreamFileState** ppFileState); - + int32_t (*streamStateGroupPut)(SStreamState* pState, int64_t groupId, void* value, int32_t vLen); SStreamStateCur* (*streamStateGroupGetCur)(SStreamState* pState); void (*streamStateGroupCurNext)(SStreamStateCur* pCur); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 8de4d16522..8eb30b8184 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -111,6 +111,16 @@ typedef struct SDatabaseOptions { SValueNode* s3KeepLocalStr; int8_t s3Compact; int8_t withArbitrator; + // for auto-compact + int8_t compactTimeOffset; // hours + int32_t compactInterval; // minutes + int32_t compactStartTime; // minutes + int32_t compactEndTime; // minutes + SValueNode* pCompactTimeOffsetNode; + SValueNode* pCompactIntervalNode; + SNodeList* pCompactTimeRangeList; + // for cache + SDbCfgInfo* pDbCfg; } SDatabaseOptions; typedef struct SCreateDatabaseStmt { @@ -160,6 +170,14 @@ typedef struct SCompactDatabaseStmt { SNode* pEnd; } SCompactDatabaseStmt; +typedef struct SCompactVgroupsStmt { + ENodeType type; + SNode* pDbName; + SNodeList* vgidList; + SNode* pStart; + SNode* pEnd; +} SCompactVgroupsStmt; + typedef struct STableOptions { ENodeType type; bool commentNull; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 74460bc911..64ef0b3829 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -481,6 +481,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_DNODE_INVALID_EN_WHITELIST TAOS_DEF_ERROR_CODE(0, 0x0428) #define TSDB_CODE_DNODE_INVALID_MONITOR_PARAS TAOS_DEF_ERROR_CODE(0, 0x0429) #define TSDB_CODE_MNODE_STOPPED TAOS_DEF_ERROR_CODE(0, 0x042A) +#define TSDB_CODE_DNODE_INVALID_COMPACT_TASKS TAOS_DEF_ERROR_CODE(0, 0x042B) // anode #define TSDB_CODE_MND_ANODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0430) @@ -904,6 +905,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_COL TAOS_DEF_ERROR_CODE(0, 0x2683) #define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT TAOS_DEF_ERROR_CODE(0, 0x2684) #define TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x2685) +#define TSDB_CODE_PAR_INVALID_VGID_LIST TAOS_DEF_ERROR_CODE(0, 0x2686) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/include/util/tdef.h b/include/util/tdef.h index c23b073d42..2ee84b42bd 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -508,6 +508,15 @@ typedef enum ELogicConditionType { #define TSDB_MIN_TABLE_TTL 0 #define TSDB_DEFAULT_TABLE_TTL 0 +#define TSDB_DEFAULT_COMPACT_INTERVAL 0 +#define TSDB_MIN_COMPACT_INTERVAL 10 // unit minute +#define TSDB_MAX_COMPACT_INTERVAL TSDB_MAX_KEEP // unit minute +#define TSDB_DEFAULT_COMPACT_START_TIME 0 +#define TSDB_DEFAULT_COMPACT_END_TIME 0 +#define TSDB_MIN_COMPACT_TIME_OFFSET 0 +#define TSDB_MAX_COMPACT_TIME_OFFSET 23 +#define TSDB_DEFAULT_COMPACT_TIME_OFFSET 0 + #define TSDB_MIN_EXPLAIN_RATIO 0 #define TSDB_MAX_EXPLAIN_RATIO 1 #define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 diff --git a/source/common/src/msg/streamMsg.c b/source/common/src/msg/streamMsg.c index 76baf6be2c..54b17b14d1 100644 --- a/source/common/src/msg/streamMsg.c +++ b/source/common/src/msg/streamMsg.c @@ -651,6 +651,7 @@ int32_t tEncodeStreamHbRsp(SEncoder* pEncoder, const SMStreamHbRspMsg* pRsp) { TAOS_CHECK_EXIT(tStartEncode(pEncoder)); TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->msgId)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pRsp->mndEpset)); tEndEncode(pEncoder); _exit: @@ -663,6 +664,7 @@ int32_t tDecodeStreamHbRsp(SDecoder* pDecoder, SMStreamHbRspMsg* pRsp) { TAOS_CHECK_EXIT(tStartDecode(pDecoder)); TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->msgId)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pRsp->mndEpset)); tEndDecode(pDecoder); _exit: diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 918c2a7330..2193c7983f 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -4039,6 +4039,12 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->s3Compact)); TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->dnodeListStr)); + // auto-compact parameters + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactInterval)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactStartTime)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactEndTime)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->compactTimeOffset)); + tEndEncode(&encoder); _exit: @@ -4106,30 +4112,44 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->tsdbPageSize)); - pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->keepTimeOffset)); + } else { + pReq->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; } DECODESQL(); - pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; - pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; - pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; - pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; - pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->withArbitrator)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->encryptAlgorithm)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->s3ChunkSize)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->s3KeepLocal)); TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->s3Compact)); + } else { + pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; + pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; + pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; } if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->dnodeListStr)); } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactInterval)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactStartTime)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime)); + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset)); + } else { + pReq->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pReq->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pReq->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pReq->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; + } + tEndDecode(&decoder); _exit: @@ -4180,6 +4200,11 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { ENCODESQL(); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->withArbitrator)); + // auto compact config + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactInterval)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactStartTime)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->compactEndTime)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->compactTimeOffset)); tEndEncode(&encoder); _exit: @@ -4247,6 +4272,19 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->withArbitrator)); } + + // auto compact config + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactInterval)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactStartTime)); + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->compactEndTime)); + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->compactTimeOffset)); + } else { + pReq->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pReq->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pReq->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pReq->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; + } tEndDecode(&decoder); _exit: @@ -4623,6 +4661,17 @@ int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->timeRange.skey)); TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->timeRange.ekey)); ENCODESQL(); + + // encode vgroup list + int32_t numOfVgroups = taosArrayGetSize(pReq->vgroupIds); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, numOfVgroups)); + if (numOfVgroups > 0) { + for (int32_t i = 0; i < numOfVgroups; ++i) { + int64_t vgid = *(int64_t *)taosArrayGet(pReq->vgroupIds, i); + TAOS_CHECK_EXIT(tEncodeI64v(&encoder, vgid)); + } + } + tEndEncode(&encoder); _exit: @@ -4646,6 +4695,26 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->timeRange.skey)); TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->timeRange.ekey)); DECODESQL(); + + // decode vgroup list + if (!tDecodeIsEnd(&decoder)) { + int32_t numOfVgroups = 0; + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &numOfVgroups)); + if (numOfVgroups > 0) { + pReq->vgroupIds = taosArrayInit(numOfVgroups, sizeof(int64_t)); + if (NULL == pReq->vgroupIds) { + TAOS_CHECK_EXIT(terrno); + } + + for (int32_t i = 0; i < numOfVgroups; ++i) { + int64_t vgid; + TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &vgid)); + if (taosArrayPush(pReq->vgroupIds, &vgid) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + } + } tEndDecode(&decoder); _exit: @@ -4653,7 +4722,11 @@ _exit: return code; } -void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); } +void tFreeSCompactDbReq(SCompactDbReq *pReq) { + FREESQL(); + taosArrayDestroy(pReq->vgroupIds); + pReq->vgroupIds = NULL; +} int32_t tSerializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) { SEncoder encoder = {0}; @@ -5345,6 +5418,10 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) { TAOS_CHECK_RETURN(tEncodeI32(encoder, pRsp->s3KeepLocal)); TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->s3Compact)); TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->hashMethod)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactInterval)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactStartTime)); + TAOS_CHECK_RETURN(tEncodeI32v(encoder, pRsp->compactEndTime)); + TAOS_CHECK_RETURN(tEncodeI8(encoder, pRsp->compactTimeOffset)); return 0; } @@ -5420,27 +5497,40 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) { } TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->schemaless)); TAOS_CHECK_RETURN(tDecodeI16(decoder, &pRsp->sstTrigger)); - pRsp->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->keepTimeOffset)); + } else { + pRsp->keepTimeOffset = TSDB_DEFAULT_KEEP_TIME_OFFSET; } - pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; - pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; - pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; - pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; - pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->withArbitrator)); TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->encryptAlgorithm)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->s3ChunkSize)); TAOS_CHECK_RETURN(tDecodeI32(decoder, &pRsp->s3KeepLocal)); TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->s3Compact)); + } else { + pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; + pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; + pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT; } if (!tDecodeIsEnd(decoder)) { TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->hashMethod)); } else { pRsp->hashMethod = 1; // default value } + if (!tDecodeIsEnd(decoder)) { + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactInterval)); + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactStartTime)); + TAOS_CHECK_RETURN(tDecodeI32v(decoder, &pRsp->compactEndTime)); + TAOS_CHECK_RETURN(tDecodeI8(decoder, &pRsp->compactTimeOffset)); + } else { + pRsp->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pRsp->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pRsp->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pRsp->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; + } return 0; } @@ -6898,6 +6988,9 @@ int32_t tSerializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryComp TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->dnodeId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->numberFileset)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->finished)); + // 1. add progress and remaining time + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pReq->progress)); + TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pReq->remainingTime)); tEndEncode(&encoder); @@ -6922,6 +7015,14 @@ int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCo TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->dnodeId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->numberFileset)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->finished)); + // 1. decode progress and remaining time + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pReq->progress)); + TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pReq->remainingTime)); + } else { + pReq->progress = 0; + pReq->remainingTime = 0; + } tEndDecode(&decoder); _exit: @@ -10280,7 +10381,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { char name[TSDB_COL_NAME_LEN] = {0}; char *tmp = NULL; TAOS_CHECK_EXIT(tDecodeCStr(pCoder, &tmp)); - strncpy(name, tmp, TSDB_COL_NAME_LEN - 1); + tstrncpy(name, tmp, TSDB_COL_NAME_LEN); if (taosArrayPush(pReq->ctb.tagName, name) == NULL) { TAOS_CHECK_EXIT(terrno); } @@ -12316,7 +12417,7 @@ void setFieldWithOptions(SFieldWithOptions *fieldWithOptions, SField *field) { fieldWithOptions->bytes = field->bytes; fieldWithOptions->flags = field->flags; fieldWithOptions->type = field->type; - strncpy(fieldWithOptions->name, field->name, TSDB_COL_NAME_LEN); + tstrncpy(fieldWithOptions->name, field->name, TSDB_COL_NAME_LEN); } int32_t tSerializeTableTSMAInfoReq(void *buf, int32_t bufLen, const STableTSMAInfoReq *pReq) { SEncoder encoder = {0}; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index fdd29bfb69..cb08046399 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -399,6 +399,8 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { {.name = "number_fileset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "progress(%)", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "remain_time(s)", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, }; static const SSysDbTableSchema anodesSchema[] = { @@ -415,6 +417,18 @@ static const SSysDbTableSchema anodesFullSchema[] = { {.name = "algo", .bytes = TSDB_ANALYTIC_ALGO_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; +static const SSysDbTableSchema filesetsFullSchema[] = { + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "fileset_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "end_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "total_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, + {.name = "last_compact", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "shold_compact", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = false}, + // {.name = "details", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, +}; + static const SSysDbTableSchema tsmaSchema[] = { {.name = "tsma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, @@ -506,6 +520,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_ANODES, anodesSchema, tListLen(anodesSchema), true}, {TSDB_INS_TABLE_ANODES_FULL, anodesFullSchema, tListLen(anodesFullSchema), true}, {TSDB_INS_DISK_USAGE, diskUsageSchema, tListLen(diskUsageSchema), false}, + {TSDB_INS_TABLE_FILESETS, filesetsFullSchema, tListLen(filesetsFullSchema), false}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a38842735c..c1ab7ccff0 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1434,7 +1434,7 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { char tmpVal[32] = {0}; - strncpy(tmpVal, val, vlen > 31 ? 31 : vlen); + tstrncpy(tmpVal, val, vlen > 31 ? 31 : vlen); printf("%s:%d type:%d vlen:%d, val:\"%s\"\n", tag, ln, (int32_t)type, vlen, tmpVal); } break; case TSDB_DATA_TYPE_FLOAT: diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d339af7718..3cee1cd7de 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" #include "cJSON.h" #include "defines.h" #include "os.h" #include "osString.h" #include "tconfig.h" -#include "tglobal.h" #include "tgrant.h" #include "tjson.h" #include "tlog.h" @@ -102,6 +102,9 @@ int32_t tsMaxStreamBackendCache = 128; // M int32_t tsPQSortMemThreshold = 16; // M int32_t tsRetentionSpeedLimitMB = 0; // unlimited +const char *tsAlterCompactTaskKeywords = "max_compact_tasks"; +int32_t tsNumOfCompactThreads = 2; + // sync raft int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; @@ -619,7 +622,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; int32_t defaultServerPort = 6030; if (taosGetFqdn(defaultFqdn) != 0) { - (void)strcpy(defaultFqdn, "localhost"); + tstrncpy(defaultFqdn, "localhost", TSDB_FQDN_LEN); } TAOS_CHECK_RETURN( @@ -772,6 +775,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); + tsNumOfCompactThreads = tsNumOfCommitThreads; + tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4); + tsNumOfSupportVnodes = tsNumOfCores * 2 + 5; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); @@ -816,6 +822,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxCompactConcurrency", tsNumOfCompactThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryUseMemoryPool", tsQueryUseMemoryPool, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "memPoolFullFunc", tsMemPoolFullFunc, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); @@ -1013,6 +1020,14 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } + pItem = cfgGetItem(pCfg, "maxCompactConcurrency"); + if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { + tsNumOfCompactThreads = numOfCores / 2; + tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4); + pItem->i32 = tsNumOfCompactThreads; + pItem->stype = stype; + } + pItem = cfgGetItem(pCfg, "numOfMnodeReadThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { tsNumOfMnodeReadThreads = numOfCores / 8; @@ -1501,6 +1516,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCommitThreads"); tsNumOfCommitThreads = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "maxCompactConcurrency"); + tsNumOfCompactThreads = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "retentionSpeedLimitMB"); tsRetentionSpeedLimitMB = pItem->i32; diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 9ced37eb38..6bd64fb5e6 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -81,7 +81,7 @@ SName* tNameDup(const SName* name) { } int32_t tNameGetDbName(const SName* name, char* dst) { - strncpy(dst, name->dbname, tListLen(name->dbname)); + tstrncpy(dst, name->dbname, tListLen(name->dbname)); return 0; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index bf163d7da6..ccc6439b5d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -475,6 +475,27 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } +extern void tsdbAlterMaxCompactTasks(); +static int32_t dmAlterMaxCompactTask(const char *value) { + int32_t max_compact_tasks; + char *endptr = NULL; + + max_compact_tasks = taosStr2Int32(value, &endptr, 10); + if (endptr == value || endptr[0] != '\0') { + return TSDB_CODE_INVALID_MSG; + } + + if (max_compact_tasks != tsNumOfCompactThreads) { + dInfo("alter max compact tasks from %d to %d", tsNumOfCompactThreads, max_compact_tasks); + tsNumOfCompactThreads = max_compact_tasks; +#ifdef TD_ENTERPRISE + tsdbAlterMaxCompactTasks(); +#endif + } + + return TSDB_CODE_SUCCESS; +} + int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t code = 0; SDCfgDnodeReq cfgReq = {0}; @@ -482,6 +503,10 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return TSDB_CODE_INVALID_MSG; } + if (strncmp(cfgReq.config, tsAlterCompactTaskKeywords, strlen(tsAlterCompactTaskKeywords) + 1) == 0) { + return dmAlterMaxCompactTask(cfgReq.value); + } + dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); SConfig *pCfg = taosGetCfg(); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 8e8d70118b..83043b4393 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -211,7 +211,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { #if defined(TD_ENTERPRISE) pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) { - strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); } #else pCfg->tdbEncryptAlgorithm = 0; diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 26f45d2fb8..d5006e1e4d 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -551,7 +551,7 @@ int32_t dmGetEncryptKey() { goto _OVER; } - strncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN + 1); + tstrncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN + 1); taosMemoryFreeClear(encryptKey); tsEncryptionKeyChksum = taosCalcChecksum(0, tsEncryptKey, strlen(tsEncryptKey)); tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED; diff --git a/source/dnode/mnode/impl/inc/mndCompactDetail.h b/source/dnode/mnode/impl/inc/mndCompactDetail.h index 601af3b64b..e99923e226 100644 --- a/source/dnode/mnode/impl/inc/mndCompactDetail.h +++ b/source/dnode/mnode/impl/inc/mndCompactDetail.h @@ -25,19 +25,17 @@ extern "C" { int32_t mndInitCompactDetail(SMnode *pMnode); void mndCleanupCompactDetail(SMnode *pMnode); -void tFreeCompactDetailObj(SCompactDetailObj *pCompact); -int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj); -int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj); +void tFreeCompactDetailObj(SCompactDetailObj *pCompact); -SSdbRaw* mndCompactDetailActionEncode(SCompactDetailObj *pCompact); -SSdbRow* mndCompactDetailActionDecode(SSdbRaw *pRaw); +SSdbRaw *mndCompactDetailActionEncode(SCompactDetailObj *pCompact); +SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw); int32_t mndCompactDetailActionInsert(SSdb *pSdb, SCompactDetailObj *pCompact); int32_t mndCompactDetailActionDelete(SSdb *pSdb, SCompactDetailObj *pCompact); int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact, SCompactDetailObj *pNewCompact); -int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup, - SVnodeGid *pVgid, int32_t index); +int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SVgObj *pVgroup, + SVnodeGid *pVgid, int32_t index); int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 0656bf8dc2..e3d2ad6d34 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -442,6 +442,10 @@ typedef struct { int8_t s3Compact; int8_t withArbitrator; int8_t encryptAlgorithm; + int8_t compactTimeOffset; // hour + int32_t compactInterval; // minute + int32_t compactStartTime; // minute + int32_t compactEndTime; // minute } SDbCfg; typedef struct { @@ -865,6 +869,8 @@ typedef struct { int64_t startTime; int32_t newNumberFileset; int32_t newFinished; + int32_t progress; + int64_t remainingTime; } SCompactDetailObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 3779bea564..50bba755c9 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -457,7 +457,7 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) { _exit: tFreeSMCfgClusterReq(&cfgReq); if (code != 0) { - mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, terrstr()); + mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, tstrerror(code)); } else { mInfo("cluster: success to config:%s %s", cfgReq.config, cfgReq.value); } diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 4c2ff9befc..ade7b63afa 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -549,6 +549,8 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c if (pDetail->compactId == compactId && pDetail->vgId == rsp->vgId && pDetail->dnodeId == rsp->dnodeId) { pDetail->newNumberFileset = rsp->numberFileset; pDetail->newFinished = rsp->finished; + pDetail->progress = rsp->progress; + pDetail->remainingTime = rsp->remainingTime; sdbCancelFetch(pMnode->pSdb, pIter); sdbRelease(pMnode->pSdb, pDetail); @@ -859,7 +861,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { return 0; } -void mndCompactPullup(SMnode *pMnode) { +static void mndCompactPullup(SMnode *pMnode) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t)); @@ -892,8 +894,113 @@ void mndCompactPullup(SMnode *pMnode) { taosArrayDestroy(pArray); } -static int32_t mndProcessCompactTimer(SRpcMsg *pReq) { - mTrace("start to process compact timer"); - mndCompactPullup(pReq->info.node); +static int32_t mndCompactDispatchAudit(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, STimeWindow *tw) { + if (!tsEnableAudit || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) { + return 0; + } + + SName name = {0}; + int32_t sqlLen = 0; + char sql[256] = {0}; + char skeyStr[40] = {0}; + char ekeyStr[40] = {0}; + char *pDbName = pDb->name; + + if (tNameFromString(&name, pDb->name, T_NAME_ACCT | T_NAME_DB) == 0) { + pDbName = name.dbname; + } + + if (taosFormatUtcTime(skeyStr, sizeof(skeyStr), tw->skey, pDb->cfg.precision) == 0 && + taosFormatUtcTime(ekeyStr, sizeof(ekeyStr), tw->ekey, pDb->cfg.precision) == 0) { + sqlLen = tsnprintf(sql, sizeof(sql), "compact db %s start with '%s' end with '%s'", pDbName, skeyStr, ekeyStr); + } else { + sqlLen = tsnprintf(sql, sizeof(sql), "compact db %s start with %" PRIi64 " end with %" PRIi64, pDbName, tw->skey, + tw->ekey); + } + auditRecord(NULL, pMnode->clusterId, "autoCompactDB", name.dbname, "", sql, sqlLen); + + return 0; +} + +extern int32_t mndCompactDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, STimeWindow tw, SArray *vgroupIds); +static int32_t mndCompactDispatch(SRpcMsg *pReq) { + int32_t code = 0; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int64_t curMs = taosGetTimestampMs(); + int64_t curMin = curMs / 60000LL; + + void *pIter = NULL; + SDbObj *pDb = NULL; + while ((pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb))) { + if (pDb->cfg.compactInterval <= 0) { + mDebug("db:%p,%s, compact interval is %dm, skip", pDb, pDb->name, pDb->cfg.compactInterval); + sdbRelease(pSdb, pDb); + continue; + } + + // daysToKeep2 would be altered + if (pDb->cfg.compactEndTime && (pDb->cfg.compactEndTime <= -pDb->cfg.daysToKeep2)) { + mWarn("db:%p,%s, compact end time:%dm <= -keep2:%dm , skip", pDb, pDb->name, pDb->cfg.compactEndTime, + -pDb->cfg.daysToKeep2); + sdbRelease(pSdb, pDb); + continue; + } + + int64_t compactStartTime = pDb->cfg.compactStartTime ? pDb->cfg.compactStartTime : -pDb->cfg.daysToKeep2; + int64_t compactEndTime = pDb->cfg.compactEndTime ? pDb->cfg.compactEndTime : -pDb->cfg.daysPerFile; + + if (compactStartTime >= compactEndTime) { + mDebug("db:%p,%s, compact start time:%" PRIi64 "m >= end time:%" PRIi64 "m, skip", pDb, pDb->name, + compactStartTime, compactEndTime); + sdbRelease(pSdb, pDb); + continue; + } + + int64_t remainder = ((curMin - (int64_t)pDb->cfg.compactTimeOffset * 60LL) % pDb->cfg.compactInterval); + if (remainder != 0) { + mDebug("db:%p,%s, current time:%" PRIi64 "m is not divisible by compact interval:%dm, offset:%" PRIi8 + "h, remainder:%" PRIi64 "m, skip", + pDb, pDb->name, curMin, pDb->cfg.compactInterval, pDb->cfg.compactTimeOffset, remainder); + sdbRelease(pSdb, pDb); + continue; + } + + if ((pDb->compactStartTime / 60000LL) == curMin) { + mDebug("db:%p:%s, compact has already been dispatched at %" PRIi64 "m(%" PRIi64 "ms), skip", pDb, pDb->name, + curMin, pDb->compactStartTime); + sdbRelease(pSdb, pDb); + continue; + } + + STimeWindow tw = { + .skey = convertTimePrecision(curMs + compactStartTime * 60000LL, TSDB_TIME_PRECISION_MILLI, pDb->cfg.precision), + .ekey = convertTimePrecision(curMs + compactEndTime * 60000LL, TSDB_TIME_PRECISION_MILLI, pDb->cfg.precision)}; + + if ((code = mndCompactDb(pMnode, NULL, pDb, tw, NULL)) == 0) { + mInfo("db:%p,%s, succeed to dispatch compact with range:[%" PRIi64 ",%" PRIi64 "], interval:%dm, start:%" PRIi64 + "m, end:%" PRIi64 "m, offset:%" PRIi8 "h", + pDb, pDb->name, tw.skey, tw.ekey, pDb->cfg.compactInterval, compactStartTime, compactEndTime, + pDb->cfg.compactTimeOffset); + } else { + mWarn("db:%p,%s, failed to dispatch compact with range:[%" PRIi64 ",%" PRIi64 "], interval:%dm, start:%" PRIi64 + "m, end:%" PRIi64 "m, offset:%" PRIi8 "h, since %s", + pDb, pDb->name, tw.skey, tw.ekey, pDb->cfg.compactInterval, compactStartTime, compactEndTime, + pDb->cfg.compactTimeOffset, tstrerror(code)); + } + + TAOS_UNUSED(mndCompactDispatchAudit(pMnode, pReq, pDb, &tw)); + + sdbRelease(pSdb, pDb); + } + return 0; +} + +static int32_t mndProcessCompactTimer(SRpcMsg *pReq) { +#ifdef TD_ENTERPRISE + mTrace("start to process compact timer"); + mndCompactPullup(pReq->info.node); + TAOS_UNUSED(mndCompactDispatch(pReq)); +#endif return 0; } diff --git a/source/dnode/mnode/impl/src/mndCompactDetail.c b/source/dnode/mnode/impl/src/mndCompactDetail.c index cbd0df7e68..9a053066b2 100644 --- a/source/dnode/mnode/impl/src/mndCompactDetail.c +++ b/source/dnode/mnode/impl/src/mndCompactDetail.c @@ -90,6 +90,14 @@ int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB TAOS_CHECK_RETURN_WITH_RELEASE(colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->startTime, false), pSdb, pCompactDetail); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_RETURN_WITH_RELEASE(colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->progress, false), + pSdb, pCompactDetail); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_RETURN_WITH_RELEASE( + colDataSetVal(pColInfo, numOfRows, (const char *)&pCompactDetail->remainingTime, false), pSdb, pCompactDetail); + numOfRows++; sdbRelease(pSdb, pCompactDetail); } @@ -101,7 +109,7 @@ int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB void tFreeCompactDetailObj(SCompactDetailObj *pCompact) {} -int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj) { +static int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDetailObj *pObj) { SEncoder encoder = {0}; int32_t code = 0; int32_t lino; @@ -118,6 +126,9 @@ int32_t tSerializeSCompactDetailObj(void *buf, int32_t bufLen, const SCompactDet TAOS_CHECK_EXIT(tEncodeI64(&encoder, pObj->startTime)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pObj->newNumberFileset)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pObj->newFinished)); + // 1. add progress and remaining time + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, pObj->progress)); + TAOS_CHECK_EXIT(tEncodeI64v(&encoder, pObj->remainingTime)); tEndEncode(&encoder); @@ -131,7 +142,7 @@ _exit: return tlen; } -int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj) { +static int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailObj *pObj) { int32_t code = 0; int32_t lino; SDecoder decoder = {0}; @@ -147,6 +158,14 @@ int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailO TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pObj->startTime)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pObj->newNumberFileset)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pObj->newFinished)); + // 1. add progress and remaining time decode + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &pObj->progress)); + TAOS_CHECK_EXIT(tDecodeI64v(&decoder, &pObj->remainingTime)); + } else { + pObj->progress = 0; + pObj->remainingTime = 0; + } tEndDecode(&decoder); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9ce5dd3dea..9d02fdf115 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -38,7 +38,7 @@ #include "tjson.h" #define DB_VER_NUMBER 1 -#define DB_RESERVE_SIZE 27 +#define DB_RESERVE_SIZE 14 static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); @@ -152,6 +152,10 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.encryptAlgorithm, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER); + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compactTimeOffset, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactStartTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactEndTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.compactInterval, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -251,6 +255,10 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.encryptAlgorithm, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER); + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compactTimeOffset, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactStartTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactEndTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.compactInterval, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); @@ -366,6 +374,10 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.s3KeepLocal = pNew->cfg.s3KeepLocal; pOld->cfg.s3Compact = pNew->cfg.s3Compact; pOld->cfg.withArbitrator = pNew->cfg.withArbitrator; + pOld->cfg.compactInterval = pNew->cfg.compactInterval; + pOld->cfg.compactStartTime = pNew->cfg.compactStartTime; + pOld->cfg.compactEndTime = pNew->cfg.compactEndTime; + pOld->cfg.compactTimeOffset = pNew->cfg.compactTimeOffset; pOld->compactStartTime = pNew->compactStartTime; pOld->tsmaVersion = pNew->tsmaVersion; taosWUnLockLatch(&pOld->lock); @@ -801,6 +813,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .tsdbPageSize = pCreate->tsdbPageSize, .withArbitrator = pCreate->withArbitrator, .encryptAlgorithm = pCreate->encryptAlgorithm, + .compactInterval = pCreate->compactInterval, + .compactStartTime = pCreate->compactStartTime, + .compactEndTime = pCreate->compactEndTime, + .compactTimeOffset = pCreate->compactTimeOffset, }; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; @@ -1144,6 +1160,34 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { code = 0; } + if (pAlter->compactInterval >= TSDB_DEFAULT_COMPACT_INTERVAL && pAlter->compactInterval != pDb->cfg.compactInterval) { + pDb->cfg.compactInterval = pAlter->compactInterval; + pDb->vgVersion++; + code = 0; + } + + if (pAlter->compactStartTime != pDb->cfg.compactStartTime && + (pAlter->compactStartTime == TSDB_DEFAULT_COMPACT_START_TIME || + pAlter->compactStartTime <= -pDb->cfg.daysPerFile)) { + pDb->cfg.compactStartTime = pAlter->compactStartTime; + pDb->vgVersion++; + code = 0; + } + + if (pAlter->compactEndTime != pDb->cfg.compactEndTime && + (pAlter->compactEndTime == TSDB_DEFAULT_COMPACT_END_TIME || pAlter->compactEndTime <= -pDb->cfg.daysPerFile)) { + pDb->cfg.compactEndTime = pAlter->compactEndTime; + pDb->vgVersion++; + code = 0; + } + + if (pAlter->compactTimeOffset >= TSDB_MIN_COMPACT_TIME_OFFSET && + pAlter->compactTimeOffset != pDb->cfg.compactTimeOffset) { + pDb->cfg.compactTimeOffset = pAlter->compactTimeOffset; + pDb->vgVersion++; + code = 0; + } + TAOS_RETURN(code); } @@ -1377,6 +1421,18 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->s3Compact = pDb->cfg.s3Compact; cfgRsp->withArbitrator = pDb->cfg.withArbitrator; cfgRsp->encryptAlgorithm = pDb->cfg.encryptAlgorithm; + cfgRsp->compactInterval = pDb->cfg.compactInterval; + cfgRsp->compactStartTime = pDb->cfg.compactStartTime; + cfgRsp->compactEndTime = pDb->cfg.compactEndTime; + if (cfgRsp->compactInterval > 0) { + if (cfgRsp->compactStartTime == 0) { + cfgRsp->compactStartTime = -cfgRsp->daysToKeep2; + } + if (cfgRsp->compactEndTime == 0) { + cfgRsp->compactEndTime = -cfgRsp->daysPerFile; + } + } + cfgRsp->compactTimeOffset = pDb->cfg.compactTimeOffset; } static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index fd520f723b..409aa4c9c9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -14,11 +14,11 @@ */ #define _DEFAULT_SOURCE +#include "mndDnode.h" #include #include "audit.h" #include "mndCluster.h" #include "mndDb.h" -#include "mndDnode.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" @@ -1077,7 +1077,6 @@ static void getSlowLogScopeString(int32_t scope, char *result) { } } - static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6ab3f3bb5c..4df5727c66 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -248,6 +248,10 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { } void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { + if (pMnode == NULL || pEpSet == NULL) { + return; + } + SSdb *pSdb = pMnode->pSdb; int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); if (totalMnodes == 0) { diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 1906835122..49dc62d471 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -144,6 +144,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_TSMAS; } else if (strncasecmp(name, TSDB_INS_DISK_USAGE, len) == 0) { type = TSDB_MGMT_TABLE_USAGE; + } else if (strncasecmp(name, TSDB_INS_TABLE_FILESETS, len) == 0) { + type = TSDB_MGMT_TABLE_FILESETS; } else { mError("invalid show name:%s len:%d", name, len); } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0a107518df..cab61648e1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2343,7 +2343,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { } int32_t total = taosArrayGetSize(*pReqTaskList); - if (total == numOfTasks) { // all tasks has send the reqs + if (total == numOfTasks) { // all tasks have sent the reqs int64_t checkpointId = mndStreamGenChkptId(pMnode, false); mInfo("stream:0x%" PRIx64 " all tasks req checkpoint, start checkpointId:%" PRId64, req.streamId, checkpointId); diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 4b3db28aa1..b7b2764442 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -15,6 +15,8 @@ #include "mndStream.h" #include "mndTrans.h" +#include "mndMnode.h" +#include "tmisce.h" typedef struct SFailedCheckpointInfo { int64_t streamUid; @@ -31,7 +33,7 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); static int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info); static bool validateHbMsg(const SArray *pNodeList, int32_t vgId); static void cleanupAfterProcessHbMsg(SStreamHbMsg *pReq, SArray *pFailedChkptList, SArray *pOrphanTasks); -static void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, int32_t vgId, int32_t msgId); +static void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, SEpSet* pEpset, int32_t vgId, int32_t msgId); static void checkforOrphanTask(SMnode* pMnode, STaskStatusEntry* p, SArray* pOrphanTasks); void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { @@ -329,6 +331,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray *pFailedChkpt = NULL; SArray *pOrphanTasks = NULL; int32_t code = 0; + SDecoder decoder = {0}; + SEpSet mnodeEpset = {0}; if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) { @@ -336,7 +340,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { } } - SDecoder decoder = {0}; tDecoderInit(&decoder, pReq->pCont, pReq->contLen); if (tDecodeStreamHbMsg(&decoder, &req) < 0) { @@ -357,13 +360,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { TAOS_RETURN(terrno); } + mndGetMnodeEpSet(pMnode, &mnodeEpset); + streamMutexLock(&execInfo.lock); mndInitStreamExecInfo(pMnode, &execInfo); if (!validateHbMsg(execInfo.pNodeList, req.vgId)) { mError("vgId:%d not exists in nodeList buf, discarded", req.vgId); - doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(terrno, &pReq->info, &mnodeEpset, req.vgId, req.msgId); streamMutexUnlock(&execInfo.lock); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); @@ -383,9 +388,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if ((pEntry->lastHbMsgId == req.msgId) && (pEntry->lastHbMsgTs == req.ts)) { mError("vgId:%d HbMsgId:%d already handled, bh msg discard, and send HbRsp", pEntry->nodeId, req.msgId); - // return directly and after the vnode to continue to send the next HbMsg. + // return directly and allow the vnode to continue to send the next HbMsg. terrno = TSDB_CODE_SUCCESS; - doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(terrno, &pReq->info, &mnodeEpset, req.vgId, req.msgId); streamMutexUnlock(&execInfo.lock); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); @@ -529,7 +534,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { streamMutexUnlock(&execInfo.lock); - doSendHbMsgRsp(TSDB_CODE_SUCCESS, &pReq->info, req.vgId, req.msgId); + doSendHbMsgRsp(TSDB_CODE_SUCCESS, &pReq->info, &mnodeEpset, req.vgId, req.msgId); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); return code; @@ -552,12 +557,13 @@ void cleanupAfterProcessHbMsg(SStreamHbMsg *pReq, SArray *pFailedChkptList, SArr taosArrayDestroy(pOrphanTasks); } -void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, int32_t vgId, int32_t msgId) { +void doSendHbMsgRsp(int32_t code, SRpcHandleInfo *pRpcInfo, SEpSet* pMndEpset, int32_t vgId, int32_t msgId) { int32_t ret = 0; int32_t tlen = 0; void *buf = NULL; - const SMStreamHbRspMsg msg = {.msgId = msgId}; + SMStreamHbRspMsg msg = {.msgId = msgId};//, .mndEpset = *pMndEpset}; + epsetAssign(&msg.mndEpset, pMndEpset); tEncodeSize(tEncodeStreamHbRsp, &msg, tlen, ret); if (ret < 0) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 68a7766370..11b3cdf3c6 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -341,6 +341,12 @@ struct SVnodeCfg { #define TABLE_IS_COL_COMPRESSED(FLG) (((FLG) & (TABLE_COL_COMPRESSED)) != 0) #define TABLE_SET_COL_COMPRESSED(FLG) ((FLG) |= TABLE_COL_COMPRESSED) +struct SFileSetReader; +int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader); +int32_t tsdbFileSetReaderNext(struct SFileSetReader *pReader); +int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *field, void *value); +void tsdbFileSetReaderClose(struct SFileSetReader **ppReader); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0080e3f7c2..f39da72507 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -379,7 +379,7 @@ struct STsdb { struct { SVHashTable *ht; SArray *arr; - } * commitInfo; + } *commitInfo; }; struct TSDBKEY { @@ -1082,6 +1082,9 @@ void tsdbRemoveFile(const char *path); } \ } while (0) +int32_t tsdbInit(); +void tsdbCleanUp(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 5351554631..cc440fd73b 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -197,7 +197,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); TSDB_CHECK_CODE(code, lino, _exit); - sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); + (void)tsnprintf(indexFullPath, sizeof(indexFullPath), "%s/%s", pMeta->path, "invert"); ret = taosMkDir(indexFullPath); SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; @@ -209,7 +209,7 @@ static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, // open pTtlMgr ("ttlv1.idx") char logPrefix[128] = {0}; - sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); + (void)tsnprintf(logPrefix, sizeof(logPrefix), "vgId:%d", TD_VID(pVnode)); code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 87beb8842b..c19a2e3ce2 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -164,7 +164,7 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { metaReaderClear(&mr); return code; } - strncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN); + tstrncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN); metaReaderClear(&mr); return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 2ff5a8ca6d..9a7cdca8f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1239,3 +1239,146 @@ void tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) { } } } + +struct SFileSetReader { + STsdb *pTsdb; + STFileSet *pFileSet; + int32_t fid; + int64_t startTime; + int64_t endTime; + int64_t lastCompactTime; + int64_t totalSize; +}; + +int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader) { + if (pVnode == NULL || ppReader == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + STsdb *pTsdb = ((SVnode *)pVnode)->pTsdb; + + (*ppReader) = taosMemoryCalloc(1, sizeof(struct SFileSetReader)); + if (*ppReader == NULL) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, __LINE__, + tstrerror(terrno)); + return terrno; + } + + (*ppReader)->pTsdb = pTsdb; + (*ppReader)->fid = INT32_MIN; + (*ppReader)->pFileSet = NULL; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) { + STsdb *pTsdb = pReader->pTsdb; + int32_t code = TSDB_CODE_SUCCESS; + + tsdbTFileSetClear(&pReader->pFileSet); + + STFileSet *fset = &(STFileSet){ + .fid = pReader->fid, + }; + + STFileSet **fsetPtr = TARRAY2_SEARCH(pReader->pTsdb->pFS->fSetArr, &fset, tsdbTFileSetCmprFn, TD_GT); + if (fsetPtr == NULL) { + pReader->fid = INT32_MAX; + return TSDB_CODE_NOT_FOUND; + } + + // ref file set + code = tsdbTFileSetInitRef(pReader->pTsdb, *fsetPtr, &pReader->pFileSet); + if (code) return code; + + // get file set details + pReader->fid = pReader->pFileSet->fid; + tsdbFidKeyRange(pReader->fid, pTsdb->keepCfg.days, pTsdb->keepCfg.precision, &pReader->startTime, &pReader->endTime); + pReader->lastCompactTime = 0; // TODO + pReader->totalSize = 0; + for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) { + STFileObj *fobj = pReader->pFileSet->farr[i]; + if (fobj) { + pReader->totalSize += fobj->f->size; + } + } + SSttLvl *lvl; + TARRAY2_FOREACH(pReader->pFileSet->lvlArr, lvl) { + STFileObj *fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { pReader->totalSize += fobj->f->size; } + } + + return code; +} + +int32_t tsdbFileSetReaderNext(struct SFileSetReader *pReader) { + int32_t code = TSDB_CODE_SUCCESS; + (void)taosThreadMutexLock(&pReader->pTsdb->mutex); + code = tsdbFileSetReaderNextNoLock(pReader); + (void)taosThreadMutexUnlock(&pReader->pTsdb->mutex); + return code; +} + +int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *field, void *value) { + const char *fieldName; + + if (pReader->fid == INT32_MIN || pReader->fid == INT32_MAX) { + return TSDB_CODE_INVALID_PARA; + } + + fieldName = "fileset_id"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int32_t *)value = pReader->fid; + return TSDB_CODE_SUCCESS; + } + + fieldName = "start_time"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int64_t *)value = pReader->startTime; + return TSDB_CODE_SUCCESS; + } + + fieldName = "end_time"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int64_t *)value = pReader->endTime; + return TSDB_CODE_SUCCESS; + } + + fieldName = "total_size"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int64_t *)value = pReader->totalSize; + return TSDB_CODE_SUCCESS; + } + + fieldName = "last_compact_time"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(int64_t *)value = pReader->lastCompactTime; + return TSDB_CODE_SUCCESS; + } + + fieldName = "should_compact"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + *(char *)value = 0; // TODO + return TSDB_CODE_SUCCESS; + } + + fieldName = "details"; + if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { + // TODO + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_INVALID_PARA; +} + +void tsdbFileSetReaderClose(struct SFileSetReader **ppReader) { + if (ppReader == NULL || *ppReader == NULL) { + return; + } + + tsdbTFileSetClear(&(*ppReader)->pFileSet); + taosMemoryFree(*ppReader); + + *ppReader = NULL; + return; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index c1f8f45d7e..b2e4621878 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -18,6 +18,22 @@ extern int32_t tsdbOpenCompMonitor(STsdb *tsdb); extern void tsdbCloseCompMonitor(STsdb *tsdb); +extern int32_t tsdbInitCompact(); +extern void tsdbCleanupCompact(); + +int32_t tsdbInit() { +#ifdef TD_ENTERPRISE + return tsdbInitCompact(); +#endif + return 0; +} + +void tsdbCleanUp() { +#ifdef TD_ENTERPRISE + tsdbCleanupCompact(); +#endif + return; +} void tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) { STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 6dba1825ad..bf79b2482d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -104,7 +104,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF } pFD->path = (char *)&pFD[1]; - strcpy(pFD->path, path); + tstrncpy(pFD->path, path, strlen(path) + 1); pFD->szPage = szPage; pFD->flag = flag; pFD->szPage = szPage; @@ -174,7 +174,6 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e opts.source = pFD->pBuf + count; opts.result = PacketData; opts.unitLen = 128; - // strncpy(opts.key, tsEncryptKey, 16); tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); NewLen = CBC_Encrypt(&opts); @@ -248,7 +247,6 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor opts.source = pFD->pBuf + count; opts.result = PacketData; opts.unitLen = 128; - // strncpy(opts.key, tsEncryptKey, 16); tstrncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN + 1); NewLen = CBC_Decrypt(&opts); diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 9e4fbd84a9..424ed0f325 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -330,7 +330,7 @@ static int32_t vnodeAsyncInit(SVAsync **async, const char *label) { return terrno; } - strcpy((char *)((*async) + 1), label); + tstrncpy((char *)((*async) + 1), label, strlen(label) + 1); (*async)->label = (const char *)((*async) + 1); (void)taosThreadMutexInit(&(*async)->mutex, NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 2ceeeca160..9c153bc8a1 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -303,7 +303,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; } else { - strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); } } #endif diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index e2bacb3dc1..df6fb17730 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -63,6 +63,12 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdSetFilesetDelimited = (void (*)(void*))tsdbSetFilesetDelimited; pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; + + // file set iterate + pReader->fileSetReaderOpen = tsdbFileSetReaderOpen; + pReader->fileSetReadNext = tsdbFileSetReaderNext; + pReader->fileSetGetEntryField = tsdbFileSetGetEntryField; + pReader->fileSetReaderClose = tsdbFileSetReaderClose; } void initMetadataAPI(SStoreMeta* pMeta) { diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 781736edba..9d326defdd 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -15,6 +15,7 @@ #include "cos.h" #include "monitor.h" +#include "tsdb.h" #include "vnd.h" static volatile int32_t VINIT = 0; @@ -26,6 +27,7 @@ int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) { TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads)); TAOS_CHECK_RETURN(walInit(stopDnodeFp)); + TAOS_CHECK_RETURN(tsdbInit()); monInitVnode(); @@ -34,6 +36,7 @@ int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) { void vnodeCleanup() { if (atomic_val_compare_exchange_32(&VINIT, 1, 0) == 0) return; + tsdbCleanUp(); vnodeAsyncClose(); walCleanUp(); smaCleanUp(); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b91abe93af..b9e686932e 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -168,7 +168,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t static int32_t vnodeVgroupIdLen(int32_t vgId) { char tmp[TSDB_FILENAME_LEN]; - sprintf(tmp, "%d", vgId); + (void)tsnprintf(tmp, TSDB_FILENAME_LEN, "%d", vgId); return strlen(tmp); } @@ -199,7 +199,7 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr char *tsdbFilePrefixPos = strstr(oldRname, tsdbFilePrefix); if (tsdbFilePrefixPos == NULL) continue; - int32_t tsdbFileVgId = 0; // atoi(tsdbFilePrefixPos + prefixLen); + int32_t tsdbFileVgId = 0; ret = taosStr2int32(tsdbFilePrefixPos + prefixLen, &tsdbFileVgId); if (ret != 0) { vError("vgId:%d, failed to get tsdb file vgid since %s", dstVgId, tstrerror(ret)); @@ -417,7 +417,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } pVnode->path = (char *)&pVnode[1]; - strcpy(pVnode->path, path); + tstrncpy(pVnode->path, path, strlen(path) + 1); pVnode->config = info.config; pVnode->state.committed = info.state.committed; pVnode->state.commitTerm = info.state.commitTerm; @@ -472,7 +472,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open wal - sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); + (void)tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); ret = taosRealPath(tdir, NULL, sizeof(tdir)); TAOS_UNUSED(ret); @@ -484,7 +484,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } // open tq - sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); + (void)tsnprintf(tdir, sizeof(tdir), "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); ret = taosRealPath(tdir, NULL, sizeof(tdir)); TAOS_UNUSED(ret); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6ca5803c19..f52a0c3aba 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -71,11 +71,11 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { if (infoReq.option == REQ_OPT_TBUID) reqTbUid = true; metaRsp.dbId = pVnode->config.dbId; - (void)strcpy(metaRsp.tbName, infoReq.tbName); + tstrncpy(metaRsp.tbName, infoReq.tbName, TSDB_TABLE_NAME_LEN); (void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); if (!reqTbUid) { - TAOS_UNUSED(sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName)); + (void)tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", infoReq.dbFName, infoReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit4; @@ -105,7 +105,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.tuid = mer1.me.uid; if (mer1.me.type == TSDB_SUPER_TABLE) { - (void)strcpy(metaRsp.stbName, mer1.me.name); + tstrncpy(metaRsp.stbName, mer1.me.name, TSDB_TABLE_NAME_LEN); schema = mer1.me.stbEntry.schemaRow; schemaTag = mer1.me.stbEntry.schemaTag; metaRsp.suid = mer1.me.uid; @@ -113,7 +113,7 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; - (void)strcpy(metaRsp.stbName, mer2.me.name); + tstrncpy(metaRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN); metaRsp.suid = mer2.me.uid; schema = mer2.me.stbEntry.schemaRow; schemaTag = mer2.me.stbEntry.schemaTag; @@ -220,10 +220,10 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { goto _exit; } - (void)strcpy(cfgRsp.tbName, cfgReq.tbName); + tstrncpy(cfgRsp.tbName, cfgReq.tbName, TSDB_TABLE_NAME_LEN); (void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); - (void)sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); + (void)tsnprintf(tableFName, TSDB_TABLE_FNAME_LEN, "%s.%s", cfgReq.dbFName, cfgReq.tbName); code = vnodeValidateTableHash(pVnode, tableFName); if (code) { goto _exit; @@ -246,7 +246,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; - (void)strcpy(cfgRsp.stbName, mer2.me.name); + tstrncpy(cfgRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN); schema = mer2.me.stbEntry.schemaRow; schemaTag = mer2.me.stbEntry.schemaTag; cfgRsp.ttl = mer1.me.ctbEntry.ttlDays; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index aff6990012..d9b41869c7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -968,7 +968,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { return; } - strcpy(pMetaRsp->dbFName, pVnode->config.dbname); + tstrncpy(pMetaRsp->dbFName, pVnode->config.dbname, TSDB_DB_FNAME_LEN); pMetaRsp->dbId = pVnode->config.dbId; pMetaRsp->vgId = TD_VID(pVnode); pMetaRsp->precision = pVnode->config.tsdbCfg.precision; @@ -1216,7 +1216,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, rcode = -1; goto _exit; } - strcpy(str, pCreateReq->name); + tstrncpy(str, pCreateReq->name, TSDB_TABLE_FNAME_LEN); if (taosArrayPush(tbNames, &str) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; rcode = -1; @@ -1225,7 +1225,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } // validate hash - sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name); + (void)tsnprintf(tbName, TSDB_TABLE_FNAME_LEN, "%s.%s", pVnode->config.dbname, pCreateReq->name); if (vnodeValidateTableHash(pVnode, tbName) < 0) { cRsp.code = TSDB_CODE_VND_HASH_MISMATCH; if (taosArrayPush(rsp.pArray, &cRsp) == NULL) { @@ -1518,7 +1518,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in pRsp->code = terrno; goto _exit; } - strcpy(str, pDropTbReq->name); + tstrncpy(str, pDropTbReq->name, TSDB_TABLE_FNAME_LEN); if (taosArrayPush(tbNames, &str) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pRsp->code = terrno; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d0656026f1..3966315c30 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -411,11 +411,17 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char keep0Str[128] = {0}; char keep1Str[128] = {0}; char keep2Str[128] = {0}; + char compactIntervalStr[13] = {0}; + char compactStartTimeStr[13] = {0}; + char compactEndTimeStr[13] = {0}; int32_t lenDuration = formatDurationOrKeep(durationStr, sizeof(durationStr), pCfg->daysPerFile); int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pCfg->daysToKeep0); int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pCfg->daysToKeep1); int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pCfg->daysToKeep2); + UNUSED(formatDurationOrKeep(compactIntervalStr, sizeof(compactIntervalStr), pCfg->compactInterval)); + UNUSED(formatDurationOrKeep(compactStartTimeStr, sizeof(compactStartTimeStr), pCfg->compactStartTime)); + UNUSED(formatDurationOrKeep(compactEndTimeStr, sizeof(compactEndTimeStr), pCfg->compactEndTime)); if (IS_SYS_DBNAME(dbName)) { len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, @@ -428,13 +434,15 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, "PRECISION '%s' REPLICA %d " "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 - " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKPAGES %d S3_KEEPLOCAL %dm S3_COMPACT %d", + " KEEP_TIME_OFFSET %dh ENCRYPT_ALGORITHM '%s' S3_CHUNKPAGES %d S3_KEEPLOCAL %dm S3_COMPACT %d " + "COMPACT_INTERVAL %s COMPACT_TIME_RANGE %s,%s COMPACT_TIME_OFFSET %"PRIi8 "h", dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, durationStr, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, keep0Str, keep1Str, keep2Str, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize, pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), - pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact); + pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact, compactIntervalStr, compactStartTimeStr, + compactEndTimeStr, pCfg->compactTimeOffset); if (pRetentions) { len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 4e949a6b8b..13ae220116 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -76,6 +76,9 @@ typedef struct SSysTableScanInfo { STableListInfo* pTableListInfo; SReadHandle* pHandle; SStorageAPI* pAPI; + + // file set iterate + struct SFileSetReader* pFileSetReader; } SSysTableScanInfo; typedef struct { @@ -2210,6 +2213,258 @@ static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } +static int32_t doSetQueryFileSetRow() { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + + // TODO + +_exit: + return code; +} + +static SSDataBlock* sysTableBuildUserFileSets(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; + SSDataBlock* p = NULL; + + // open cursor if not opened + if (pInfo->pFileSetReader == NULL) { + code = pAPI->tsdReader.fileSetReaderOpen(pInfo->readHandle.vnode, &pInfo->pFileSetReader); + QUERY_CHECK_CODE(code, lino, _end); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + code = tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + QUERY_CHECK_CODE(code, lino, _end); + + code = tNameGetDbName(&sn, varDataVal(dbname)); + QUERY_CHECK_CODE(code, lino, _end); + + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_FILESETS); + QUERY_CHECK_NULL(p, code, lino, _end, terrno); + + code = blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + QUERY_CHECK_CODE(code, lino, _end); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t ret = 0; + + // loop to query each entry + for (;;) { + int32_t ret = pAPI->tsdReader.fileSetReadNext(pInfo->pFileSetReader); + if (ret) { + if (ret == TSDB_CODE_NOT_FOUND) { + // no more scan entry + break; + } else { + code = ret; + QUERY_CHECK_CODE(code, lino, _end); + } + } + + // fill the data block + { + SColumnInfoData* pColInfoData; + int32_t index = 0; + + // db_name + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, db, false); + QUERY_CHECK_CODE(code, lino, _end); + + // vgroup_id + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); + QUERY_CHECK_CODE(code, lino, _end); + + // fileset_id + int32_t filesetId = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "fileset_id", &filesetId); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&filesetId, false); + QUERY_CHECK_CODE(code, lino, _end); + + // start_time + int64_t startTime = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "start_time", &startTime); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&startTime, false); + QUERY_CHECK_CODE(code, lino, _end); + + // end_time + int64_t endTime = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "end_time", &endTime); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&endTime, false); + QUERY_CHECK_CODE(code, lino, _end); + + // total_size + int64_t totalSize = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "total_size", &totalSize); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&totalSize, false); + QUERY_CHECK_CODE(code, lino, _end); + + // last_compact + int64_t lastCompacat = 0; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "last_compact_time", &lastCompacat); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&lastCompacat, false); + QUERY_CHECK_CODE(code, lino, _end); + + // shold_compact + bool shouldCompact = false; + code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "should_compact", &shouldCompact); + QUERY_CHECK_CODE(code, lino, _end); + pColInfoData = taosArrayGet(p->pDataBlock, index++); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, (char*)&shouldCompact, false); + QUERY_CHECK_CODE(code, lino, _end); + + // // details + // const char* details = NULL; + // code = pAPI->tsdReader.fileSetGetEntryField(pInfo->pFileSetReader, "details", &details); + // QUERY_CHECK_CODE(code, lino, _end); + // pColInfoData = taosArrayGet(p->pDataBlock, index++); + // QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + // code = colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); + // QUERY_CHECK_CODE(code, lino, _end); + } + + // check capacity + if (++numOfRows >= pOperator->resultInfo.capacity) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + code = relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + QUERY_CHECK_CODE(code, lino, _end); + + code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + QUERY_CHECK_CODE(code, lino, _end); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + code = relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + QUERY_CHECK_CODE(code, lino, _end); + + code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + QUERY_CHECK_CODE(code, lino, _end); + + blockDataCleanup(p); + numOfRows = 0; + } + + blockDataDestroy(p); + p = NULL; + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + blockDataDestroy(p); + pTaskInfo->code = code; + pAPI->tsdReader.fileSetReaderClose(&pInfo->pFileSetReader); + T_LONG_JMP(pTaskInfo->env, code); + } + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableScanUserFileSets(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSysTableScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SNode* pCondition = pInfo->pCondition; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pInfo->readHandle.mnd != NULL) { + // do nothing on mnode + qTrace("This operator do nothing on mnode, task id:%s", GET_TASKID(pTaskInfo)); + return NULL; + } else { +#if 0 + if (pInfo->showRewrite == false) { + if (pCondition != NULL && pInfo->pIdx == NULL) { + SSTabFltArg arg = { + .pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; + + SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); + QUERY_CHECK_NULL(idx, code, lino, _end, terrno); + idx->init = 0; + idx->uids = taosArrayInit(128, sizeof(int64_t)); + QUERY_CHECK_NULL(idx->uids, code, lino, _end, terrno); + idx->lastIdx = 0; + + pInfo->pIdx = idx; // set idx arg + + int flt = optSysTabFilte(&arg, pCondition, idx->uids); + if (flt == 0) { + pInfo->pIdx->init = 1; + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } else if ((flt == -1) || (flt == -2)) { + qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); + } + } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } + } +#endif + + return sysTableBuildUserFileSets(pOperator); + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } + return NULL; +} + static int32_t getSysTableDbNameColId(const char* pTable) { // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { // return 1; @@ -2309,6 +2564,8 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) pBlock = sysTableScanUserSTables(pOperator); } else if (strncasecmp(name, TSDB_INS_DISK_USAGE, TSDB_TABLE_FNAME_LEN) == 0) { pBlock = sysTableScanUsage(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_FILESETS, TSDB_TABLE_FNAME_LEN) == 0) { + pBlock = sysTableScanUserFileSets(pOperator); } else { // load the meta from mnode of the given epset pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo); } @@ -2529,7 +2786,8 @@ int32_t createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNo QUERY_CHECK_CODE(code, lino, _error); if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_FILESETS, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*)readHandle; } else { if (tsem_init(&pInfo->ready, 0, 0) != TSDB_CODE_SUCCESS) { diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 8789762a85..9fed5c2a6e 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -99,8 +99,8 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData return terrno; } - (void)sprintf( - wktPatternWithSpace, + (void)tsnprintf( + wktPatternWithSpace, 4 * 1024, "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\)))|linestring( *)z?m?( " "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c12cef9feb..6d4d89607f 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -183,6 +183,8 @@ const char* nodesNodeName(ENodeType type) { return "DescribeStmt"; case QUERY_NODE_COMPACT_DATABASE_STMT: return "CompactDatabaseStmt"; + case QUERY_NODE_COMPACT_VGROUPS_STMT: + return "CompactVgroupsStmt"; case QUERY_NODE_CREATE_STREAM_STMT: return "CreateStreamStmt"; case QUERY_NODE_DROP_STREAM_STMT: @@ -1262,7 +1264,7 @@ static const char* jkStreamOption_destHasPrimaryKey = "StreamOptionDestHasPrimar static int32_t streamNodeOptionToJson(const void* pObj, SJson* pJson) { const SStreamNodeOption* pNode = (const SStreamNodeOption*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkStreamOption_triggerType, pNode->triggerType); + int32_t code = tjsonAddIntegerToObject(pJson, jkStreamOption_triggerType, pNode->triggerType); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkStreamOption_watermark, pNode->watermark); @@ -1284,7 +1286,7 @@ static int32_t streamNodeOptionToJson(const void* pObj, SJson* pJson) { static int32_t jsonToStreamNodeOption(const SJson* pJson, void* pObj) { SStreamNodeOption* pNode = (SStreamNodeOption*)pObj; - int32_t code = tjsonGetTinyIntValue(pJson, jkStreamOption_triggerType, &pNode->triggerType); + int32_t code = tjsonGetTinyIntValue(pJson, jkStreamOption_triggerType, &pNode->triggerType); if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkStreamOption_watermark, &pNode->watermark); @@ -1349,7 +1351,8 @@ static int32_t logicInterpFuncNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkInterpFuncLogicPlanTimeSeries, nodeToJson, pNode->pTimeSeries); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); + code = + tjsonAddObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanRangeInterval, pNode->rangeInterval); @@ -1393,7 +1396,8 @@ static int32_t jsonToLogicInterpFuncNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkInterpFuncLogicPlanTimeSeries, &pNode->pTimeSeries); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonToObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); + code = + tjsonToObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkInterpFuncLogicPlanRangeInterval, &pNode->rangeInterval); @@ -3394,7 +3398,8 @@ static int32_t physiInterpFuncNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanTimeSeries, nodeToJson, pNode->pTimeSeries); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); + code = + tjsonAddObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanRangeInterval, pNode->rangeInterval); @@ -3441,7 +3446,8 @@ static int32_t jsonToPhysiInterpFuncNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkInterpFuncPhysiPlanTimeSeries, &pNode->pTimeSeries); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonToObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); + code = + tjsonToObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkInterpFuncPhysiPlanRangeInterval, &pNode->rangeInterval); @@ -4240,7 +4246,7 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { case TSDB_DATA_TYPE_NCHAR: { // cJSON only support utf-8 encoding. Convert memory content to hex string. int32_t bufSize = varDataLen(pNode->datum.p) * 2 + 1; - char* buf = taosMemoryCalloc(bufSize, sizeof(char)); + char* buf = taosMemoryCalloc(bufSize, sizeof(char)); if (!buf) return terrno; code = taosHexEncode(varDataVal(pNode->datum.p), buf, varDataLen(pNode->datum.p), bufSize); if (code != TSDB_CODE_SUCCESS) { @@ -5386,13 +5392,13 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { static const char* jkWindowOffsetStartOffset = "StartOffset"; static const char* jkWindowOffsetEndOffset = "EndOffset"; static int32_t windowOffsetNodeToJson(const void* pObj, SJson* pJson) { - const SWindowOffsetNode* pNode = (const SWindowOffsetNode*)pObj; + const SWindowOffsetNode* pNode = (const SWindowOffsetNode*)pObj; - int32_t code = tjsonAddObject(pJson, jkWindowOffsetStartOffset, nodeToJson, pNode->pStartOffset); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkWindowOffsetEndOffset, nodeToJson, pNode->pEndOffset); + int32_t code = tjsonAddObject(pJson, jkWindowOffsetStartOffset, nodeToJson, pNode->pStartOffset); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkWindowOffsetEndOffset, nodeToJson, pNode->pEndOffset); } - return code; + return code; } static int32_t jsonToWindowOffsetNode(const SJson* pJson, void* pObj) { @@ -5428,6 +5434,9 @@ static const char* jkDatabaseOptionsS3ChunkSize = "S3ChunkSize"; static const char* jkDatabaseOptionsS3KeepLocalNode = "S3KeepLocalNode"; static const char* jkDatabaseOptionsS3KeepLocal = "S3KeepLocal"; static const char* jkDatabaseOptionsS3Compact = "S3Compact"; +static const char* jkDatabaseOptionsCompactIntervalNode = "compactIntervalNode"; +static const char* jkDatabaseOptionsCompactTimeRange = "compactTimeRange"; +static const char* jkDatabaseOptionsCompactTimeOffsetNode = "compactTimeOffsetNode"; static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { const SDatabaseOptions* pNode = (const SDatabaseOptions*)pObj; @@ -5499,6 +5508,15 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3Compact, pNode->s3Compact); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDatabaseOptionsCompactIntervalNode, nodeToJson, pNode->pCompactIntervalNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkDatabaseOptionsCompactTimeRange, pNode->pCompactTimeRangeList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDatabaseOptionsCompactTimeOffsetNode, nodeToJson, pNode->pCompactTimeOffsetNode); + } return code; } @@ -5573,6 +5591,15 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsS3Compact, &pNode->s3Compact); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDatabaseOptionsCompactIntervalNode, (SNode**)&pNode->pCompactIntervalNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkDatabaseOptionsCompactTimeRange, &pNode->pCompactTimeRangeList); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDatabaseOptionsCompactTimeOffsetNode, (SNode**)&pNode->pCompactTimeOffsetNode); + } return code; } @@ -5641,11 +5668,11 @@ static const char* jkColumnOptionsEncode = "encode"; static const char* jkColumnOptionsCompress = "compress"; static const char* jkColumnOptionsLevel = "level"; static int32_t columnOptionsToJson(const void* pObj, SJson* pJson) { - const SColumnOptions* pNode = (const SColumnOptions*)pObj; - int32_t code = tjsonAddStringToObject(pJson, jkColumnOptionsEncode, pNode->encode); - code = tjsonAddStringToObject(pJson, jkColumnOptionsCompress, pNode->compress); - code = tjsonAddStringToObject(pJson, jkColumnOptionsLevel, pNode->compressLevel); - return code; + const SColumnOptions* pNode = (const SColumnOptions*)pObj; + int32_t code = tjsonAddStringToObject(pJson, jkColumnOptionsEncode, pNode->encode); + code = tjsonAddStringToObject(pJson, jkColumnOptionsCompress, pNode->compress); + code = tjsonAddStringToObject(pJson, jkColumnOptionsLevel, pNode->compressLevel); + return code; } static int32_t jsonToColumnOptions(const SJson* pJson, void* pObj) { @@ -6816,7 +6843,7 @@ static int32_t dropQnodeStmtToJson(const void* pObj, SJson* pJson) { return drop static int32_t jsonToDropQnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); } static const char* jkCreateAnodeStmtUrl = "Url"; -static const char* jkUpdateDropANodeStmtId = "AnodeId"; +static const char* jkUpdateDropANodeStmtId = "AnodeId"; static int32_t createAnodeStmtToJson(const void* pObj, SJson* pJson) { const SCreateAnodeStmt* pNode = (const SCreateAnodeStmt*)pObj; @@ -7158,6 +7185,16 @@ static int32_t jsonToCompactDatabaseStmt(const SJson* pJson, void* pObj) { return tjsonGetStringValue(pJson, jkCompactDatabaseStmtDbName, pNode->dbName); } +static int32_t compactVgroupsStmtToJson(const void* pObj, SJson* pJson) { + const SCompactVgroupsStmt* pNode = (const SCompactVgroupsStmt*)pObj; + return 0; +} + +static int32_t jsonToCompactVgroupsStmt(const SJson* pJson, void* pObj) { + SCompactVgroupsStmt* pNode = (SCompactVgroupsStmt*)pObj; + return 0; +} + static const char* jkCreateStreamStmtStreamName = "StreamName"; static const char* jkCreateStreamStmtTargetDbName = "TargetDbName"; static const char* jkCreateStreamStmtTargetTabName = "TargetTabName"; @@ -7490,7 +7527,7 @@ static int32_t jsonToShowClusterMachinesStmt(const SJson* pJson, void* pObj) { r static int32_t jsonToShowEncryptionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } static int32_t showUsageStmtStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } -static int32_t jsonToShowUsageStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } +static int32_t jsonToShowUsageStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId"; static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern"; @@ -8042,6 +8079,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return describeStmtToJson(pObj, pJson); case QUERY_NODE_COMPACT_DATABASE_STMT: return compactDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_COMPACT_VGROUPS_STMT: + return compactVgroupsStmtToJson(pObj, pJson); case QUERY_NODE_CREATE_STREAM_STMT: return createStreamStmtToJson(pObj, pJson); case QUERY_NODE_DROP_STREAM_STMT: @@ -8407,6 +8446,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDescribeStmt(pJson, pObj); case QUERY_NODE_COMPACT_DATABASE_STMT: return jsonToCompactDatabaseStmt(pJson, pObj); + case QUERY_NODE_COMPACT_VGROUPS_STMT: + return jsonToCompactVgroupsStmt(pJson, pObj); case QUERY_NODE_CREATE_STREAM_STMT: return jsonToCreateStreamStmt(pJson, pObj); case QUERY_NODE_DROP_STREAM_STMT: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 4baddf5362..7beaeaa46c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -471,7 +471,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) { code = makeNode(type, sizeof(SSetOperator), &pNode); break; case QUERY_NODE_RANGE_AROUND: - code = makeNode(type, sizeof(SRangeAroundNode), &pNode); break; + code = makeNode(type, sizeof(SRangeAroundNode), &pNode); + break; case QUERY_NODE_SELECT_STMT: code = makeNode(type, sizeof(SSelectStmt), &pNode); break; @@ -593,6 +594,9 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) { case QUERY_NODE_COMPACT_DATABASE_STMT: code = makeNode(type, sizeof(SCompactDatabaseStmt), &pNode); break; + case QUERY_NODE_COMPACT_VGROUPS_STMT: + code = makeNode(type, sizeof(SCompactVgroupsStmt), &pNode); + break; case QUERY_NODE_CREATE_FUNCTION_STMT: code = makeNode(type, sizeof(SCreateFunctionStmt), &pNode); break; @@ -1156,6 +1160,9 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode((SNode*)pOptions->s3KeepLocalStr); nodesDestroyList(pOptions->pKeep); nodesDestroyList(pOptions->pRetentions); + nodesDestroyNode((SNode*)pOptions->pCompactIntervalNode); + nodesDestroyList(pOptions->pCompactTimeRangeList); + nodesDestroyNode((SNode*)pOptions->pCompactTimeOffsetNode); break; } case QUERY_NODE_TABLE_OPTIONS: { @@ -1320,14 +1327,20 @@ void nodesDestroyNode(SNode* pNode) { } break; } - case QUERY_NODE_CREATE_DATABASE_STMT: - nodesDestroyNode((SNode*)((SCreateDatabaseStmt*)pNode)->pOptions); + case QUERY_NODE_CREATE_DATABASE_STMT: { + SDatabaseOptions* pOptions = ((SCreateDatabaseStmt*)pNode)->pOptions; + taosMemoryFreeClear(pOptions->pDbCfg); + nodesDestroyNode((SNode*)pOptions); break; + } case QUERY_NODE_DROP_DATABASE_STMT: // no pointer field break; - case QUERY_NODE_ALTER_DATABASE_STMT: - nodesDestroyNode((SNode*)((SAlterDatabaseStmt*)pNode)->pOptions); + case QUERY_NODE_ALTER_DATABASE_STMT: { + SDatabaseOptions* pOptions = ((SAlterDatabaseStmt*)pNode)->pOptions; + taosMemoryFreeClear(pOptions->pDbCfg); + nodesDestroyNode((SNode*)pOptions); break; + } case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field case QUERY_NODE_TRIM_DATABASE_STMT: // no pointer field break; @@ -1443,6 +1456,14 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pEnd); break; } + case QUERY_NODE_COMPACT_VGROUPS_STMT: { + SCompactVgroupsStmt* pStmt = (SCompactVgroupsStmt*)pNode; + nodesDestroyNode(pStmt->pDbName); + nodesDestroyList(pStmt->vgidList); + nodesDestroyNode(pStmt->pStart); + nodesDestroyNode(pStmt->pEnd); + break; + } case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field break; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index a7453a17b1..e69a3da4a9 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -70,6 +70,9 @@ typedef enum EDatabaseOptionType { DB_OPTION_KEEP_TIME_OFFSET, DB_OPTION_ENCRYPT_ALGORITHM, DB_OPTION_DNODES, + DB_OPTION_COMPACT_INTERVAL, + DB_OPTION_COMPACT_TIME_RANGE, + DB_OPTION_COMPACT_TIME_OFFSET, } EDatabaseOptionType; typedef enum ETableOptionType { @@ -200,6 +203,8 @@ SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed); SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd); +SNode* createCompactVgroupsStmt(SAstCreateContext* pCxt, SNode* pDbName, SNodeList* vgidList, SNode* pStart, + SNode* pEnd); SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createAlterTableOptions(SAstCreateContext* pCxt); SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9e45c5f75e..63eb09d509 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -229,6 +229,7 @@ cmd ::= FLUSH DATABASE db_name(A). cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); } cmd ::= S3MIGRATE DATABASE db_name(A). { pCxt->pRootNode = createS3MigrateDatabaseStmt(pCxt, &A); } cmd ::= COMPACT DATABASE db_name(A) start_opt(B) end_opt(C). { pCxt->pRootNode = createCompactStmt(pCxt, &A, B, C); } +cmd ::= COMPACT db_name_cond_opt(A) VGROUPS IN NK_LP integer_list(B) NK_RP start_opt(C) end_opt(D). { pCxt->pRootNode = createCompactVgroupsStmt(pCxt, A, B, C, D); } %type not_exists_opt { bool } %destructor not_exists_opt { } @@ -287,6 +288,12 @@ db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); } db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_ENCRYPT_ALGORITHM, &C); } db_options(A) ::= db_options(B) DNODES NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DNODES, &C); } +db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_INTEGER (C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } +db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } +db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } +db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); } @@ -322,17 +329,33 @@ alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; } alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; } alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; } +alter_db_option(A) ::= COMPACT_INTERVAL NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } +alter_db_option(A) ::= COMPACT_INTERVAL NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_integer_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } +alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } %type integer_list { SNodeList* } %destructor integer_list { nodesDestroyList($$); } integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); } +%type signed_integer_list { SNodeList* } +%destructor signed_integer_list { nodesDestroyList($$); } +signed_integer_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, B); } +signed_integer_list(A) ::= signed_integer_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, C); } + %type variable_list { SNodeList* } %destructor variable_list { nodesDestroyList($$); } variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); } variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); } +%type signed_variable_list { SNodeList* } +%destructor signed_variable_list { nodesDestroyList($$); } +signed_variable_list(A) ::= signed_variable(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +signed_variable_list(A) ::= signed_variable_list(B) NK_COMMA signed_variable(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } + %type retention_list { SNodeList* } %destructor retention_list { nodesDestroyList($$); } retention_list(A) ::= retention(B). { A = createNodeList(pCxt, B); } @@ -1044,21 +1067,33 @@ literal(A) ::= NK_QUESTION(B). duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } -signed(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } -signed(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } -signed(A) ::= NK_MINUS(B) NK_INTEGER(C). { +signed_variable(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +signed_variable(A) ::= NK_PLUS NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } +signed_variable(A) ::= NK_MINUS(B) NK_VARIABLE(C). { + SToken t = B; + t.n = (C.z + C.n) - B.z; + A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &t)); + } + +signed_integer(A) ::= NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } +signed_integer(A) ::= NK_PLUS NK_INTEGER(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B); } +signed_integer(A) ::= NK_MINUS(B) NK_INTEGER(C). { SToken t = B; t.n = (C.z + C.n) - B.z; A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } -signed(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } -signed(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } -signed(A) ::= NK_MINUS(B) NK_FLOAT(C). { + +signed_float(A) ::= NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +signed_float(A) ::= NK_PLUS NK_FLOAT(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +signed_float(A) ::= NK_MINUS(B) NK_FLOAT(C). { SToken t = B; t.n = (C.z + C.n) - B.z; A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } +signed(A) ::= signed_integer(B). { A = B; } +signed(A) ::= signed_float(B). { A = B; } + signed_literal(A) ::= signed(B). { A = B; } signed_literal(A) ::= NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } signed_literal(A) ::= NK_BOOL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a42e01b234..a13472620b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1838,6 +1838,10 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; pOptions->dnodeListStr[0] = 0; + pOptions->compactInterval = TSDB_DEFAULT_COMPACT_INTERVAL; + pOptions->compactStartTime = TSDB_DEFAULT_COMPACT_START_TIME; + pOptions->compactEndTime = TSDB_DEFAULT_COMPACT_END_TIME; + pOptions->compactTimeOffset = TSDB_DEFAULT_COMPACT_TIME_OFFSET; return (SNode*)pOptions; _err: return NULL; @@ -1882,6 +1886,10 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->withArbitrator = -1; pOptions->encryptAlgorithm = -1; pOptions->dnodeListStr[0] = 0; + pOptions->compactInterval = -1; + pOptions->compactStartTime = -1; + pOptions->compactEndTime = -1; + pOptions->compactTimeOffset = -1; return (SNode*)pOptions; _err: return NULL; @@ -2029,6 +2037,24 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED } else { COPY_STRING_FORM_STR_TOKEN(pDbOptions->dnodeListStr, (SToken*)pVal); } + break; + case DB_OPTION_COMPACT_INTERVAL: + if (TK_NK_INTEGER == ((SToken*)pVal)->type) { + pDbOptions->compactInterval = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + } else { + pDbOptions->pCompactIntervalNode = (SValueNode*)createDurationValueNode(pCxt, (SToken*)pVal); + } + break; + case DB_OPTION_COMPACT_TIME_RANGE: + pDbOptions->pCompactTimeRangeList = pVal; + break; + case DB_OPTION_COMPACT_TIME_OFFSET: + if (TK_NK_INTEGER == ((SToken*)pVal)->type) { + pDbOptions->compactTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + } else { + pDbOptions->pCompactTimeOffsetNode = (SValueNode*)createDurationValueNode(pCxt, (SToken*)pVal); + } + break; default: break; } @@ -2038,6 +2064,8 @@ _err: return NULL; } + + SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { return setDatabaseOptionImpl(pCxt, pOptions, type, pVal, false); } @@ -2047,6 +2075,7 @@ SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOp switch (pAlterOption->type) { case DB_OPTION_KEEP: case DB_OPTION_RETENTIONS: + case DB_OPTION_COMPACT_TIME_RANGE: return setDatabaseOptionImpl(pCxt, pOptions, pAlterOption->type, pAlterOption->pList, true); default: break; @@ -2153,6 +2182,30 @@ _err: return NULL; } +SNode* createCompactVgroupsStmt(SAstCreateContext* pCxt, SNode* pDbName, SNodeList* vgidList, SNode* pStart, + SNode* pEnd) { + CHECK_PARSER_STATUS(pCxt); + if (NULL == pDbName) { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "database not specified"); + pCxt->errCode = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + CHECK_PARSER_STATUS(pCxt); + } + SCompactVgroupsStmt* pStmt = NULL; + pCxt->errCode = nodesMakeNode(QUERY_NODE_COMPACT_VGROUPS_STMT, (SNode**)&pStmt); + CHECK_MAKE_NODE(pStmt); + pStmt->pDbName = pDbName; + pStmt->vgidList = vgidList; + pStmt->pStart = pStart; + pStmt->pEnd = pEnd; + return (SNode*)pStmt; +_err: + nodesDestroyNode(pDbName); + nodesDestroyList(vgidList); + nodesDestroyNode(pStart); + nodesDestroyNode(pEnd); + return NULL; +} + SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); STableOptions* pOptions = NULL; diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index e122c922ba..1687916cb0 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -184,7 +184,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c } if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES) || - 0 == strcmp(pTable, TSDB_INS_TABLE_COLS) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE)) && + 0 == strcmp(pTable, TSDB_INS_TABLE_COLS) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE) || + 0 == strcmp(pTable, TSDB_INS_TABLE_FILESETS)) && QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { code = collectMetaKeyFromInsTags(pCxt); } @@ -641,6 +642,20 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* return code; } +static int32_t collectMetaKeyFromShowFilesets(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_FILESETS, + pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); + } + if (TSDB_CODE_SUCCESS == code) { + code = + reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal, + NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); + } + return code; +} + static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS, pCxt->pMetaCache); @@ -859,6 +874,10 @@ static int32_t collectMetaKeyFromCompactDatabase(SCollectMetaKeyCxt* pCxt, SComp return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } +static int32_t collectMetaKeyFromCompactVgroups(SCollectMetaKeyCxt* pCxt, SCompactVgroupsStmt* pStmt) { + return reserveDbCfgInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromGrant(SCollectMetaKeyCxt* pCxt, SGrantStmt* pStmt) { if ('\0' == pStmt->tabName[0]) { return TSDB_CODE_SUCCESS; @@ -999,6 +1018,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromDescribe(pCxt, (SDescribeStmt*)pStmt); case QUERY_NODE_COMPACT_DATABASE_STMT: return collectMetaKeyFromCompactDatabase(pCxt, (SCompactDatabaseStmt*)pStmt); + case QUERY_NODE_COMPACT_VGROUPS_STMT: + return collectMetaKeyFromCompactVgroups(pCxt, (SCompactVgroupsStmt*)pStmt); case QUERY_NODE_CREATE_STREAM_STMT: return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt); case QUERY_NODE_GRANT_STMT: @@ -1037,6 +1058,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowStreams(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TABLES_STMT: return collectMetaKeyFromShowTables(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_FILESETS_STMT: + return collectMetaKeyFromShowFilesets(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TAGS_STMT: return collectMetaKeyFromShowTags(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TABLE_TAGS_STMT: diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index b38e2f05f0..ea2e9d712f 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -72,6 +72,9 @@ static SKeyword keywordTable[] = { {"COMP", TK_COMP}, {"COMPACT", TK_COMPACT}, {"COMPACTS", TK_COMPACTS}, + {"COMPACT_INTERVAL", TK_COMPACT_INTERVAL}, + {"COMPACT_TIME_OFFSET", TK_COMPACT_TIME_OFFSET}, + {"COMPACT_TIME_RANGE", TK_COMPACT_TIME_RANGE}, {"CONNECTION", TK_CONNECTION}, {"CONNECTIONS", TK_CONNECTIONS}, {"CONNS", TK_CONNS}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d1675ee1cc..f191080512 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -370,6 +370,13 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = { .numOfShowCols = 1, .pShowCols = {"*"} }, + { + .showType = QUERY_NODE_SHOW_FILESETS_STMT, + .pDbName = TSDB_INFORMATION_SCHEMA_DB, + .pTableName = TSDB_INS_TABLE_FILESETS, + .numOfShowCols = 1, + .pShowCols = {"*"} + }, }; // clang-format on @@ -3809,7 +3816,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); FOREACH(pPartKey, pSelect->pPartitionByList) { if (nodesEqualNode(pPartKey, *pNode)) { - return (pSelect->hasAggFuncs || pSelect->pWindow) ? rewriteExprToGroupKeyFunc(pCxt, pNode) : DEAL_RES_IGNORE_CHILD; + return (pSelect->hasAggFuncs || pSelect->pWindow) ? rewriteExprToGroupKeyFunc(pCxt, pNode) + : DEAL_RES_IGNORE_CHILD; } if ((partionByTbname) && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { @@ -4122,7 +4130,8 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { static bool sysTableFromVnode(const char* pTable) { return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) || - (0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE)); + (0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) || 0 == strcmp(pTable, TSDB_INS_DISK_USAGE) || + (0 == strcmp(pTable, TSDB_INS_TABLE_FILESETS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -4195,7 +4204,8 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, if (TSDB_CODE_SUCCESS == code && ((0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) || 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS) || - (0 == strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE) && !hasUserDbCond))) { + (0 == strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE) && !hasUserDbCond) || + 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_FILESETS))) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs); } @@ -4295,7 +4305,8 @@ static bool isSingleTable(SRealTableNode* pRealTable) { return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE); + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_DISK_USAGE) && + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_FILESETS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); } @@ -5439,7 +5450,8 @@ static int32_t doCheckFillValues(STranslateContext* pCxt, SFillNode* pFill, SNod int32_t fillNo = 0; SNodeListNode* pFillValues = (SNodeListNode*)pFill->pValues; SNode* pProject = NULL; - if (!pFillValues) return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); + if (!pFillValues) + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); FOREACH(pProject, pProjectionList) { if (needFill(pProject)) { if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { @@ -6292,7 +6304,7 @@ static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) { } static EDealRes hasRowTsOriginFuncWalkNode(SNode* pNode, void* ctx) { - bool *hasRowTsOriginFunc = ctx; + bool* hasRowTsOriginFunc = ctx; if (nodeType(pNode) == QUERY_NODE_FUNCTION) { SFunctionNode* pFunc = (SFunctionNode*)pNode; if (fmIsRowTsOriginFunc(pFunc->funcId)) { @@ -6307,10 +6319,12 @@ static int32_t checkInterpForStream(STranslateContext* pCxt, SSelectStmt* pSelec if (pCxt->createStream) { SFillNode* pFill = (SFillNode*)pSelect->pFill; if (pFill->mode == FILL_MODE_NEAR) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "FILL NEAR is not supported by stream"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "FILL NEAR is not supported by stream"); } if (pSelect->pRangeAround) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "RANGE with around is not supported by stream"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "RANGE with around is not supported by stream"); } } return TSDB_CODE_SUCCESS; @@ -6335,24 +6349,25 @@ static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect nodesWalkExprs(pSelect->pProjectionList, hasRowTsOriginFuncWalkNode, &hasRowTsOriginFunc); if (hasRowTsOriginFunc && pCxt->createStream) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "_irowts_origin is not supported by stream"); + "_irowts_origin is not supported by stream"); } if (TSDB_CODE_SUCCESS == code) { SFillNode* pFill = (SFillNode*)pSelect->pFill; if (pSelect->pRangeAround) { if (pFill->mode != FILL_MODE_PREV && pFill->mode != FILL_MODE_NEXT && pFill->mode != FILL_MODE_NEAR) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, - "Range with interval can only used with fill PREV/NEXT/NEAR"); + "Range with interval can only used with fill PREV/NEXT/NEAR"); } - if (TSDB_CODE_SUCCESS == code) - code = doCheckFillValues(pCxt, pFill, pSelect->pProjectionList); + if (TSDB_CODE_SUCCESS == code) code = doCheckFillValues(pCxt, pFill, pSelect->pProjectionList); } else { if (FILL_MODE_PREV == pFill->mode || FILL_MODE_NEXT == pFill->mode || FILL_MODE_NEAR == pFill->mode) { if (pFill->pValues) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Can't specify fill values"); } } else { - if (hasRowTsOriginFunc) return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "_irowts_origin can only be used with FILL PREV/NEXT/NEAR"); + if (hasRowTsOriginFunc) + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, + "_irowts_origin can only be used with FILL PREV/NEXT/NEAR"); } code = checkFillValues(pCxt, pFill, pSelect->pProjectionList); } @@ -6370,7 +6385,8 @@ static int32_t translateInterpAround(STranslateContext* pCxt, SSelectStmt* pSele if (nodeType(pAround->pInterval) == QUERY_NODE_VALUE) { SValueNode* pVal = (SValueNode*)pAround->pInterval; if (pVal->datum.i == 0) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, "Range interval cannot be 0"); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, + "Range interval cannot be 0"); } int8_t unit = pVal->unit; if (unit == TIME_UNIT_YEAR || unit == TIME_UNIT_MONTH) { @@ -6423,7 +6439,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { // single point interp every can be omitted } else { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, - "Missing RANGE clause, EVERY clause or FILL clause"); + "Missing RANGE clause, EVERY clause or FILL clause"); } } } else { @@ -6432,8 +6448,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { "Range clause with around interval can't be used with EVERY clause"); } if (!pSelect->pFill) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, - "Missing FILL clause"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, "Missing FILL clause"); } } } @@ -7755,6 +7770,12 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->encryptAlgorithm = pStmt->pOptions->encryptAlgorithm; tstrncpy(pReq->dnodeListStr, pStmt->pOptions->dnodeListStr, TSDB_DNODE_LIST_LEN); + // auto-compact options + pReq->compactInterval = pStmt->pOptions->compactInterval; + pReq->compactStartTime = pStmt->pOptions->compactStartTime; + pReq->compactEndTime = pStmt->pOptions->compactEndTime; + pReq->compactTimeOffset = pStmt->pOptions->compactTimeOffset; + return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); } @@ -7768,8 +7789,8 @@ static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const cha return TSDB_CODE_SUCCESS; } -static int32_t checkDbRangeOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t minVal, - int32_t maxVal) { +static int32_t checkDbRangeOption(STranslateContext* pCxt, const char* pName, int64_t val, int64_t minVal, + int64_t maxVal) { return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal, true); } @@ -8081,6 +8102,16 @@ static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbP return TSDB_CODE_SUCCESS; } +static FORCE_INLINE int32_t translateGetDbCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo** ppDbCfg) { + if (*ppDbCfg) { + return TSDB_CODE_SUCCESS; + } + if (!(*ppDbCfg = taosMemoryCalloc(1, sizeof(SDbCfgInfo)))) { + return terrno; + } + return getDBCfg(pCxt, pDbName, *ppDbCfg); +} + static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t daysPerFile = pOptions->daysPerFile; int32_t s3KeepLocal = pOptions->s3KeepLocal; @@ -8088,13 +8119,9 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa if (-1 == daysPerFile && -1 == daysToKeep0) { return TSDB_CODE_SUCCESS; } else if (-1 == daysPerFile || -1 == daysToKeep0) { - SDbCfgInfo dbCfg = {0}; - int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - daysPerFile = (-1 == daysPerFile ? dbCfg.daysPerFile : daysPerFile); - daysToKeep0 = (-1 == daysToKeep0 ? dbCfg.daysToKeep0 : daysToKeep0); + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); + daysPerFile = (-1 == daysPerFile ? pOptions->pDbCfg->daysPerFile : daysPerFile); + daysToKeep0 = (-1 == daysToKeep0 ? pOptions->pDbCfg->daysToKeep0 : daysToKeep0); } if (daysPerFile > daysToKeep0 / 3) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, @@ -8118,6 +8145,126 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa return TSDB_CODE_SUCCESS; } +static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { + int32_t code = 0; + int64_t interval = 0; + int32_t keep2 = pOptions->keep[2]; + + if (NULL != pOptions->pCompactIntervalNode) { + if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pCompactIntervalNode)) { + return pCxt->errCode; + } + if (TIME_UNIT_MINUTE != pOptions->pCompactIntervalNode->unit && + TIME_UNIT_HOUR != pOptions->pCompactIntervalNode->unit && + TIME_UNIT_DAY != pOptions->pCompactIntervalNode->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_interval unit: %c, only %c, %c, %c allowed", + pOptions->pCompactIntervalNode->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, + TIME_UNIT_DAY); + } + interval = getBigintFromValueNode(pOptions->pCompactIntervalNode); + if (interval != 0) { + if (keep2 == -1) { // alter db + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); + keep2 = pOptions->pDbCfg->daysToKeep2; + } + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); + } + } else if (pOptions->compactInterval > 0) { + interval = pOptions->compactInterval * 1440; // convert to minutes + if (keep2 == -1) { // alter db + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); + keep2 = pOptions->pDbCfg->daysToKeep2; + } + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); + } + if (code == 0) pOptions->compactInterval = interval; + return code; +} + +static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { + if (NULL == pOptions->pCompactTimeRangeList) { + return TSDB_CODE_SUCCESS; + } + + if (LIST_LENGTH(pOptions->pCompactTimeRangeList) != 2) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range, should have 2 values"); + } + + SValueNode* pStart = (SValueNode*)nodesListGetNode(pOptions->pCompactTimeRangeList, 0); + SValueNode* pEnd = (SValueNode*)nodesListGetNode(pOptions->pCompactTimeRangeList, 1); + if (DEAL_RES_ERROR == translateValue(pCxt, pStart)) { + return pCxt->errCode; + } + if (DEAL_RES_ERROR == translateValue(pCxt, pEnd)) { + return pCxt->errCode; + } + if (IS_DURATION_VAL(pStart->flag)) { + if (TIME_UNIT_MINUTE != pStart->unit && TIME_UNIT_HOUR != pStart->unit && TIME_UNIT_DAY != pStart->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range start unit: %c, only %c, %c, %c allowed", + pStart->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY); + } + } else { + pStart->datum.i *= 1440; + } + if (IS_DURATION_VAL(pEnd->flag)) { + if (TIME_UNIT_MINUTE != pEnd->unit && TIME_UNIT_HOUR != pEnd->unit && TIME_UNIT_DAY != pEnd->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range end unit: %c, only %c, %c, %c allowed", + pEnd->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY); + } + } else { + pEnd->datum.i *= 1440; + } + pOptions->compactStartTime = getBigintFromValueNode(pStart); + pOptions->compactEndTime = getBigintFromValueNode(pEnd); + + if (pOptions->compactStartTime >= pOptions->compactEndTime) { + return generateSyntaxErrMsgExt( + &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range: %dm,%dm, start time should be less than end time", + pOptions->compactStartTime, pOptions->compactEndTime); + } + + int32_t keep2 = pOptions->keep[2]; + int32_t days = pOptions->daysPerFile; + if (keep2 == -1 || days == -1) { // alter db + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); + keep2 = pOptions->pDbCfg->daysToKeep2; + days = pOptions->pDbCfg->daysPerFile; + } + if (pOptions->compactStartTime < -keep2 || pOptions->compactStartTime > -days) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range: %dm, start_time should be in range: [%dm, %dm]", + pOptions->compactStartTime, -keep2, -days); + } + if (pOptions->compactEndTime < -keep2 || pOptions->compactEndTime > -days) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_range: %dm, end time should be in range: [%dm, %dm]", + pOptions->compactEndTime, -keep2, -days); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t checkDbCompactTimeOffsetOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { + if (pOptions->pCompactTimeOffsetNode) { + if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pCompactTimeOffsetNode)) { + return pCxt->errCode; + } + if (TIME_UNIT_HOUR != pOptions->pCompactTimeOffsetNode->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option compact_time_offset unit: %c, only %c allowed", + pOptions->pCompactTimeOffsetNode->unit, TIME_UNIT_HOUR); + } + pOptions->compactTimeOffset = getBigintFromValueNode(pOptions->pCompactTimeOffsetNode) / 60; + } + return checkDbRangeOption(pCxt, "compact_time_offset", pOptions->compactTimeOffset, TSDB_MIN_COMPACT_TIME_OFFSET, + TSDB_MAX_COMPACT_TIME_OFFSET); +} + static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t code = checkDbRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); @@ -8234,6 +8381,15 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT); } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbCompactIntervalOption(pCxt, pDbName, pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbCompactTimeRangeOption(pCxt, pDbName, pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbCompactTimeOffsetOption(pCxt, pOptions); + } return code; } @@ -8470,14 +8626,17 @@ static int32_t buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStm pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal; pReq->s3Compact = pStmt->pOptions->s3Compact; pReq->withArbitrator = pStmt->pOptions->withArbitrator; + pReq->compactInterval = pStmt->pOptions->compactInterval; + pReq->compactStartTime = pStmt->pOptions->compactStartTime; + pReq->compactEndTime = pStmt->pOptions->compactEndTime; + pReq->compactTimeOffset = pStmt->pOptions->compactTimeOffset; return code; } static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { if (pStmt->pOptions->walLevel == 0) { - SDbCfgInfo dbCfg = {0}; - int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); - if (TSDB_CODE_SUCCESS == code && dbCfg.replications > 1) { + TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pStmt->dbName, &pStmt->pOptions->pDbCfg)); + if (pStmt->pOptions->pDbCfg->replications > 1) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option, wal_level 0 should be used with replica 1"); } @@ -9913,6 +10072,9 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt return code; } +#define MIN_MAX_COMPACT_TASKS 1 +#define MAX_MAX_COMPACT_TASKS 100 + static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) { SMCfgDnodeReq cfgReq = {0}; cfgReq.dnodeId = pStmt->dnodeId; @@ -9920,7 +10082,12 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt tstrncpy(cfgReq.value, pStmt->value, TSDB_DNODE_VALUE_LEN); int32_t code = 0; - if (0 == strncasecmp(cfgReq.config, "encrypt_key", 12)) { + + const char* validConfigs[] = { + "encrypt_key", + tsAlterCompactTaskKeywords, + }; + if (0 == strncasecmp(cfgReq.config, validConfigs[0], strlen(validConfigs[0]) + 1)) { int32_t klen = strlen(cfgReq.value); if (klen > ENCRYPT_KEY_LEN || klen < ENCRYPT_KEY_LEN_MIN) { tFreeSMCfgDnodeReq(&cfgReq); @@ -9929,6 +10096,28 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN); } code = buildCmdMsg(pCxt, TDMT_MND_CREATE_ENCRYPT_KEY, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); + } else if (0 == strncasecmp(cfgReq.config, validConfigs[1], strlen(validConfigs[1]) + 1)) { + char* endptr = NULL; + int32_t maxCompactTasks = taosStr2Int32(cfgReq.value, &endptr, 10); + int32_t minMaxCompactTasks = MIN_MAX_COMPACT_TASKS; + int32_t maxMaxCompactTasks = MAX_MAX_COMPACT_TASKS; + + // check format + if (endptr == cfgReq.value || endptr[0] != '\0') { + tFreeSMCfgDnodeReq(&cfgReq); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, + "Invalid max compact tasks: %s", cfgReq.value); + } + + // check range + if (maxCompactTasks < minMaxCompactTasks || maxCompactTasks > maxMaxCompactTasks) { + tFreeSMCfgDnodeReq(&cfgReq); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, + "Invalid max compact tasks: %d, valid range [%d,%d]", maxCompactTasks, + minMaxCompactTasks, maxMaxCompactTasks); + } + + code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); } else { code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); } @@ -10620,27 +10809,28 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) return code; } -static int32_t translateCompactRange(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt, SCompactDbReq* pReq) { +static int32_t translateCompactRange(STranslateContext* pCxt, const char* dbName, SNode* pStart, SNode* pEnd, + STimeWindow* timeRange) { SDbCfgInfo dbCfg = {0}; - int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); - if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pStart) { - ((SValueNode*)pStmt->pStart)->node.resType.precision = dbCfg.precision; - ((SValueNode*)pStmt->pStart)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; - code = doTranslateValue(pCxt, (SValueNode*)pStmt->pStart); + int32_t code = getDBCfg(pCxt, dbName, &dbCfg); + if (TSDB_CODE_SUCCESS == code && NULL != pStart) { + ((SValueNode*)pStart)->node.resType.precision = dbCfg.precision; + ((SValueNode*)pStart)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + code = doTranslateValue(pCxt, (SValueNode*)pStart); } - if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pEnd) { - ((SValueNode*)pStmt->pEnd)->node.resType.precision = dbCfg.precision; - ((SValueNode*)pStmt->pEnd)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; - code = doTranslateValue(pCxt, (SValueNode*)pStmt->pEnd); + if (TSDB_CODE_SUCCESS == code && NULL != pEnd) { + ((SValueNode*)pEnd)->node.resType.precision = dbCfg.precision; + ((SValueNode*)pEnd)->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + code = doTranslateValue(pCxt, (SValueNode*)pEnd); } if (TSDB_CODE_SUCCESS == code) { - pReq->timeRange.skey = NULL != pStmt->pStart ? ((SValueNode*)pStmt->pStart)->datum.i : INT64_MIN; - pReq->timeRange.ekey = NULL != pStmt->pEnd ? ((SValueNode*)pStmt->pEnd)->datum.i : INT64_MAX; + timeRange->skey = NULL != pStart ? ((SValueNode*)pStart)->datum.i : INT64_MIN; + timeRange->ekey = NULL != pEnd ? ((SValueNode*)pEnd)->datum.i : INT64_MAX; } return code; } -static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt) { +static int32_t translateCompactDb(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt) { SCompactDbReq compactReq = {0}; SName name; int32_t code = TSDB_CODE_SUCCESS; @@ -10648,7 +10838,7 @@ static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* p if (TSDB_CODE_SUCCESS != code) return code; (void)tNameGetFullDbName(&name, compactReq.db); - code = translateCompactRange(pCxt, pStmt, &compactReq); + code = translateCompactRange(pCxt, pStmt->dbName, pStmt->pStart, pStmt->pEnd, &compactReq.timeRange); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &compactReq); } @@ -10656,6 +10846,85 @@ static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* p return code; } +static int32_t translateVgroupList(STranslateContext* pCxt, SNodeList* vgroupList, SArray** ppVgroups) { + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pHash = NULL; + int32_t numOfVgroups = LIST_LENGTH(vgroupList); + + (*ppVgroups) = taosArrayInit(numOfVgroups, sizeof(int64_t)); + if (NULL == *ppVgroups) { + return terrno; + } + + pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (NULL == pHash) { + code = terrno; + } + + if (TSDB_CODE_SUCCESS == code) { + SNode* pNode = NULL; + FOREACH(pNode, vgroupList) { + SValueNode* pVal = (SValueNode*)pNode; + if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { + code = TSDB_CODE_VND_INVALID_VGROUP_ID; + break; + } + + int64_t vgroupId = getBigintFromValueNode(pVal); + + if (NULL != taosHashGet(pHash, &vgroupId, sizeof(vgroupId))) { + code = TSDB_CODE_PAR_INVALID_VGID_LIST; + break; + } + + code = taosHashPut(pHash, &vgroupId, sizeof(vgroupId), NULL, 0); + if (code) { + break; + } + + if (NULL == taosArrayPush(*ppVgroups, &vgroupId)) { + code = terrno; + break; + } + } + } + + taosHashCleanup(pHash); + if (code) { + taosArrayDestroy(*ppVgroups); + *ppVgroups = NULL; + } + return code; +} + +static int32_t translateCompactVgroups(STranslateContext* pCxt, SCompactVgroupsStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + SName name; + SCompactDbReq req = {0}; + + code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, + strlen(((SValueNode*)pStmt->pDbName)->literal)); + if (TSDB_CODE_SUCCESS == code) { + (void)tNameGetFullDbName(&name, req.db); + } + + if (TSDB_CODE_SUCCESS == code) { + code = + translateCompactRange(pCxt, ((SValueNode*)pStmt->pDbName)->literal, pStmt->pStart, pStmt->pEnd, &req.timeRange); + } + + if (TSDB_CODE_SUCCESS == code) { + code = translateVgroupList(pCxt, pStmt->vgidList, &req.vgroupIds); + } + + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &req); + } + + tFreeSCompactDbReq(&req); + return code; +} + static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt) { SKillConnReq killReq = {0}; killReq.connId = pStmt->targetId; @@ -13079,7 +13348,10 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { code = translateDescribe(pCxt, (SDescribeStmt*)pNode); break; case QUERY_NODE_COMPACT_DATABASE_STMT: - code = translateCompact(pCxt, (SCompactDatabaseStmt*)pNode); + code = translateCompactDb(pCxt, (SCompactDatabaseStmt*)pNode); + break; + case QUERY_NODE_COMPACT_VGROUPS_STMT: + code = translateCompactVgroups(pCxt, (SCompactVgroupsStmt*)pNode); break; case QUERY_NODE_ALTER_CLUSTER_STMT: code = translateAlterCluster(pCxt, (SAlterClusterStmt*)pNode); @@ -13406,6 +13678,7 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS case QUERY_NODE_SHOW_VARIABLES_STMT: return extractShowVariablesResultSchema(numOfCols, pSchema); case QUERY_NODE_COMPACT_DATABASE_STMT: + case QUERY_NODE_COMPACT_VGROUPS_STMT: return extractCompactDbResultSchema(numOfCols, pSchema); default: break; @@ -16800,6 +17073,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { break; case QUERY_NODE_SHOW_VARIABLES_STMT: case QUERY_NODE_COMPACT_DATABASE_STMT: + case QUERY_NODE_COMPACT_VGROUPS_STMT: pQuery->haveResultSet = true; pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 00d897cf8b..11cf926081 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3524,9 +3524,10 @@ static void eliminateProjPushdownProjIdx(SNodeList* pParentProjects, SNodeList* } } -static int32_t eliminateProjOptFindProjPrefixWithOrderCheck(SProjectLogicNode* pProj, SProjectLogicNode* pChild, SNodeList** pNewChildTargets, bool *orderMatch) { +static int32_t eliminateProjOptFindProjPrefixWithOrderCheck(SProjectLogicNode* pProj, SProjectLogicNode* pChild, + SNodeList** pNewChildTargets, bool* orderMatch) { int32_t code = 0; - SNode* pProjection = NULL, *pChildTarget = NULL; + SNode * pProjection = NULL, *pChildTarget = NULL; *orderMatch = true; FORBOTH(pProjection, pProj->pProjections, pChildTarget, pChild->node.pTargets) { if (!pProjection) break; @@ -3557,7 +3558,7 @@ static int32_t eliminateProjOptPushTargetsToSetOpChildren(SProjectLogicNode* pSe FOREACH(pChildProj, pSetOp->node.pChildren) { if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChildProj)) { SProjectLogicNode* pChildLogic = (SProjectLogicNode*)pChildProj; - SNodeList* pNewChildTargetsForChild = NULL; + SNodeList* pNewChildTargetsForChild = NULL; code = eliminateProjOptFindProjPrefixWithOrderCheck(pSetOp, pChildLogic, &pNewChildTargetsForChild, &orderMatch); if (TSDB_CODE_SUCCESS != code) break; nodesDestroyList(pChildLogic->node.pTargets); @@ -3584,8 +3585,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* if (NULL == pProjectNode->node.pParent) { SNodeList* pNewChildTargets = NULL; SNode * pProjection = NULL, *pChildTarget = NULL; - isSetOpProj = - QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj; + isSetOpProj = QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj; if (isSetOpProj) { // For sql: select ... from (select ... union all select ...); // When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and @@ -3649,8 +3649,10 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* // if pChild is a project logic node, remove its projection which is not reference by its target. if (needReplaceTargets) { alignProjectionWithTarget(pChild); - // Since we have eliminated the outer proj, we need to push down the new targets to the children of the set operation. - if (isSetOpProj && orderMatch && !sizeMatch) code = eliminateProjOptPushTargetsToSetOpChildren((SProjectLogicNode*)pChild); + // Since we have eliminated the outer proj, we need to push down the new targets to the children of the set + // operation. + if (isSetOpProj && orderMatch && !sizeMatch) + code = eliminateProjOptPushTargetsToSetOpChildren((SProjectLogicNode*)pChild); } } pCxt->optimized = true; @@ -4326,7 +4328,8 @@ static EDealRes lastRowScanOptSetColDataType(SNode* pNode, void* pContext) { return lastRowScanOptGetColAndSetDataType(pNode, pContext, true); } -static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, SNodeList* pLastRowCols, bool erase, int32_t pkBytes) { +static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, SNodeList* pLastRowCols, bool erase, + int32_t pkBytes) { SNode* pTarget = NULL; WHERE_EACH(pTarget, pTargets) { bool found = false; @@ -5442,7 +5445,8 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit static bool tbCntScanOptIsEligibleScan(STbCntScanOptInfo* pInfo) { if (0 != strcmp(pInfo->pScan->tableName.dbname, TSDB_INFORMATION_SCHEMA_DB) || 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_TABLES) || NULL != pInfo->pScan->pGroupTags || - 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_DISK_USAGE)) { + 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_DISK_USAGE) || + 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_FILESETS)) { return false; } if (1 == pInfo->pScan->pVgroupList->numOfVgroups && MNODE_HANDLE == pInfo->pScan->pVgroupList->vgroups[0].vgId) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0540020d85..c60024b323 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -811,7 +811,8 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) || 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_COLS) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_DISK_USAGE)) { + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_DISK_USAGE) || + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_FILESETS)) { if (pScanLogicNode->pVgroupList) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 3a4e3eef89..53b6a38b35 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -127,6 +127,67 @@ static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* p return tmsgSendReq(pEpset, &msg); } +static int32_t streamTaskGetMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask* pTask = NULL; + + int32_t code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); + if (code != 0) { + continue; + } + + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + epsetAssign(pEpSet, &pTask->info.mnodeEpset); + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_FAILED; +} + +static void streamTaskUpdateMndEpset(SStreamMeta* pMeta, SEpSet* pEpSet) { + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask* pTask = NULL; + + int32_t code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); + if (code != 0) { + stError("vgId:%d s-task:0x%x failed to acquire it for updating mnode epset, code:%s", pMeta->vgId, pId->taskId, + tstrerror(code)); + continue; + } + + // ignore this error since it is only for log file + char buf[256] = {0}; + int32_t ret = epsetToStr(&pTask->info.mnodeEpset, buf, tListLen(buf)); + if (ret != 0) { // print error and continue + stError("failed to convert epset to str, code:%s", tstrerror(ret)); + } + + char newBuf[256] = {0}; + ret = epsetToStr(pEpSet, newBuf, tListLen(newBuf)); + if (ret != 0) { + stError("failed to convert epset to str, code:%s", tstrerror(ret)); + } + + epsetAssign(&pTask->info.mnodeEpset, pEpSet); + stInfo("s-task:0x%x update mnd epset, from %s to %s", pId->taskId, buf, newBuf); + streamMetaReleaseTask(pMeta, pTask); + } + + stDebug("vgId:%d update mnd epset for %d tasks completed", pMeta->vgId, numOfTasks); +} + // NOTE: this task should be executed within the SStreamMeta lock region. int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { SEpSet epset = {0}; @@ -140,24 +201,11 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { stDebug("vgId:%d hbMsg rsp not recv, send current hbMsg, msgId:%d, total:%d again", pMeta->vgId, pInfo->hbMsg.msgId, pInfo->hbCount); - for(int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); - STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; - SStreamTask* pTask = NULL; - - code = streamMetaAcquireTaskUnsafe(pMeta, &id, &pTask); - if (code != 0) { - continue; - } - - if (pTask->info.fillHistory == 1) { - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - epsetAssign(&epset, &pTask->info.mnodeEpset); - streamMetaReleaseTask(pMeta, pTask); - break; + code = streamTaskGetMndEpset(pMeta, &epset); + if (code != 0) { + stError("vgId:%d failed to get the mnode epset, not retrying sending hbMsg, msgId:%d", pMeta->vgId, + pInfo->hbMsg.msgId); + return code; } pInfo->msgSendTs = taosGetTimestampMs(); @@ -384,9 +432,11 @@ void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSe } int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) { - stDebug("vgId:%d process hbMsg rsp, msgId:%d rsp confirmed", pMeta->vgId, pRsp->msgId); SMetaHbInfo* pInfo = pMeta->pHbInfo; + SEpSet epset = {0}; + int32_t code = 0; + stDebug("vgId:%d process hbMsg rsp, msgId:%d rsp confirmed", pMeta->vgId, pRsp->msgId); streamMetaWLock(pMeta); // current waiting rsp recved @@ -396,6 +446,13 @@ int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp) { pInfo->hbCount += 1; pInfo->msgSendTs = -1; + + code = streamTaskGetMndEpset(pMeta, &epset); + if (!isEpsetEqual(&pRsp->mndEpset, &epset) && (code == 0)) { + // we need to update the mnode epset for each tasks + stInfo("vgId:%d mnode epset updated, update mnode epset for all tasks", pMeta->vgId); + streamTaskUpdateMndEpset(pMeta, &pRsp->mndEpset); + } } else { stWarn("vgId:%d recv expired hb rsp, msgId:%d, discarded", pMeta->vgId, pRsp->msgId); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b9e2d3435f..fbde104f4e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -692,7 +692,7 @@ int32_t syncGetArbToken(int64_t rid, char* outToken) { memset(outToken, 0, TSDB_ARB_TOKEN_SIZE); (void)taosThreadMutexLock(&pSyncNode->arbTokenMutex); - strncpy(outToken, pSyncNode->arbToken, TSDB_ARB_TOKEN_SIZE); + tstrncpy(outToken, pSyncNode->arbToken, TSDB_ARB_TOKEN_SIZE); (void)taosThreadMutexUnlock(&pSyncNode->arbTokenMutex); syncNodeRelease(pSyncNode); diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index c2f97982f5..ca39bd1a23 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -262,7 +262,7 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { */ // Open the journal char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pTxn->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); @@ -365,7 +365,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); // remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { @@ -595,7 +595,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { } char jTxnFileName[TDB_FILENAME_LEN]; - sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + (void)tsnprintf(jTxnFileName, TDB_FILENAME_LEN, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); @@ -1179,7 +1179,7 @@ int tdbPagerRestoreJournals(SPager *pPager) { int dirLen = strlen(pPager->pEnv->dbName); memcpy(jname, pPager->pEnv->dbName, dirLen); jname[dirLen] = '/'; - sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); + (void)tsnprintf(jname + dirLen + 1, TD_PATH_MAX - dirLen - 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); code = tdbPagerRestore(pPager, jname); if (code) { taosArrayDestroy(pTxnList); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index da5e1f47e9..43a2ff6a23 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -539,7 +539,7 @@ int32_t decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const opts.source = pHead->head.body; opts.result = newBody; opts.unitLen = 16; - TAOS_UNUSED(strncpy((char *)opts.key, cfg->encryptKey, 16)); + tstrncpy((char *)opts.key, cfg->encryptKey, sizeof(opts.key)); int32_t count = CBC_Decrypt(&opts); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 109bb16f51..a2e874aabf 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -1395,14 +1395,14 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { int32_t len = 0; char newValue[1024] = {0}; - strcpy(newValue, value); + tstrncpy(newValue, value, sizeof(newValue)); int32_t count = 1; while (vlen < 1024) { (void)paGetToken(value + vlen + 1 * count, &tmp, &len); if (len == 0) break; tmp[len] = 0; - strcpy(newValue + vlen, tmp); + tstrncpy(newValue + vlen, tmp, sizeof(newValue) - vlen); vlen += len; count++; } diff --git a/source/util/src/tenv.c b/source/util/src/tenv.c index 539687878b..2108f05c5b 100644 --- a/source/util/src/tenv.c +++ b/source/util/src/tenv.c @@ -24,9 +24,6 @@ int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t c char *p = cfgNameStr; if (envNameStr[0] != 'T' || envNameStr[1] != 'A' || envNameStr[2] != 'O' || envNameStr[3] != 'S' || envNameStr[4] != '_') { - // if(p != envNameStr) strncpy(p, envNameStr, cfgNameMaxLen - 1); - // p[cfgNameMaxLen - 1] = '\0'; - // return strlen(cfgNameStr); cfgNameStr[0] = '\0'; return TSDB_CODE_INVALID_PARA; } @@ -57,7 +54,9 @@ int32_t taosEnvToCfg(const char *envStr, char *cfgStr) { if (envStr == NULL || cfgStr == NULL) { return TSDB_CODE_INVALID_PARA; } - if (cfgStr != envStr) strcpy(cfgStr, envStr); + if (cfgStr != envStr) { + tstrncpy(cfgStr, envStr, strlen(envStr) + 1); + } char *p = strchr(cfgStr, '='); if (p != NULL) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a2a94f053d..195cb21618 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -196,7 +196,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_DROPPING, "Object is dropping") // mnode-dnode-part1 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "Dnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "Dnode does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "Vgroup does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "Vgroup does not exist or not in db") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CANT_DROP_LEADER, "Cannot drop mnode which is leader") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of dnodes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_CFG, "Cluster cfg inconsistent") @@ -419,7 +419,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_CHARSET, "charset not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_LOCALE, "locale not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR, "ttlChangeOnWrite not match") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_EN_WHITELIST, "enableWhiteList not match") -TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "Mnode stopped") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_STOPPED, "Mnode stopped") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, "Invalid max compact tasks") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") @@ -748,6 +749,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ANOMALY_WIN_COL, "ANOMALY_WINDOW not TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT, "ANOMALY_WINDOW option should include algo field") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE, "Invalid forecast clause") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR, "Syntax error in regular expression") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VGID_LIST, "Invalid vgid list") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c3d78b7c17..a9eef1bfc9 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,8 +19,8 @@ #include "tconfig.h" #include "tglobal.h" #include "tjson.h" -#include "tutil.h" #include "ttime.h" +#include "tutil.h" #define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) @@ -166,9 +166,9 @@ static int32_t taosStartLog() { } static int32_t getDay(char *buf, int32_t bufSize) { - time_t t; + time_t t; int32_t code = taosTime(&t); - if(code != 0) { + if (code != 0) { return code; } struct tm tmInfo; @@ -204,8 +204,8 @@ int32_t taosInitSlowLog() { getFullPathName(tsLogObj.slowLogName, logFileName); - char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; - char day[TD_TIME_STR_LEN] = {0}; + char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; + char day[TD_TIME_STR_LEN] = {0}; int32_t code = getDay(day, sizeof(day)); if (code != 0) { (void)printf("failed to get day, reason:%s\n", tstrerror(code)); @@ -299,9 +299,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { return 0; } -void taosSetNoNewFile() { - tsLogObj.openInProgress = 1; -} +void taosSetNoNewFile() { tsLogObj.openInProgress = 1; } static void taosStopLog() { if (tsLogObj.logHandle) { @@ -503,7 +501,7 @@ static void taosOpenNewSlowLogFile() { taosWriteLog(tsLogObj.slowHandle); atomic_store_32(&tsLogObj.slowHandle->lock, 0); - char day[TD_TIME_STR_LEN] = {0}; + char day[TD_TIME_STR_LEN] = {0}; int32_t code = getDay(day, sizeof(day)); if (code != 0) { uError("failed to get day, reason:%s", tstrerror(code)); @@ -882,32 +880,6 @@ void taosPrintSlowLog(const char *format, ...) { taosMemoryFree(buffer); } -#if 0 -void taosDumpData(unsigned char *msg, int32_t len) { - if (!osLogSpaceAvailable()) return; - taosUpdateLogNums(DEBUG_DUMP); - - char temp[256] = {0}; - int32_t i, pos = 0, c = 0; - - for (i = 0; i < len; ++i) { - sprintf(temp + pos, "%02x ", msg[i]); - c++; - pos += 3; - if (c >= 16) { - temp[pos++] = '\n'; - TAOS_UNUSED((taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos) <= 0)); - c = 0; - pos = 0; - } - } - - temp[pos++] = '\n'; - - TAOS_UNUSED(taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos)); -} -#endif - static void taosCloseLogByFd(TdFilePtr pFile) { if (pFile != NULL) { taosUnLockLogFile(pFile); diff --git a/source/util/src/tunit.c b/source/util/src/tunit.c index e73045cc89..357e942faa 100644 --- a/source/util/src/tunit.c +++ b/source/util/src/tunit.c @@ -95,33 +95,14 @@ static int32_t parseCfgIntWithUnit(const char* str, int64_t* res) { } int32_t taosStrHumanToInt64(const char* str, int64_t* out) { - int64_t res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) *out = (int64_t)res; return code; } -#ifdef BUILD_NO_CALL -void taosInt64ToHumanStr(int64_t val, char* outStr) { - if (((val >= UNIT_ONE_EXBIBYTE) || (-val >= UNIT_ONE_EXBIBYTE)) && ((val % UNIT_ONE_EXBIBYTE) == 0)) { - sprintf(outStr, "%qdE", (long long)val / UNIT_ONE_EXBIBYTE); - } else if (((val >= UNIT_ONE_PEBIBYTE) || (-val >= UNIT_ONE_PEBIBYTE)) && ((val % UNIT_ONE_PEBIBYTE) == 0)) { - sprintf(outStr, "%qdP", (long long)val / UNIT_ONE_PEBIBYTE); - } else if (((val >= UNIT_ONE_TEBIBYTE) || (-val >= UNIT_ONE_TEBIBYTE)) && ((val % UNIT_ONE_TEBIBYTE) == 0)) { - sprintf(outStr, "%qdT", (long long)val / UNIT_ONE_TEBIBYTE); - } else if (((val >= UNIT_ONE_GIBIBYTE) || (-val >= UNIT_ONE_GIBIBYTE)) && ((val % UNIT_ONE_GIBIBYTE) == 0)) { - sprintf(outStr, "%qdG", (long long)val / UNIT_ONE_GIBIBYTE); - } else if (((val >= UNIT_ONE_MEBIBYTE) || (-val >= UNIT_ONE_MEBIBYTE)) && ((val % UNIT_ONE_MEBIBYTE) == 0)) { - sprintf(outStr, "%qdM", (long long)val / UNIT_ONE_MEBIBYTE); - } else if (((val >= UNIT_ONE_KIBIBYTE) || (-val >= UNIT_ONE_KIBIBYTE)) && ((val % UNIT_ONE_KIBIBYTE) == 0)) { - sprintf(outStr, "%qdK", (long long)val / UNIT_ONE_KIBIBYTE); - } else - sprintf(outStr, "%qd", (long long)val); -} -#endif - int32_t taosStrHumanToInt32(const char* str, int32_t* out) { - int64_t res; + int64_t res; int32_t code = parseCfgIntWithUnit(str, &res); if (code == TSDB_CODE_SUCCESS) { if (res < INT32_MIN || res > INT32_MAX) { @@ -131,16 +112,3 @@ int32_t taosStrHumanToInt32(const char* str, int32_t* out) { } return code; } - -#ifdef BUILD_NO_CALL -void taosInt32ToHumanStr(int32_t val, char* outStr) { - if (((val >= UNIT_ONE_GIBIBYTE) || (-val >= UNIT_ONE_GIBIBYTE)) && ((val % UNIT_ONE_GIBIBYTE) == 0)) { - sprintf(outStr, "%qdG", (long long)val / UNIT_ONE_GIBIBYTE); - } else if (((val >= UNIT_ONE_MEBIBYTE) || (-val >= UNIT_ONE_MEBIBYTE)) && ((val % UNIT_ONE_MEBIBYTE) == 0)) { - sprintf(outStr, "%qdM", (long long)val / UNIT_ONE_MEBIBYTE); - } else if (((val >= UNIT_ONE_KIBIBYTE) || (-val >= UNIT_ONE_KIBIBYTE)) && ((val % UNIT_ONE_KIBIBYTE) == 0)) { - sprintf(outStr, "%qdK", (long long)val / UNIT_ONE_KIBIBYTE); - } else - sprintf(outStr, "%qd", (long long)val); -} -#endif diff --git a/tests/army/whole/checkErrorCode.py b/tests/army/whole/checkErrorCode.py index b24c2ae7b5..b2aa1fce1b 100644 --- a/tests/army/whole/checkErrorCode.py +++ b/tests/army/whole/checkErrorCode.py @@ -39,7 +39,7 @@ ignoreCodes = [ '0x800003A7', '0x800003AA', '0x800003AB', '0x800003AC', '0x800003AD', '0x800003B0', '0x800003B2', '0x800003B4', '0x800003B5', '0x800003BA', '0x800003C0', '0x800003C1', '0x800003D0', '0x800003D8', '0x800003D9', '0x800003E2', '0x800003F4', '0x800003F8', '0x80000412', '0x80000413', '0x80000414', '0x80000415', '0x80000416', '0x80000417', '0x80000418', '0x80000419', '0x80000420', '0x80000421', '0x80000422', '0x80000423', - '0x80000424', '0x80000425', '0x80000426', '0x80000427', '0x80000428', '0x80000429', '0x8000042A', '0x80000430', '0x80000431', '0x80000432', + '0x80000424', '0x80000425', '0x80000426', '0x80000427', '0x80000428', '0x80000429', '0x8000042A', '0x8000042B','0x80000430', '0x80000431', '0x80000432', '0x80000433', '0x80000434', '0x80000435', '0x80000436', '0x80000437', '0x80000438', '0x80000440', '0x80000441', '0x80000442', '0x80000443', '0x80000444', '0x80000445', '0x80000446', '0x80000447', '0x80000485', '0x80000486', '0x800004A0', '0x800004A1', '0x800004B1', '0x800004B2', '0x800004B3', '0x80000504', '0x80000528', '0x80000532', '0x80000533', '0x80000534', '0x80000535', '0x80000536', '0x80000537', '0x80000538', @@ -55,7 +55,7 @@ ignoreCodes = [ '0x80002207', '0x80002406', '0x80002407', '0x80002503', '0x80002506', '0x80002507', '0x8000261B', '0x80002653', '0x80002668', '0x80002669', '0x8000266A', '0x8000266B', '0x8000266C', '0x8000266D', '0x8000266E', '0x8000266F', '0x80002670', '0x80002671', '0x80002672', '0x80002673', '0x80002674', '0x80002675', '0x80002676', '0x80002677', '0x80002678', '0x80002679', '0x8000267A', '0x8000267B', '0x8000267C', '0x8000267D', - '0x8000267E', '0x8000267F', '0x80002680', '0x80002681', '0x80002682', '0x80002683', '0x80002684', '0x80002685', '0x80002703', '0x80002806', + '0x8000267E', '0x8000267F', '0x80002680', '0x80002681', '0x80002682', '0x80002683', '0x80002684', '0x80002685', '0x80002686', '0x80002703', '0x80002806', '0x80002807', '0x80002808', '0x80002809', '0x8000280A', '0x8000280B', '0x8000280C', '0x8000280D', '0x8000280E', '0x8000280F', '0x80002810', '0x80002811', '0x80002812', '0x80002813', '0x80002814', '0x80002815', '0x8000290B', '0x80002920', '0x80003003', '0x80003006', '0x80003106', '0x80003107', '0x80003108', '0x80003109', '0x80003110', '0x80003111', '0x80003112', '0x80003250', '0x80004003', '0x80004004', '0x80004005', diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index 9589b6dc6f..e59121f631 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -25,7 +25,7 @@ class TDTestCase: def run(self): print("running {}".format(__file__)) tdSql.execute("drop database if exists scd") - tdSql.execute("create database if not exists scd") + tdSql.execute("create database if not exists scd compact_interval 0") tdSql.execute('use scd') tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') @@ -35,24 +35,24 @@ class TDTestCase: tdSql.execute("create table tb3 using stb1 tags(3,'3',3.0);") - tdSql.execute('create database scd2 stt_trigger 3;') + tdSql.execute('create database scd2 stt_trigger 3 compact_interval 1;') - tdSql.execute('create database scd4 stt_trigger 13;') + tdSql.execute('create database scd4 stt_trigger 13 compact_interval 12h compact_time_range -60,-10 compact_time_offset 23h;') tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 0d COMPACT_TIME_RANGE 0d,0d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE -3650d,-10d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 12h COMPACT_TIME_RANGE -60d,-10d COMPACT_TIME_OFFSET 23h") self.restartTaosd(1, dbname='scd') @@ -60,16 +60,16 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 0d COMPACT_TIME_RANGE 0d,0d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE -3650d,-10d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 12h COMPACT_TIME_RANGE -60d,-10d COMPACT_TIME_OFFSET 23h") tdSql.execute('drop database scd') diff --git a/tests/develop-test/2-query/table_count_scan.py b/tests/develop-test/2-query/table_count_scan.py index 30672cabf2..9725c36bee 100644 --- a/tests/develop-test/2-query/table_count_scan.py +++ b/tests/develop-test/2-query/table_count_scan.py @@ -63,22 +63,25 @@ class TDTestCase: tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:06\',true,7,7,7,7,7,7,"777","7777",7,7,7,7);') - - #tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') - #tdSql.checkRows(3) - #tdSql.checkData(0, 0, 35) - #tdSql.checkData(0, 1, 'information_schema') - #tdSql.checkData(0, 2, None) - #tdSql.checkData(1, 0, 3) - #tdSql.checkData(1, 1, 'tbl_count') - #tdSql.checkData(1, 2, 'stb1') - #tdSql.checkData(2, 0, 5) - #tdSql.checkData(2, 1, 'performance_schema') - #tdSql.checkData(2, 2, None) - + tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(3) + for i in range(0, 3): + db_name = tdSql.getData(i, 1) + if db_name == 'information_schema': + tdSql.checkData(i, 0, 36) + tdSql.checkData(i, 2, None) + elif db_name == 'performance_schema': + tdSql.checkData(i, 0, 5) + tdSql.checkData(i, 2, None) + elif db_name == 'tbl_count': + tdSql.checkData(i, 0, 3) + tdSql.checkData(i, 2, 'stb1') + else: + raise Exception("unexpected db_name: %s" % db_name) + tdSql.query('select count(1) v,db_name, stable_name from information_schema.ins_tables group by db_name, stable_name order by v desc;') tdSql.checkRows(3) - tdSql.checkData(0, 0, 35) + tdSql.checkData(0, 0, 36) tdSql.checkData(0, 1, 'information_schema') tdSql.checkData(0, 2, None) tdSql.checkData(1, 0, 5) @@ -94,7 +97,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'performance_schema') tdSql.checkData(0, 0, 3) tdSql.checkData(0, 1, 'tbl_count') - tdSql.checkData(2, 0, 35) + tdSql.checkData(2, 0, 36) tdSql.checkData(2, 1, 'information_schema') tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") @@ -107,7 +110,7 @@ class TDTestCase: tdSql.query('select count(*) from information_schema.ins_tables') tdSql.checkRows(1) - tdSql.checkData(0, 0, 43) + tdSql.checkData(0, 0, 44) tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') @@ -190,7 +193,7 @@ class TDTestCase: tdSql.checkData(2, 0, 5) tdSql.checkData(2, 1, 'performance_schema') tdSql.checkData(2, 2, None) - tdSql.checkData(3, 0, 35) + tdSql.checkData(3, 0, 36) tdSql.checkData(3, 1, 'information_schema') tdSql.checkData(3, 2, None) @@ -205,7 +208,7 @@ class TDTestCase: tdSql.checkData(2, 0, 5) tdSql.checkData(2, 1, 'performance_schema') tdSql.checkData(2, 2, None) - tdSql.checkData(3, 0, 35) + tdSql.checkData(3, 0, 36) tdSql.checkData(3, 1, 'information_schema') tdSql.checkData(3, 2, None) @@ -216,7 +219,7 @@ class TDTestCase: tdSql.checkData(0, 1, 'tbl_count') tdSql.checkData(1, 0, 5) tdSql.checkData(1, 1, 'performance_schema') - tdSql.checkData(2, 0, 35) + tdSql.checkData(2, 0, 36) tdSql.checkData(2, 1, 'information_schema') tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") @@ -229,7 +232,7 @@ class TDTestCase: tdSql.query('select count(*) from information_schema.ins_tables') tdSql.checkRows(1) - tdSql.checkData(0, 0, 44) + tdSql.checkData(0, 0, 45) tdSql.execute('drop database tbl_count') diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 562aec88ec..4aedc0991e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -400,6 +400,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim index a9b27f1389..f8c1d6a986 100644 --- a/tests/script/tsim/query/sys_tbname.sim +++ b/tests/script/tsim/query/sys_tbname.sim @@ -58,7 +58,7 @@ endi sql select tbname from information_schema.ins_tables; print $rows $data00 -if $rows != 44 then +if $rows != 45 then return -1 endi if $data00 != @ins_tables@ then diff --git a/tests/script/tsim/query/tableCount.sim b/tests/script/tsim/query/tableCount.sim index a22bdc8b24..49e9126361 100644 --- a/tests/script/tsim/query/tableCount.sim +++ b/tests/script/tsim/query/tableCount.sim @@ -53,7 +53,7 @@ sql select stable_name,count(table_name) from information_schema.ins_tables grou if $rows != 3 then return -1 endi -if $data01 != 41 then +if $data01 != 42 then return -1 endi if $data11 != 10 then @@ -72,7 +72,7 @@ endi if $data11 != 5 then return -1 endi -if $data21 != 35 then +if $data21 != 36 then return -1 endi if $data31 != 5 then @@ -97,7 +97,7 @@ endi if $data42 != 3 then return -1 endi -if $data52 != 35 then +if $data52 != 36 then return -1 endi if $data62 != 5 then diff --git a/tests/system-test/0-others/compact_vgroups.py b/tests/system-test/0-others/compact_vgroups.py new file mode 100644 index 0000000000..18ad36359f --- /dev/null +++ b/tests/system-test/0-others/compact_vgroups.py @@ -0,0 +1,157 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.autogen import * + +import taos + +class TDTestCase: + # Test cases ====================== + def compactVgroupsErrorTest(self): + # invalid sql + sql = "compact vgroups;" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # invalid sql + sql = "compact vgroups in" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # invalid sql + sql = "compact vgroups in ()" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error without using database + sql = "compact vgroups in (2)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error with duplicate vgroup + sql = "compact db1.vgroups in (2, 2)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error with invalid vgroup id + sql = "compact db1.vgroups in (0, -1)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error to compact vgroups not in the same dat + sql = "compact db1.vgroups in (7, 8)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + # error to compact vgroups not in the same database + sql = "compact db1.vgroups in (2, 5, 8)" + tdLog.info(f"expect error SQL: {sql}") + tdSql.error(sql) + + def waitCompactFinish(self): + while True: + sql = 'show compacts' + rows = tdSql.query(sql) + if rows == 0: + break + time.sleep(1) + + def compactVgroupsSqlTest(self): + # make sure there is no compacts + sql = 'show compacts' + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 0) + + # use db1 and compact with db name should be ok + sql = 'use db1' + tdLog.info(f'expect success SQL: {sql}') + tdSql.execute(sql) + + sql = 'compact vgroups in (2)' + tdLog.info(f'expect success SQL: {sql}') + tdSql.execute(sql) + + # check there should be one row in compacts + sql = 'show compacts' + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 1) + + compactId = tdSql.getData(0, 0) + + # query the compact status + sql = f'show compact {compactId}' + tdLog.info(f'expect success SQL: {sql}') + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 1) + tdSql.checkEqual(tdSql.getData(0, 0), compactId) # compact_id + tdSql.checkEqual(tdSql.getData(0, 1), 2) # vgroup_id + + # wait for compact finish + self.waitCompactFinish() + + # start a new compact + sql = 'compact db2.vgroups in (7, 10)' + tdLog.info(f'expect success SQL: {sql}') + tdSql.execute(sql) + + sql = 'show compacts' + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 1) + + compactId = tdSql.getData(0, 0) + sql = f'show compact {compactId}' + tdLog.info(f'expect success SQL: {sql}') + rows = tdSql.query(sql) + tdSql.checkEqual(rows, 2) + tdSql.checkEqual(tdSql.getData(0, 1) in (7, 10), True) + tdSql.checkEqual(tdSql.getData(1, 1) in (7, 10), True) + tdSql.checkEqual(tdSql.getData(0, 1) != tdSql.getData(1, 1), True) + + # wait for compact finish + self.waitCompactFinish() + + + # Test Framework Apis + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to execute {__file__}") + + # init sql + tdSql.init(conn.cursor(), True) + + def run(self): + # create database db1 + sql = "create database db1 vgroups 5" + tdLog.info(sql) + tdSql.execute(sql) + + # create database db2 + sql = "create database db2 vgroups 5" + tdLog.info(sql) + tdSql.execute(sql) + + # error test + self.compactVgroupsErrorTest() + + # success to compact vgroups + self.compactVgroupsSqlTest() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 39a7c96537..af0dd6d949 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -61,7 +61,7 @@ class TDTestCase: self.ins_list = ['ins_dnodes','ins_mnodes','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\ 'ins_indexes','ins_stables','ins_tables','ins_tags','ins_columns','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\ 'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges','ins_views', - 'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usage"] + 'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usagea", "ins_filesets"] self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps'] def insert_data(self,column_dict,tbname,row_num): insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) @@ -222,7 +222,8 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(293, 294)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(303, 304)) + tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(60, len(tdSql.queryResult)) diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index 24ccac5d90..a6cccb8aaa 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -434,7 +434,7 @@ class TDTestCase: tdSql.checkRows(2) sql = "select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, stable_name `TABLE_NAME`, 'TABLE' `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_stables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, table_name `TABLE_NAME`, case when `type`='SYSTEM_TABLE' then 'TABLE' when `type`='NORMAL_TABLE' then 'TABLE' when `type`='CHILD_TABLE' then 'TABLE' else 'UNKNOWN' end `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_tables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, view_name `TABLE_NAME`, 'VIEW' `TABLE_TYPE`, NULL `REMARKS` from information_schema.ins_views" tdSql.query(sql, queryTimes=1) - tdSql.checkRows(48) + tdSql.checkRows(49) def stop(self): tdSql.close()