From a73ad9642a3c4352e6443a60c043532ed8b71858 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 26 Dec 2024 17:24:17 +0800 Subject: [PATCH 01/22] fix(stream): check if the altered tag value table should be added into stream/tq scan table list. --- source/dnode/vnode/src/vnd/vnodeSvr.c | 31 ++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index f8b75eb15c..4eac1cd5c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1415,7 +1415,8 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i SVAlterTbReq vAlterTbReq = {0}; SVAlterTbRsp vAlterTbRsp = {0}; SDecoder dc = {0}; - int32_t rcode = 0; + int32_t code = 0; + int32_t lino = 0; int32_t ret; SEncoder ec = {0}; STableMetaRsp vMetaRsp = {0}; @@ -1431,7 +1432,6 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i if (tDecodeSVAlterTbReq(&dc, &vAlterTbReq) < 0) { vAlterTbRsp.code = TSDB_CODE_INVALID_MSG; tDecoderClear(&dc); - rcode = -1; goto _exit; } @@ -1439,7 +1439,6 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) { vAlterTbRsp.code = terrno; tDecoderClear(&dc); - rcode = -1; goto _exit; } tDecoderClear(&dc); @@ -1449,6 +1448,31 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i vAlterTbRsp.pMeta = &vMetaRsp; } + if (vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL || vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL) { + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, vAlterTbReq.tbName); + if (uid == 0) { + vError("vgId:%d, %s failed at %s:%d since table %s not found", TD_VID(pVnode), __func__, __FILE__, __LINE__, + vAlterTbReq.tbName); + goto _exit; + } + + SArray* tbUids = taosArrayInit(4, sizeof(int64_t)); + void* p = taosArrayPush(tbUids, &uid); + TSDB_CHECK_NULL(p, code, lino, _exit, terrno); + + vDebug("vgId:%d, remove tags value altered table:%s from query table list", TD_VID(pVnode), vAlterTbReq.tbName); + if ((code = tqUpdateTbUidList(pVnode->pTq, tbUids, false)) < 0) { + vError("vgId:%d, failed to remove tbUid list since %s", TD_VID(pVnode), tstrerror(code)); + } + + vDebug("vgId:%d, try to add table:%s in query table list", TD_VID(pVnode), vAlterTbReq.tbName); + if ((code = tqUpdateTbUidList(pVnode->pTq, tbUids, true)) < 0) { + vError("vgId:%d, failed to add tbUid list since %s", TD_VID(pVnode), tstrerror(code)); + } + + taosArrayDestroy(tbUids); + } + _exit: taosArrayDestroy(vAlterTbReq.pMultiTag); tEncodeSize(tEncodeSVAlterTbRsp, &vAlterTbRsp, pRsp->contLen, ret); @@ -1457,6 +1481,7 @@ _exit: if (tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp) != 0) { vError("vgId:%d, failed to encode alter table response", TD_VID(pVnode)); } + tEncoderClear(&ec); if (vMetaRsp.pSchemas) { taosMemoryFree(vMetaRsp.pSchemas); From ba4d47b8b680723748170ac2344058a7c4812d6a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 29 Dec 2024 00:15:58 +0800 Subject: [PATCH 02/22] enh(stream): extend wait time upto 300s --- source/libs/stream/src/streamCheckStatus.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 118cb1cfb6..5250e96a73 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -18,7 +18,7 @@ #include "streamBackendRocksdb.h" #include "streamInt.h" -#define CHECK_NOT_RSP_DURATION 10 * 1000 // 10 sec +#define CHECK_NOT_RSP_DURATION 30 * 1000 // 10 sec static void processDownstreamReadyRsp(SStreamTask* pTask); static void rspMonitorFn(void* param, void* tmrId); @@ -660,7 +660,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { pInfo->timeoutRetryCount += 1; - // timeout more than 100 sec, add into node update list + // timeout more than 300 sec, add into node update list if (pInfo->timeoutRetryCount > 10) { pInfo->timeoutRetryCount = 0; @@ -674,7 +674,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { findCheckRspStatus(pInfo, *pTaskId, &p); if (p != NULL) { code = streamTaskAddIntoNodeUpdateList(pTask, p->vgId); - stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpdate list", + stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 300sec, add into nodeUpdate list", id, vgId, p->taskId, p->vgId); } } From 0c8ac0f2ecd2b202432b4e41b626b5aeebffbf7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 29 Dec 2024 00:42:39 +0800 Subject: [PATCH 03/22] enh(stream): inc the timeout threshold to be 600s. --- source/libs/stream/src/streamCheckStatus.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 5250e96a73..f880526541 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -18,7 +18,7 @@ #include "streamBackendRocksdb.h" #include "streamInt.h" -#define CHECK_NOT_RSP_DURATION 30 * 1000 // 10 sec +#define CHECK_NOT_RSP_DURATION 60 * 1000 // 60 sec static void processDownstreamReadyRsp(SStreamTask* pTask); static void rspMonitorFn(void* param, void* tmrId); @@ -660,7 +660,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { pInfo->timeoutRetryCount += 1; - // timeout more than 300 sec, add into node update list + // timeout more than 600 sec, add into node update list if (pInfo->timeoutRetryCount > 10) { pInfo->timeoutRetryCount = 0; @@ -674,7 +674,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { findCheckRspStatus(pInfo, *pTaskId, &p); if (p != NULL) { code = streamTaskAddIntoNodeUpdateList(pTask, p->vgId); - stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 300sec, add into nodeUpdate list", + stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 600sec, add into nodeUpdate list", id, vgId, p->taskId, p->vgId); } } From b0fa490dd6cf25c126914aa7910a3afbceeb6f03 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 15:13:38 +0800 Subject: [PATCH 04/22] test: add test case for auto compact --- include/libs/nodes/cmdnodes.h | 2 +- source/common/src/systable.c | 3 + source/dnode/mnode/impl/src/mndDb.c | 75 ++++++-- source/libs/parser/inc/sql.y | 24 ++- source/libs/parser/src/parTranslater.c | 99 +++++----- tests/parallel_test/cases.task | 1 + tests/system-test/0-others/compact_auto.py | 170 ++++++++++++++++++ .../0-others/information_schema.py | 2 +- tests/system-test/1-insert/alter_database.py | 4 +- 9 files changed, 303 insertions(+), 77 deletions(-) create mode 100644 tests/system-test/0-others/compact_auto.py diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 8eb30b8184..2d329ec879 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -112,7 +112,7 @@ typedef struct SDatabaseOptions { int8_t s3Compact; int8_t withArbitrator; // for auto-compact - int8_t compactTimeOffset; // hours + int32_t compactTimeOffset; // hours int32_t compactInterval; // minutes int32_t compactStartTime; // minutes int32_t compactEndTime; // minutes diff --git a/source/common/src/systable.c b/source/common/src/systable.c index cb08046399..8bed8d23eb 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -123,6 +123,9 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "encrypt_algorithm", .bytes = TSDB_ENCRYPT_ALGO_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "compact_interval", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "compact_time_range", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "compact_time_offset", .bytes = 4 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema userFuncSchema[] = { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c28fd343f0..1d1f5744d4 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -499,6 +499,20 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return code; if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return code; + if (pCfg->compactInterval != 0 && + (pCfg->compactInterval < TSDB_MIN_COMPACT_INTERVAL || pCfg->compactInterval > pCfg->daysToKeep2)) + return code; + if (pCfg->compactStartTime != 0 && + (pCfg->compactStartTime < -pCfg->daysToKeep2 || pCfg->compactStartTime > -pCfg->daysPerFile)) + return code; + if (pCfg->compactEndTime != 0 && + (pCfg->compactEndTime < -pCfg->daysToKeep2 || pCfg->compactEndTime > -pCfg->daysPerFile)) + return code; + if (pCfg->compactStartTime != 0 && pCfg->compactEndTime != 0 && pCfg->compactStartTime >= pCfg->compactEndTime) + return code; + if (pCfg->compactTimeOffset < TSDB_MIN_COMPACT_TIME_OFFSET || pCfg->compactTimeOffset > TSDB_MAX_COMPACT_TIME_OFFSET) + return code; + code = 0; TAOS_RETURN(code); } @@ -564,6 +578,22 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN if (pNewCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pNewCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return code; if (pNewCfg->s3Compact < TSDB_MIN_S3_COMPACT || pNewCfg->s3Compact > TSDB_MAX_S3_COMPACT) return code; + if (pNewCfg->compactInterval != 0 && + (pNewCfg->compactInterval < TSDB_MIN_COMPACT_INTERVAL || pNewCfg->compactInterval > pNewCfg->daysToKeep2)) + return code; + if (pNewCfg->compactStartTime != 0 && + (pNewCfg->compactStartTime < -pNewCfg->daysToKeep2 || pNewCfg->compactStartTime > -pNewCfg->daysPerFile)) + return code; + if (pNewCfg->compactEndTime != 0 && + (pNewCfg->compactEndTime < -pNewCfg->daysToKeep2 || pNewCfg->compactEndTime > -pNewCfg->daysPerFile)) + return code; + if (pNewCfg->compactStartTime != 0 && pNewCfg->compactEndTime != 0 && + pNewCfg->compactStartTime >= pNewCfg->compactEndTime) + return code; + if (pNewCfg->compactTimeOffset < TSDB_MIN_COMPACT_TIME_OFFSET || + pNewCfg->compactTimeOffset > TSDB_MAX_COMPACT_TIME_OFFSET) + return code; + code = 0; TAOS_RETURN(code); } @@ -1150,20 +1180,24 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { code = 0; } + bool compactTimeRangeChanged = false; if (pAlter->compactStartTime != pDb->cfg.compactStartTime && (pAlter->compactStartTime == TSDB_DEFAULT_COMPACT_START_TIME || pAlter->compactStartTime <= -pDb->cfg.daysPerFile)) { pDb->cfg.compactStartTime = pAlter->compactStartTime; - pDb->vgVersion++; + compactTimeRangeChanged = true; code = 0; } if (pAlter->compactEndTime != pDb->cfg.compactEndTime && (pAlter->compactEndTime == TSDB_DEFAULT_COMPACT_END_TIME || pAlter->compactEndTime <= -pDb->cfg.daysPerFile)) { pDb->cfg.compactEndTime = pAlter->compactEndTime; - pDb->vgVersion++; + compactTimeRangeChanged = true; code = 0; } + if(compactTimeRangeChanged) { + pDb->vgVersion++; + } if (pAlter->compactTimeOffset >= TSDB_MIN_COMPACT_TIME_OFFSET && pAlter->compactTimeOffset != pDb->cfg.compactTimeOffset) { @@ -1408,14 +1442,6 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->compactInterval = pDb->cfg.compactInterval; cfgRsp->compactStartTime = pDb->cfg.compactStartTime; cfgRsp->compactEndTime = pDb->cfg.compactEndTime; - if (cfgRsp->compactInterval > 0) { - if (cfgRsp->compactStartTime == 0) { - cfgRsp->compactStartTime = -cfgRsp->daysToKeep2; - } - if (cfgRsp->compactEndTime == 0) { - cfgRsp->compactEndTime = -cfgRsp->daysPerFile; - } - } cfgRsp->compactTimeOffset = pDb->cfg.compactTimeOffset; } @@ -2432,6 +2458,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)strictVstr, false), &lino, _OVER); + char durationStr[128] = {0}; char durationVstr[128] = {0}; int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE, pDb->cfg.daysPerFile); @@ -2440,10 +2467,10 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER); - char keepVstr[512] = {0}; - char keep0Str[128] = {0}; - char keep1Str[128] = {0}; - char keep2Str[128] = {0}; + char keepVstr[128] = {0}; + char keep0Str[32] = {0}; + char keep1Str[32] = {0}; + char keep2Str[32] = {0}; int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pDb->cfg.daysToKeep0); int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pDb->cfg.daysToKeep1); @@ -2556,6 +2583,26 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, STR_WITH_MAXSIZE_TO_VARSTR(encryptAlgorithmVStr, encryptAlgorithmStr, 24); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)encryptAlgorithmVStr, false), &lino, _OVER); + + TAOS_UNUSED(formatDurationOrKeep(durationStr, sizeof(durationStr), pDb->cfg.compactInterval)); + STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr)); + if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) { + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER); + } + + len = formatDurationOrKeep(durationStr, sizeof(durationStr), pDb->cfg.compactStartTime); + TAOS_UNUSED(formatDurationOrKeep(durationVstr, sizeof(durationVstr), pDb->cfg.compactEndTime)); + TAOS_UNUSED(snprintf(durationStr + len, sizeof(durationStr) - len, ",%s", durationVstr)); + STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr)); + if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) { + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER); + } + + TAOS_UNUSED(snprintf(durationStr, sizeof(durationStr), "%dh", pDb->cfg.compactTimeOffset)); + STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr)); + if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) { + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER); + } } _OVER: if (code != 0) mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code)); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 63eb09d509..f52d85eea7 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -290,8 +290,7 @@ db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). db_options(A) ::= db_options(B) DNODES NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DNODES, &C); } db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_INTEGER (C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } -db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } -db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_duration_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } @@ -331,8 +330,7 @@ alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; } alter_db_option(A) ::= COMPACT_INTERVAL NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } alter_db_option(A) ::= COMPACT_INTERVAL NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } -alter_db_option(A) ::= COMPACT_TIME_RANGE signed_integer_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } -alter_db_option(A) ::= COMPACT_TIME_RANGE signed_variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_duration_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } @@ -341,20 +339,17 @@ alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_VARIABLE(B). integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); } -%type signed_integer_list { SNodeList* } -%destructor signed_integer_list { nodesDestroyList($$); } -signed_integer_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, B); } -signed_integer_list(A) ::= signed_integer_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, C); } - %type variable_list { SNodeList* } %destructor variable_list { nodesDestroyList($$); } variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); } variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); } -%type signed_variable_list { SNodeList* } -%destructor signed_variable_list { nodesDestroyList($$); } -signed_variable_list(A) ::= signed_variable(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } -signed_variable_list(A) ::= signed_variable_list(B) NK_COMMA signed_variable(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } +%type signed_duration_list { SNodeList* } +%destructor signed_duration_list { nodesDestroyList($$); } +signed_duration_list(A) ::= signed_variable(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +signed_duration_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, B); } +signed_duration_list(A) ::= signed_duration_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, C); } +signed_duration_list(A) ::= signed_duration_list(B) NK_COMMA signed_variable(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } %type retention_list { SNodeList* } %destructor retention_list { nodesDestroyList($$); } @@ -594,6 +589,7 @@ cmd ::= SHOW BNODES. cmd ::= SHOW SNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } cmd ::= SHOW CLUSTER. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } cmd ::= SHOW TRANSACTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } +cmd ::= SHOW TRANSACTION NK_INTEGER(A). { pCxt->pRootNode = createShowTransactionDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); } cmd ::= SHOW TABLE DISTRIBUTED full_table_name(A). { pCxt->pRootNode = createShowTableDistributedStmt(pCxt, A); } cmd ::= SHOW CONSUMERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } cmd ::= SHOW SUBSCRIPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } @@ -1709,4 +1705,4 @@ null_ordering_opt(A) ::= NULLS LAST. column_options(A) ::= . { A = createDefaultColumnOptions(pCxt); } column_options(A) ::= column_options(B) PRIMARY KEY. { A = setColumnOptionsPK(pCxt, B); } -column_options(A) ::= column_options(B) NK_ID(C) NK_STRING(D). { A = setColumnOptions(pCxt, B, &C, &D); } +column_options(A) ::= column_options(B) NK_ID(C) NK_STRING(D). { A = setColumnOptions(pCxt, B, &C, &D); } \ No newline at end of file diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9f411c4296..178471f813 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2005,7 +2005,7 @@ static EDealRes translateDurationValue(STranslateContext* pCxt, SValueNode* pVal pVal->datum.i = AUTO_DURATION_VALUE; pVal->unit = getPrecisionUnit(pVal->node.resType.precision); } else if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, - pVal->node.resType.precision, false) != TSDB_CODE_SUCCESS) { + pVal->node.resType.precision, true) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; @@ -7781,23 +7781,23 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS } static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int64_t val, int64_t minVal, - int64_t maxVal, bool skipUndef) { - if (skipUndef ? ((val >= 0) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { + int64_t maxVal, int8_t unit, bool skipUndef) { + if (skipUndef ? ((val >= 0 | val < -2) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, - "Invalid option %s: %" PRId64 ", valid range: [%" PRId64 ", %" PRId64 "]", pName, - val, minVal, maxVal); + "Invalid option %s: %" PRId64 "%c, valid range: [%" PRId64 "%c, %" PRId64 "%c]", + pName, val, unit, minVal, unit, maxVal, unit); } return TSDB_CODE_SUCCESS; } static int32_t checkDbRangeOption(STranslateContext* pCxt, const char* pName, int64_t val, int64_t minVal, - int64_t maxVal) { - return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal, true); + int64_t maxVal, int8_t unit) { + return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal, unit, true); } static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName, int64_t val, int64_t minVal, int64_t maxVal) { - return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal, true); + return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal, 0, true); } static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { @@ -7813,7 +7813,8 @@ static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOption } pOptions->s3KeepLocal = getBigintFromValueNode(pOptions->s3KeepLocalStr); } - return checkDbRangeOption(pCxt, "s3KeepLocal", pOptions->s3KeepLocal, TSDB_MIN_S3_KEEP_LOCAL, TSDB_MAX_S3_KEEP_LOCAL); + return checkDbRangeOption(pCxt, "s3KeepLocal", pOptions->s3KeepLocal, TSDB_MIN_S3_KEEP_LOCAL, TSDB_MAX_S3_KEEP_LOCAL, + 0); } static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { @@ -7829,7 +7830,8 @@ static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOpt } pOptions->daysPerFile = getBigintFromValueNode(pOptions->pDaysPerFile); } - return checkDbRangeOption(pCxt, "daysPerFile", pOptions->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); + return checkDbRangeOption(pCxt, "daysPerFile", pOptions->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE, + 0); } static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { @@ -8148,7 +8150,6 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t code = 0; - int64_t interval = 0; int32_t keep2 = pOptions->keep[2]; if (NULL != pOptions->pCompactIntervalNode) { @@ -8163,23 +8164,26 @@ static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, const char* pOptions->pCompactIntervalNode->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY); } - interval = getBigintFromValueNode(pOptions->pCompactIntervalNode); + int64_t interval = getBigintFromValueNode(pOptions->pCompactIntervalNode); if (interval != 0) { if (keep2 == -1) { // alter db TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); keep2 = pOptions->pDbCfg->daysToKeep2; } - code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2, TIME_UNIT_MINUTE); + TAOS_CHECK_RETURN(code); } + pOptions->compactInterval = (int32_t)interval; } else if (pOptions->compactInterval > 0) { - interval = pOptions->compactInterval * 1440; // convert to minutes - if (keep2 == -1) { // alter db + int64_t interval = (int64_t)pOptions->compactInterval * 1440; // convert to minutes + if (keep2 == -1) { // alter db TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); keep2 = pOptions->pDbCfg->daysToKeep2; } - code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2, TIME_UNIT_MINUTE); + TAOS_CHECK_RETURN(code); + pOptions->compactInterval = (int32_t)interval; } - if (code == 0) pOptions->compactInterval = interval; return code; } @@ -8222,13 +8226,16 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char pOptions->compactStartTime = getBigintFromValueNode(pStart); pOptions->compactEndTime = getBigintFromValueNode(pEnd); + if (pOptions->compactStartTime == 0 && pOptions->compactEndTime == 0) { + return TSDB_CODE_SUCCESS; + } + if (pOptions->compactStartTime >= pOptions->compactEndTime) { return generateSyntaxErrMsgExt( &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option compact_time_range: %dm,%dm, start time should be less than end time", pOptions->compactStartTime, pOptions->compactEndTime); } - int32_t keep2 = pOptions->keep[2]; int32_t days = pOptions->daysPerFile; if (keep2 == -1 || days == -1) { // alter db @@ -8238,7 +8245,7 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char } if (pOptions->compactStartTime < -keep2 || pOptions->compactStartTime > -days) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option compact_time_range: %dm, start_time should be in range: [%dm, %dm]", + "Invalid option compact_time_range: %dm, start time should be in range: [%dm, %dm]", pOptions->compactStartTime, -keep2, -days); } if (pOptions->compactEndTime < -keep2 || pOptions->compactEndTime > -days) { @@ -8246,7 +8253,6 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char "Invalid option compact_time_range: %dm, end time should be in range: [%dm, %dm]", pOptions->compactEndTime, -keep2, -days); } - return TSDB_CODE_SUCCESS; } @@ -8263,12 +8269,12 @@ static int32_t checkDbCompactTimeOffsetOption(STranslateContext* pCxt, SDatabase pOptions->compactTimeOffset = getBigintFromValueNode(pOptions->pCompactTimeOffsetNode) / 60; } return checkDbRangeOption(pCxt, "compact_time_offset", pOptions->compactTimeOffset, TSDB_MIN_COMPACT_TIME_OFFSET, - TSDB_MAX_COMPACT_TIME_OFFSET); + TSDB_MAX_COMPACT_TIME_OFFSET, TIME_UNIT_HOUR); } static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t code = - checkDbRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); + checkDbRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE, 0); if (TSDB_CODE_SUCCESS == code) { code = checkDbCacheModelOption(pCxt, pOptions); } @@ -8276,26 +8282,27 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbEncryptAlgorithmOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = - checkDbRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, TSDB_MAX_DB_CACHE_SIZE); + code = checkDbRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, + TSDB_MAX_DB_CACHE_SIZE, 0); } if (TSDB_CODE_SUCCESS == code) { - code = - checkDbRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); + code = checkDbRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL, + 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbDaysOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "fsyncPeriod", pOptions->fsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); + code = + checkDbRangeOption(pCxt, "fsyncPeriod", pOptions->fsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "maxRowsPerBlock", pOptions->maxRowsPerBlock, TSDB_MIN_MAXROWS_FBLOCK, - TSDB_MAX_MAXROWS_FBLOCK); + TSDB_MAX_MAXROWS_FBLOCK, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "minRowsPerBlock", pOptions->minRowsPerBlock, TSDB_MIN_MINROWS_FBLOCK, - TSDB_MAX_MINROWS_FBLOCK); + TSDB_MAX_MINROWS_FBLOCK, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbPrecisionOption(pCxt, pOptions); @@ -8307,18 +8314,18 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbKeepTimeOffsetOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE); + code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "pagesize", pOptions->pagesize, TSDB_MIN_PAGESIZE_PER_VNODE, - TSDB_MAX_PAGESIZE_PER_VNODE); + TSDB_MAX_PAGESIZE_PER_VNODE, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "tsdbPagesize", pOptions->tsdbPageSize, TSDB_MIN_TSDB_PAGESIZE, - TSDB_MAX_TSDB_PAGESIZE); + TSDB_MAX_TSDB_PAGESIZE, 0); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); + code = checkDbRangeOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbStrictOption(pCxt, pOptions); @@ -8328,7 +8335,8 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName TSDB_MAX_WAL_LEVEL); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "vgroups", pOptions->numOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); + code = + checkDbRangeOption(pCxt, "vgroups", pOptions->numOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "singleStable", pOptions->singleStable, TSDB_DB_SINGLE_STABLE_ON, @@ -8342,21 +8350,22 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "walRetentionPeriod", pOptions->walRetentionPeriod, - TSDB_DB_MIN_WAL_RETENTION_PERIOD, INT32_MAX); + TSDB_DB_MIN_WAL_RETENTION_PERIOD, INT32_MAX, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "walRetentionSize", pOptions->walRetentionSize, TSDB_DB_MIN_WAL_RETENTION_SIZE, - INT32_MAX); - } - if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "walRollPeriod", pOptions->walRollPeriod, TSDB_DB_MIN_WAL_ROLL_PERIOD, INT32_MAX); + INT32_MAX, 0); } if (TSDB_CODE_SUCCESS == code) { code = - checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX); + checkDbRangeOption(pCxt, "walRollPeriod", pOptions->walRollPeriod, TSDB_DB_MIN_WAL_ROLL_PERIOD, INT32_MAX, 0); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER); + code = checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX, + 0); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "withArbitrator", pOptions->withArbitrator, TSDB_MIN_DB_WITH_ARBITRATOR, @@ -9933,7 +9942,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pStmt) { int32_t code = 0; SCreateUserReq createReq = {0}; - if ((code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, false))) { + if ((code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, 0, false))) { return code; } tstrncpy(createReq.user, pStmt->userName, TSDB_USER_LEN); @@ -9962,13 +9971,13 @@ static int32_t checkAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt) { int32_t code = 0; switch (pStmt->alterType) { case TSDB_ALTER_USER_ENABLE: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "enable", pStmt->enable, 0, 1, false); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "enable", pStmt->enable, 0, 1, 0, false); break; case TSDB_ALTER_USER_SYSINFO: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, false); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, 0, false); break; case TSDB_ALTER_USER_CREATEDB: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "createdb", pStmt->createdb, 0, 1, false); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "createdb", pStmt->createdb, 0, 1, 0, false); break; } return code; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ed73a2742c..0f82efc61b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -411,6 +411,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_auto.py ,,n,system-test,python3 ./test.py -f 0-others/dumpsdb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact.py -N 3 diff --git a/tests/system-test/0-others/compact_auto.py b/tests/system-test/0-others/compact_auto.py new file mode 100644 index 0000000000..27c24087c1 --- /dev/null +++ b/tests/system-test/0-others/compact_auto.py @@ -0,0 +1,170 @@ +import taos +import sys +import time +import socket +import os +import threading +import psutil +import platform +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.default_compact_options = [ "0d", "0d,0d", "0h"] + self.compact_options = [["db00", "0m", "-0d,0", "0", "0d", "0d,0d", "0h"], + ["db01", "0m", "-2d,-1", "0", "0d", "-2d,-1d", "0h"], + ["db02", "2880m", "-61d,-1", "0", "2d", "-61d,-1d", "0h"], + ["db03", "48h", "-87840m,-60", "1h", "2d", "-61d,-60d", "1h"], + ["db04", "2d", "-87840m,-1440h", "12", "2d", "-61d,-60d", "12h"], + ["db05", "2", "-61,-1440h", "23h", "2d", "-61d,-60d", "23h"], + ] + + def create_db_compact(self): + tdLog.info("create db compact options") + for item in self.compact_options: + tdSql.execute(f'create database {item[0]} compact_interval {item[1]} compact_time_range {item[2]} compact_time_offset {item[3]} duration 1d') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + tdSql.query(f'show create database {item[0]}') + tdSql.checkEqual(tdSql.queryResult[0][0], item[0]) + tdSql.checkEqual(True, f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}' in tdSql.queryResult[0][1]) + tdSql.execute(f'drop database {item[0]}') + + def checkShowCreateWithTimeout(self, db, expectResult, timeout=30): + result = False + for i in range(timeout): + tdSql.query(f'show create database `%s`' %(db)) + tdSql.checkEqual(tdSql.queryResult[0][0], db) + if expectResult in tdSql.queryResult[0][1]: + result = True + break + time.sleep(1) + if result == False: + raise Exception(f"Unexpected result of 'show create database `{db}`':{tdSql.queryResult[0][1]}, expect:{expectResult}") + + def alter_db_compact(self): + tdLog.info("alter db compact options together") + for item in self.compact_options: + tdSql.execute(f'create database {item[0]} duration 1d') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], self.default_compact_options[0]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[2]) + tdSql.query(f'show create database {item[0]}') + tdSql.checkEqual(tdSql.queryResult[0][0], item[0]) + tdSql.checkEqual(True, f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {self.default_compact_options[2]}' in tdSql.queryResult[0][1]) + tdSql.execute(f'alter database {item[0]} compact_interval {item[1]} compact_time_range {item[2]} compact_time_offset {item[3]}') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + for item in self.compact_options: + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'drop database {item[0]}') + + tdLog.info("alter db compact options separately") + compact_separate_options = [["db100", "0m", "-0d,0", "0", "0d", "0d,0d", "0h"], + ["db101", "10m", "-2d,-1", "1", "10m", "-2d,-1d", "1h"]] + for item in compact_separate_options: + tdSql.execute(f'create database {item[0]} duration 1d') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], self.default_compact_options[0]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[2]) + tdSql.query(f'show create database {item[0]}') + tdSql.checkEqual(tdSql.queryResult[0][0], item[0]) + tdSql.checkEqual(True, f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {self.default_compact_options[2]}' in tdSql.queryResult[0][1]) + tdSql.execute(f'alter database {item[0]} compact_time_offset {item[3]}', queryTimes=10) + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], self.default_compact_options[0]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'alter database {item[0]} compact_interval {item[1]}', queryTimes=10) + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'alter database {item[0]} compact_time_range {item[2]}', queryTimes=10) + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'alter database {item[0]} compact_time_offset {item[3]}', queryTimes=10) + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}') + for item in compact_separate_options: + tdSql.execute(f'drop database {item[0]}', queryTimes=10) + + def compact_error(self): + compact_err_list = [["compact_time_range 86400m,61d", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range 60,61", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range 60d,61d", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -60,-60", "Invalid option compact_time_range: -86400m,-86400m, start time should be less than end time"], + ["compact_time_range -60,-1440h", "Invalid option compact_time_range: -86400m,-86400m, start time should be less than end time"], + ["compact_time_range -60d,-61d", "Invalid option compact_time_range: -86400m,-87840m, start time should be less than end time"], + ["compact_time_range -5256001m,-1", "Invalid option compact_time_range: -5256001m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -199999999999m,-199999999998m", "start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -99999999999999999999m,-99999999999999999998m", "Invalid value type: -99999999999999999999"], + ["compact_time_range -60d,-1", "Invalid option compact_time_range: -1440m, end time should be in range: [-5256000m, -14400m]"], + ["compact_interval 24h compact_time_range -60,61", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_interval 100 compact_time_range -60d,61d", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -60d,87840m", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -100,-90s", "Invalid option compact_time_range end unit: s, only m, h, d allowed"], + ["compact_interval 10m compact_time_range -120d,-14400m compact_time_offset -1", "syntax error near"], + ["compact_time_range -100,-99d compact_interval -1", "syntax error near"], + ["compact_time_range 0", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100,-90,-80", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100;-90", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100:-90", "syntax error near"], + ["compact_time_range -100 -90", "syntax error near"], + ["compact_interval 1m", "Invalid option compact_interval: 1m, valid range: [10m, 5256000m]"], + ["compact_interval 199999999999m", "valid range: [10m, 5256000m]"], + ["compact_interval 9999999999999m", "Invalid option compact_interval: 9999999999999m, valid range: [10m, 5256000m]"], + ["compact_interval 5256001m", "Invalid option compact_interval: 5256001m, valid range: [10m, 5256000m]"], + ["compact_interval 3651", "Invalid option compact_interval: 5257440m, valid range: [10m, 5256000m]"], + ["compact_interval 86400s", "Invalid option compact_interval unit: s, only m, h, d allowed"], + ["compact_interval -1", "syntax error near"], + ["compact_time_offset -1", "syntax error near"], + ["compact_time_offset 3600s", "Invalid option compact_time_offset unit: s, only h allowed"], + ["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ["compact_time_offset 24", "Invalid option compact_time_offset: 24h, valid range: [0h, 23h]"], + ["compact_time_offset 24h", "Invalid option compact_time_offset: 24h, valid range: [0h, 23h]"], + ["compact_time_offset 9999999999999", "valid range: [0h, 23h]"], + ["compact_time_offset 199999999999", "valid range: [0h, 23h]"], + ["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ["compact_interval 10m compact_time_range -120d,-60 compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ] + tdSql.execute('create database if not exists db') + for item in compact_err_list: + tdSql.error(f"create database db {item[0]}", expectErrInfo=item[1], fullMatched=False) + tdSql.error(f"alter database db {item[0]}", expectErrInfo=item[1], fullMatched=False) + tdSql.execute('drop database db') + + def run(self): + self.create_db_compact() + self.alter_db_compact() + self.compact_error() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index af0dd6d949..130085b91d 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(303, 304)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(312, 313)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(60, len(tdSql.queryResult)) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index f58bb7517e..17a0664926 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -81,8 +81,8 @@ class TDTestCase: tdSql.query('select * from information_schema.ins_databases where name = "db"') db_options_items = ["replica","keep","buffer","pages","minrows","cachemodel","cachesize","wal_level","wal_fsync_period", - "wal_retention_period","wal_retention_size","stt_trigger"] - db_options_result_idx = [4,7,8,10,11,18,19,20,21,22,23,24] + "wal_retention_period","wal_retention_size","stt_trigger", "compact_interval", "compact_time_range", "compact_time_offset"] + db_options_result_idx = [4,7,8,10,11,18,19,20,21,22,23,24,34,35,36] self.option_result = [] for idx in db_options_result_idx: From de3558a4a7bb624f3dc3b37a19de9487d832a6d3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 15:14:51 +0800 Subject: [PATCH 05/22] test: add test case for auto compact --- source/libs/parser/inc/sql.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index f52d85eea7..63a6b97bbb 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1705,4 +1705,4 @@ null_ordering_opt(A) ::= NULLS LAST. column_options(A) ::= . { A = createDefaultColumnOptions(pCxt); } column_options(A) ::= column_options(B) PRIMARY KEY. { A = setColumnOptionsPK(pCxt, B); } -column_options(A) ::= column_options(B) NK_ID(C) NK_STRING(D). { A = setColumnOptions(pCxt, B, &C, &D); } \ No newline at end of file +column_options(A) ::= column_options(B) NK_ID(C) NK_STRING(D). { A = setColumnOptions(pCxt, B, &C, &D); } From 4e211e72218a8c1c707eb5031dc9139188f41db2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 15:16:13 +0800 Subject: [PATCH 06/22] test: add test case for auto compact --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 178471f813..a6c8f04d4a 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8386,10 +8386,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "s3_chunkpages", pOptions->s3ChunkSize, TSDB_MIN_S3_CHUNK_SIZE, - TSDB_MAX_S3_CHUNK_SIZE); + TSDB_MAX_S3_CHUNK_SIZE, 0); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT); + code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbCompactIntervalOption(pCxt, pDbName, pOptions); From 243195c37afc6bb193069144a80b3d5747e5a93a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 15:21:49 +0800 Subject: [PATCH 07/22] test: add test case for auto compact --- source/libs/parser/inc/sql.y | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 63a6b97bbb..fda49e7ee2 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -589,7 +589,6 @@ cmd ::= SHOW BNODES. cmd ::= SHOW SNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } cmd ::= SHOW CLUSTER. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } cmd ::= SHOW TRANSACTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } -cmd ::= SHOW TRANSACTION NK_INTEGER(A). { pCxt->pRootNode = createShowTransactionDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); } cmd ::= SHOW TABLE DISTRIBUTED full_table_name(A). { pCxt->pRootNode = createShowTableDistributedStmt(pCxt, A); } cmd ::= SHOW CONSUMERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } cmd ::= SHOW SUBSCRIPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } From 52eab6c1bc800562db96077ab90e3c4aabeced59 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 15:34:37 +0800 Subject: [PATCH 08/22] test: add test case for auto compact --- source/libs/parser/src/parTranslater.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a6c8f04d4a..52f1554dfb 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8236,6 +8236,7 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char "Invalid option compact_time_range: %dm,%dm, start time should be less than end time", pOptions->compactStartTime, pOptions->compactEndTime); } + int32_t keep2 = pOptions->keep[2]; int32_t days = pOptions->daysPerFile; if (keep2 == -1 || days == -1) { // alter db @@ -8253,6 +8254,7 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char "Invalid option compact_time_range: %dm, end time should be in range: [%dm, %dm]", pOptions->compactEndTime, -keep2, -days); } + return TSDB_CODE_SUCCESS; } From baa2d6e505e2167440b2f70beb542341ff37e0a3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 15:38:08 +0800 Subject: [PATCH 09/22] test: add test case for auto compact --- source/libs/parser/src/parTranslater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 52f1554dfb..a95b0fed3c 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8367,7 +8367,8 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName 0); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER, 0); + code = + checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER, 0); } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "withArbitrator", pOptions->withArbitrator, TSDB_MIN_DB_WITH_ARBITRATOR, From 1e95823d4f46038d9bf15f55cb9e2f9ea48194ef Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 15:49:16 +0800 Subject: [PATCH 10/22] test: add test case for auto compact --- source/libs/parser/src/parTranslater.c | 77 ++++++++++++-------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a95b0fed3c..be87f6cbc0 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7781,23 +7781,23 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS } static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int64_t val, int64_t minVal, - int64_t maxVal, int8_t unit, bool skipUndef) { + int64_t maxVal, bool skipUndef) { if (skipUndef ? ((val >= 0 | val < -2) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, - "Invalid option %s: %" PRId64 "%c, valid range: [%" PRId64 "%c, %" PRId64 "%c]", - pName, val, unit, minVal, unit, maxVal, unit); + "Invalid option %s: %" PRId64 ", valid range: [%" PRId64 ", %" PRId64 "]", + pName, val, minVal, maxVal); } return TSDB_CODE_SUCCESS; } static int32_t checkDbRangeOption(STranslateContext* pCxt, const char* pName, int64_t val, int64_t minVal, - int64_t maxVal, int8_t unit) { - return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal, unit, true); + int64_t maxVal) { + return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal, true); } static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName, int64_t val, int64_t minVal, int64_t maxVal) { - return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal, 0, true); + return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal, true); } static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { @@ -7813,8 +7813,7 @@ static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOption } pOptions->s3KeepLocal = getBigintFromValueNode(pOptions->s3KeepLocalStr); } - return checkDbRangeOption(pCxt, "s3KeepLocal", pOptions->s3KeepLocal, TSDB_MIN_S3_KEEP_LOCAL, TSDB_MAX_S3_KEEP_LOCAL, - 0); + return checkDbRangeOption(pCxt, "s3KeepLocal", pOptions->s3KeepLocal, TSDB_MIN_S3_KEEP_LOCAL, TSDB_MAX_S3_KEEP_LOCAL); } static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { @@ -7830,8 +7829,7 @@ static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOpt } pOptions->daysPerFile = getBigintFromValueNode(pOptions->pDaysPerFile); } - return checkDbRangeOption(pCxt, "daysPerFile", pOptions->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE, - 0); + return checkDbRangeOption(pCxt, "daysPerFile", pOptions->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); } static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { @@ -8170,7 +8168,7 @@ static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, const char* TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); keep2 = pOptions->pDbCfg->daysToKeep2; } - code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2, TIME_UNIT_MINUTE); + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); TAOS_CHECK_RETURN(code); } pOptions->compactInterval = (int32_t)interval; @@ -8180,7 +8178,7 @@ static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, const char* TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); keep2 = pOptions->pDbCfg->daysToKeep2; } - code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2, TIME_UNIT_MINUTE); + code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); TAOS_CHECK_RETURN(code); pOptions->compactInterval = (int32_t)interval; } @@ -8271,12 +8269,12 @@ static int32_t checkDbCompactTimeOffsetOption(STranslateContext* pCxt, SDatabase pOptions->compactTimeOffset = getBigintFromValueNode(pOptions->pCompactTimeOffsetNode) / 60; } return checkDbRangeOption(pCxt, "compact_time_offset", pOptions->compactTimeOffset, TSDB_MIN_COMPACT_TIME_OFFSET, - TSDB_MAX_COMPACT_TIME_OFFSET, TIME_UNIT_HOUR); + TSDB_MAX_COMPACT_TIME_OFFSET); } static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t code = - checkDbRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE, 0); + checkDbRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); if (TSDB_CODE_SUCCESS == code) { code = checkDbCacheModelOption(pCxt, pOptions); } @@ -8284,27 +8282,26 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbEncryptAlgorithmOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, - TSDB_MAX_DB_CACHE_SIZE, 0); + code = + checkDbRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, TSDB_MAX_DB_CACHE_SIZE); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL, - 0); + code = + checkDbRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); } if (TSDB_CODE_SUCCESS == code) { code = checkDbDaysOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = - checkDbRangeOption(pCxt, "fsyncPeriod", pOptions->fsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD, 0); + code = checkDbRangeOption(pCxt, "fsyncPeriod", pOptions->fsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "maxRowsPerBlock", pOptions->maxRowsPerBlock, TSDB_MIN_MAXROWS_FBLOCK, - TSDB_MAX_MAXROWS_FBLOCK, 0); + TSDB_MAX_MAXROWS_FBLOCK); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "minRowsPerBlock", pOptions->minRowsPerBlock, TSDB_MIN_MINROWS_FBLOCK, - TSDB_MAX_MINROWS_FBLOCK, 0); + TSDB_MAX_MINROWS_FBLOCK); } if (TSDB_CODE_SUCCESS == code) { code = checkDbPrecisionOption(pCxt, pOptions); @@ -8316,18 +8313,18 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbKeepTimeOffsetOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE, 0); + code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "pagesize", pOptions->pagesize, TSDB_MIN_PAGESIZE_PER_VNODE, - TSDB_MAX_PAGESIZE_PER_VNODE, 0); + TSDB_MAX_PAGESIZE_PER_VNODE); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "tsdbPagesize", pOptions->tsdbPageSize, TSDB_MIN_TSDB_PAGESIZE, - TSDB_MAX_TSDB_PAGESIZE, 0); + TSDB_MAX_TSDB_PAGESIZE); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA, 0); + code = checkDbRangeOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); } if (TSDB_CODE_SUCCESS == code) { code = checkDbStrictOption(pCxt, pOptions); @@ -8337,8 +8334,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName TSDB_MAX_WAL_LEVEL); } if (TSDB_CODE_SUCCESS == code) { - code = - checkDbRangeOption(pCxt, "vgroups", pOptions->numOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB, 0); + code = checkDbRangeOption(pCxt, "vgroups", pOptions->numOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "singleStable", pOptions->singleStable, TSDB_DB_SINGLE_STABLE_ON, @@ -8352,23 +8348,20 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "walRetentionPeriod", pOptions->walRetentionPeriod, - TSDB_DB_MIN_WAL_RETENTION_PERIOD, INT32_MAX, 0); + TSDB_DB_MIN_WAL_RETENTION_PERIOD, INT32_MAX); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "walRetentionSize", pOptions->walRetentionSize, TSDB_DB_MIN_WAL_RETENTION_SIZE, - INT32_MAX, 0); + INT32_MAX); } if (TSDB_CODE_SUCCESS == code) { - code = - checkDbRangeOption(pCxt, "walRollPeriod", pOptions->walRollPeriod, TSDB_DB_MIN_WAL_ROLL_PERIOD, INT32_MAX, 0); + code = checkDbRangeOption(pCxt, "walRollPeriod", pOptions->walRollPeriod, TSDB_DB_MIN_WAL_ROLL_PERIOD, INT32_MAX); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX, - 0); + code = checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX); } if (TSDB_CODE_SUCCESS == code) { - code = - checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER, 0); + code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER); } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "withArbitrator", pOptions->withArbitrator, TSDB_MIN_DB_WITH_ARBITRATOR, @@ -8389,10 +8382,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "s3_chunkpages", pOptions->s3ChunkSize, TSDB_MIN_S3_CHUNK_SIZE, - TSDB_MAX_S3_CHUNK_SIZE, 0); + TSDB_MAX_S3_CHUNK_SIZE); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT, 0); + code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT); } if (TSDB_CODE_SUCCESS == code) { code = checkDbCompactIntervalOption(pCxt, pDbName, pOptions); @@ -9945,7 +9938,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pStmt) { int32_t code = 0; SCreateUserReq createReq = {0}; - if ((code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, 0, false))) { + if ((code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, false))) { return code; } tstrncpy(createReq.user, pStmt->userName, TSDB_USER_LEN); @@ -9974,13 +9967,13 @@ static int32_t checkAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt) { int32_t code = 0; switch (pStmt->alterType) { case TSDB_ALTER_USER_ENABLE: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "enable", pStmt->enable, 0, 1, 0, false); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "enable", pStmt->enable, 0, 1, false); break; case TSDB_ALTER_USER_SYSINFO: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, 0, false); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, false); break; case TSDB_ALTER_USER_CREATEDB: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "createdb", pStmt->createdb, 0, 1, 0, false); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "createdb", pStmt->createdb, 0, 1, false); break; } return code; From c5b0f6da6ccd416332d93b2fd68ba7f116f92302 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 16:02:29 +0800 Subject: [PATCH 11/22] test: add test case for auto compact --- tests/system-test/0-others/compact_auto.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/system-test/0-others/compact_auto.py b/tests/system-test/0-others/compact_auto.py index 27c24087c1..813be3e9f0 100644 --- a/tests/system-test/0-others/compact_auto.py +++ b/tests/system-test/0-others/compact_auto.py @@ -133,20 +133,20 @@ class TDTestCase: ["compact_time_range -100;-90", "Invalid option compact_time_range, should have 2 value"], ["compact_time_range -100:-90", "syntax error near"], ["compact_time_range -100 -90", "syntax error near"], - ["compact_interval 1m", "Invalid option compact_interval: 1m, valid range: [10m, 5256000m]"], - ["compact_interval 199999999999m", "valid range: [10m, 5256000m]"], - ["compact_interval 9999999999999m", "Invalid option compact_interval: 9999999999999m, valid range: [10m, 5256000m]"], - ["compact_interval 5256001m", "Invalid option compact_interval: 5256001m, valid range: [10m, 5256000m]"], - ["compact_interval 3651", "Invalid option compact_interval: 5257440m, valid range: [10m, 5256000m]"], + ["compact_interval 1m", "Invalid option compact_interval: 1, valid range: [10, 5256000]"], + ["compact_interval 199999999999m", "valid range: [10, 5256000]"], + ["compact_interval 9999999999999m", "Invalid option compact_interval: 9999999999999, valid range: [10, 5256000]"], + ["compact_interval 5256001m", "Invalid option compact_interval: 5256001, valid range: [10, 5256000]"], + ["compact_interval 3651", "Invalid option compact_interval: 5257440, valid range: [10, 5256000]"], ["compact_interval 86400s", "Invalid option compact_interval unit: s, only m, h, d allowed"], ["compact_interval -1", "syntax error near"], ["compact_time_offset -1", "syntax error near"], ["compact_time_offset 3600s", "Invalid option compact_time_offset unit: s, only h allowed"], ["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], - ["compact_time_offset 24", "Invalid option compact_time_offset: 24h, valid range: [0h, 23h]"], - ["compact_time_offset 24h", "Invalid option compact_time_offset: 24h, valid range: [0h, 23h]"], - ["compact_time_offset 9999999999999", "valid range: [0h, 23h]"], - ["compact_time_offset 199999999999", "valid range: [0h, 23h]"], + ["compact_time_offset 24", "Invalid option compact_time_offset: 24, valid range: [0, 23]"], + ["compact_time_offset 24h", "Invalid option compact_time_offset: 24, valid range: [0, 23]"], + ["compact_time_offset 9999999999999", "valid range: [0, 23]"], + ["compact_time_offset 199999999999", "valid range: [0, 23]"], ["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], ["compact_interval 10m compact_time_range -120d,-60 compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], ] @@ -167,4 +167,4 @@ class TDTestCase: tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From 92df4a39db54c88d61c2660e07c0f0ca76d92769 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 16:08:09 +0800 Subject: [PATCH 12/22] test: add test case for auto compact --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index be87f6cbc0..371e29bc9a 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7784,8 +7784,8 @@ static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const cha int64_t maxVal, bool skipUndef) { if (skipUndef ? ((val >= 0 | val < -2) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, - "Invalid option %s: %" PRId64 ", valid range: [%" PRId64 ", %" PRId64 "]", - pName, val, minVal, maxVal); + "Invalid option %s: %" PRId64 ", valid range: [%" PRId64 ", %" PRId64 "]", pName, + val, minVal, maxVal); } return TSDB_CODE_SUCCESS; } From 5f539d41f6da006741025b2772d2a6d6d320cd90 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 16:24:12 +0800 Subject: [PATCH 13/22] test: add test case for auto compact --- source/libs/parser/src/parTranslater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 371e29bc9a..87c648989e 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8358,7 +8358,8 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRangeOption(pCxt, "walRollPeriod", pOptions->walRollPeriod, TSDB_DB_MIN_WAL_ROLL_PERIOD, INT32_MAX); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX); + code = + checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER); From bad49e31ef2ef064352a5f35f0f4391ec9d5e004 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 16:32:07 +0800 Subject: [PATCH 14/22] test: add test case for auto compact --- tests/develop-test/2-query/show_create_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index e59121f631..16e9b1d62c 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -47,7 +47,7 @@ class TDTestCase: tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE -3650d,-10d COMPACT_TIME_OFFSET 0h") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE 0d,0d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd4') tdSql.checkRows(1) @@ -65,7 +65,7 @@ class TDTestCase: tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE -3650d,-10d COMPACT_TIME_OFFSET 0h") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE 0d,0d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') From 70a3e969fa18fef822868b99df474867c8327575 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 17:25:41 +0800 Subject: [PATCH 15/22] enh: robustness of test case --- Jenkinsfile2 | 2 +- tests/system-test/2-query/elapsed.py | 2 +- tests/system-test/2-query/statecount.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 1b2f28908c..85c54e7e90 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -508,7 +508,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 "} options { skipDefaultCheckout() } when { changeRequest() diff --git a/tests/system-test/2-query/elapsed.py b/tests/system-test/2-query/elapsed.py index dc89cd513d..c1191c3441 100644 --- a/tests/system-test/2-query/elapsed.py +++ b/tests/system-test/2-query/elapsed.py @@ -1542,7 +1542,7 @@ class TDTestCase: def query_precision(self): def generate_data(precision="ms"): - tdSql.execute("create database if not exists db_%s precision '%s';" %(precision, precision)) + tdSql.execute("create database if not exists db_%s keep 36500 precision '%s';" %(precision, precision)) tdSql.execute("use db_%s;" %precision) tdSql.execute("create stable db_%s.st (ts timestamp , id int) tags(ind int);"%precision) tdSql.execute("create table db_%s.tb1 using st tags(1);"%precision) diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 006215956b..f5dba2df4f 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -395,7 +395,7 @@ class TDTestCase: def query_precision(self): def generate_data(precision="ms"): dbname = f"db_{precision}" - tdSql.execute(f"create database if not exists db_%s precision '%s';" %(precision, precision)) + tdSql.execute(f"create database if not exists db_%s keep 36500 precision '%s';" %(precision, precision)) tdSql.execute("use db_%s;" %precision) tdSql.execute(f"create stable db_%s.st (ts timestamp , id int) tags(ind int);"%precision) tdSql.execute(f"create table db_%s.tb1 using {dbname}.st tags(1);"%precision) @@ -449,7 +449,7 @@ class TDTestCase: def check_boundary_values(self, dbname="bound_test"): tdSql.execute(f"drop database if exists {dbname}") - tdSql.execute(f"create database if not exists {dbname}") + tdSql.execute(f"create database if not exists {dbname} keep 36500") tdSql.execute( f"create table {dbname}.stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" ) From 8978b7d97feb557922f57e4157905bb444ba935d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 29 Dec 2024 17:52:43 +0800 Subject: [PATCH 16/22] test: add test cases. --- tests/parallel_test/cases.task | 2 + tests/script/tsim/stream/tag.sim | 110 +++++++++++++++++++++++++++++++ tests/script/tsim/testsuit.sim | 4 ++ 3 files changed, 116 insertions(+) create mode 100644 tests/script/tsim/stream/tag.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ed73a2742c..8cfb37a189 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1382,6 +1382,8 @@ ,,y,script,./test.sh -f tsim/stream/basic2.sim ,,y,script,./test.sh -f tsim/stream/basic3.sim ,,y,script,./test.sh -f tsim/stream/basic4.sim +,,y,script,./test.sh -f tsim/stream/basic5.sim +,,y,script,./test.sh -f tsim/stream/tag.sim ,,y,script,./test.sh -f tsim/stream/snodeCheck.sim ,,y,script,./test.sh -f tsim/stream/concurrentcheckpt.sim ,,y,script,./test.sh -f tsim/stream/checkpointInterval0.sim diff --git a/tests/script/tsim/stream/tag.sim b/tests/script/tsim/stream/tag.sim new file mode 100644 index 0000000000..f293f4ac05 --- /dev/null +++ b/tests/script/tsim/stream/tag.sim @@ -0,0 +1,110 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect + +print step1 +print =============== create database +sql create database test vgroups 2; +sql use test; + +sql create table st1(ts timestamp, a int, b int , c int, d double) tags(x int); +sql create table t1 using st1 tags(1); +sql create table t2 using st1 tags(2); +sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1 from st1 where x>=2 interval(60s) ; + +run tsim/stream/checkTaskStatus.sim + + +sql insert into t2 values(1648791213000,0,1,1,1.0); +sql insert into t2 values(1648791213001,9,2,2,1.1); +sql insert into t2 values(1648791213009,0,3,3,1.0); + +sql insert into t1 values(1648791223000,0,1,1,1.0); +sql insert into t1 values(1648791223001,9,2,2,1.1); +sql insert into t1 values(1648791223009,0,3,3,1.0); + +sleep 300 + +sql select * from streamt; +if $data01 != 3 then + return -1 +endi + +sql alter table t1 set tag x=3; + +sql insert into t1 values(1648791233000,0,1,1,1.0); +sql insert into t1 values(1648791233001,9,2,2,1.1); +sql insert into t1 values(1648791233009,0,3,3,1.0); + +sleep 1000 +sql select * from streamt; + +if $data01 != 6 then + return -1 +endi + +sql alter table t1 set tag x=1; +sql alter table t2 set tag x=1; + +sql insert into t1 values(1648791243000,0,1,1,1.0); +sql insert into t1 values(1648791243001,9,2,2,1.1); + + +sql select * from streamt; +if $data01 != 6 then + return -1 +endi + +#$loop_count = 0 +#loop2: +# +#sleep 300 +#print 1 sql select * from streamt; +#sql select * from streamt; +# +#print $data00 $data01 $data02 $data03 +#print $data10 $data11 $data12 $data13 +# +#$loop_count = $loop_count + 1 +#if $loop_count == 10 then +# return -1 +#endi +# +## row 0 +#if $data01 != 3 then +# print ======data01=$data01 +# goto loop2 +#endi +# +#if $data02 != 6 then +# print ======data02=$data02 +# goto loop2 +#endi +# +#if $data03 != 3 then +# print ======data03=$data03 +# goto loop2 +#endi +# +## row 1 +#if $data11 != 3 then +# print ======data11=$data11 +# goto loop2 +#endi +# +#if $data12 != 6 then +# print ======data12=$data12 +# goto loop2 +#endi +# +#if $data13 != 3 then +# print ======data13=$data13 +# goto loop2 +#endi +# + +print tag end +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index ec52b8c234..67b84c24df 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -111,6 +111,10 @@ run tsim/stream/distributeInterval0.sim run tsim/stream/distributeSession0.sim run tsim/stream/state0.sim run tsim/stream/basic2.sim +run tsim/stream/basic3.sim +run tsim/stream/basic4.sim +run tsim/stream/basic5.sim +run tsim/stream/tag.sim run tsim/stream/concurrentcheckpt.sim run tsim/insert/basic1.sim run tsim/insert/commit-merge0.sim From 7715c6755d59b6113bca3f7e8da8141a2c67e2d0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 29 Dec 2024 18:20:51 +0800 Subject: [PATCH 17/22] chore: adjust test case of information_schema --- tests/system-test/0-others/information_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 130085b91d..d4a1aa938d 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(312, 313)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(306, 307)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(60, len(tdSql.queryResult)) From 1669b66f18bacd64e50f46fd7f9a25ee6c5abfe7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 29 Dec 2024 19:54:05 +0800 Subject: [PATCH 18/22] git: recover ci --- Jenkinsfile2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 85c54e7e90..94c96e86d8 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -508,8 +508,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 "} - options { skipDefaultCheckout() } + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() } From d0939a4202d31ab450bb253892dc59369847e94b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 29 Dec 2024 20:04:51 +0800 Subject: [PATCH 19/22] minor changes --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 94c96e86d8..b2b42d6ecd 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -508,7 +508,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} when { changeRequest() } From 91ec961b5451663c8e5b02899a5ae17b79dc346c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 29 Dec 2024 20:05:41 +0800 Subject: [PATCH 20/22] git: set tags --- Jenkinsfile2 | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index b2b42d6ecd..1b2f28908c 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -509,6 +509,7 @@ pipeline { } stage('linux test') { agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} + options { skipDefaultCheckout() } when { changeRequest() } From c2c0f274605095ae5815f3ed5784e38e976ae6fd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 29 Dec 2024 23:44:18 +0800 Subject: [PATCH 21/22] minor changes --- tests/system-test/0-others/information_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 5179111ce5..390bf3d9dd 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(306, 307)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(312, 313)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(61, len(tdSql.queryResult)) From 8f8b70fc2cc3ccea6a59b15b5b4a71d6d63a96f0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 30 Dec 2024 09:52:55 +0800 Subject: [PATCH 22/22] fix: child table limit offset case --- .../5-taos-tools/taosbenchmark/json/custom_col_tag.json | 2 +- .../5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/custom_col_tag.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/custom_col_tag.json index fec5775cd6..b844f7b1b8 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/custom_col_tag.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/custom_col_tag.json @@ -36,7 +36,7 @@ "insert_mode": "taosc", "line_protocol": "line", "childtable_limit": -10, - "childtable_offset": 10, + "childtable_offset": 0, "insert_rows": 20, "insert_interval": 0, "interlace_rows": 0, diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json index 5694b58407..5ba870a3bd 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json @@ -36,7 +36,7 @@ "insert_mode": "taosc", "line_protocol": "line", "childtable_limit": -10, - "childtable_offset": 10, + "childtable_offset": 0, "insert_rows": 20, "insert_interval": 0, "interlace_rows": 0,