From 64765579e94cf3e66dc6413a50f045b74cb51f41 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 25 Oct 2023 22:42:01 +0800 Subject: [PATCH 1/9] enh: check interval and keep for retentions --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 3 +- source/libs/parser/src/parTranslater.c | 55 +++++++++++++++++++--- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 3b4827a6be..76177766ed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1521,7 +1521,8 @@ ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { return pVnode->pTsdb->pFS->fsstate; } for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) { - if (SMA_RSMA_GET_TSDB(pVnode, lvl)->pFS->fsstate != TSDB_FS_STATE_NORMAL) { + STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl); + if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) { return TSDB_FS_STATE_INCOMPLETE; } } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 72293e2f8c..989ae5d00e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4576,7 +4576,7 @@ static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int return TSDB_CODE_SUCCESS; } -static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions) { +static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions, int8_t precision) { if (NULL == pRetentions) { return TSDB_CODE_SUCCESS; } @@ -4588,6 +4588,12 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete SValueNode* pPrevFreq = NULL; SValueNode* pPrevKeep = NULL; SNode* pRetention = NULL; + int64_t tsdbMinKeep = TSDB_MIN_KEEP; + int64_t tsdbMaxKeep = TSDB_MAX_KEEP; + if (precision == TSDB_TIME_PRECISION_NANO) { + tsdbMaxKeep = TSDB_MAX_KEEP_NS; + } + FOREACH(pRetention, pRetentions) { SNode* pNode = NULL; FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) { @@ -4599,11 +4605,48 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete SValueNode* pFreq = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 0); SValueNode* pKeep = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 1); - if (pFreq->datum.i <= 0 || 'n' == pFreq->unit || 'y' == pFreq->unit || pFreq->datum.i >= pKeep->datum.i || - (NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) || - (NULL != pPrevKeep && pPrevKeep->datum.i > pKeep->datum.i)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option retentions"); + + ASSERTS(pFreq->isDuration && pKeep->isDuration, "Retentions freq/keep should have unit"); + + // check unit + if (pFreq->isDuration && TIME_UNIT_SECOND != pFreq->unit && TIME_UNIT_MINUTE != pFreq->unit && + TIME_UNIT_HOUR != pFreq->unit && TIME_UNIT_DAY != pFreq->unit && TIME_UNIT_WEEK != pFreq->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option retentions(freq): %s, only s, m, h, d, w allowed", pFreq->literal); } + + if (pKeep->isDuration && TIME_UNIT_MINUTE != pKeep->unit && TIME_UNIT_HOUR != pKeep->unit && + TIME_UNIT_DAY != pKeep->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option retentions(keep): %s, only m, h, d allowed", pKeep->literal); + } + + // check value range + if (pFreq->datum.i <= 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option retentions(freq): %s should larger than 0", pFreq->literal); + } + int64_t keepMinute = pKeep->datum.i / 60000; // ms to minute + if (keepMinute < tsdbMinKeep || keepMinute > tsdbMaxKeep) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option retentions(keep): %" PRId64 "m, valid range: [%" PRIi64 + "m, %" PRId64 "m]", + keepMinute, tsdbMinKeep, tsdbMaxKeep); + } + + // check relationships + if (NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option retentions(freq): %s should larger than %s", pFreq->literal, + pPrevFreq->literal); + } + + if (NULL != pPrevKeep && pPrevKeep->datum.i > pKeep->datum.i) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option retentions(keep): %s should not larger than %s", + pPrevKeep->literal, pKeep->literal); + } + pPrevFreq = pFreq; pPrevKeep = pKeep; } @@ -4723,7 +4766,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName TSDB_DB_SINGLE_STABLE_OFF); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRetentionsOption(pCxt, pOptions->pRetentions); + code = checkDbRetentionsOption(pCxt, pOptions->pRetentions, pOptions->precision); } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "schemaless", pOptions->schemaless, TSDB_DB_SCHEMALESS_ON, TSDB_DB_SCHEMALESS_OFF); From a83b3053f43e4802230bb55c72512b43f071be36 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 25 Oct 2023 22:52:03 +0800 Subject: [PATCH 2/9] enh: check interval and keep for retentions --- source/libs/parser/src/parTranslater.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 989ae5d00e..41c8b69e5d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4588,12 +4588,6 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete SValueNode* pPrevFreq = NULL; SValueNode* pPrevKeep = NULL; SNode* pRetention = NULL; - int64_t tsdbMinKeep = TSDB_MIN_KEEP; - int64_t tsdbMaxKeep = TSDB_MAX_KEEP; - if (precision == TSDB_TIME_PRECISION_NANO) { - tsdbMaxKeep = TSDB_MAX_KEEP_NS; - } - FOREACH(pRetention, pRetentions) { SNode* pNode = NULL; FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) { @@ -4626,12 +4620,13 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option retentions(freq): %s should larger than 0", pFreq->literal); } - int64_t keepMinute = pKeep->datum.i / 60000; // ms to minute - if (keepMinute < tsdbMinKeep || keepMinute > tsdbMaxKeep) { + int64_t keepMinute = pKeep->datum.i / getUnitPerMinute(pKeep->node.resType.precision); + int64_t tsdbMaxKeep = TSDB_TIME_PRECISION_NANO == precision ? TSDB_MAX_KEEP_NS : TSDB_MAX_KEEP; + if (keepMinute < TSDB_MIN_KEEP || keepMinute > tsdbMaxKeep) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option retentions(keep): %" PRId64 "m, valid range: [%" PRIi64 "m, %" PRId64 "m]", - keepMinute, tsdbMinKeep, tsdbMaxKeep); + keepMinute, TSDB_MIN_KEEP, tsdbMaxKeep); } // check relationships From e387abcbd3a7e9f81f9d1407b14e911c7e7ba745 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 26 Oct 2023 06:34:25 +0800 Subject: [PATCH 3/9] enh: test case for retentions --- .../system-test/1-insert/create_retentions.py | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 0090a7124f..5b2aa4d0d6 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -52,13 +52,40 @@ class TDTestCase: @property def create_databases_sql_err(self): return [ - "create database db1 retentions 0s:1d", - "create database db3 retentions 1s:0d", - "create database db1 retentions 1s:1y", + # check grammar + "create database db1 retentions", + "create database db1 retentions 1s:1d,2s:2d,3s:3d,4s:4d", + # check unit + "create database db1 retentions 1b:1d", + "create database db1 retentions 1u:1d", + "create database db1 retentions 1a:1d", + "create database db1 retentions 1n:1d", + "create database db1 retentions 1y:1d", + "create database db1 retentions 1s:86400s", + "create database db1 retentions 1s:86400000a", + "create database db1 retentions 1s:86400000000u", + "create database db1 retentions 1s:86400000000000b", + "create database db1 retentions 1s:1w", "create database db1 retentions 1s:1n", - "create database db2 retentions 1w:1d ;", - "create database db5 retentions 1s:1d,3s:3d,2s:2d", - "create database db1 retentions 1s:1n,2s:2d,3s:3d,4s:4d", + "create database db1 retentions 1s:1y", + # check value range + "create database db1 retentions -1s:1d", + "create database db1 retentions 0s:1d", + "create database db3 retentions 1s:-1d", + "create database db3 retentions 1s:0d", + "create database db3 retentions 1s:1439m", + "create database db3 retentions 1s:365001d", + "create database db3 retentions 1s:8760001h", + "create database db3 retentions 1s:525600001m", + "create database db3 retentions 1s:106581d precision 'ns'", + "create database db3 retentions 1s:2557921h precision 'ns'", + "create database db3 retentions 1s:153475201m precision 'ns'", + # check relationships + "create database db5 retentions 2m:1d,1s:2d", + "create database db5 retentions 1s:2880m,2s:2879m", + "create database db5 retentions 1s:1d,2s:2d,2s:3d", + "create database db5 retentions 1s:1d,3s:2d,2s:3d", + "create database db1 retentions 1s:1d,2s:3d,3s:2d", ] @property From 4097b53bfa74d1ba948d9b7701e30407a3ef2672 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 26 Oct 2023 11:21:56 +0800 Subject: [PATCH 4/9] enh: only float/double allowed for sum/avg of rsma --- source/libs/parser/src/parTranslater.c | 34 +++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 41c8b69e5d..7a06d27354 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5059,7 +5059,7 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS } static bool validRollupFunc(const char* pFunc) { - static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"}; + static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"}; static const int32_t numOfRollupFuncs = (sizeof(rollupFuncs) / sizeof(char*)); for (int i = 0; i < numOfRollupFuncs; ++i) { if (0 == strcmp(rollupFuncs[i], pFunc)) { @@ -5069,6 +5069,17 @@ static bool validRollupFunc(const char* pFunc) { return false; } +static bool caclRollupFunc(const char* pFunc) { + static const char* calcRollupFuncs[] = {"avg", "sum"}; + static const int32_t numOfCalcRollupFuncs = (sizeof(calcRollupFuncs) / sizeof(char*)); + for (int i = 0; i < numOfCalcRollupFuncs; ++i) { + if (0 == strcmp(calcRollupFuncs[i], pFunc)) { + return true; + } + } + return false; +} + static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs, bool createStable, SDbCfgInfo* pDbCfg) { if (NULL == pFuncs) { @@ -5142,7 +5153,7 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN return code; } -static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, int32_t ntags, SNodeList* pCols) { +static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, int32_t ntags, SNodeList* pCols, SNodeList* pRollupFuncs) { int32_t ncols = LIST_LENGTH(pCols); if (ncols < TSDB_MIN_COLUMNS) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); @@ -5153,8 +5164,16 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in int32_t code = TSDB_CODE_SUCCESS; bool first = true; + bool isCalcRollup = false; int32_t rowSize = 0; SNode* pNode = NULL; + char* pFunc = NULL; + + // if (pRollupFuncs) { + // pFunc = ((SFunctionNode*)nodesListGetNode(pRollupFuncs, 0))->functionName; + // isCalcRollup = caclRollupFunc(pFunc); + // } + FOREACH(pNode, pCols) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; if (first) { @@ -5178,6 +5197,15 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } } + + // if (TSDB_CODE_SUCCESS == code && isCalcRollup) { + // if (pCol->dataType.type != TSDB_DATA_TYPE_FLOAT && pCol->dataType.type != TSDB_DATA_TYPE_DOUBLE) { + // code = + // generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, + // "Invalid column type: %s, only float/double allowed for %s", pCol->colName, pFunc); + // } + // } + if (TSDB_CODE_SUCCESS == code) { code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES); } @@ -5204,7 +5232,7 @@ static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt int32_t code = checkTableTagsSchema(pCxt, pHash, pStmt->pTags); if (TSDB_CODE_SUCCESS == code) { - code = checkTableColsSchema(pCxt, pHash, LIST_LENGTH(pStmt->pTags), pStmt->pCols); + code = checkTableColsSchema(pCxt, pHash, LIST_LENGTH(pStmt->pTags), pStmt->pCols, pStmt->pOptions->pRollupFuncs); } taosHashCleanup(pHash); From 2ad3b9834448808031df179a8c93b26afcd458cf Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 27 Oct 2023 13:49:39 +0800 Subject: [PATCH 5/9] chore: test case for rollup --- source/libs/parser/src/parTranslater.c | 29 ++++++++-------- .../system-test/1-insert/create_retentions.py | 33 +++++++++++++++---- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7a06d27354..f08ae78e3c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5163,21 +5163,20 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in int32_t code = TSDB_CODE_SUCCESS; - bool first = true; + int32_t colIndex = 0; bool isCalcRollup = false; int32_t rowSize = 0; SNode* pNode = NULL; char* pFunc = NULL; - // if (pRollupFuncs) { - // pFunc = ((SFunctionNode*)nodesListGetNode(pRollupFuncs, 0))->functionName; - // isCalcRollup = caclRollupFunc(pFunc); - // } + if (pRollupFuncs) { + pFunc = ((SFunctionNode*)nodesListGetNode(pRollupFuncs, 0))->functionName; + isCalcRollup = caclRollupFunc(pFunc); + } FOREACH(pNode, pCols) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; - if (first) { - first = false; + if (0 == colIndex) { if (TSDB_DATA_TYPE_TIMESTAMP != pCol->dataType.type) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN); } @@ -5198,13 +5197,13 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } } - // if (TSDB_CODE_SUCCESS == code && isCalcRollup) { - // if (pCol->dataType.type != TSDB_DATA_TYPE_FLOAT && pCol->dataType.type != TSDB_DATA_TYPE_DOUBLE) { - // code = - // generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, - // "Invalid column type: %s, only float/double allowed for %s", pCol->colName, pFunc); - // } - // } + if (TSDB_CODE_SUCCESS == code && isCalcRollup && 0 != colIndex) { + if (pCol->dataType.type != TSDB_DATA_TYPE_FLOAT && pCol->dataType.type != TSDB_DATA_TYPE_DOUBLE) { + code = + generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, + "Invalid column type: %s, only float/double allowed for %s", pCol->colName, pFunc); + } + } if (TSDB_CODE_SUCCESS == code) { code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES); @@ -5214,6 +5213,8 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } else { break; } + + ++colIndex; } if (TSDB_CODE_SUCCESS == code && rowSize > TSDB_MAX_BYTES_PER_ROW) { diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 5b2aa4d0d6..2d502b1e92 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -119,6 +119,16 @@ class TDTestCase: f"create stable {dbname}.stb24 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " , f"create stable {dbname}.stb25 ({PRIMARY_COL} timestamp, {INT_COL} int) " , f"create stable {dbname}.stb26 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " , + # only float/double allowd for avg/sum + f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)", + f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BINT_COL} bigint) tags (tag1 int) rollup(avg)", + f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BOOL_COL} bool) tags (tag1 int) rollup(avg)", + f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BINARY_COL} binary(10)) tags (tag1 int) rollup(avg)", + f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum)", + f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BINT_COL} bigint) tags (tag1 int) rollup(sum)", + f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BOOL_COL} bool) tags (tag1 int) rollup(sum)", + f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BINARY_COL} binary(10)) tags (tag1 int) rollup(sum)", + # watermark, max_delay: [0, 900000], [ms, s, m, ?] f"create stable stb17 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u", @@ -135,10 +145,10 @@ class TDTestCase: @property def create_stable_sql_current(self): return [ - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)", + f"create stable stb1 ({PRIMARY_COL} timestamp, {FLOAT_COL} float) tags (tag1 int) rollup(avg)", f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s max_delay 1m", f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s max_delay 1m", - f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m", + f"create stable stb4 ({PRIMARY_COL} timestamp, {DOUBLE_COL} double) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m", f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s max_delay 1m", f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m", f"create stable stb7 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m sma({INT_COL})", @@ -181,6 +191,12 @@ class TDTestCase: {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned, {BINARY_COL} binary(16) ) tags ({INT_TAG} int) rollup({rsma_type}) watermark 5s,5s max_delay 5s,5s ''' + elif rsma_type.lower().strip() in ("sum", "avg"): + create_stb_sql = f'''create table {dbname}.{stb}( + ts timestamp, {DOUBLE_COL} double, {DOUBLE_COL}_1 double, {DOUBLE_COL}_2 double, {DOUBLE_COL}_3 double, + {FLOAT_COL} float, {DOUBLE_COL}_4 double, {FLOAT_COL}_1 float, {FLOAT_COL}_2 float, {FLOAT_COL}_3 float, + {DOUBLE_COL}_5 double) tags ({INT_TAG} int) rollup({rsma_type}) watermark 5s,5s max_delay 5s,5s + ''' else: create_stb_sql = f'''create table {dbname}.{stb}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, @@ -227,11 +243,16 @@ class TDTestCase: {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, {data.utint_data[i]}, {data.usint_data[i]}, {data.uint_data[i]}, {data.ubint_data[i]}, '{data.vchar_data[i]}' ''' - else: + elif rsma_type.lower().strip() in ("sum", "avg"): row_data = f''' {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, {data.utint_data[i]}, {data.usint_data[i]}, {data.uint_data[i]}, {data.ubint_data[i]} ''' + else: + row_data = f''' + {data.double_data[i]}, {data.double_data[i]}, {data.double_data[i]}, {data.double_data[i]}, {data.float_data[i]}, {data.double_data[i]}, + {data.float_data[i]}, {data.float_data[i]}, {data.float_data[i]}, {data.double_data[i]} + ''' else: row_data = f''' {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, @@ -272,17 +293,17 @@ class TDTestCase: tdSql.query(f"select count(*) from {DB3}.{STBNAME} where ts > now()-5m") tdSql.checkData(0, 0, self.rows * db3_ctb_num) tdSql.checkRows(1) - tdSql.query(f"select {INT_COL} from {DB3}.{CTBNAME} where ts > now()-4d") + tdSql.query(f"select {FLOAT_COL} from {DB3}.{CTBNAME} where ts > now()-4d") # not stable #tdSql.checkData(0, 0, self.rows-1) - tdSql.query(f"select {INT_COL} from {DB3}.{CTBNAME} where ts > now()-6d") + tdSql.query(f"select {DOUBLE_COL} from {DB3}.{CTBNAME} where ts > now()-6d") # not stable # tdSql.checkData(0, 0, self.rows-1) # from ...pytest.util.sql import tdSql tdLog.printNoPrefix("==========step2.1.1 : alter stb schemaL drop column") - tdSql.query(f"select {BINT_COL} from {DB3}.{STBNAME}") + tdSql.query(f"select {FLOAT_COL} from {DB3}.{STBNAME}") #tdSql.execute(f"alter stable {DB3}.stb1 drop column {BINT_COL}") # not support alter stable schema anymore tdSql.error(f"alter stable {DB3}.stb1 drop column {BINT_COL}") From dc8cb7a99e1f4fd39e4045a08b29e02ee11811a4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 27 Oct 2023 14:36:01 +0800 Subject: [PATCH 6/9] fix: memory leak --- source/dnode/mnode/impl/src/mndSync.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 7f6a0397ad..f46f33ac22 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -95,7 +95,11 @@ static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STr } _OUT: - taosMemoryFreeClear(pRow); + if (pRow) { + SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type]; + if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false); + taosMemoryFreeClear(pRow); + } return code; } From 4ea44c429d4e03ebf1f667f816b3d086ecf1466d Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 27 Oct 2023 14:41:17 +0800 Subject: [PATCH 7/9] chore: code optimization --- source/libs/parser/src/parTranslater.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f08ae78e3c..316f5c54d3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5164,16 +5164,15 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in int32_t code = TSDB_CODE_SUCCESS; int32_t colIndex = 0; - bool isCalcRollup = false; int32_t rowSize = 0; SNode* pNode = NULL; char* pFunc = NULL; + bool isCalcRollup = false; if (pRollupFuncs) { pFunc = ((SFunctionNode*)nodesListGetNode(pRollupFuncs, 0))->functionName; isCalcRollup = caclRollupFunc(pFunc); } - FOREACH(pNode, pCols) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; if (0 == colIndex) { @@ -5213,7 +5212,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } else { break; } - + // next column ++colIndex; } From ac7246642972245f277ebea9a7acf89724e15628 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 27 Oct 2023 15:11:35 +0800 Subject: [PATCH 8/9] chore: naming optimization --- source/libs/parser/src/parTranslater.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 316f5c54d3..b8433a4472 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5069,11 +5069,11 @@ static bool validRollupFunc(const char* pFunc) { return false; } -static bool caclRollupFunc(const char* pFunc) { - static const char* calcRollupFuncs[] = {"avg", "sum"}; - static const int32_t numOfCalcRollupFuncs = (sizeof(calcRollupFuncs) / sizeof(char*)); - for (int i = 0; i < numOfCalcRollupFuncs; ++i) { - if (0 == strcmp(calcRollupFuncs[i], pFunc)) { +static bool aggrRollupFunc(const char* pFunc) { + static const char* aggrRollupFuncs[] = {"avg", "sum"}; + static const int32_t numOfAggrRollupFuncs = (sizeof(aggrRollupFuncs) / sizeof(char*)); + for (int i = 0; i < numOfAggrRollupFuncs; ++i) { + if (0 == strcmp(aggrRollupFuncs[i], pFunc)) { return true; } } @@ -5153,7 +5153,8 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN return code; } -static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, int32_t ntags, SNodeList* pCols, SNodeList* pRollupFuncs) { +static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, int32_t ntags, SNodeList* pCols, + SNodeList* pRollupFuncs) { int32_t ncols = LIST_LENGTH(pCols); if (ncols < TSDB_MIN_COLUMNS) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); @@ -5167,11 +5168,11 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in int32_t rowSize = 0; SNode* pNode = NULL; char* pFunc = NULL; - bool isCalcRollup = false; + bool isAggrRollup = false; if (pRollupFuncs) { pFunc = ((SFunctionNode*)nodesListGetNode(pRollupFuncs, 0))->functionName; - isCalcRollup = caclRollupFunc(pFunc); + isAggrRollup = aggrRollupFunc(pFunc); } FOREACH(pNode, pCols) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; @@ -5196,7 +5197,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } } - if (TSDB_CODE_SUCCESS == code && isCalcRollup && 0 != colIndex) { + if (TSDB_CODE_SUCCESS == code && isAggrRollup && 0 != colIndex) { if (pCol->dataType.type != TSDB_DATA_TYPE_FLOAT && pCol->dataType.type != TSDB_DATA_TYPE_DOUBLE) { code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, From eae1b4eb031b316f7764786eee36e38961f13632 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 27 Oct 2023 19:15:23 +0800 Subject: [PATCH 9/9] chore: relationship between interval/keep --- source/libs/parser/src/parTranslater.c | 6 ++++++ tests/system-test/1-insert/create_retentions.py | 1 + 2 files changed, 7 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b8433a4472..ece2c82748 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4630,6 +4630,12 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete } // check relationships + if (pFreq->datum.i >= pKeep->datum.i) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option retentions(freq/keep): %s should larger than %s", pKeep->literal, + pFreq->literal); + } + if (NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option retentions(freq): %s should larger than %s", pFreq->literal, diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 2d502b1e92..9435dcd081 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -81,6 +81,7 @@ class TDTestCase: "create database db3 retentions 1s:2557921h precision 'ns'", "create database db3 retentions 1s:153475201m precision 'ns'", # check relationships + "create database db5 retentions 1441m:1440m,2d:3d", "create database db5 retentions 2m:1d,1s:2d", "create database db5 retentions 1s:2880m,2s:2879m", "create database db5 retentions 1s:1d,2s:2d,2s:3d",