From e3098accc4c7dc222670311e887d01437d63907c Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 13 Dec 2023 23:25:37 +0800 Subject: [PATCH 01/93] support count empty table --- .gitignore | 4 +- include/libs/nodes/plannodes.h | 1 + include/libs/nodes/querynodes.h | 1 + source/dnode/vnode/src/sma/smaRollup.c | 1 + source/libs/executor/inc/executorInt.h | 4 +- source/libs/executor/src/scanoperator.c | 102 +++++++++- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 9 +- source/libs/nodes/src/nodesMsgFuncs.c | 7 +- source/libs/parser/src/parTranslater.c | 1 + source/libs/planner/src/planPhysiCreater.c | 22 +++ tests/parallel_test/cases.task | 1 + tests/system-test/2-query/count.py | 33 ++-- tests/system-test/2-query/group_partition.py | 183 ++++++++++++++++++ .../2-query/nestedQueryInterval.py | 26 +-- 15 files changed, 359 insertions(+), 37 deletions(-) create mode 100644 tests/system-test/2-query/group_partition.py diff --git a/.gitignore b/.gitignore index 08e3d57717..f8b42f9176 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -build/ +*build/ compile_commands.json CMakeSettings.json .cache @@ -132,3 +132,5 @@ tools/taos-tools tools/taosws-rs tags .clangd +*CMakeCache* +*CMakeFiles* diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b99a97a194..4b3c846389 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -434,6 +434,7 @@ typedef struct STableScanPhysiNode { bool assignBlockUid; int8_t igCheckUpdate; bool filesetDelimited; + bool needCountEmptyTable; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5c5172b9cd..fdf598153f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -363,6 +363,7 @@ typedef struct SSelectStmt { bool hasLastRowFunc; bool hasLastFunc; bool hasTimeLineFunc; + bool hasCountFunc; bool hasUdaf; bool hasStateKey; bool onlyHasKeepOrderFunc; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index abe4c3f2fc..5b33451be5 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -750,6 +750,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); + pReq = NULL; TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e3e504cdbc..2523b87cfb 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -262,6 +262,7 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; + SHashObj* pValuedTables; // non empty table uids SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; @@ -269,8 +270,9 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - // TsdReader readerAPI; bool filesetDelimited; + bool needCountEmptyTable; + bool processingEmptyTable; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..1889492aa0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,6 +655,50 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } + +// record processed (non empty) table +static int32_t insertTableToProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { + if (!pTableScanInfo->needCountEmptyTable) { + return TSDB_CODE_SUCCESS; + } + if (NULL == pTableScanInfo->pValuedTables) { + int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); + pTableScanInfo->pValuedTables = + taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (NULL == pTableScanInfo->pValuedTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + taosHashPut(pTableScanInfo->pValuedTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, + sizeof(pTableScanInfo->scanTimes)); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { + STableScanInfo* pTableScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSDataBlock* pBlock = pTableScanInfo->pResBlock; + + blockDataEmpty(pBlock); + pBlock->info.rows = 1; + pBlock->info.id.uid = tbInfo->uid; + pBlock->info.id.groupId = pOperator->dynamicTask ? tbInfo->uid : tbInfo->groupId; + + // only one row: set all col data to null & hasNull + int32_t col_num = blockDataGetNumOfCols(pBlock); + for (int32_t i = 0; i < col_num; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + colDataSetNULL(pColInfoData, 0); + } + + // set tag/tbname + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); + + pOperator->resultInfo.totalRows++; + return pBlock; +} + static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -722,7 +766,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { return NULL; } -static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { +static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKeyInfo* pList, int32_t num) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -736,6 +780,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { + insertTableToProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -764,6 +809,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { + insertTableToProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -780,6 +826,39 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } } + if (pTableScanInfo->needCountEmptyTable) { + if (num == 0 && 0 == taosHashGetSize(pTableScanInfo->pValuedTables)) { + // table by table, num is 0 + if (!pTableScanInfo->processingEmptyTable) { + pTableScanInfo->processingEmptyTable = true; + // current table is empty, fill result block info & return + const STableKeyInfo* info = tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->currentTable); + return getBlockForEmptyTable(pOperator, info); + } + + } else if (num > taosHashGetSize(pTableScanInfo->pValuedTables)) { + // group by group, num >= 1 + if (!pTableScanInfo->processingEmptyTable) { + pTableScanInfo->processingEmptyTable = true; + pTableScanInfo->currentTable = 0; + } + if (pTableScanInfo->currentTable < num) { + // loop: get empty table uid & process + while (pTableScanInfo->currentTable < num) { + const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; + if (pTableScanInfo->pValuedTables && + NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { + } else { + return getBlockForEmptyTable(pOperator, info); + } + } + } + } + + pTableScanInfo->processingEmptyTable = false; + } + taosHashClear(pTableScanInfo->pValuedTables); + return NULL; } @@ -861,7 +940,7 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -876,15 +955,16 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int32_t num = 0; + STableKeyInfo* pList = NULL; if (pInfo->currentGroupId == -1) { - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo) || numOfTables == 0) { setOperatorCompleted(pOperator); return NULL; } - int32_t num = 0; - STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); @@ -899,9 +979,11 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; } + } else { + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); } - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -923,7 +1005,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - + if (pOperator->pOperatorGetParam) { pOperator->dynamicTask = true; int32_t code = createTableListInfoFromParam(pOperator); @@ -952,7 +1034,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo tInfo = {0}; while (1) { - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) { return result; } @@ -1012,6 +1094,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); + taosHashCleanup(pTableScanInfo->pValuedTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1075,6 +1158,9 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; + pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; + pInfo->processingEmptyTable = false; + pInfo->base.pTableListInfo = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 97438b84a6..8a154dcf00 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -652,6 +652,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(igExpired); COPY_SCALAR_FIELD(filesetDelimited); + COPY_SCALAR_FIELD(needCountEmptyTable); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c445af61cc..402a6c6e3d 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1841,6 +1841,7 @@ static const char* jkTableScanPhysiPlanSubtable = "Subtable"; static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate"; static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited"; +static const char* jkTableScanPhysiPlanNeedCountEmptyTable = "NeedCountEmptyTable"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1912,7 +1913,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanFilesetDelimited, pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, pNode->needCountEmptyTable); + } return code; } @@ -1986,7 +1989,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanFilesetDelimited, &pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, &pNode->needCountEmptyTable); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index d6eb3360aa..b36e2695f6 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2170,6 +2170,9 @@ static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEnc if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueBool(pEncoder, pNode->filesetDelimited); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->needCountEmptyTable); + } return code; } @@ -2251,7 +2254,9 @@ static int32_t msgToPhysiTableScanNodeInline(STlvDecoder* pDecoder, void* pObj) if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueBool(pDecoder, &pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->needCountEmptyTable); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..98799d3d1d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1902,6 +1902,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { if (NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt)) { SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt; pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); + pSelect->hasCountFunc = pSelect->hasCountFunc ? true : (FUNCTION_TYPE_COUNT == pFunc->funcType); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); if (fmIsIndefiniteRowsFunc(pFunc->funcId)) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index d1fbd0681d..901927b1d1 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,6 +587,27 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } +static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + // refuse interval + if (pScanLogicNode->interval > 0) { + return false; + } + SNode* pRoot = pCxt->pPlanCxt->pAstRoot; + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + SSelectStmt* pSelect = (SSelectStmt*)pRoot; + // select & count + if (pSelect->hasCountFunc) { + // key only accept tag/tbname + if (NULL != pSelect->pGroupByList) { + return !keysHasCol(pSelect->pGroupByList); + } else if (NULL != pSelect->pPartitionByList) { + return !keysHasCol(pSelect->pPartitionByList); + } + } + } + return false; +} + static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, @@ -623,6 +644,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; + pTableScan->needCountEmptyTable = calcNeedCountEmpty(pCxt, pScanLogicNode); int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..e63f86743b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -347,6 +347,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 40d9b3ff8b..7e99f9e4dd 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -57,8 +57,11 @@ class TDTestCase: tdSql.query(f'select count(*) from {self.stbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + rows = [2, 0] function_names = ['count', 'hyperloglog'] - for function_name in function_names: + for i in range(2): + function_name = function_names[i] + row = rows[i] tdSql.query(f'select {function_name}(tbname) from {self.stbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -93,17 +96,17 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.stbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by t0') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by t0') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.stbname} group by c1)') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -113,17 +116,24 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1 interval(1s)') tdSql.checkRows(0) - tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname)') + tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + if 'count' == function_name: + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + elif 'hyperloglog' == function_name: + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 0, 0) def query_empty_ntb(self): tdSql.query(f'select count(*) from {self.ntbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + rows = [1, 0] function_names = ['count', 'hyperloglog'] - for function_name in function_names: + for i in range(2): + function_name = function_names[i] + row = rows[i] tdSql.query(f'select {function_name}(tbname) from {self.ntbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -158,7 +168,7 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.ntbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.ntbname} group by c1)') @@ -170,10 +180,11 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by c1 interval(1s)') tdSql.checkRows(0) - tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) tdSql.checkData(0, 1, None) + def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') tdSql.checkEqual(tdSql.queryResult[0][0],tbnum*rownum) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py new file mode 100644 index 0000000000..eb5f069b3d --- /dev/null +++ b/tests/system-test/2-query/group_partition.py @@ -0,0 +1,183 @@ +# author : bobliu +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.row_nums = 10 + self.tb_nums = 10 + self.ts = 1537146000000 + self.dbname = "db" + self.stable = "stb" + + def prepare_db(self): + tdSql.execute(f" use {self.dbname} ") + tdSql.execute(f" create stable {self.dbname}.{self.stable} (ts timestamp , c1 int , c2 bigint , c3 float , c4 double , c5 smallint , c6 tinyint , c7 bool , c8 binary(36) , c9 nchar(36) , uc1 int unsigned,\ + uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags(t1 timestamp , t2 int , t3 bigint , t4 float , t5 double , t6 smallint , t7 tinyint , t8 bool , t9 binary(36)\ + , t10 nchar(36) , t11 int unsigned , t12 bigint unsigned ,t13 smallint unsigned , t14 tinyint unsigned ) ") + + for i in range(self.tb_nums): + tbname = f"{self.dbname}.sub_{self.stable}_{i}" + ts = self.ts + i*10000 + tdSql.execute(f"create table {tbname} using {self.dbname}.{self.stable} tags ({ts} , {i} , {i}*10 ,{i}*1.0,{i}*1.0 , 1 , 2, 'true', 'binary_{i}' ,'nchar_{i}',{i},{i},10,20 )") + + def insert_db(self, tb_nums, row_nums): + for i in range(tb_nums): + tbname = f"{self.dbname}.sub_{self.stable}_{i}" + ts_base = self.ts + i*10000 + for row in range(row_nums): + ts = ts_base + row*1000 + tdSql.execute(f"insert into {tbname} values({ts} , {row} , {row} , {row} , {row} , 1 , 2 , 'true' , 'binary_{row}' , 'nchar_{row}' , {row} , {row} , 1 ,2 )") + + + def test_groupby(self, check_num, real_num): + # tbname + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + # having filter out empty + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # tag + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by t2 ") + tdSql.checkRows(check_num) + + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") + tdSql.checkRows(check_num) + + # having + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # col where filter nothing + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + # tdSql.checkRows(check_num) + + ############### same with old ############### + # col where filter all + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + # tdSql.checkRows(0) + + # col where filter part + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") + tdSql.checkRows(real_num) + + # col + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(real_num) + + # count + sum(col) + tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(real_num) + + tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") + num = 0 + if real_num > 0: + num = self.row_nums + tdSql.checkRows(num) + + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} group by ts ") + tdSql.checkRows(real_num * self.row_nums) + + # col + tag + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") + tdSql.checkRows(real_num * self.row_nums) + + + def test_partitionby(self, check_num, real_num): + tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + # having filter out empty + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + #tag + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} partition by t2 ") + tdSql.checkRows(check_num) + + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") + tdSql.checkRows(check_num) + + # having + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # col where filter nothing + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") + # tdSql.checkRows(check_num) + + ############### same with old ############### + # col where filter all + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") + # tdSql.checkRows(0) + + # col where filter part + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") + tdSql.checkRows(real_num) + + #col + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(real_num) + + tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") + num = 0 + if real_num > 0: + num = self.row_nums + tdSql.checkRows(num) + + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} partition by ts ") + tdSql.checkRows(real_num * self.row_nums) + + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") + tdSql.checkRows(real_num * self.row_nums) + + def test_error(self): + tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") + tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") + + + def run(self): + tdSql.prepare() + self.prepare_db() + check_num = self.tb_nums + self.test_groupby(check_num, 0) + self.test_partitionby(check_num, 0) + # insert into half of tables + real_num = 5 + self.insert_db(real_num, self.row_nums) + self.test_groupby(check_num, real_num) + self.test_partitionby(check_num, real_num) + + # test old version before changed + # self.test_groupby(0, 0) + # self.test_partitionby(0, 0) + # self.insert_db(5, self.row_nums) + # self.test_groupby(5, 5) + # self.test_partitionby(5, 5) + + self.test_error() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index c16fc03c27..b6ef50dcda 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -1112,13 +1112,13 @@ class TDTestCase: def TS_3932(self): tdLog.debug("test insert data into stable") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 100) tdSql.checkData(1, 1, 200) tdSql.query(f"insert into nested.stable_1 (ts,tbname) values(now,'stable_1_1');") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 101) tdSql.checkData(1, 1, 200) @@ -1127,7 +1127,7 @@ class TDTestCase: coulmn_name = qlist[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;",queryTimes=5) - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 111) tdSql.checkData(1, 1, 200) @@ -1136,7 +1136,7 @@ class TDTestCase: coulmn_name = q_null_list[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;",queryTimes=5) - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 121) tdSql.checkData(1, 1, 200) @@ -1184,7 +1184,7 @@ class TDTestCase: def TS_3932_flushdb(self): tdLog.debug("test flush db and insert data into stable") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 121) tdSql.checkData(1, 1, 200) @@ -1192,7 +1192,7 @@ class TDTestCase: q_null_list = ['q_int_null', 'q_bigint_null', 'q_smallint_null', 'q_tinyint_null', 'q_float_null', 'q_double_null', 'q_bool_null', 'q_binary_null', 'q_nchar_null', 'q_ts_null'] tdSql.query(f"insert into nested.stable_1 (ts,tbname) values(now,'stable_1_1');") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 122) tdSql.checkData(1, 1, 200) @@ -1200,7 +1200,7 @@ class TDTestCase: coulmn_name = qlist[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 132) tdSql.checkData(1, 1, 200) @@ -1208,7 +1208,7 @@ class TDTestCase: coulmn_name = q_null_list[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 142) tdSql.checkData(1, 1, 200) @@ -1223,7 +1223,7 @@ class TDTestCase: nested.stable_1 (ts,tbname,q_nchar) values(now+8a,'stable_1_1',1)\ nested.stable_1 (ts,tbname,q_ts) values(now+9a,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 152); tdSql.checkData(1, 1, 200); @@ -1330,7 +1330,7 @@ class TDTestCase: nested.stable_null_childtable (ts,tbname,q_ts) values(now+9a,'stable_null_childtable_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 162); tdSql.checkData(1, 1, 200); @@ -1349,7 +1349,7 @@ class TDTestCase: nested.stable_null_childtable (ts,tbname,q_int) values(now,'$^%$%^&',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); @@ -1387,7 +1387,7 @@ class TDTestCase: nested.stable_null_childtable(tbname,ts,q_int,q_binary) file '{self.testcasePath}/stable_null_childtable.csv';") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); @@ -1423,7 +1423,7 @@ class TDTestCase: tdSql.query(f"insert into nested.stable_null_childtable(tbname,ts,q_int,q_binary) file '{self.testcasePath}/stable_null_childtable.csv';") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); From 3bfc7b34862cbea233e33b9b5617f154bf28f295 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 00:47:15 +0800 Subject: [PATCH 02/93] adjust case --- tests/system-test/2-query/group_partition.py | 44 +++++++++++++------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index eb5f069b3d..58be8563fa 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -46,6 +46,12 @@ class TDTestCase: tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname order by count(*)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname having count(*)>=0") + tdSql.checkRows(check_num) + # having filter out empty tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -62,26 +68,26 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # col where filter nothing - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") - # tdSql.checkRows(check_num) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + tdSql.checkRows(check_num) - ############### same with old ############### # col where filter all - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") - # tdSql.checkRows(0) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + tdSql.checkRows(check_num) # col where filter part tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) # col tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) # count + sum(col) tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") num = 0 if real_num > 0: @@ -103,6 +109,12 @@ class TDTestCase: tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname having count(*)>=0") + tdSql.checkRows(check_num) + # having filter out empty tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -119,22 +131,25 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # col where filter nothing - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") - # tdSql.checkRows(check_num) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") + tdSql.checkRows(check_num) - ############### same with old ############### # col where filter all - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") - # tdSql.checkRows(0) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") + tdSql.checkRows(check_num) # col where filter part tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) #col tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") tdSql.checkRows(real_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") num = 0 if real_num > 0: @@ -150,6 +165,7 @@ class TDTestCase: def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") + tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 interval(1d)") def run(self): From 3b0d1480f7783d05b4cab34a4548fd95c436c476 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 11:28:26 +0800 Subject: [PATCH 03/93] adjust case --- tests/script/tsim/field/3.sim | 14 ++++---- tests/script/tsim/field/4.sim | 18 +++++----- tests/script/tsim/field/6.sim | 38 ++++++++++---------- tests/script/tsim/field/bigint.sim | 4 +-- tests/script/tsim/field/double.sim | 4 +-- tests/script/tsim/field/smallint.sim | 4 +-- tests/script/tsim/field/unsigined_bigint.sim | 4 +-- tests/script/tsim/parser/mixed_blocks.sim | 3 +- tests/system-test/2-query/count_partition.py | 4 +-- tests/system-test/2-query/last_row.py | 2 +- 10 files changed, 48 insertions(+), 47 deletions(-) diff --git a/tests/script/tsim/field/3.sim b/tests/script/tsim/field/3.sim index 8b428febcd..4d4801c54a 100644 --- a/tests/script/tsim/field/3.sim +++ b/tests/script/tsim/field/3.sim @@ -447,44 +447,44 @@ if $data00 != 100 then endi print =============== step17 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step18 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/4.sim b/tests/script/tsim/field/4.sim index 361ca4c326..d83de81c88 100644 --- a/tests/script/tsim/field/4.sim +++ b/tests/script/tsim/field/4.sim @@ -619,56 +619,56 @@ if $data00 != 100 then endi print =============== step22 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step23 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/6.sim b/tests/script/tsim/field/6.sim index 52fd0b3780..2d2b5623c7 100644 --- a/tests/script/tsim/field/6.sim +++ b/tests/script/tsim/field/6.sim @@ -824,117 +824,117 @@ if $data00 != 25 then endi print =============== step28 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol3 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol3 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol4 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol4 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol5 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol5 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol6 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol6 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step29 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step30 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/bigint.sim b/tests/script/tsim/field/bigint.sim index ce35cacd84..a791d6a1cb 100644 --- a/tests/script/tsim/field/bigint.sim +++ b/tests/script/tsim/field/bigint.sim @@ -127,7 +127,7 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 print $data10 $data11 $data12 $data13 $data14 $data15 $data16 if $data00 != 100 then @@ -136,7 +136,7 @@ if $data00 != 100 then endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/double.sim b/tests/script/tsim/field/double.sim index 1f0cea4be8..2daa78eb7a 100644 --- a/tests/script/tsim/field/double.sim +++ b/tests/script/tsim/field/double.sim @@ -127,14 +127,14 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/smallint.sim b/tests/script/tsim/field/smallint.sim index 66bfee5838..980d69297a 100644 --- a/tests/script/tsim/field/smallint.sim +++ b/tests/script/tsim/field/smallint.sim @@ -128,14 +128,14 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/unsigined_bigint.sim b/tests/script/tsim/field/unsigined_bigint.sim index baa57ce1f6..de8098c297 100644 --- a/tests/script/tsim/field/unsigined_bigint.sim +++ b/tests/script/tsim/field/unsigined_bigint.sim @@ -131,7 +131,7 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 print $data10 $data11 $data12 $data13 $data14 $data15 $data16 if $data00 != 100 then @@ -140,7 +140,7 @@ if $data00 != 100 then endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/parser/mixed_blocks.sim b/tests/script/tsim/parser/mixed_blocks.sim index 04e9df3ff4..8716192858 100644 --- a/tests/script/tsim/parser/mixed_blocks.sim +++ b/tests/script/tsim/parser/mixed_blocks.sim @@ -100,7 +100,8 @@ if $data05 != 1 then endi sql select max(c1), min(c1), sum(c1), avg(c1), count(c1), t1 from $stb where c1 > 0 group by t1 -if $rows != 1 then +if $rows != 2 then + print === rows $rows return -1 endi if $data00 != 319 then diff --git a/tests/system-test/2-query/count_partition.py b/tests/system-test/2-query/count_partition.py index f59376a979..e970b00cec 100644 --- a/tests/system-test/2-query/count_partition.py +++ b/tests/system-test/2-query/count_partition.py @@ -66,9 +66,9 @@ class TDTestCase: tdSql.checkRows(self.row_nums+1) tdSql.query(f"select count(c1) , count(t2) from {dbname}.stb where abs(c1+t2)=1 partition by tbname") - tdSql.checkRows(2) + tdSql.checkRows(10) tdSql.query(f"select count(c1) from {dbname}.stb where abs(c1+t2)=1 partition by tbname") - tdSql.checkRows(2) + tdSql.checkRows(10) tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname order by tbname") tdSql.checkRows(self.tb_nums) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index a6bcc2c5f1..5b989eb456 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -436,7 +436,7 @@ class TDTestCase: tdSql.checkData(1,1,None) tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 ") - tdSql.checkRows(2) + tdSql.checkRows(4) # filter by tbname tdSql.query(f"select last_row(c1) from {dbname}.stb1 where tbname = 'ct1' ") From ef5b5786b38e3d14465ae16780b47ed4f5935265 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 16:03:09 +0800 Subject: [PATCH 04/93] adjust --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1889492aa0..6df46ef4d0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -683,7 +683,7 @@ static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STable blockDataEmpty(pBlock); pBlock->info.rows = 1; pBlock->info.id.uid = tbInfo->uid; - pBlock->info.id.groupId = pOperator->dynamicTask ? tbInfo->uid : tbInfo->groupId; + pBlock->info.id.groupId = tbInfo->groupId; // only one row: set all col data to null & hasNull int32_t col_num = blockDataGetNumOfCols(pBlock); From 6e4703e5b43af47d83aca6acf4e5d8fe1dc16d7d Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 22:58:40 +0800 Subject: [PATCH 05/93] adjust ts --- .../2-query/nestedQueryInterval.py | 100 ++++++++++-------- 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index b6ef50dcda..185848f6e0 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -5,6 +5,8 @@ import socket import os import threading +from datetime import timezone, datetime + from util.log import * from util.sql import * from util.cases import * @@ -1196,9 +1198,14 @@ class TDTestCase: tdSql.checkData(0, 1, 122) tdSql.checkData(1, 1, 200) + pd = datetime.datetime.now() + ts = int(datetime.datetime.timestamp(pd)*1000 - 10000) + print(f"start time {ts}") + for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values({ts+i},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 132) @@ -1206,92 +1213,101 @@ class TDTestCase: for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values({ts+i},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 142) tdSql.checkData(1, 1, 200) - tdSql.execute(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'stable_1_1',1) \ - nested.stable_1 (ts,tbname,q_bigint) values(now+1a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_smallint) values(now+2a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_tinyint) values(now+3a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_float) values(now+4a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_double) values(now+5a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_bool) values(now+6a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_binary) values(now+7a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_nchar) values(now+8a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_ts) values(now+9a,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts,tbname,q_int) values({ts},'stable_1_1',1) \ + nested.stable_1 (ts,tbname,q_bigint) values({ts+1},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_smallint) values({ts+2},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_tinyint) values({ts+3},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_float) values({ts+4},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_double) values({ts+5},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_bool) values({ts+6},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_binary) values({ts+7},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_nchar) values({ts+8},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_ts) values({ts+9},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 152); tdSql.checkData(1, 1, 200); - tdSql.query(f"insert into nested.stable_null_data (ts,tbname) values(now,'stable_null_data_1');") + tdSql.query(f"insert into nested.stable_null_data (ts,tbname) values({ts},'stable_null_data_1');") + ts = ts + 1 tdSql.query(f"select tbname,count(*) from nested.stable_null_data_1 group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 22); for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 32) for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") - + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 42) - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values(now,'stable_null_data_1',1) \ - nested.stable_null_data (ts,tbname,q_bigint) values(now+1a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_smallint) values(now+2a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_tinyint) values(now+3a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_float) values(now+4a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_double) values(now+5a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_bool) values(now+6a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_binary) values(now+7a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_nchar) values(now+8a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_ts) values(now+9a,'stable_null_data_1',1);") + tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values({ts},'stable_null_data_1',1) \ + nested.stable_null_data (ts,tbname,q_bigint) values({ts+1},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_smallint) values({ts+2},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_tinyint) values({ts+3},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_float) values({ts+4},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_double) values({ts+5},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_bool) values({ts+6},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_binary) values({ts+7},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_nchar) values({ts+8},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_ts) values({ts+9},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 52); - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname) values(now,'stable_null_childtable_1');") + tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname) values({ts},'stable_null_childtable_1');") + ts = ts + 1 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 22) for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 32) for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 42); - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int) values(now,'stable_null_childtable_1',1) \ - nested.stable_null_childtable (ts,tbname,q_bigint) values(now+1a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_smallint) values(now+2a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_tinyint) values(now+3a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_float) values(now+4a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_double) values(now+5a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_bool) values(now+6a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_binary) values(now+7a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_nchar) values(now+8a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_ts) values(now+9a,'stable_null_childtable_1',1);") + tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int) values({ts},'stable_null_childtable_1',1) \ + nested.stable_null_childtable (ts,tbname,q_bigint) values({ts+1},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_smallint) values({ts+2},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_tinyint) values({ts+3},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_float) values({ts+4},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_double) values({ts+5},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_bool) values({ts+6},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_binary) values({ts+7},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_nchar) values({ts+8},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_ts) values({ts+9},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 52); @@ -1344,9 +1360,9 @@ class TDTestCase: #test special character - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'!@!@$$^$',1) \ - nested.stable_null_data (ts,tbname,q_int) values(now,'%^$^&^&',1) \ - nested.stable_null_childtable (ts,tbname,q_int) values(now,'$^%$%^&',1);") + tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now+10a,'!@!@$$^$',1) \ + nested.stable_null_data (ts,tbname,q_int) values(now+10a,'%^$^&^&',1) \ + nested.stable_null_childtable (ts,tbname,q_int) values(now+10a,'$^%$%^&',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(7) From c9b69316c1f8f71281d1506ed681989b5f4e4990 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 01:19:16 +0800 Subject: [PATCH 06/93] calcNeedCountEmpty --- include/libs/nodes/plannodes.h | 1 + source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/planner/src/planLogicCreater.c | 9 +++++++++ source/libs/planner/src/planOptimizer.c | 1 + source/libs/planner/src/planPhysiCreater.c | 22 +++++++++++----------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 4b3c846389..55ac3b86bf 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -147,6 +147,7 @@ typedef struct SAggLogicNode { bool isGroupTb; bool isPartTb; // true if partition keys has tbname bool hasGroup; + bool isCountByTag; // true if hasCountFunc & part by tag/tbname } SAggLogicNode; typedef struct SProjectLogicNode { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 8a154dcf00..2af31a8c7a 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -453,6 +453,7 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { COPY_SCALAR_FIELD(isGroupTb); COPY_SCALAR_FIELD(isPartTb); COPY_SCALAR_FIELD(hasGroup); + COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2adc5b3072..17b1b272ba 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -748,6 +748,15 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; + bool isCountByTag = false; + if (pSelect->hasCountFunc) { + if (pSelect->pGroupByList) { + isCountByTag = !keysHasCol(pSelect->pGroupByList); + } else if (pSelect->pPartitionByList) { + isCountByTag = !keysHasCol(pSelect->pPartitionByList); + } + } + pAgg->isCountByTag = isCountByTag; if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index aa3181e166..9ecbd3ae9f 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2783,6 +2783,7 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, pNew->isGroupTb = pAgg->isGroupTb; pNew->isPartTb = pAgg->isPartTb; pNew->hasGroup = pAgg->hasGroup; + pNew->isCountByTag = pAgg->isCountByTag; pNew->node.pChildren = nodesCloneList(pAgg->node.pChildren); int32_t code = 0; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 901927b1d1..9c4e9f97ab 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -37,6 +37,7 @@ typedef struct SPhysiPlanContext { SArray* pLocationHelper; bool hasScan; bool hasSysScan; + SLogicNode* pNode; // tmp record LogicSubplan->pNode } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey, int32_t keyBufSize) { @@ -592,17 +593,13 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog if (pScanLogicNode->interval > 0) { return false; } + // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { - SSelectStmt* pSelect = (SSelectStmt*)pRoot; - // select & count - if (pSelect->hasCountFunc) { - // key only accept tag/tbname - if (NULL != pSelect->pGroupByList) { - return !keysHasCol(pSelect->pGroupByList); - } else if (NULL != pSelect->pPartitionByList) { - return !keysHasCol(pSelect->pPartitionByList); - } + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) + && pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { + SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; + if (pNode->isCountByTag) { + return true; } } return false; @@ -2302,7 +2299,9 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } + pCxt->pNode = pLogicSubplan->pNode; code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); + pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); } @@ -2454,7 +2453,8 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), .hasScan = false, - .hasSysScan = false}; + .hasSysScan = false, + .pNode = NULL}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } From 651b681d4c95879c5a20668c8b85c45583383e98 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 18:45:51 +0800 Subject: [PATCH 07/93] adjust case --- tests/system-test/2-query/count.py | 4 ++-- tests/system-test/2-query/last_row.py | 4 ++-- tests/system-test/2-query/nestedQueryInterval.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 7e99f9e4dd..0e1d332d89 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -119,8 +119,8 @@ class TDTestCase: tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname order by tbname)') tdSql.checkRows(1) if 'count' == function_name: - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 2) elif 'hyperloglog' == function_name: tdSql.checkData(0, 0, 0) tdSql.checkData(0, 0, 0) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 5b989eb456..6469b3ea54 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -435,8 +435,8 @@ class TDTestCase: tdSql.checkData(1,0,'ct4') tdSql.checkData(1,1,None) - tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 ") - tdSql.checkRows(4) + tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 having count(c1)>0") + tdSql.checkRows(2) # filter by tbname tdSql.query(f"select last_row(c1) from {dbname}.stb1 where tbname = 'ct1' ") diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index 185848f6e0..07b5519432 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -1113,8 +1113,8 @@ class TDTestCase: def TS_3932(self): tdLog.debug("test insert data into stable") - tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(6) + tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname having count(*)>0 order by tbname;") + tdSql.checkRows(2) tdSql.checkData(0, 1, 100) tdSql.checkData(1, 1, 200) From 52c244d9ec0a9ce286a25379b3516159f0380222 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 23:48:37 +0800 Subject: [PATCH 08/93] adjust --- source/libs/planner/src/planPhysiCreater.c | 22 +++++++++++++++----- tests/system-test/2-query/count.py | 2 +- tests/system-test/2-query/group_partition.py | 10 ++++++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 9c4e9f97ab..5c78ff4432 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -595,13 +595,25 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog } // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) - && pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { - SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; - if (pNode->isCountByTag) { - return true; + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + // case1 root select + SSelectStmt* pSelect = (SSelectStmt*)pRoot; + if (pSelect->hasCountFunc) { + if (NULL != pSelect->pGroupByList) { + return !keysHasCol(pSelect->pGroupByList); + } else if (NULL != pSelect->pPartitionByList) { + return !keysHasCol(pSelect->pPartitionByList); + } + } + // case2 inner agg + if (pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { + SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; + if (pNode->isCountByTag) { + return true; + } } } + return false; } diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 0e1d332d89..a80c6f07e9 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -123,7 +123,7 @@ class TDTestCase: tdSql.checkData(0, 1, 2) elif 'hyperloglog' == function_name: tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) def query_empty_ntb(self): tdSql.query(f'select count(*) from {self.ntbname}') diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 58be8563fa..c08ea98ef6 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -83,6 +83,10 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + #inner select + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") + tdSql.checkRows(check_num) + # count + sum(col) tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) @@ -146,6 +150,10 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + #inner select + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") + tdSql.checkRows(check_num) + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") tdSql.checkRows(real_num) @@ -196,4 +204,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From da0ecc18b95af94abff05cece0acd002dfd38bd0 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 02:51:52 +0800 Subject: [PATCH 09/93] test --- source/libs/planner/src/planPhysiCreater.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 5c78ff4432..397258d6ca 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2311,9 +2311,11 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - pCxt->pNode = pLogicSubplan->pNode; + + if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pLogicSubplan->pNode)) { + pCxt->pNode = pLogicSubplan->pNode; + } code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); - pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); } @@ -2414,6 +2416,7 @@ static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogi break; } } + pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code) { *pPhysiPlan = pPlan; From 15694df001da02ab54ced68b014d6e48fe4a1cff Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 11:54:10 +0800 Subject: [PATCH 10/93] adjust --- source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/planner/src/planLogicCreater.c | 20 ++++++++-------- source/libs/planner/src/planOptimizer.c | 1 - source/libs/planner/src/planPhysiCreater.c | 27 +++------------------- tests/parallel_test/cases.task | 4 ++++ 5 files changed, 19 insertions(+), 35 deletions(-) diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 2af31a8c7a..c68fd81d22 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -424,6 +424,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(groupOrderScan); COPY_SCALAR_FIELD(onlyMetaCtbIdx); COPY_SCALAR_FIELD(filesetDelimited); + COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } @@ -453,7 +454,6 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { COPY_SCALAR_FIELD(isGroupTb); COPY_SCALAR_FIELD(isPartTb); COPY_SCALAR_FIELD(hasGroup); - COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 17b1b272ba..f55689c472 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -486,6 +486,16 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = tagScanSetExecutionMode(pScan); } + bool isCountByTag = false; + if (pSelect->hasCountFunc) { + if (pSelect->pGroupByList) { + isCountByTag = !keysHasCol(pSelect->pGroupByList); + } else if (pSelect->pPartitionByList) { + isCountByTag = !keysHasCol(pSelect->pPartitionByList); + } + } + pScan->isCountByTag = isCountByTag; + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pScan; } else { @@ -748,15 +758,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; - bool isCountByTag = false; - if (pSelect->hasCountFunc) { - if (pSelect->pGroupByList) { - isCountByTag = !keysHasCol(pSelect->pGroupByList); - } else if (pSelect->pPartitionByList) { - isCountByTag = !keysHasCol(pSelect->pPartitionByList); - } - } - pAgg->isCountByTag = isCountByTag; + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 9ecbd3ae9f..aa3181e166 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2783,7 +2783,6 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, pNew->isGroupTb = pAgg->isGroupTb; pNew->isPartTb = pAgg->isPartTb; pNew->hasGroup = pAgg->hasGroup; - pNew->isCountByTag = pAgg->isCountByTag; pNew->node.pChildren = nodesCloneList(pAgg->node.pChildren); int32_t code = 0; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 397258d6ca..0db0fd77fb 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -37,7 +37,6 @@ typedef struct SPhysiPlanContext { SArray* pLocationHelper; bool hasScan; bool hasSysScan; - SLogicNode* pNode; // tmp record LogicSubplan->pNode } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey, int32_t keyBufSize) { @@ -595,23 +594,8 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog } // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { - // case1 root select - SSelectStmt* pSelect = (SSelectStmt*)pRoot; - if (pSelect->hasCountFunc) { - if (NULL != pSelect->pGroupByList) { - return !keysHasCol(pSelect->pGroupByList); - } else if (NULL != pSelect->pPartitionByList) { - return !keysHasCol(pSelect->pPartitionByList); - } - } - // case2 inner agg - if (pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { - SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; - if (pNode->isCountByTag) { - return true; - } - } + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) && pScanLogicNode->isCountByTag) { + return true; } return false; @@ -2312,9 +2296,6 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pLogicSubplan->pNode)) { - pCxt->pNode = pLogicSubplan->pNode; - } code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); @@ -2416,7 +2397,6 @@ static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogi break; } } - pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code) { *pPhysiPlan = pPlan; @@ -2468,8 +2448,7 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), .hasScan = false, - .hasSysScan = false, - .pNode = NULL}; + .hasSysScan = false}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e63f86743b..5e7ef407f0 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -348,6 +348,10 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py From baee536cb06976332fe4d22305305edc0ee1d013 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 11:54:23 +0800 Subject: [PATCH 11/93] adjust --- include/libs/nodes/plannodes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 55ac3b86bf..f0383e8168 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -119,6 +119,7 @@ typedef struct SScanLogicNode { bool groupOrderScan; bool onlyMetaCtbIdx; // for tag scan with no tbname bool filesetDelimited; // returned blocks delimited by fileset + bool isCountByTag; // true if selectstmt hasCountFunc & part by tag/tbname } SScanLogicNode; typedef struct SJoinLogicNode { @@ -147,7 +148,6 @@ typedef struct SAggLogicNode { bool isGroupTb; bool isPartTb; // true if partition keys has tbname bool hasGroup; - bool isCountByTag; // true if hasCountFunc & part by tag/tbname } SAggLogicNode; typedef struct SProjectLogicNode { From 31c54e5edabc71924d72befc3194942b8b40fb73 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 12:42:25 +0800 Subject: [PATCH 12/93] adjust --- tests/system-test/2-query/count.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index a80c6f07e9..9bf9a4b9cc 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -182,8 +182,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 2) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') From f672df239e176a5256e76f9937d9e19a8da5b2cc Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 13:33:45 +0800 Subject: [PATCH 13/93] adjust --- tests/system-test/2-query/count.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 9bf9a4b9cc..3ba651622a 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -182,8 +182,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') From 2c9fa56e9c3ea6a0bbca237800da86995aa98cee Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 26 Dec 2023 10:24:50 +0800 Subject: [PATCH 14/93] calcNeedCountEmpty --- source/libs/planner/src/planPhysiCreater.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0db0fd77fb..444bda84cd 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,14 +587,12 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - // refuse interval +static bool calcNeedCountEmpty(SScanLogicNode* pScanLogicNode) { if (pScanLogicNode->interval > 0) { return false; } - // limit: root node is select - SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) && pScanLogicNode->isCountByTag) { + + if (pScanLogicNode->isCountByTag) { return true; } @@ -637,7 +635,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; - pTableScan->needCountEmptyTable = calcNeedCountEmpty(pCxt, pScanLogicNode); + pTableScan->needCountEmptyTable = calcNeedCountEmpty(pScanLogicNode); int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { From 3faf1514f43d1e03cb6557349a644ffcc598f93b Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 26 Dec 2023 20:47:15 +0800 Subject: [PATCH 15/93] doGroupedTableScan --- source/libs/executor/src/scanoperator.c | 45 +++++++++++----------- source/libs/planner/src/planLogicCreater.c | 1 - source/libs/planner/src/planPhysiCreater.c | 1 - 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6df46ef4d0..0f9b2f1eac 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -657,7 +657,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, // record processed (non empty) table -static int32_t insertTableToProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { +static int32_t markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (!pTableScanInfo->needCountEmptyTable) { return TSDB_CODE_SUCCESS; } @@ -770,6 +770,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; + bool outputAll = pTableScanInfo->base.pTableListInfo->oneTableForEachGroup; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { @@ -780,7 +781,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - insertTableToProcessed(pTableScanInfo, p->info.id.uid); + markTableProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -809,7 +810,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - insertTableToProcessed(pTableScanInfo, p->info.id.uid); + markTableProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -827,30 +828,30 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->needCountEmptyTable) { - if (num == 0 && 0 == taosHashGetSize(pTableScanInfo->pValuedTables)) { - // table by table, num is 0 - if (!pTableScanInfo->processingEmptyTable) { - pTableScanInfo->processingEmptyTable = true; - // current table is empty, fill result block info & return - const STableKeyInfo* info = tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->currentTable); - return getBlockForEmptyTable(pOperator, info); - } - - } else if (num > taosHashGetSize(pTableScanInfo->pValuedTables)) { - // group by group, num >= 1 + // pList is NULL in mode TABLE_SCAN__TABLE_ORDER for streamscan, no need to process + // pList not NULL, group by group, num >= 1 + int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pValuedTables); + if (pList && num > tb_cnt) { if (!pTableScanInfo->processingEmptyTable) { pTableScanInfo->processingEmptyTable = true; pTableScanInfo->currentTable = 0; } if (pTableScanInfo->currentTable < num) { - // loop: get empty table uid & process - while (pTableScanInfo->currentTable < num) { - const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; - if (pTableScanInfo->pValuedTables && - NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { - } else { - return getBlockForEmptyTable(pOperator, info); + if (outputAll) { + // loop: get empty table uid & process + while (pTableScanInfo->currentTable < num) { + const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; + if (pTableScanInfo->pValuedTables && + NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { + } else { + return getBlockForEmptyTable(pOperator, info); + } } + } else if (tb_cnt == 0) { + // only need one & all empty table in this group + // output first one + pTableScanInfo->currentTable = num; + return getBlockForEmptyTable(pOperator, pList); } } } @@ -1005,7 +1006,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - + if (pOperator->pOperatorGetParam) { pOperator->dynamicTask = true; int32_t code = createTableListInfoFromParam(pOperator); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f55689c472..e642b5bd5f 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -758,7 +758,6 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; - if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 444bda84cd..96205904e3 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2293,7 +2293,6 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); From 42f9d54a70b67170278adf3dac701ec10742d4a9 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 00:28:42 +0800 Subject: [PATCH 16/93] adjust --- source/libs/executor/src/scanoperator.c | 3 +- tests/system-test/2-query/group_partition.py | 42 +++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0f9b2f1eac..c69d963c79 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -770,7 +770,8 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - bool outputAll = pTableScanInfo->base.pTableListInfo->oneTableForEachGroup; + // Only when all tables are scanned can you determine how many groups the tag has + bool outputAll = true; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index c08ea98ef6..3d9874bfe2 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -40,9 +40,6 @@ class TDTestCase: tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) @@ -57,12 +54,13 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # tag - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by t2 ") - tdSql.checkRows(check_num) - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") tdSql.checkRows(check_num) + # multi tag + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") + tdSql.checkRows(check_num) + # having tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -87,10 +85,16 @@ class TDTestCase: tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") tdSql.checkRows(check_num) - # count + sum(col) + # multi agg tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(real_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") num = 0 @@ -105,14 +109,20 @@ class TDTestCase: tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} group by t2, t3, c1 ") + tdSql.checkRows(real_num * self.row_nums) + def test_partitionby(self, check_num, real_num): tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + tdSql.query(f" select count(c5), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(real_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") tdSql.checkRows(check_num) @@ -124,12 +134,12 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) #tag - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} partition by t2 ") - tdSql.checkRows(check_num) - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") tdSql.checkRows(check_num) + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} partition by t2, t3, tbname") + tdSql.checkRows(check_num) + # having tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -150,6 +160,13 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + #multi agg + tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(real_num) + #inner select tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) @@ -170,6 +187,9 @@ class TDTestCase: tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} partition by t2, t3, c1 ") + tdSql.checkRows(real_num * self.row_nums) + def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") From 86351d5487ae55010b6922d58e058c8a58420173 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 01:05:04 +0800 Subject: [PATCH 17/93] check count & window --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e642b5bd5f..ea8bb666b1 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -487,7 +487,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } bool isCountByTag = false; - if (pSelect->hasCountFunc) { + if (pSelect->hasCountFunc && !pSelect->pWindow) { if (pSelect->pGroupByList) { isCountByTag = !keysHasCol(pSelect->pGroupByList); } else if (pSelect->pPartitionByList) { From 158c5996dfb98aadb767847fce4d63555d63f957 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 15:51:36 +0800 Subject: [PATCH 18/93] adjust case --- source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/planner/src/planPhysiCreater.c | 14 +- tests/system-test/2-query/group_partition.py | 219 +++++++++---------- 3 files changed, 101 insertions(+), 134 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ea8bb666b1..12b7360165 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -487,7 +487,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } bool isCountByTag = false; - if (pSelect->hasCountFunc && !pSelect->pWindow) { + if (pSelect->hasCountFunc && NULL == pSelect->pWindow) { if (pSelect->pGroupByList) { isCountByTag = !keysHasCol(pSelect->pGroupByList); } else if (pSelect->pPartitionByList) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 96205904e3..e266c55425 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,18 +587,6 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static bool calcNeedCountEmpty(SScanLogicNode* pScanLogicNode) { - if (pScanLogicNode->interval > 0) { - return false; - } - - if (pScanLogicNode->isCountByTag) { - return true; - } - - return false; -} - static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, @@ -635,7 +623,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; - pTableScan->needCountEmptyTable = calcNeedCountEmpty(pScanLogicNode); + pTableScan->needCountEmptyTable = pScanLogicNode->isCountByTag; int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 3d9874bfe2..0ac1be8482 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -35,161 +35,132 @@ class TDTestCase: tdSql.execute(f"insert into {tbname} values({ts} , {row} , {row} , {row} , {row} , 1 , 2 , 'true' , 'binary_{row}' , 'nchar_{row}' , {row} , {row} , 1 ,2 )") - def test_groupby(self, check_num, real_num): - # tbname - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") + def test_groupby(self, keyword, check_num, nonempty_tb_num): + ####### by tbname + tdSql.query(f"select count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} {keyword} by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname order by count(*)") + tdSql.query(f"select tbname from {self.dbname}.{self.stable} {keyword} by tbname order by count(*)") tdSql.checkRows(check_num) - tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname having count(*)>=0") + # last + tdSql.query(f"select last(ts), count(*) from {self.dbname}.{self.stable} {keyword} by tbname order by last(ts)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} {keyword} by tbname having count(*)>=0") tdSql.checkRows(check_num) # having filter out empty - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") - tdSql.checkRows(check_num - real_num) + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} {keyword} by tbname having count(*) <= 0") + tdSql.checkRows(check_num - nonempty_tb_num) - # tag - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") + ####### by tag + tdSql.query(f"select t2, count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by t2 ") tdSql.checkRows(check_num) - # multi tag - tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} {keyword} by t2 having count(*) <= 0") + tdSql.checkRows(check_num - nonempty_tb_num) + + # where + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now {keyword} by t2 ") tdSql.checkRows(check_num) - # having - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") - tdSql.checkRows(check_num - real_num) - - # col where filter nothing - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 {keyword} by t2 ") tdSql.checkRows(check_num) - # col where filter all - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 {keyword} by t2 ") tdSql.checkRows(check_num) - # col where filter part - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") - tdSql.checkRows(check_num) - - # col - tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - #inner select - tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") - tdSql.checkRows(check_num) - - # multi agg - tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") - tdSql.checkRows(real_num) - - ############### same with old ############### - tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") + ####### by col + tdSql.query(f"select c1, count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by c1 ") num = 0 - if real_num > 0: + if nonempty_tb_num > 0: num = self.row_nums tdSql.checkRows(num) - tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} group by ts ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} {keyword} by ts ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) # col + tag - tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} {keyword} by t2, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} group by t2, t3, c1 ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} {keyword} by t2, t3, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - - def test_partitionby(self, check_num, real_num): - tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) - tdSql.query(f" select count(c5), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") - tdSql.checkRows(real_num) - - tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") - tdSql.checkRows(check_num) - - tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname having count(*)>=0") - tdSql.checkRows(check_num) - - # having filter out empty - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") - tdSql.checkRows(check_num - real_num) - - #tag - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") + def test_multi_group_key(self, check_num, nonempty_tb_num): + # multi tag/tbname + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") tdSql.checkRows(check_num) tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} partition by t2, t3, tbname") tdSql.checkRows(check_num) - # having - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") - tdSql.checkRows(check_num - real_num) + # multi tag + col + tdSql.query(f"select t1, t2, c1, count(*) from {self.dbname}.{self.stable} partition by t1, t2, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - # col where filter nothing - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") - tdSql.checkRows(check_num) + # tag + multi col + tdSql.query(f"select t2, c1, c2, count(*) from {self.dbname}.{self.stable} partition by t2, c1, c2 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - # col where filter all - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") - tdSql.checkRows(check_num) - # col where filter part - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") - tdSql.checkRows(check_num) + def test_multi_agg(self, all_tb_num, nonempty_tb_num): + tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(all_tb_num) - #col - tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) + tdSql.query(f"select count(1), sum(1), avg(c1), apercentile(c1, 50), spread(ts) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(all_tb_num) - #multi agg - tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) + tdSql.query(f"select count(c1), sum(c1), min(c1), mode(c1), stddev(c1), spread(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(all_tb_num) + + # elapsed: continuous duration in a statistical period, table merge scan + tdSql.query(f" select count(c1), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(nonempty_tb_num) tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") - tdSql.checkRows(real_num) + tdSql.checkRows(nonempty_tb_num) + + + def test_innerSelect(self, check_num): + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") + tdSql.checkRows(check_num) - #inner select tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) + tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.stable} partition by t1)") + tdSql.checkRows(check_num) + + + def test_window(self, nonempty_tb_num): + # empty group optimization condition is not met + # time window tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") - tdSql.checkRows(real_num) + tdSql.checkRows(nonempty_tb_num) - ############### same with old ############### - tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") - num = 0 - if real_num > 0: - num = self.row_nums - tdSql.checkRows(num) - - tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} partition by ts ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select _wstart, _wend, count(c1), max(c1), apercentile(c1, 50) from {self.dbname}.{self.stable} partition by tbname interval(1d)") + tdSql.checkRows(nonempty_tb_num) - tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") - tdSql.checkRows(real_num * self.row_nums) + # state window + tdSql.query(f"select tbname, count(*), c1 from {self.dbname}.{self.stable} partition by tbname state_window(c1)") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} partition by t2, t3, c1 ") - tdSql.checkRows(real_num * self.row_nums) + # session window + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname session(ts, 5s)") + tdSql.checkRows(nonempty_tb_num) + # event window + tdSql.query(f"select tbname, count(*) from {self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") + tdSql.checkRows(nonempty_tb_num) + + + def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") @@ -199,21 +170,29 @@ class TDTestCase: def run(self): tdSql.prepare() self.prepare_db() - check_num = self.tb_nums - self.test_groupby(check_num, 0) - self.test_partitionby(check_num, 0) - # insert into half of tables - real_num = 5 - self.insert_db(real_num, self.row_nums) - self.test_groupby(check_num, real_num) - self.test_partitionby(check_num, real_num) + # empty table only + self.test_groupby('group', self.tb_nums, 0) + self.test_groupby('partition', self.tb_nums, 0) + self.test_innerSelect(self.tb_nums) + self.test_multi_group_key(self.tb_nums, 0) + self.test_multi_agg(self.tb_nums, 0) + self.test_window(0) - # test old version before changed - # self.test_groupby(0, 0) - # self.test_partitionby(0, 0) + # insert data to 5 tables + nonempty_tb_num = 5 + self.insert_db(nonempty_tb_num, self.row_nums) + + self.test_groupby('group', self.tb_nums, nonempty_tb_num) + self.test_groupby('partition', self.tb_nums, nonempty_tb_num) + self.test_innerSelect(self.tb_nums) + self.test_multi_group_key(self.tb_nums, nonempty_tb_num) + self.test_multi_agg(self.tb_nums, nonempty_tb_num) + self.test_window(nonempty_tb_num) + + ## test old version before changed + # self.test_groupby('group', 0, 0) # self.insert_db(5, self.row_nums) - # self.test_groupby(5, 5) - # self.test_partitionby(5, 5) + # self.test_groupby('group', 5, 5) self.test_error() From f6e55d358b375e86c5d82aca6ed089303d4b9347 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 16:12:50 +0800 Subject: [PATCH 19/93] =?UTF-8?q?=C2=A0adjust=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/system-test/2-query/group_partition.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 0ac1be8482..8b06f0d6fd 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -134,7 +134,7 @@ class TDTestCase: tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) - tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.stable} partition by t1)") + tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.dbname}.{self.stable} partition by t1)") tdSql.checkRows(check_num) @@ -156,7 +156,7 @@ class TDTestCase: tdSql.checkRows(nonempty_tb_num) # event window - tdSql.query(f"select tbname, count(*) from {self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") tdSql.checkRows(nonempty_tb_num) From d795798648ac0d1945e90bbbf39d445ed0f88679 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 21:37:00 +0800 Subject: [PATCH 20/93] count state --- source/libs/executor/inc/executorInt.h | 11 ++- source/libs/executor/src/scanoperator.c | 94 ++++++++++++++----------- 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 2523b87cfb..8b0bef5fa6 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -217,6 +217,12 @@ enum { TABLE_SCAN__BLOCK_ORDER = 2, }; +typedef enum ETableCountState { + TABLE_COUNT_STATE_NONE = 0, // before start scan + TABLE_COUNT_STATE_SCAN = 1, // scanning + TABLE_COUNT_STATE_END = 2, // finish or noneed to process +} ETableCountState; + typedef struct SAggSupporter { SSHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer @@ -262,17 +268,18 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; - SHashObj* pValuedTables; // non empty table uids + SHashObj* pRemainTables; // remain table to process SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; int8_t scanMode; int8_t assignBlockUid; + uint8_t countState; // empty table count state + bool isOneGroup; // whether or not only one group in this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; bool needCountEmptyTable; - bool processingEmptyTable; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c69d963c79..31fe31007c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,26 +655,44 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } - -// record processed (non empty) table -static int32_t markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { +static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { return TSDB_CODE_SUCCESS; } - if (NULL == pTableScanInfo->pValuedTables) { + pTableScanInfo->isOneGroup = true; + if (NULL == pTableScanInfo->pRemainTables) { int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); - pTableScanInfo->pValuedTables = + pTableScanInfo->pRemainTables = taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (NULL == pTableScanInfo->pValuedTables) { + if (NULL == pTableScanInfo->pRemainTables) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_OUT_OF_MEMORY; } } - - taosHashPut(pTableScanInfo->pValuedTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, - sizeof(pTableScanInfo->scanTimes)); + uint64_t groupId = 0; + for (int32_t i = 0; i < num; i++) { + const STableKeyInfo* pInfo = pList + i; + if (pTableScanInfo->isOneGroup) { + if (i == 0) { + groupId = pInfo->groupId; + } else if (groupId != pInfo->groupId) { + pTableScanInfo->isOneGroup = false; + } + } + taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); + } return TSDB_CODE_SUCCESS; } +static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { + // case0 group scanning, mark + // case1 stream scan: no need to mark + if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { + return; + } + taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); +} + static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -770,14 +788,17 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - // Only when all tables are scanned can you determine how many groups the tag has - bool outputAll = true; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; } + if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { + initRemainTable(pTableScanInfo, pList, num); + pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; + } + // do the ascending order traverse in the first place. while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); @@ -829,37 +850,28 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->needCountEmptyTable) { - // pList is NULL in mode TABLE_SCAN__TABLE_ORDER for streamscan, no need to process - // pList not NULL, group by group, num >= 1 - int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pValuedTables); - if (pList && num > tb_cnt) { - if (!pTableScanInfo->processingEmptyTable) { - pTableScanInfo->processingEmptyTable = true; - pTableScanInfo->currentTable = 0; - } - if (pTableScanInfo->currentTable < num) { - if (outputAll) { - // loop: get empty table uid & process - while (pTableScanInfo->currentTable < num) { - const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; - if (pTableScanInfo->pValuedTables && - NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { - } else { - return getBlockForEmptyTable(pOperator, info); - } - } - } else if (tb_cnt == 0) { - // only need one & all empty table in this group - // output first one - pTableScanInfo->currentTable = num; - return getBlockForEmptyTable(pOperator, pList); + int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); + if (tb_cnt) { + if (!pTableScanInfo->isOneGroup) { + // get first empty table uid, mark processed & rm from hash + void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); + if (pIte != NULL) { + size_t keySize = 0; + uint64_t* pUid = taosHashGetKey(pIte, &keySize); + STableKeyInfo info = {.uid = *pUid, .groupId = *(uint64_t*)pIte}; + taosHashCancelIterate(pTableScanInfo->pRemainTables, pIte); + markTableProcessed(pTableScanInfo, *pUid); + return getBlockForEmptyTable(pOperator, &info); } + } else { + // output one table for this group + taosHashClear(pTableScanInfo->pRemainTables); + return getBlockForEmptyTable(pOperator, pList); } } - pTableScanInfo->processingEmptyTable = false; + pTableScanInfo->countState = TABLE_COUNT_STATE_END; } - taosHashClear(pTableScanInfo->pValuedTables); return NULL; } @@ -937,7 +949,8 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - + pInfo->countState = TABLE_COUNT_STATE_NONE; + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; @@ -968,6 +981,7 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { } tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); + pInfo->countState = TABLE_COUNT_STATE_NONE; ASSERT(pInfo->base.dataReader == NULL); int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, @@ -1034,6 +1048,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { int32_t numOfTables = 0; // tableListGetSize(pTaskInfo->pTableListInfo); STableKeyInfo tInfo = {0}; + pInfo->countState = TABLE_COUNT_STATE_END; while (1) { SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); @@ -1096,7 +1111,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - taosHashCleanup(pTableScanInfo->pValuedTables); + taosHashCleanup(pTableScanInfo->pRemainTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1161,7 +1176,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->exprSupp.numOfExprs = numOfCols; pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; - pInfo->processingEmptyTable = false; pInfo->base.pTableListInfo = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); From abed781bd324dda7c636e162a25646292214004a Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 23:15:11 +0800 Subject: [PATCH 21/93] opt --- source/libs/executor/src/scanoperator.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 31fe31007c..0714019352 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -657,6 +657,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_SUCCESS; } pTableScanInfo->isOneGroup = true; @@ -681,6 +682,7 @@ static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyIn } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); } + pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; return TSDB_CODE_SUCCESS; } @@ -690,6 +692,11 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { return; } + // case2 only one group, uid ready + if (pTableScanInfo->isOneGroup) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; + return; + } taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); } @@ -796,7 +803,6 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { initRemainTable(pTableScanInfo, pList, num); - pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; } // do the ascending order traverse in the first place. @@ -849,7 +855,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } } - if (pTableScanInfo->needCountEmptyTable) { + if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); if (tb_cnt) { if (!pTableScanInfo->isOneGroup) { @@ -865,14 +871,14 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } } else { // output one table for this group - taosHashClear(pTableScanInfo->pRemainTables); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return getBlockForEmptyTable(pOperator, pList); } } - pTableScanInfo->countState = TABLE_COUNT_STATE_END; } + taosHashClear(pTableScanInfo->pRemainTables); return NULL; } From a5f88a86e9b18ac7b3a3a61592659fa72a40c9a1 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 09:45:01 +0800 Subject: [PATCH 22/93] adjust --- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/scanoperator.c | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 8b0bef5fa6..b87dee475d 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -275,7 +275,7 @@ typedef struct STableScanInfo { int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state - bool isOneGroup; // whether or not only one group in this scan + bool isSameGroup; // whether all tables are in the same group this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0714019352..dd6dff0301 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,12 +655,12 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } -static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { +static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_SUCCESS; } - pTableScanInfo->isOneGroup = true; + pTableScanInfo->isSameGroup = true; if (NULL == pTableScanInfo->pRemainTables) { int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); pTableScanInfo->pRemainTables = @@ -673,11 +673,11 @@ static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyIn uint64_t groupId = 0; for (int32_t i = 0; i < num; i++) { const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isOneGroup) { + if (pTableScanInfo->isSameGroup) { if (i == 0) { groupId = pInfo->groupId; } else if (groupId != pInfo->groupId) { - pTableScanInfo->isOneGroup = false; + pTableScanInfo->isSameGroup = false; } } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); @@ -692,8 +692,8 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { return; } - // case2 only one group, uid ready - if (pTableScanInfo->isOneGroup) { + // case2 if all table in same group, process only once + if (pTableScanInfo->isSameGroup) { pTableScanInfo->countState = TABLE_COUNT_STATE_END; return; } @@ -802,7 +802,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { - initRemainTable(pTableScanInfo, pList, num); + initTableCountEnv(pTableScanInfo, pList, num); } // do the ascending order traverse in the first place. @@ -858,7 +858,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); if (tb_cnt) { - if (!pTableScanInfo->isOneGroup) { + if (!pTableScanInfo->isSameGroup) { // get first empty table uid, mark processed & rm from hash void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); if (pIte != NULL) { From 424ab1bbe32672570d4b8549b9b8f375e55bcbb5 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 16:18:38 +0800 Subject: [PATCH 23/93] support table merge scan --- source/libs/executor/inc/executorInt.h | 3 ++- source/libs/executor/src/scanoperator.c | 27 ++++++++++++++------ tests/system-test/2-query/group_partition.py | 17 +++++++++--- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index b87dee475d..ae38d4940c 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -307,7 +307,8 @@ typedef struct STableMergeScanInfo { SHashObj* mSkipTables; int64_t mergeLimit; SSortExecInfo sortExecInfo; - + bool needCountEmptyTable; + bool bGroupProcessed; // the group return data means processed bool filesetDelimited; bool bNewFilesetEvent; bool bNextDurationBlockEvent; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index dd6dff0301..a1f9baa082 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -700,17 +700,14 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); } -static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { - STableScanInfo* pTableScanInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - +static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, + const STableKeyInfo* tbInfo) { blockDataEmpty(pBlock); pBlock->info.rows = 1; pBlock->info.id.uid = tbInfo->uid; pBlock->info.id.groupId = tbInfo->groupId; - // only one row: set all col data to null & hasNull + // only one row: set all col data to null & hasNull int32_t col_num = blockDataGetNumOfCols(pBlock); for (int32_t i = 0; i < col_num; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); @@ -718,7 +715,14 @@ static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STable } // set tag/tbname - doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); + doSetTagColumnData(pBase, pBlock, pTaskInfo, pBlock->info.rows); + return pBlock; +} + +static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { + STableScanInfo* pTableScanInfo = pOperator->info; + SSDataBlock* pBlock = + getOneRowResultBlock(pOperator->pTaskInfo, &pTableScanInfo->base, pTableScanInfo->pResBlock, tbInfo); pOperator->resultInfo.totalRows++; return pBlock; @@ -3585,7 +3589,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { } pInfo->tableEndIndex = i - 1; } - + pInfo->bGroupProcessed = false; int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; @@ -3707,9 +3711,14 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity, pOperator); + if (pBlock == NULL && !pInfo->bGroupProcessed && pInfo->needCountEmptyTable) { + STableKeyInfo* tbInfo = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + pBlock = getOneRowResultBlock(pTaskInfo, &pInfo->base, pInfo->pResBlock, tbInfo); + } if (pBlock != NULL) { pBlock->info.id.groupId = pInfo->groupId; pOperator->resultInfo.totalRows += pBlock->info.rows; + pInfo->bGroupProcessed = true; return pBlock; } else { if (pInfo->bNewFilesetEvent) { @@ -3864,6 +3873,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN } else { pInfo->filesetDelimited = pTableScanNode->filesetDelimited; } + pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; + setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 8b06f0d6fd..e228351f0e 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -120,12 +120,21 @@ class TDTestCase: tdSql.checkRows(all_tb_num) # elapsed: continuous duration in a statistical period, table merge scan - tdSql.query(f" select count(c1), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") - tdSql.checkRows(nonempty_tb_num) - - tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.query(f" select count(c1), max(c5), last_row(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select count(c1), min(c1), avg(c1), elapsed(ts), mode(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select count(c1), elapsed(ts), twa(c1), irate(c1), leastsquares(c1,1,1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select avg(c1), elapsed(ts), twa(c1), irate(c1) from {self.dbname}.{self.stable} partition by tbname") tdSql.checkRows(nonempty_tb_num) + # if nonempty_tb_num > 0: + # tdSql.query(f" select avg(c1), percentile(c1, 50) from {self.dbname}.sub_{self.stable}_1") + # tdSql.checkRows(1) def test_innerSelect(self, check_num): tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") From 2526df745ee62db8786adcd732e8b0cd4646a723 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:15:09 +0800 Subject: [PATCH 24/93] enh: adjust threshold of snap replication timeout --- include/libs/sync/sync.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e54237fe8b..cb053d2548 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -46,8 +46,8 @@ extern "C" { #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 -#define SYNC_SNAP_RESEND_MS 1000 * 300 -#define SYNC_SNAP_TIMEOUT_MS 1000 * 1800 +#define SYNC_SNAP_RESEND_MS 1000 * 60 +#define SYNC_SNAP_TIMEOUT_MS 1000 * 300 #define SYNC_VND_COMMIT_MIN_MS 3000 From d83f85bf2642982d8fdcb5bd53b58011cc8f81e1 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:26:46 +0800 Subject: [PATCH 25/93] enh: send rsp msg on rejecting snap replication due to smaller term --- source/libs/sync/src/syncSnapshot.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 10a8734617..f0e457ef8d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1001,6 +1001,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { sRError(pReceiver, "reject snap replication with smaller term. msg term:%" PRId64 ", seq:%d", pMsg->term, pMsg->seq); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, terrno); return -1; } From baa7f9c895faf0ed12d59c5ff02460ae8eb3f1da Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:29:49 +0800 Subject: [PATCH 26/93] enh: adjust error msg as warn for not ready to propose etc --- source/dnode/vnode/src/vnd/vnodeSync.c | 8 ++++---- source/libs/sync/src/syncMain.c | 6 +++--- source/libs/sync/src/syncPipeline.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 048092131d..0f491e9a58 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -220,8 +220,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) isWeak, isBlock, msg, numOfMsgs, arrayPos, pMsg->info.handle); if (!pVnode->restored) { - vGError("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, - TMSG_INFO(pMsg->msgType)); + vGWarn("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, + TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_SYN_RESTORING; vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); @@ -284,8 +284,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle); if (!pVnode->restored) { - vGError("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, - TMSG_INFO(pMsg->msgType)); + vGWarn("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, + TMSG_INFO(pMsg->msgType)); vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 6f3b3fdf98..52557f7b9c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -662,14 +662,14 @@ ESyncRole syncGetRole(int64_t rid) { int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { terrno = TSDB_CODE_SYN_NOT_LEADER; - sNError(pSyncNode, "sync propose not leader, type:%s", TMSG_INFO(pMsg->msgType)); + sNWarn(pSyncNode, "sync propose not leader, type:%s", TMSG_INFO(pMsg->msgType)); return -1; } if (!pSyncNode->restoreFinish) { terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; - sNError(pSyncNode, "failed to sync propose since not ready, type:%s, last:%" PRId64 ", cmt:%" PRId64, - TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); + sNWarn(pSyncNode, "failed to sync propose since not ready, type:%s, last:%" PRId64 ", cmt:%" PRId64, + TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); return -1; } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 28ee5ba841..70bdd4a837 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -206,7 +206,7 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { } if (pLogStore->syncLogGetEntry(pLogStore, index, &pEntry) < 0) { - sError("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); + sWarn("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); break; } @@ -1237,7 +1237,7 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, } else { *pInBuf = false; if (pNode->pLogStore->syncLogGetEntry(pNode->pLogStore, index, &pEntry) < 0) { - sError("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); + sWarn("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); } } return pEntry; @@ -1253,7 +1253,7 @@ int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex ind pEntry = syncLogBufferGetOneEntry(pBuf, pNode, index, &inBuf); if (pEntry == NULL) { - sError("vgId:%d, failed to get raft entry for index:%" PRId64 "", pNode->vgId, index); + sWarn("vgId:%d, failed to get raft entry for index:%" PRId64 "", pNode->vgId, index); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); if (pMgr) { From ef22830ca39a70be588db086c6d769d49159e444 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 18:01:30 +0800 Subject: [PATCH 27/93] adjust --- tests/system-test/2-query/irate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index d976edb49c..d882c9b8fb 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -209,7 +209,7 @@ class TDTestCase: tdSql.error(f"select irate(c1), abs(c1) from {dbname}.ct4 ") # agg functions mix with agg functions - tdSql.query(f"select irate(c1), count(c5) from {dbname}.stb1 partition by tbname order by tbname") + tdSql.query(f"select irate(c1), count(c5) from {dbname}.stb1 partition by tbname having count(c5)>0 order by tbname") tdSql.checkData(0, 0, 0.000000000) tdSql.checkData(1, 0, 0.000000000) tdSql.checkData(0, 1, 13) From 9052ca6eded8c879f3c6a9728ce308e3bd2eac15 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 28 Dec 2023 18:05:28 +0800 Subject: [PATCH 28/93] add test case for ts-4403 --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/test_ts4403.py | 77 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 tests/system-test/2-query/test_ts4403.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e7c26b9833..a04a9f6d0a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -519,6 +519,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py diff --git a/tests/system-test/2-query/test_ts4403.py b/tests/system-test/2-query/test_ts4403.py new file mode 100644 index 0000000000..7f0561d084 --- /dev/null +++ b/tests/system-test/2-query/test_ts4403.py @@ -0,0 +1,77 @@ +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TS-4403 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db' + self.stbname = 'st' + + def prepareData(self): + # db + tdSql.execute("create database {};".format(self.dbname)) + tdSql.execute("use {};".format(self.dbname)) + tdLog.debug("Create database %s" % self.dbname) + + # super table + tdSql.execute("create table {} (ts timestamp, col1 int) tags (t1 int, t2 binary(16), t3 nchar(16));".format(self.stbname)) + tdLog.debug("Create super table %s" % self.stbname) + + # create index for all tags + tdSql.execute("create index t2_st on {} (t2);".format(self.stbname)) + tdSql.execute("create index t3_st on {} (t3);".format(self.stbname)) + + def run(self): + self.prepareData() + # check index number + tdSql.query("show indexes from {};".format(self.stbname)) + assert(3 == len(tdSql.queryResult)) + tdLog.debug("All tags of super table have index successfully") + + # drop default first tag index + tdSql.execute("drop index t1_st;") + tdSql.query("show indexes from {};".format(self.stbname)) + assert(2 == len(tdSql.queryResult)) + tdLog.debug("Delete the default index of first tag successfully") + + # create index for first tag + tdSql.execute("create index t1_st on {} (t1);".format(self.stbname)) + tdSql.query("show indexes from {};".format(self.stbname)) + assert(3 == len(tdSql.queryResult)) + tdLog.debug("Create index for first tag successfully") + + # null as index value to create child table + tdSql.execute("create table ct1 using {} tags(null, null, null);".format(self.stbname)) + tdSql.query("show tables;") + assert(1 == len(tdSql.queryResult)) + tdLog.debug("Create child table with tags value as 'null' successfully") + + # redundant index with different index name for some tag + tdSql.error("create index t2_ct1 on st (t2);") + tdLog.debug("Verify redundant index with different index name for some tag successfully") + + # same index name for different tag + tdSql.error("create index t2_st on st (t3);") + tdLog.debug("Verify same index name for some tag successfully") + + # add index for multiple tags(TD-28078) + tdSql.error("create index tt on {} (t2, t3);".format(self.stbname)) + tdLog.debug("Verify add index for multiple tags successfully") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 9f8f69a791f9544e11475af4f0f7c0fa04663b56 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 21:54:16 +0800 Subject: [PATCH 29/93] adjust --- source/libs/executor/src/scanoperator.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a1f9baa082..7b2df4e206 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -670,15 +670,11 @@ static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKey return TSDB_CODE_OUT_OF_MEMORY; } } - uint64_t groupId = 0; + uint64_t groupId = pList->groupId; for (int32_t i = 0; i < num; i++) { const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isSameGroup) { - if (i == 0) { - groupId = pInfo->groupId; - } else if (groupId != pInfo->groupId) { - pTableScanInfo->isSameGroup = false; - } + if (pTableScanInfo->isSameGroup && groupId != pInfo->groupId) { + pTableScanInfo->isSameGroup = false; } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); } From e2b61a55fc13aee349cb63abb3c7d003c1de783f Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 29 Dec 2023 01:40:17 +0800 Subject: [PATCH 30/93] refactor --- source/libs/executor/inc/executorInt.h | 6 +- source/libs/executor/src/executor.c | 12 +-- source/libs/executor/src/scanoperator.c | 129 ++++++++---------------- 3 files changed, 52 insertions(+), 95 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index ae38d4940c..9acef69f9c 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -268,14 +268,12 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; - SHashObj* pRemainTables; // remain table to process SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; - int32_t currentTable; + int32_t tableStartIndex; // current group scan start + int32_t tableEndIndex; // current group scan end int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state - bool isSameGroup; // whether all tables are in the same group this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6cee79bff2..fb39de484f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1209,7 +1209,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0); uid = pTableInfo->uid; ts = INT64_MIN; - pScanInfo->currentTable = 0; + pScanInfo->tableEndIndex = 0; } else { taosRUnLockLatch(&pTaskInfo->lock); qError("no table in table list, %s", id); @@ -1223,16 +1223,16 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pInfo->pTableScanOp->resultInfo.totalRows = 0; // start from current accessed position - // we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start + // we cannot start from the pScanInfo->tableEndIndex, since the commit offset may cause the rollback of the start // position, let's find it from the beginning. index = tableListFind(pTableListInfo, uid, 0); taosRUnLockLatch(&pTaskInfo->lock); if (index >= 0) { - pScanInfo->currentTable = index; + pScanInfo->tableEndIndex = index; } else { qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, - numOfTables, pScanInfo->currentTable, id); + numOfTables, pScanInfo->tableEndIndex, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1255,12 +1255,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", - uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id); } else { pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1); pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", - uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id); } // restore the key value diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7b2df4e206..e990d3d975 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,45 +655,29 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } -static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { - if (!pTableScanInfo->needCountEmptyTable) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return TSDB_CODE_SUCCESS; - } - pTableScanInfo->isSameGroup = true; - if (NULL == pTableScanInfo->pRemainTables) { - int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); - pTableScanInfo->pRemainTables = - taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (NULL == pTableScanInfo->pRemainTables) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return TSDB_CODE_OUT_OF_MEMORY; - } - } - uint64_t groupId = pList->groupId; - for (int32_t i = 0; i < num; i++) { - const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isSameGroup && groupId != pInfo->groupId) { - pTableScanInfo->isSameGroup = false; - } - taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); - } - pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; - return TSDB_CODE_SUCCESS; -} -static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { - // case0 group scanning, mark - // case1 stream scan: no need to mark - if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { - return; +static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) { + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + int32_t i = pInfo->tableStartIndex + 1; + for (; i < numOfTables; ++i) { + STableKeyInfo* pCur = tableListGetInfo(pInfo->base.pTableListInfo, i); + if (pCur->groupId != pStart->groupId) { + break; + } } - // case2 if all table in same group, process only once - if (pTableScanInfo->isSameGroup) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return; + + pInfo->tableEndIndex = i - 1; + if (!pInfo->needCountEmptyTable) { + pInfo->countState = TABLE_COUNT_STATE_END; + } else { + pInfo->countState = TABLE_COUNT_STATE_SCAN; } - taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); + + *pKeyInfo = pStart; + *size = i - pInfo->tableStartIndex; } static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, @@ -791,7 +775,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { return NULL; } -static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKeyInfo* pList, int32_t num) { +static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -801,15 +785,11 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey return NULL; } - if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { - initTableCountEnv(pTableScanInfo, pList, num); - } - // do the ascending order traverse in the first place. while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - markTableProcessed(pTableScanInfo, p->info.id.uid); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return p; } @@ -838,7 +818,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - markTableProcessed(pTableScanInfo, p->info.id.uid); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return p; } @@ -856,29 +836,13 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { - int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); - if (tb_cnt) { - if (!pTableScanInfo->isSameGroup) { - // get first empty table uid, mark processed & rm from hash - void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); - if (pIte != NULL) { - size_t keySize = 0; - uint64_t* pUid = taosHashGetKey(pIte, &keySize); - STableKeyInfo info = {.uid = *pUid, .groupId = *(uint64_t*)pIte}; - taosHashCancelIterate(pTableScanInfo->pRemainTables, pIte); - markTableProcessed(pTableScanInfo, *pUid); - return getBlockForEmptyTable(pOperator, &info); - } - } else { - // output one table for this group - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return getBlockForEmptyTable(pOperator, pList); - } - } + // output once for this group pTableScanInfo->countState = TABLE_COUNT_STATE_END; + STableKeyInfo* pStart = + (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); + return getBlockForEmptyTable(pOperator, pStart); } - taosHashClear(pTableScanInfo->pRemainTables); return NULL; } @@ -938,8 +902,8 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + if (pInfo->tableEndIndex + 1 >= numOfTables) { setOperatorCompleted(pOperator); if (pOperator->dynamicTask) { taosArrayClear(pInfo->base.pTableListInfo->pTableList); @@ -954,14 +918,13 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pInfo->countState = TABLE_COUNT_STATE_NONE; + initNextGroupScan(pInfo, &pList, &num); pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; - SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -979,15 +942,14 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - if (pInfo->currentGroupId == -1) { + if (pInfo->tableEndIndex == -1) { int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo) || numOfTables == 0) { + if (pInfo->tableEndIndex + 1 == numOfTables) { setOperatorCompleted(pOperator); return NULL; } - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pInfo->countState = TABLE_COUNT_STATE_NONE; + initNextGroupScan(pInfo, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, @@ -1001,11 +963,9 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; } - } else { - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); } - SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -1038,7 +998,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } if (pOperator->status == OP_EXEC_DONE) { - pInfo->currentGroupId = -1; + pInfo->tableEndIndex = -1; pOperator->status = OP_OPENED; SSDataBlock* result = NULL; while (true) { @@ -1057,29 +1017,29 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->countState = TABLE_COUNT_STATE_END; while (1) { - SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) { return result; } // if no data, switch to next table and continue scan - pInfo->currentTable++; + pInfo->tableEndIndex++; taosRLockLatch(&pTaskInfo->lock); numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if (pInfo->currentTable >= numOfTables) { + if (pInfo->tableEndIndex >= numOfTables) { qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); taosRUnLockLatch(&pTaskInfo->lock); return NULL; } - tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); + tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableEndIndex); taosRUnLockLatch(&pTaskInfo->lock); pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, - pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); + pInfo->tableEndIndex, numOfTables, GET_TASKID(pTaskInfo)); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; @@ -1117,7 +1077,6 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - taosHashCleanup(pTableScanInfo->pRemainTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1173,7 +1132,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } - pInfo->currentGroupId = -1; + pInfo->tableEndIndex = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; @@ -1268,7 +1227,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.startVersion = 0; pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; - pTableScanInfo->currentGroupId = -1; + pTableScanInfo->tableEndIndex = -1; pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -2170,7 +2129,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->pTableScanOp->status = OP_OPENED; pTSInfo->scanTimes = 0; - pTSInfo->currentGroupId = -1; + pTSInfo->tableEndIndex = -1; } if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { From 90a8fe558196d58198221a46b1c08f0e27203bf3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 29 Dec 2023 15:08:56 +0800 Subject: [PATCH 31/93] fix: fill prev not working when desc fill --- source/libs/executor/src/tfill.c | 13 +++-- tests/system-test/2-query/fill_with_group.py | 51 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 6c537d7b98..7b63bc8720 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -169,13 +169,14 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* setNotFillColumn(pFillInfo, pDstCol, index, i); } } else { - SGroupKeys* pKey = taosArrayGet(pFillInfo->prev.pRowVal, i); + SRowVal* pRVal = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; + SGroupKeys* pKey = taosArrayGet(pRVal->pRowVal, i); if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { colDataSetNULL(pDstCol, index); continue; } - SGroupKeys* pKey1 = taosArrayGet(pFillInfo->prev.pRowVal, pFillInfo->tsSlotId); + SGroupKeys* pKey1 = taosArrayGet(pRVal->pRowVal, pFillInfo->tsSlotId); int64_t prevTs = *(int64_t*)pKey1->pData; int32_t srcSlotId = GET_DEST_SLOT_ID(pCol); @@ -346,9 +347,10 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t char* src = colDataGetData(pSrc, pFillInfo->index); if (!colDataIsNull_s(pSrc, pFillInfo->index)) { colDataSetVal(pDst, index, src, false); - saveColData(pFillInfo->prev.pRowVal, i, src, false); + SRowVal* pRVal = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; + saveColData(pRVal->pRowVal, i, src, false); if (pFillInfo->srcTsSlotId == dstSlotId) { - pFillInfo->prev.key = *(int64_t*)src; + pRVal->key = *(int64_t*)src; } } else { // the value is null if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { @@ -361,7 +363,8 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t } else if (pFillInfo->type == TSDB_FILL_LINEAR) { bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); colDataSetVal(pDst, index, src, isNull); - saveColData(pFillInfo->prev.pRowVal, i, src, isNull); // todo: + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; + saveColData(p, i, src, isNull); // todo: } else if (pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { colDataSetNULL(pDst, index); } else if (pFillInfo->type == TSDB_FILL_NEXT) { diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index b442647ff4..fce504d0f7 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -144,10 +144,61 @@ class TDTestCase: tdSql.query(sql) tdSql.checkRows(6) + def test_fill_with_order_by2(self): + ## window size: 5 minutes, with 6 rows in meters every 10 minutes + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart asc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, 10) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(linear) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + def run(self): self.prepareTestEnv() self.test_partition_by_with_interval_fill_prev_new_group_fill_error() self.test_fill_with_order_by() + self.test_fill_with_order_by2() def stop(self): tdSql.close() From 7cb12db8ce4b4eaa3e1446fdb0eca21c74376c8c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 29 Dec 2023 15:20:27 +0800 Subject: [PATCH 32/93] remove more useless code --- source/dnode/vnode/src/inc/tsdb.h | 18 - .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 796 ------------------ 2 files changed, 814 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ebaf0851e6..75f90af378 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -258,18 +258,6 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback); int32_t tsdbFSClose(STsdb *pTsdb); void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); // tsdbReaderWriter.c ============================================================================================== -// SDataFWriter -int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); -int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); -int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); -int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx); -int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast); -int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo); - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -281,12 +269,6 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *p int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); -// SDelFWriter -int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); -int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx); -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx); -int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e6f419362c..babf8c75fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -433,635 +433,6 @@ _exit: return code; } -// SDataFWriter ==================================================== -int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { - int32_t code = 0; - int32_t flag; - int64_t n; - int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; - SDataFWriter *pWriter = NULL; - char fname[TSDB_FILENAME_LEN]; - char hdr[TSDB_FHDR_SIZE] = {0}; - - // alloc - pWriter = taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pWriter->pTsdb = pTsdb; - pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, - .fid = pSet->fid, - .pHeadF = &pWriter->fHead, - .pDataF = &pWriter->fData, - .pSmaF = &pWriter->fSma, - .nSttF = pSet->nSttF}; - pWriter->fHead = *pSet->pHeadF; - pWriter->fData = *pSet->pDataF; - pWriter->fSma = *pSet->pSmaF; - for (int8_t iStt = 0; iStt < pSet->nSttF; iStt++) { - pWriter->wSet.aSttF[iStt] = &pWriter->fStt[iStt]; - pWriter->fStt[iStt] = *pSet->aSttF[iStt]; - } - - // head - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pHeadFD); - if (code) goto _err; - - code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fHead.size += TSDB_FHDR_SIZE; - - // data - if (pWriter->fData.size == 0) { - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_READ | TD_FILE_WRITE; - } - tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pDataFD); - if (code) goto _err; - if (pWriter->fData.size == 0) { - code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fData.size += TSDB_FHDR_SIZE; - } - - // sma - if (pWriter->fSma.size == 0) { - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_READ | TD_FILE_WRITE; - } - tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pSmaFD); - if (code) goto _err; - if (pWriter->fSma.size == 0) { - code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - pWriter->fSma.size += TSDB_FHDR_SIZE; - } - - // stt - ASSERT(pWriter->fStt[pSet->nSttF - 1].size == 0); - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbSttFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fStt[pSet->nSttF - 1], fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pSttFD); - if (code) goto _err; - code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fStt[pWriter->wSet.nSttF - 1].size += TSDB_FHDR_SIZE; - - *ppWriter = pWriter; - return code; - -_err: - tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; - return code; -} - -int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - STsdb *pTsdb = NULL; - - if (*ppWriter == NULL) goto _exit; - - pTsdb = (*ppWriter)->pTsdb; - if (sync) { - code = tsdbFsyncFile((*ppWriter)->pHeadFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pDataFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pSmaFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pSttFD); - if (code) goto _err; - } - - tsdbCloseFile(&(*ppWriter)->pHeadFD); - tsdbCloseFile(&(*ppWriter)->pDataFD); - tsdbCloseFile(&(*ppWriter)->pSmaFD); - tsdbCloseFile(&(*ppWriter)->pSttFD); - - for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree((*ppWriter)->aBuf[iBuf]); - } - taosMemoryFree(*ppWriter); -_exit: - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { - int32_t code = 0; - int64_t n; - char hdr[TSDB_FHDR_SIZE]; - - // head ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutHeadFile(hdr, &pWriter->fHead); - code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // data ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutDataFile(hdr, &pWriter->fData); - code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // sma ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSmaFile(hdr, &pWriter->fSma); - code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // stt ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSttFile(hdr, &pWriter->fStt[pWriter->wSet.nSttF - 1]); - code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; - int64_t n; - - // check - if (taosArrayGetSize(aBlockIdx) == 0) { - pHeadFile->offset = pHeadFile->size; - goto _exit; - } - - // prepare - size = 0; - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pHeadFile->offset = pHeadFile->size; - pHeadFile->size += size; - -_exit: - // tsdbTrace("vgId:%d, write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", - // TD_VID(pWriter->pTsdb->pVnode), - // pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); - return code; - -_err: - tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; - int64_t n; - - ASSERT(mDataBlk->nItem > 0); - - // alloc - size = tPutMapData(NULL, mDataBlk); - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = tPutMapData(pWriter->aBuf[0], mDataBlk); - - // write - code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pBlockIdx->offset = pHeadFile->size; - pBlockIdx->size = size; - pHeadFile->size += size; - - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%" PRId64 " suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 - " size:%" PRId64 " nItem:%d", - TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, - pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); - return code; - -_err: - tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { - int32_t code = 0; - SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; - int64_t size = 0; - int64_t n; - - // check - if (taosArrayGetSize(aSttBlk) == 0) { - pSttFile->offset = pSttFile->size; - goto _exit; - } - - // size - size = 0; - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { - size += tPutSttBlk(NULL, taosArrayGet(aSttBlk, iBlockL)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // encode - n = 0; - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { - n += tPutSttBlk(pWriter->aBuf[0] + n, taosArrayGet(aSttBlk, iBlockL)); - } - - // write - code = tsdbWriteFile(pWriter->pSttFD, pSttFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pSttFile->offset = pSttFile->size; - pSttFile->size += size; - -_exit: - tsdbTrace("vgId:%d, tsdb write stt block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pSttFile->offset, size); - return code; - -_err: - tsdbError("vgId:%d, tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { - int32_t code = 0; - - pSmaInfo->offset = 0; - pSmaInfo->size = 0; - - // encode - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; - - SColumnDataAgg sma = {.colId = pColData->cid}; - tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); - - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); - if (code) goto _err; - pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); - } - - // write - if (pSmaInfo->size) { - code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); - if (code) goto _err; - - pSmaInfo->offset = pWriter->fSma.size; - pWriter->fSma.size += pSmaInfo->size; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast) { - int32_t code = 0; - - ASSERT(pBlockData->nRow > 0); - - if (toLast) { - pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; - } else { - pBlkInfo->offset = pWriter->fData.size; - } - pBlkInfo->szBlock = 0; - pBlkInfo->szKey = 0; - - int32_t aBufN[4] = {0}; - code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); - if (code) goto _err; - - // write ================= - STsdbFD *pFD = toLast ? pWriter->pSttFD : pWriter->pDataFD; - - pBlkInfo->szKey = aBufN[3] + aBufN[2]; - pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - - int64_t offset = pBlkInfo->offset; - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[3], aBufN[3]); - if (code) goto _err; - offset += aBufN[3]; - - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[2], aBufN[2]); - if (code) goto _err; - offset += aBufN[2]; - - if (aBufN[1]) { - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[1], aBufN[1]); - if (code) goto _err; - offset += aBufN[1]; - } - - if (aBufN[0]) { - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], aBufN[0]); - if (code) goto _err; - } - - // update info - if (toLast) { - pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; - } else { - pWriter->fData.size += pBlkInfo->szBlock; - } - - // ================= SMA ==================== - if (pSmaInfo) { - code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); - if (code) goto _err; - } - -_exit: - tsdbTrace("vgId:%d, tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", - TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, - pBlkInfo->szBlock); - return code; - -_err: - tsdbError("vgId:%d, tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) { - int32_t code = 0; - int32_t lino = 0; - - STsdbFD *pFD = NULL; - if (pSmaInfo) { - pFD = pWriter->pDataFD; - pBlkInfo->offset = pWriter->fData.size; - } else { - pFD = pWriter->pSttFD; - pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; - } - pBlkInfo->szBlock = 0; - pBlkInfo->szKey = 0; - - // hdr - int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr); - code = tRealloc(&pWriter->aBuf[0], n); - TSDB_CHECK_CODE(code, lino, _exit); - - tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr); - - code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += n; - pBlkInfo->szBlock += n; - - // uid + ver + key - if (pDiskData->pUid) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szUid; - pBlkInfo->szBlock += pDiskData->hdr.szUid; - } - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szVer; - pBlkInfo->szBlock += pDiskData->hdr.szVer; - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szKey; - pBlkInfo->szBlock += pDiskData->hdr.szKey; - - // aBlockCol - if (pDiskData->hdr.szBlkCol) { - code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol); - TSDB_CHECK_CODE(code, lino, _exit); - - n = 0; - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol); - } - ASSERT(n == pDiskData->hdr.szBlkCol); - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskData->hdr.szBlkCol; - } - - // aDiskCol - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - - if (pDiskCol->pBit) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szBitmap; - } - - if (pDiskCol->pOff) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szOffset; - } - - if (pDiskCol->pVal) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szValue; - } - } - - if (pSmaInfo) { - pWriter->fData.size += pBlkInfo->szBlock; - } else { - pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; - goto _exit; - } - - pSmaInfo->offset = 0; - pSmaInfo->size = 0; - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - - if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue; - if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue; - if (!pDiskCol->bCol.smaOn) continue; - - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg)); - TSDB_CHECK_CODE(code, lino, _exit); - pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg); - } - - if (pSmaInfo->size) { - pSmaInfo->offset = pWriter->fSma.size; - - code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->fSma.size += pSmaInfo->size; - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { - int32_t code = 0; - int64_t n; - int64_t size; - TdFilePtr pOutFD = NULL; - TdFilePtr PInFD = NULL; - int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; - char fNameFrom[TSDB_FILENAME_LEN]; - char fNameTo[TSDB_FILENAME_LEN]; - - // head - tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); - tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // data - tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); - tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // sma - tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); - tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // stt - for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { - tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); - tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - // SDataFReader ==================================================== int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; @@ -1478,173 +849,6 @@ _exit: return code; } -// SDelFWriter ==================================================== -int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { - int32_t code = 0; - int32_t lino = 0; - char fname[TSDB_FILENAME_LEN]; - uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter = NULL; - int64_t n; - - // alloc - pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); - if (pDelFWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pDelFWriter->pTsdb = pTsdb; - pDelFWriter->fDel = *pFile; - - tsdbDelFileName(pTsdb, pFile, fname); - code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); - TSDB_CHECK_CODE(code, lino, _exit); - - // update header - code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); - TSDB_CHECK_CODE(code, lino, _exit); - - pDelFWriter->fDel.size = TSDB_FHDR_SIZE; - pDelFWriter->fDel.offset = 0; - - *ppWriter = pDelFWriter; - -_exit: - if (code) { - if (pDelFWriter) { - tsdbCloseFile(&pDelFWriter->pWriteH); - taosMemoryFree(pDelFWriter); - } - *ppWriter = NULL; - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); - } else { - *ppWriter = pDelFWriter; - } - return code; -} - -int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - SDelFWriter *pWriter = *ppWriter; - STsdb *pTsdb = pWriter->pTsdb; - - // sync - if (sync) { - code = tsdbFsyncFile(pWriter->pWriteH); - if (code) goto _err; - } - - // close - tsdbCloseFile(&pWriter->pWriteH); - - for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree(pWriter->aBuf[iBuf]); - } - taosMemoryFree(pWriter); - - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { - int32_t code = 0; - int64_t size; - int64_t n; - - // prepare - size = 0; - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pDelIdx->offset = pWriter->fDel.size; - pDelIdx->size = size; - pWriter->fDel.size += size; - - return code; - -_err: - tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { - int32_t code = 0; - int64_t size; - int64_t n; - SDelIdx *pDelIdx; - - // prepare - size = 0; - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pWriter->fDel.offset = pWriter->fDel.size; - pWriter->fDel.size += size; - - return code; - -_err: - tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) { - int32_t code = 0; - char hdr[TSDB_FHDR_SIZE] = {0}; - int64_t size = TSDB_FHDR_SIZE; - int64_t n; - - // build - tPutDelFile(hdr, &pWriter->fDel); - - // write - code = tsdbWriteFile(pWriter->pWriteH, 0, hdr, size); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} // SDelFReader ==================================================== struct SDelFReader { STsdb *pTsdb; From 7e1d59c565e408dc952b26ecdcd349c070cd3f9f Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 29 Dec 2023 16:22:41 +0800 Subject: [PATCH 33/93] sort --- source/libs/executor/inc/executorInt.h | 5 ++-- source/libs/executor/src/executil.c | 4 +++- source/libs/executor/src/operator.c | 2 +- source/libs/executor/src/scanoperator.c | 32 ++++++++++++++++++------- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 9acef69f9c..c09f615735 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -269,8 +269,9 @@ typedef struct STableScanInfo { SSDataBlock* pResBlock; SHashObj* pIgnoreTables; SSampleExecInfo sample; // sample execution info - int32_t tableStartIndex; // current group scan start - int32_t tableEndIndex; // current group scan end + int32_t tableStartIndex; // current group scan start + int32_t tableEndIndex; // current group scan end + int32_t currentGroupIndex; // current group index of groupOffset int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 377de99fc0..4b3b1ffead 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2122,6 +2122,9 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool groupByTbname = groupbyTbname(group); size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); + if (!numOfTables) { + return code; + } if (group == NULL || groupByTbname) { for (int32_t i = 0; i < numOfTables; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); @@ -2143,7 +2146,6 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* if (code != TSDB_CODE_SUCCESS) { return code; } - if (pScanNode->groupOrderScan) pTableListInfo->numOfOuputGroups = taosArrayGetSize(pTableListInfo->pTableList); if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 69a8acb3d7..0b5e7f51ed 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -302,7 +302,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pTableListInfo->idInfo.suid = pTableScanNode->scan.suid; pTableListInfo->idInfo.tableType = pTableScanNode->scan.tableType; } else { - code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, + code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e990d3d975..27c9a76757 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -659,17 +659,30 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) { pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); - int32_t i = pInfo->tableStartIndex + 1; - for (; i < numOfTables; ++i) { - STableKeyInfo* pCur = tableListGetInfo(pInfo->base.pTableListInfo, i); - if (pCur->groupId != pStart->groupId) { - break; + STableListInfo* pTableListInfo = pInfo->base.pTableListInfo; + int32_t numOfTables = tableListGetSize(pTableListInfo); + STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pTableListInfo, pInfo->tableStartIndex); + + if (pTableListInfo->oneTableForEachGroup) { + pInfo->tableEndIndex = pInfo->tableStartIndex; + } else if (pTableListInfo->groupOffset) { + pInfo->currentGroupIndex++; + if (pInfo->currentGroupIndex + 1 < pTableListInfo->numOfOuputGroups) { + pInfo->tableEndIndex = pTableListInfo->groupOffset[pInfo->currentGroupIndex + 1] - 1; + } else { + pInfo->tableEndIndex = numOfTables - 1; } + } else { + int32_t i = pInfo->tableStartIndex + 1; + for (; i < numOfTables; ++i) { + STableKeyInfo* pCur = tableListGetInfo(pTableListInfo, i); + if (pCur->groupId != pStart->groupId) { + break; + } + } + pInfo->tableEndIndex = i - 1; } - pInfo->tableEndIndex = i - 1; if (!pInfo->needCountEmptyTable) { pInfo->countState = TABLE_COUNT_STATE_END; } else { @@ -677,7 +690,7 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i } *pKeyInfo = pStart; - *size = i - pInfo->tableStartIndex; + *size = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; } static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, @@ -1133,6 +1146,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->tableEndIndex = -1; + pInfo->currentGroupIndex = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; From 87b7a9a980233880deda02ef05a0f3915560379c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 29 Dec 2023 17:51:35 +0800 Subject: [PATCH 34/93] fix: stream test case --- tests/script/tsim/stream/basic0.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 917a0a73ab..a525594861 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -33,6 +33,8 @@ if $rows != 3 then return -1 endi +sleep 1000 + sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) sql show stables From 673d3bc376ad8955bd4b3b9163853eebb983b915 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 31 Dec 2023 13:15:05 +0800 Subject: [PATCH 35/93] arrangeTableGroup --- source/libs/executor/src/executil.c | 70 +++++++++++++++++++++++++++++ source/libs/executor/src/operator.c | 2 +- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 4b3b1ffead..f3e2e99332 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2116,6 +2116,70 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TSDB_CODE_SUCCESS; } +static int32_t arrangeTableGroup(STableListInfo* pTableListInfo) { + int32_t code = TSDB_CODE_SUCCESS; + size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); + SArray* pDup = NULL; + SHashObj* pHashObj = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (!pHashObj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t num = 1; + // first: get the number of tables per group + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, i); + int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); + // update each group's table count + if (!pVal) { + taosHashPut(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId), &num, sizeof(num)); + } else { + (*pVal)++; + } + } + pTableListInfo->numOfOuputGroups = taosHashGetSize(pHashObj); + pTableListInfo->oneTableForEachGroup = (pTableListInfo->numOfOuputGroups == numOfTables); + + if (pTableListInfo->numOfOuputGroups > 1 && pTableListInfo->numOfOuputGroups < numOfTables) { + pDup = taosArrayDup(pTableListInfo->pTableList, NULL); + pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups); + if (pDup == NULL || pTableListInfo->groupOffset == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + int32_t idx = 0, sum = 0; + + void* pIter = taosHashIterate(pHashObj, NULL); + while (pIter != NULL) { + // record each group's offset + pTableListInfo->groupOffset[idx] = sum; + int32_t* pData = (int32_t*)pIter; + sum += *pData; + // change value to record group item's first position + *pData = pTableListInfo->groupOffset[idx++]; + pIter = taosHashIterate(pHashObj, pIter); + } + + // second: arrange the tables and put the items with same groupId together + STableKeyInfo* pStart = taosArrayGet(pTableListInfo->pTableList, 0); + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pInfo = taosArrayGet(pDup, i); + int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); + if (*pVal != i) { + pStart[*pVal] = *pInfo; + } + (*pVal)++; // update to next item's position + } + } + +_exit: + taosHashCleanup(pHashObj); + taosArrayDestroy(pDup); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pTableListInfo->groupOffset); + } + return code; +} + int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, bool groupSort, uint8_t* digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; @@ -2149,6 +2213,12 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; + if (pTableScanNode->needCountEmptyTable) { + // only put together tables with the same groupid + arrangeTableGroup(pTableListInfo); + } } } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 0b5e7f51ed..69a8acb3d7 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -302,7 +302,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pTableListInfo->idInfo.suid = pTableScanNode->scan.suid; pTableListInfo->idInfo.tableType = pTableScanNode->scan.tableType; } else { - code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, + code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; From 40205541a67f8db225254d111aef6390abd9a650 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:19:23 +0800 Subject: [PATCH 36/93] fix: check in client when create index --- source/libs/parser/src/parTranslater.c | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bdee4d54ac..42aebb3806 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6929,11 +6929,45 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde } static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + SName name; + STableMeta* pMeta = NULL; + int32_t code = + getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + if (code) { + taosMemoryFree(pMeta); + return code; + } + if (TSDB_SUPER_TABLE != pMeta->tableType) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + } + + if (LIST_LENGTH(pStmt->pCols) != 1) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only 1 tag is allowed"); + } + + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pCols) { + SColumnNode* p = (SColumnNode*)pNode; + const SSchema* pSchema = getTagSchema(pMeta, p->colName); + if (!pSchema) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, p->colName); + } + if (IS_IDX_ON(pSchema)) { + code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + goto _exit; + } + } + SCreateTagIndexReq createTagIdxReq = {0}; - int32_t code = buildCreateTagIndexReq(pCxt, pStmt, &createTagIdxReq); + code = buildCreateTagIndexReq(pCxt, pStmt, &createTagIdxReq); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSCreateTagIdxReq, &createTagIdxReq); } +_exit: + taosMemoryFree(pMeta); return code; } From 0d26c186f0d3f11818264498e9e5b40b97626881 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:23:19 +0800 Subject: [PATCH 37/93] chore: code optimization --- source/libs/parser/src/parTranslater.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42aebb3806..d980c0f5ae 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6949,11 +6949,10 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS SNode* pNode = NULL; FOREACH(pNode, pStmt->pCols) { - SColumnNode* p = (SColumnNode*)pNode; - const SSchema* pSchema = getTagSchema(pMeta, p->colName); + const SSchema* pSchema = getTagSchema(pMeta, ((SColumnNode*)pNode)->colName); if (!pSchema) { taosMemoryFree(pMeta); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, p->colName); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, ((SColumnNode*)pNode)->colName); } if (IS_IDX_ON(pSchema)) { code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; From ecf3a7b6d350f5e3f40ae9736f84e1a6f2b8c9d3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:28:30 +0800 Subject: [PATCH 38/93] fix: code optimization --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d980c0f5ae..c3b9f9fc85 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6939,12 +6939,12 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS } if (TSDB_SUPER_TABLE != pMeta->tableType) { taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only super table is allowed"); } if (LIST_LENGTH(pStmt->pCols) != 1) { taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only 1 tag is allowed"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only one tag is allowed"); } SNode* pNode = NULL; From 0e9a1b22edf3a71aab960b48f7d02b77b2336ec0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 2 Jan 2024 10:10:44 +0800 Subject: [PATCH 39/93] fix:error msg is not specific in create stream --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bdee4d54ac..c9345dd00e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7446,7 +7446,7 @@ static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStream } } if (!found) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, ((SColumnDefNode*)pTag)->colName); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, getTagNameForCreateStreamTag(pTag)); } } return TSDB_CODE_SUCCESS; From be0d35a5af93f21e1ee774b440d0a093577432f3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 2 Jan 2024 10:20:04 +0800 Subject: [PATCH 40/93] fix: last table scan reported invalid param when col was dropped which should report invalid col --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- tests/system-test/2-query/last_cache_scan.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 1f54f13592..4ca7881814 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -120,7 +120,7 @@ static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInf } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing i += 1; } else { - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; } } diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 39271318ba..23433277c2 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -413,8 +413,7 @@ class TDTestCase: tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) - - + tdSql.query_success_failed('select last(c2), c2, c3 from meters', queryTimes=10, expectErrInfo="Invalid column name: c2") def run(self): self.prepareTestEnv() From 85ba9a4a5fe2332dd3fc889ee71e8f4e0b96fc20 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 2 Jan 2024 15:45:03 +0800 Subject: [PATCH 41/93] remainGroups --- source/libs/executor/inc/executil.h | 1 + source/libs/executor/inc/executorInt.h | 7 +- source/libs/executor/src/executil.c | 103 +++++++----------------- source/libs/executor/src/scanoperator.c | 51 ++++++++---- 4 files changed, 72 insertions(+), 90 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 946d3311a4..640ed2f2f9 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -105,6 +105,7 @@ typedef struct STableListInfo { int32_t* groupOffset; // keep the offset value for each group in the tableList SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid + SHashObj* remainGroups; // remaining group has not yet processed the empty group STableListIdInfo idInfo; // this maybe the super table or ordinary table } STableListInfo; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index c09f615735..bee378effc 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -218,9 +218,10 @@ enum { }; typedef enum ETableCountState { - TABLE_COUNT_STATE_NONE = 0, // before start scan - TABLE_COUNT_STATE_SCAN = 1, // scanning - TABLE_COUNT_STATE_END = 2, // finish or noneed to process + TABLE_COUNT_STATE_NONE = 0, // before start scan + TABLE_COUNT_STATE_SCAN = 1, // cur group scanning + TABLE_COUNT_STATE_PROCESSED = 2, // cur group processed + TABLE_COUNT_STATE_END = 3, // finish or noneed to process } ETableCountState; typedef struct SAggSupporter { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f3e2e99332..fe3528cce6 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -456,7 +456,7 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C } int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest, - SStorageAPI* pAPI) { + SStorageAPI* pAPI, bool initRemainGroups) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -590,6 +590,15 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf goto end; } + if (initRemainGroups) { + pTableListInfo->remainGroups = + taosHashInit(rows, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pTableListInfo->remainGroups == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + } + for (int i = 0; i < rows; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); @@ -631,6 +640,14 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf int32_t len = (int32_t)(pStart - (char*)keyBuf); info->groupId = calcGroupId(keyBuf, len); + if (initRemainGroups) { + // groupId ~ table uid + taosHashPut(pTableListInfo->remainGroups, &(info->groupId), sizeof(info->groupId), &(info->uid), sizeof(info->uid)); + } + } + + if (initRemainGroups) { + pTableListInfo->numOfOuputGroups = taosHashGetSize(pTableListInfo->remainGroups); } if (tsTagFilterCache) { @@ -2025,6 +2042,7 @@ STableListInfo* tableListCreate() { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pListInfo->remainGroups = NULL; pListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); if (pListInfo->pTableList == NULL) { @@ -2054,7 +2072,7 @@ void* tableListDestroy(STableListInfo* pTableListInfo) { taosMemoryFreeClear(pTableListInfo->groupOffset); taosHashCleanup(pTableListInfo->map); - + taosHashCleanup(pTableListInfo->remainGroups); pTableListInfo->pTableList = NULL; pTableListInfo->map = NULL; taosMemoryFree(pTableListInfo); @@ -2068,6 +2086,7 @@ void tableListClear(STableListInfo* pTableListInfo) { taosArrayClear(pTableListInfo->pTableList); taosHashClear(pTableListInfo->map); + taosHashClear(pTableListInfo->remainGroups); taosMemoryFree(pTableListInfo->groupOffset); pTableListInfo->numOfOuputGroups = 1; pTableListInfo->oneTableForEachGroup = false; @@ -2116,70 +2135,6 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TSDB_CODE_SUCCESS; } -static int32_t arrangeTableGroup(STableListInfo* pTableListInfo) { - int32_t code = TSDB_CODE_SUCCESS; - size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); - SArray* pDup = NULL; - SHashObj* pHashObj = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (!pHashObj) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int32_t num = 1; - // first: get the number of tables per group - for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, i); - int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); - // update each group's table count - if (!pVal) { - taosHashPut(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId), &num, sizeof(num)); - } else { - (*pVal)++; - } - } - pTableListInfo->numOfOuputGroups = taosHashGetSize(pHashObj); - pTableListInfo->oneTableForEachGroup = (pTableListInfo->numOfOuputGroups == numOfTables); - - if (pTableListInfo->numOfOuputGroups > 1 && pTableListInfo->numOfOuputGroups < numOfTables) { - pDup = taosArrayDup(pTableListInfo->pTableList, NULL); - pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups); - if (pDup == NULL || pTableListInfo->groupOffset == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - int32_t idx = 0, sum = 0; - - void* pIter = taosHashIterate(pHashObj, NULL); - while (pIter != NULL) { - // record each group's offset - pTableListInfo->groupOffset[idx] = sum; - int32_t* pData = (int32_t*)pIter; - sum += *pData; - // change value to record group item's first position - *pData = pTableListInfo->groupOffset[idx++]; - pIter = taosHashIterate(pHashObj, pIter); - } - - // second: arrange the tables and put the items with same groupId together - STableKeyInfo* pStart = taosArrayGet(pTableListInfo->pTableList, 0); - for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pInfo = taosArrayGet(pDup, i); - int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); - if (*pVal != i) { - pStart[*pVal] = *pInfo; - } - (*pVal)++; // update to next item's position - } - } - -_exit: - taosHashCleanup(pHashObj); - taosArrayDestroy(pDup); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pTableListInfo->groupOffset); - } - return code; -} - int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, bool groupSort, uint8_t* digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; @@ -2206,19 +2161,21 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI); + bool initRemainGroups = false; + if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; + if (pTableScanNode->needCountEmptyTable) { + initRemainGroups = true; + } + } + + code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI, initRemainGroups); if (code != TSDB_CODE_SUCCESS) { return code; } if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { - STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable) { - // only put together tables with the same groupid - arrangeTableGroup(pTableListInfo); - } } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 27c9a76757..8999340ddc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -673,14 +673,7 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i pInfo->tableEndIndex = numOfTables - 1; } } else { - int32_t i = pInfo->tableStartIndex + 1; - for (; i < numOfTables; ++i) { - STableKeyInfo* pCur = tableListGetInfo(pTableListInfo, i); - if (pCur->groupId != pStart->groupId) { - break; - } - } - pInfo->tableEndIndex = i - 1; + pInfo->tableEndIndex = numOfTables - 1; } if (!pInfo->needCountEmptyTable) { @@ -693,6 +686,17 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i *size = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; } +void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { + if (pInfo->countState == TABLE_COUNT_STATE_END) { + return; + } + if (pInfo->base.pTableListInfo->oneTableForEachGroup) { + pInfo->countState = TABLE_COUNT_STATE_PROCESSED; + } else { + taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId)); + } +} + static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, const STableKeyInfo* tbInfo) { blockDataEmpty(pBlock); @@ -802,7 +806,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; + markGroupProcessed(pTableScanInfo, p->info.id.groupId); return p; } @@ -831,7 +835,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; + markGroupProcessed(pTableScanInfo, p->info.id.groupId); return p; } @@ -849,11 +853,30 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { - // output once for this group + STableListInfo* pTableListInfo = pTableScanInfo->base.pTableListInfo; + if (pTableListInfo->oneTableForEachGroup) { // group by tbname + if (pTableScanInfo->countState < TABLE_COUNT_STATE_PROCESSED) { + pTableScanInfo->countState = TABLE_COUNT_STATE_PROCESSED; + STableKeyInfo* pStart = + (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); + return getBlockForEmptyTable(pOperator, pStart); + } + } else { // group by tag + int32_t numOfTables = tableListGetSize(pTableListInfo); + if (pTableScanInfo->tableEndIndex + 1 >= numOfTables) { + // get empty group, mark processed & rm from hash + void* pIte = taosHashIterate(pTableListInfo->remainGroups, NULL); + if (pIte != NULL) { + size_t keySize = 0; + uint64_t* pGroupId = taosHashGetKey(pIte, &keySize); + STableKeyInfo info = {.uid = *(uint64_t*)pIte, .groupId = *pGroupId}; + taosHashCancelIterate(pTableListInfo->remainGroups, pIte); + markGroupProcessed(pTableScanInfo, *pGroupId); + return getBlockForEmptyTable(pOperator, &info); + } + } + } pTableScanInfo->countState = TABLE_COUNT_STATE_END; - STableKeyInfo* pStart = - (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); - return getBlockForEmptyTable(pOperator, pStart); } return NULL; From 1a265a2e9dfbf1c38f4262b9f8e5b839bca71b6e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:26:46 +0800 Subject: [PATCH 42/93] test: modify testcase of hot refresh config --- tests/pytest/util/common.py | 16 ++++++++++++++++ .../test_hot_refresh_configurations.py | 18 ++++++++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 9c45c09715..2e7bd2a89c 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -506,7 +506,23 @@ class TDCom: if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] break + if platform.system().lower() == 'windows': + win_sep = "\\" + buildPath = buildPath.replace(win_sep,'/') + return buildPath + + def getTaosdPath(self, dnodeID="dnode1"): + buildPath = self.getBuildPath() + print("123456",buildPath) + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + taosdPath = buildPath + "/../sim/" + dnodeID + tdLog.info("taosdPath: %s" % taosdPath) + return taosdPath + def getClientCfgPath(self): buildPath = self.getBuildPath() diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 09e634974b..d281b9c1f7 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -201,6 +201,17 @@ class TDTestCase: tdSql.error(f'alter {dnode} "{name} {v}";') def run(self): + + # reset log + taosdPath = tdCom.getTaosdPath() + print(taosdPath,"123") + taosdLogAbsoluteFilename = taosdPath + "/log/" + "taosdlog*" + print(taosdLogAbsoluteFilename) + tdSql.execute("alter all dnodes 'resetlog';") + r = subprocess.Popen("cat {} | grep 'reset log file'".format(taosdLogAbsoluteFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = r.communicate() + assert('reset log file' in stdout.decode()) + for key in self.configration_dic: if "cli" == key: for item in self.configration_dic[key]: @@ -214,12 +225,7 @@ class TDTestCase: self.svr_check(item["name"], item["alias"], item["except_values"], True) else: raise Exception(f"unknown key: {key}") - # reset log - path = os.sep.join([tdDnodes.getDnodesRootDir(), "dnode1", "log", "taosdlog.*"]) - tdSql.execute("alter all dnodes 'resetlog';") - r = subprocess.Popen("cat {} | grep 'reset log file'".format(path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = r.communicate() - assert('reset log file' in stdout.decode()) + def stop(self): tdSql.close() From 480f571c2bda07f02a7b77cd3b4d2beff8763b29 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:29:01 +0800 Subject: [PATCH 43/93] test: modify testcase of hot refresh config --- tests/pytest/util/common.py | 1 - .../system-test/0-others/test_hot_refresh_configurations.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 2e7bd2a89c..010f45a573 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -514,7 +514,6 @@ class TDCom: def getTaosdPath(self, dnodeID="dnode1"): buildPath = self.getBuildPath() - print("123456",buildPath) if (buildPath == ""): tdLog.exit("taosd not found!") else: diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index d281b9c1f7..71f6290469 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -203,10 +203,7 @@ class TDTestCase: def run(self): # reset log - taosdPath = tdCom.getTaosdPath() - print(taosdPath,"123") - taosdLogAbsoluteFilename = taosdPath + "/log/" + "taosdlog*" - print(taosdLogAbsoluteFilename) + taosdLogAbsoluteFilename = tdCom.getTaosdPath() + "/log/" + "taosdlog*" tdSql.execute("alter all dnodes 'resetlog';") r = subprocess.Popen("cat {} | grep 'reset log file'".format(taosdLogAbsoluteFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = r.communicate() From 7333fe54052ac97ac306c727499cf3a3162353bd Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:35:01 +0800 Subject: [PATCH 44/93] test: add script loop.bat --- tests/system-test/loop.bat | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/system-test/loop.bat diff --git a/tests/system-test/loop.bat b/tests/system-test/loop.bat new file mode 100644 index 0000000000..41e3901b3f --- /dev/null +++ b/tests/system-test/loop.bat @@ -0,0 +1,13 @@ +@echo off +set LoopNumner=%1 +set cmd=%2 + +echo "Loop Number: %LoopNumner%" +echo "command : %cmd:"=%" + +For /l %%i in (1,1,%LoopNumner% ) do ( + echo "execute times:"%%i + %cmd:"=% +) + +% case loop.bat 100 "python3 .\test.py -f 2-query\query_cols_tags_and_or.py" % \ No newline at end of file From 93fe9558289052437385d5b24fe339593dcfd944 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 17:48:05 +0800 Subject: [PATCH 45/93] test: modify testcase of windows --- tests/script/win-test-file | 2 +- tests/system-test/win-test-file | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/script/win-test-file b/tests/script/win-test-file index d9ff09f468..ada8574165 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -15,7 +15,6 @@ ./test.sh -f tsim/dnode/balance2.sim ./test.sh -f tsim/vnode/replica3_repeat.sim ./test.sh -f tsim/parser/col_arithmetic_operation.sim -./test.sh -f tsim/trans/create_db.sim ./test.sh -f tsim/dnode/balance3.sim ./test.sh -f tsim/vnode/replica3_many.sim ./test.sh -f tsim/stable/metrics_idx.sim @@ -201,6 +200,7 @@ ./test.sh -f tsim/query/show_db_table_kind.sim ./test.sh -f tsim/query/bi_star_table.sim ./test.sh -f tsim/query/bi_tag_scan.sim +./test.sh -f tsim/query/bi_tbname_col.sim ./test.sh -f tsim/query/tag_scan.sim ./test.sh -f tsim/query/nullColSma.sim ./test.sh -f tsim/query/bug3398.sim diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index aefdb1e824..0db83e71a3 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -149,7 +149,7 @@ python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -python3 ./test.py -f 7-tmq/tmq_taosx.py +# python3 ./test.py -f 7-tmq/tmq_taosx.py python3 ./test.py -f 7-tmq/tmq_replay.py python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py python3 ./test.py -f 7-tmq/tmq_offset.py @@ -163,6 +163,7 @@ python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 @@ -196,6 +197,7 @@ python3 ./test.py -f 0-others/ttlChangeOnWrite.py python3 ./test.py -f 0-others/compress_tsz1.py python3 ./test.py -f 0-others/compress_tsz2.py python3 ./test.py -f 0-others/view/non_marterial_view/test_view.py +python3 ./test.py -f 0-others/test_show_table_distributed.py python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 0-others/tag_index_basic.py python3 ./test.py -N 3 -f 0-others/walRetention.py @@ -204,6 +206,7 @@ python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 python3 ./test.py -f 0-others/timeRangeWise.py -N 3 python3 ./test.py -f 0-others/delete_check.py python3 ./test.py -f 0-others/test_hot_refresh_configurations.py +python3 ./test.py -f 1-insert/insert_double.py python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/alter_replica.py -N 3 python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py @@ -252,6 +255,10 @@ python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 python3 ./test.py -f 1-insert/precisionUS.py python3 ./test.py -f 1-insert/precisionNS.py python3 ./test.py -f 1-insert/test_ts4219.py +python3 ./test.py -f 1-insert/ts-4272.py +python3 ./test.py -f 1-insert/test_ts4295.py +python3 ./test.py -f 1-insert/test_td27388.py +python3 ./test.py -f 1-insert/insert_timestamp.py python3 ./test.py -f 0-others/show.py python3 ./test.py -f 0-others/show_tag_index.py python3 ./test.py -f 0-others/information_schema.py @@ -795,6 +802,7 @@ python3 ./test.py -f 2-query/blockSMA.py -Q 4 python3 ./test.py -f 2-query/projectionDesc.py -Q 4 python3 ./test.py -f 2-query/odbc.py python3 ./test.py -f 2-query/fill_with_group.py +python3 ./test.py -f 2-query/state_window.py -Q 3 python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 python3 ./test.py -f 99-TDcase/TD-20582.py python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 From 5bbf12c2c9240df4677fa3ba00325ce01b8d4b38 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 18:35:17 +0800 Subject: [PATCH 46/93] fix(tsdb): check if stt blocks are clean before building composed data blocks. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 1f54f13592..0aef1efc7b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1713,6 +1713,9 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* } else if ((!dataInDataFile) && dataInSttFile) { // no data ile block exists return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + } else if (pBlockScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) { + // opt model for count data in stt file, which is not overlap with data blocks in files. + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else { // row in both stt file blocks and data file blocks TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); From 866fac624fa9985bd3b0d90c371cae19bdc83bc4 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Tue, 2 Jan 2024 19:26:59 +0800 Subject: [PATCH 47/93] Add test cases for TS-4348 and TD-27939 --- tests/system-test/2-query/ts-4348-td-27939.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/system-test/2-query/ts-4348-td-27939.py diff --git a/tests/system-test/2-query/ts-4348-td-27939.py b/tests/system-test/2-query/ts-4348-td-27939.py new file mode 100644 index 0000000000..2a861899d5 --- /dev/null +++ b/tests/system-test/2-query/ts-4348-td-27939.py @@ -0,0 +1,46 @@ +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + tdSql.execute("drop database if exists ts_4338;") + tdSql.execute("create database ts_4338;") + tdSql.execute("drop table if exists ts_4338.t;") + tdSql.execute("create database if not exists ts_4338;") + tdSql.execute("create table ts_4338.t (ts timestamp, i8 tinyint);") + tdSql.execute("insert into ts_4338.t (ts, i8) values (now(), 1) (now()+1s, 2);") + + def run(self): + # TS-4348 + tdSql.query(f'select i8 from ts_4338.t;') + tdSql.checkRows(2) + + tdSql.query(f'select i8 from ts_4338.t where 1 = 1;') + tdSql.checkRows(2) + + tdSql.query(f'select i8 from ts_4338.t where i8 = 1;') + tdSql.checkRows(1) + + tdSql.query(f'select * from (select * from ts_4338.t where i8 = 3);') + tdSql.checkRows(0) + + # TD-27939 + tdSql.query(f'select * from (select * from ts_4338.t where 1 = 100);') + tdSql.checkRows(0) + + tdSql.query(f'select * from (select * from (select * from ts_4338.t where 1 = 200));') + tdSql.checkRows(0) + + tdSql.execute("drop database if exists ts_4338;") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From b48df60fdd157ac058f1d4e4e2731304210c5872 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 20:18:10 +0800 Subject: [PATCH 48/93] test: add testcase that merge same ver stt file after split db --- tests/system-test/0-others/splitVGroupRep1.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupRep1.py index f799c9dbd8..1745b34cc4 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupRep1.py @@ -164,7 +164,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 1 wal_retention_size 1" + sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 0 wal_retention_size 1" self.exeDouble(sql) # create super talbe st @@ -221,14 +221,14 @@ class TDTestCase: sql1 = sql.replace('@db_name', self.db1) tdLog.info(sql1) start1 = time.time() - rows1 = tdSql.query(sql1) + rows1 = tdSql.query(sql1,queryTimes=2) spend1 = time.time() - start1 res1 = copy.deepcopy(tdSql.queryResult) sql2 = sql.replace('@db_name', self.db2) tdLog.info(sql2) start2 = time.time() - tdSql.query(sql2) + tdSql.query(sql2,queryTimes=2) spend2 = time.time() - start2 res2 = tdSql.queryResult @@ -269,7 +269,7 @@ class TDTestCase: def checkResult(self): # check vgroupid sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{self.db2}'" - tdSql.query(sql) + tdSql.query(sql,queryTimes=2) tdSql.checkRows(self.vgroups2) # check child table count same @@ -412,8 +412,16 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - + tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver3.stt to db2 and db1 ") + for dbname in [self.db2, self.db1]: + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') tdLog.info("check db1 and db2 same after creating ...") + self.checkResult() for i in range(3): @@ -423,6 +431,15 @@ class TDTestCase: end = time.time() self.vgroups2 += 1 + # insert the same data per tables into splited vgroups + tdLog.info("insert the same data per tables into splited vgroups(3,4)") + for dbname in [self.db2, self.db1]: + for tableid in range(self.childCnt): + tdSql.execute(f'insert into {dbname}.t{tableid} values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.ta values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999",0);') + tdSql.execute(f'flush database {dbname}') + # check two db query result same self.checkResult() spend = "%.3f"%(end-start) From 7b805d2b002f49496d591bb2a2276f05442cc465 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 20:19:09 +0800 Subject: [PATCH 49/93] test: add testcase that merge same ver stt file after split db --- tests/system-test/0-others/splitVGroupRep1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupRep1.py index 1745b34cc4..f7dab4fcf6 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupRep1.py @@ -412,7 +412,7 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver3.stt to db2 and db1 ") + tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver4.stt to db2 and db1 ") for dbname in [self.db2, self.db1]: tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') tdSql.execute(f'flush database {dbname}') From c0186a214d9dc11faf56fedd2b483ce228ff91b5 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 3 Jan 2024 14:50:05 +0800 Subject: [PATCH 50/93] test: add testcase for TD-28106 of copy file in snapshot --- tests/parallel_test/cases.task | 6 +- tests/script/win-test-file | 1 + tests/system-test/0-others/splitVGroup.py | 464 ++++++++++++++++++ .../{splitVGroupRep1.py => splitVGroupWal.py} | 8 +- tests/system-test/win-test-file | 8 +- 5 files changed, 479 insertions(+), 8 deletions(-) create mode 100644 tests/system-test/0-others/splitVGroup.py rename tests/system-test/0-others/{splitVGroupRep1.py => splitVGroupWal.py} (99%) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 093fec78ab..edf28b6518 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -260,8 +260,10 @@ e ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 3 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py diff --git a/tests/script/win-test-file b/tests/script/win-test-file index ada8574165..744415c89f 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -189,6 +189,7 @@ ./test.sh -f tsim/query/unionall_as_table.sim ./test.sh -f tsim/query/multi_order_by.sim ./test.sh -f tsim/query/sys_tbname.sim +./test.sh -f tsim/query/sort-pre-cols.sim ./test.sh -f tsim/query/groupby.sim ./test.sh -f tsim/query/groupby_distinct.sim ./test.sh -f tsim/query/event.sim diff --git a/tests/system-test/0-others/splitVGroup.py b/tests/system-test/0-others/splitVGroup.py new file mode 100644 index 0000000000..c49713fc6d --- /dev/null +++ b/tests/system-test/0-others/splitVGroup.py @@ -0,0 +1,464 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time +import copy +import string + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # random string + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into @db_name.t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + + # insert nomal talbe + for i in range(20): + self.ts += 1000 + name = self.random_string(20) + sql = f"insert into @db_name.ta values({self.ts}, {i}, {self.ts%100000}, '{name}', false)" + self.exeDouble(sql) + + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + def exeDouble(self, sql): + # dbname replace + sql1 = sql.replace("@db_name", self.db1) + + if len(sql1) > 100: + tdLog.info(sql1[:100]) + else: + tdLog.info(sql1) + tdSql.execute(sql1) + + sql2 = sql.replace("@db_name", self.db2) + if len(sql2) > 100: + tdLog.info(sql2[:100]) + else: + tdLog.info(sql2) + tdSql.execute(sql2) + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000 + self.childCnt = 4 + self.childRow = 10000 + self.batchSize = 50000 + self.vgroups1 = 1 + self.vgroups2 = 1 + self.db1 = "db1" + self.db2 = "db2" + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db wal_retention_period 0 + sql = f"create database @db_name vgroups {self.vgroups1} replica {self.replicaVar} wal_retention_period 0 wal_retention_size 1" + self.exeDouble(sql) + + # create super talbe st + sql = f"create table @db_name.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + self.exeDouble(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table @db_name.t{i} using @db_name.st tags({i}) " + self.exeDouble(sql) + + # create normal table + sql = f"create table @db_name.ta(ts timestamp, c1 int, c2 bigint, c3 binary(32), c4 bool)" + self.exeDouble(sql) + + # insert data + self.insertData() + + # update + self.ts = 1680000000000 + 20000 + self.childRow = 1000 + + + # delete data + sql = "delete from @db_name.st where ts > 1680000019000 and ts < 1680000062000" + self.exeDouble(sql) + sql = "delete from @db_name.st where ts > 1680000099000 and ts < 1680000170000" + self.exeDouble(sql) + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.time() % 10000 + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # check query result same + def queryDouble(self, sql): + # sql + sql1 = sql.replace('@db_name', self.db1) + tdLog.info(sql1) + start1 = time.time() + rows1 = tdSql.query(sql1,queryTimes=2) + spend1 = time.time() - start1 + res1 = copy.deepcopy(tdSql.queryResult) + + sql2 = sql.replace('@db_name', self.db2) + tdLog.info(sql2) + start2 = time.time() + tdSql.query(sql2,queryTimes=2) + spend2 = time.time() - start2 + res2 = tdSql.queryResult + + rowlen1 = len(res1) + rowlen2 = len(res2) + errCnt = 0 + + if rowlen1 != rowlen2: + tdLog.exit(f"both row count not equal. rowlen1={rowlen1} rowlen2={rowlen2} ") + return False + + for i in range(rowlen1): + row1 = res1[i] + row2 = res2[i] + collen1 = len(row1) + collen2 = len(row2) + if collen1 != collen2: + tdLog.exit(f"both col count not equal. collen1={collen1} collen2={collen2}") + return False + for j in range(collen1): + if row1[j] != row2[j]: + tdLog.info(f"error both column value not equal. row={i} col={j} col1={row1[j]} col2={row2[j]} .") + errCnt += 1 + + if errCnt > 0: + tdLog.exit(f" db2 column value different with db2. different count ={errCnt} ") + + # warning performance + diff = (spend2 - spend1)*100/spend1 + tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff)) + if spend2 > spend1 and diff > 20: + tdLog.info("warning: the diff for performance after spliting is over 20%") + + return True + + + # check result + def checkResult(self): + # check vgroupid + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{self.db2}'" + tdSql.query(sql,queryTimes=2) + tdSql.checkRows(self.vgroups2) + + # check child table count same + sql = "select table_name from information_schema.ins_tables where db_name='@db_name' order by table_name" + self.queryDouble(sql) + + # check row value is ok + sql = "select * from @db_name.st order by ts, tbname" + self.queryDouble(sql) + + # where + sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts, tbname" + self.queryDouble(sql) + + # max + sql = "select max(c1) from @db_name.st" + self.queryDouble(sql) + + # min + sql = "select min(c2) from @db_name.st" + self.queryDouble(sql) + + # sum + sql = "select sum(c1) from @db_name.st" + self.queryDouble(sql) + + # normal table + + # count + sql = "select count(*) from @db_name.ta" + self.queryDouble(sql) + + # all rows + sql = "select * from @db_name.ta" + self.queryDouble(sql) + + # sum + sql = "select sum(c1) from @db_name.ta" + self.queryDouble(sql) + + + # get vgroup list + def getVGroup(self, db_name): + vgidList = [] + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + res = tdSql.getResult(sql) + rows = len(res) + for i in range(rows): + vgidList.append(res[i][0]) + + return vgidList; + + # split vgroup on db2 + def splitVGroup(self, db_name): + vgids = self.getVGroup(db_name) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdLog.info(sql) + tdSql.execute(sql) + + # wait end + seconds = 300 + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("split vgroup finished.") + return True + #tdLog.info(f"i={i} wait split vgroup ...") + time.sleep(1) + + tdLog.exit(f"split vgroup transaction is not finished after executing {seconds}s") + return False + + # split error + def expectSplitError(self, dbName): + vgids = self.getVGroup(dbName) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdLog.info(sql) + tdSql.error(sql) + + # expect split ok + def expectSplitOk(self, dbName): + # split vgroup + vgList1 = self.getVGroup(dbName) + self.splitVGroup(dbName) + vgList2 = self.getVGroup(dbName) + vgNum1 = len(vgList1) + 1 + vgNum2 = len(vgList2) + if vgNum1 != vgNum2: + tdLog.exit(f" vglist len={vgNum1} is not same for expect {vgNum2}") + return + + # split empty database + def splitEmptyDB(self): + dbName = "emptydb" + vgNum = 2 + # create database + sql = f"create database {dbName} vgroups {vgNum} replica {self.replicaVar }" + tdLog.info(sql) + tdSql.execute(sql) + + # split vgroup + self.expectSplitOk(dbName) + + + # forbid + def checkForbid(self): + # stream + tdLog.info("check forbid split having stream...") + tdSql.execute("create database streamdb;") + tdSql.execute("use streamdb;") + tdSql.execute("create table ta(ts timestamp, age int);") + tdSql.execute("create stream ma into sta as select count(*) from ta interval(1s);") + self.expectSplitError("streamdb") + tdSql.execute("drop stream ma;") + self.expectSplitOk("streamdb") + + # topic + tdLog.info("check forbid split having topic...") + tdSql.execute("create database topicdb wal_retention_period 10;") + tdSql.execute("use topicdb;") + tdSql.execute("create table ta(ts timestamp, age int);") + tdSql.execute("create topic toa as select * from ta;") + + #self.expectSplitError("topicdb") + tdSql.execute("drop topic toa;") + self.expectSplitOk("topicdb") + + # compact and check db2 + def compactAndCheck(self): + tdLog.info("compact db2 and check result ...") + # compact + tdSql.execute(f"compact database {self.db2};") + # check result + self.checkResult() + + # run + def run(self): + # prepare env + self.prepareEnv() + tdLog.info("generate at least two stt files of the same fileset (e.g. v4f1944) for db2 and db1 ") + for dbname in [self.db2, self.db1]: + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdLog.info("check db1 and db2 same after creating ...") + + self.checkResult() + + for i in range(3): + # split vgroup on db2 + start = time.time() + self.splitVGroup(self.db2) + end = time.time() + self.vgroups2 += 1 + + # insert the same data per tables into splited vgroups + tdLog.info("insert the same data per tables into splited vgroups(3,4)") + for dbname in [self.db2, self.db1]: + for tableid in range(self.childCnt): + tdSql.execute(f'insert into {dbname}.t{tableid} values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.ta values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999",0);') + tdSql.execute(f'flush database {dbname}') + + # check two db query result same + self.checkResult() + spend = "%.3f"%(end-start) + tdLog.info(f"split vgroup i={i} passed. spend = {spend}s") + + # split empty db + self.splitEmptyDB() + + # check topic and stream forib + self.checkForbid() + + # compact database + self.compactAndCheck() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupWal.py similarity index 99% rename from tests/system-test/0-others/splitVGroupRep1.py rename to tests/system-test/0-others/splitVGroupWal.py index f7dab4fcf6..dfb2281e5e 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupWal.py @@ -164,7 +164,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 0 wal_retention_size 1" + sql = f"create database @db_name vgroups {self.vgroups1} replica {self.replicaVar}" self.exeDouble(sql) # create super talbe st @@ -277,11 +277,11 @@ class TDTestCase: self.queryDouble(sql) # check row value is ok - sql = "select * from @db_name.st order by ts" + sql = "select * from @db_name.st order by ts, tbname" self.queryDouble(sql) # where - sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts" + sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts, tbname" self.queryDouble(sql) # max @@ -369,7 +369,7 @@ class TDTestCase: dbName = "emptydb" vgNum = 2 # create database - sql = f"create database {dbName} vgroups {vgNum} replica 1" + sql = f"create database {dbName} vgroups {vgNum} replica {self.replicaVar}" tdLog.info(sql) tdSql.execute(sql) diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index e261e18456..1ab0fee7bf 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -1,4 +1,5 @@ python3 ./test.py -f 2-query/tbname_vgroup.py +python3 ./test.py -f 2-query/compact-col.py python3 ./test.py -f 2-query/stbJoin.py python3 ./test.py -f 2-query/stbJoin.py -Q 2 python3 ./test.py -f 2-query/stbJoin.py -Q 3 @@ -205,8 +206,10 @@ python3 ./test.py -f 0-others/test_show_table_distributed.py python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 0-others/tag_index_basic.py python3 ./test.py -N 3 -f 0-others/walRetention.py -python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 -python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 +python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 1 +python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 1 +python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 3 +python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 3 python3 ./test.py -f 0-others/timeRangeWise.py -N 3 python3 ./test.py -f 0-others/delete_check.py python3 ./test.py -f 0-others/test_hot_refresh_configurations.py @@ -467,6 +470,7 @@ python3 ./test.py -f 2-query/concat2.py python3 ./test.py -f 2-query/json_tag.py python3 ./test.py -f 2-query/nestedQueryInterval.py python3 ./test.py -f 2-query/systable_func.py +python3 ./test.py -f 2-query/test_ts4382.py python3 ./test.py -f 2-query/stablity.py python3 ./test.py -f 2-query/stablity_1.py python3 ./test.py -f 2-query/elapsed.py From 35b8683262ecfbdcd3a7af15fd80bd74a39136b3 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 3 Jan 2024 14:52:24 +0800 Subject: [PATCH 51/93] test: add testcase for TD-28106 of copy file in snapshot --- tests/system-test/0-others/splitVGroupWal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/splitVGroupWal.py b/tests/system-test/0-others/splitVGroupWal.py index dfb2281e5e..d11d63b5a9 100644 --- a/tests/system-test/0-others/splitVGroupWal.py +++ b/tests/system-test/0-others/splitVGroupWal.py @@ -412,7 +412,7 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver4.stt to db2 and db1 ") + tdLog.info("generate at least two stt files of the same fileset (e.g. v4f1944) for db2 and db1 ") for dbname in [self.db2, self.db1]: tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') tdSql.execute(f'flush database {dbname}') From e67c6764664b8f37504b7cc86f61555bade36601 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 3 Jan 2024 15:19:27 +0800 Subject: [PATCH 52/93] TS-4423: add test case --- tests/parallel_test/cases.task | 1 + .../system-test/1-insert/stt_blocks_check.py | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tests/system-test/1-insert/stt_blocks_check.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 093fec78ab..0e826304c1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -109,6 +109,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_stb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/stt_blocks_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py diff --git a/tests/system-test/1-insert/stt_blocks_check.py b/tests/system-test/1-insert/stt_blocks_check.py new file mode 100644 index 0000000000..5de6414453 --- /dev/null +++ b/tests/system-test/1-insert/stt_blocks_check.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from random import randint +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.boundary import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def stt_block_check(self): + tdSql.prepare() + tdSql.execute('use db') + + tdSql.execute('create table meters (ts timestamp, c1 int, c2 float) tags(t1 int)') + tdSql.execute("create table d0 using meters tags(1)") + + ts = 1704261670000 + + sql = "insert into d0 values " + for i in range(100): + sql = sql + f"({ts + i}, 1, 0.1)" + tdSql.execute(sql) + tdSql.execute("flush database db") + + ts = 1704261670099 + + sql = "insert into d0 values " + for i in range(100): + sql = sql + f"({ts + i}, 1, 0.1)" + tdSql.execute(sql) + tdSql.execute("flush database db") + + tdSql.execute(f"insert into d0 values({ts + 100}, 2, 1.0)") + tdSql.execute("flush database db") + + time.sleep(2) + + tdSql.query("show table distributed db.meters") + tdSql.query("select count(*) from db.meters") + tdSql.checkData(0, 0, 200) + + def run(self): + self.stt_block_check() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 589b046ecb366471018830268defd73b2735ce93 Mon Sep 17 00:00:00 2001 From: charles Date: Wed, 3 Jan 2024 17:05:55 +0800 Subject: [PATCH 53/93] update alter_database test case for arm64 ci --- tests/system-test/1-insert/alter_database.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 8de43b8ee8..6a831b88ff 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -4,7 +4,8 @@ import time import socket import os import threading - +import psutil +import platform from util.log import * from util.sql import * from util.cases import * @@ -17,6 +18,14 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.buffer_boundary = [3, 4097, 8193, 12289, 16384] + # remove the value > free_memory, 70% is the weight to calculate the max value + if platform.system() == "Linux" and platform.machine() == "aarch64": + mem = psutil.virtual_memory() + free_memory = mem.free * 0.7 / 1024 / 1024 + for item in self.buffer_boundary: + if item > free_memory: + self.buffer_boundary.remove(item) + self.buffer_error = [self.buffer_boundary[0] - 1, self.buffer_boundary[-1]+1] # pages_boundary >= 64 From b818faaa5eba3bf6fc28a4e059ea81d86a3b54c3 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 3 Jan 2024 17:51:27 +0800 Subject: [PATCH 54/93] remainGroups --- source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/scanoperator.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fe3528cce6..2e93dbd803 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2164,7 +2164,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool initRemainGroups = false; if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable) { + if (pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { initRemainGroups = true; } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8999340ddc..c2a194d150 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -690,7 +690,7 @@ void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { if (pInfo->countState == TABLE_COUNT_STATE_END) { return; } - if (pInfo->base.pTableListInfo->oneTableForEachGroup) { + if (pInfo->base.pTableListInfo->oneTableForEachGroup || pInfo->base.pTableListInfo->groupOffset) { pInfo->countState = TABLE_COUNT_STATE_PROCESSED; } else { taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId)); @@ -854,14 +854,14 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { STableListInfo* pTableListInfo = pTableScanInfo->base.pTableListInfo; - if (pTableListInfo->oneTableForEachGroup) { // group by tbname + if (pTableListInfo->oneTableForEachGroup || pTableListInfo->groupOffset) { // group by tbname, group by tag + sort if (pTableScanInfo->countState < TABLE_COUNT_STATE_PROCESSED) { pTableScanInfo->countState = TABLE_COUNT_STATE_PROCESSED; STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); return getBlockForEmptyTable(pOperator, pStart); } - } else { // group by tag + } else { // group by tag + no sort int32_t numOfTables = tableListGetSize(pTableListInfo); if (pTableScanInfo->tableEndIndex + 1 >= numOfTables) { // get empty group, mark processed & rm from hash From de631e20a5469ff80e638e2e99e742bd6b924aff Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 3 Jan 2024 19:57:22 +0800 Subject: [PATCH 55/93] fix: refactor logdebugs --- include/common/tglobal.h | 4 +- include/util/tconfig.h | 2 +- include/util/tdef.h | 7 ++- include/util/tlog.h | 1 + source/common/src/tglobal.c | 98 +++++++++++++++++++++------------- source/dnode/mgmt/exe/dmMain.c | 2 +- source/util/src/tconfig.c | 37 ++++++++++++- source/util/src/tlog.c | 1 + utils/tsim/inc/simInt.h | 1 - utils/tsim/src/simSystem.c | 2 - 10 files changed, 110 insertions(+), 45 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 887f4cb6dc..f006779a48 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -228,8 +228,8 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer); struct SConfig *taosGetCfg(); -void taosSetAllDebugFlag(int32_t flag, bool rewrite); -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite); +void taosSetAllDebugFlag(int32_t flag); +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(); diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 07abdd223d..f2a9446700 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -94,7 +94,7 @@ typedef struct SConfigItem { int64_t imax; double fmax; }; - SArray *array; // SDiskCfg + SArray *array; // SDiskCfg/SLogVar } SConfigItem; typedef struct { diff --git a/include/util/tdef.h b/include/util/tdef.h index 51b0b63da2..875a6f5738 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -260,7 +260,7 @@ typedef enum ELogicConditionType { #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 32 -#define TSDB_LABEL_LEN 12 +#define TSDB_LABEL_LEN 12 #define TSDB_JOB_STATUS_LEN 32 #define TSDB_CLUSTER_ID_LEN 40 @@ -272,6 +272,7 @@ typedef enum ELogicConditionType { #define TSDB_SHOW_SCHEMA_JSON_LEN TSDB_MAX_COLUMNS * 256 #define TSDB_SLOW_QUERY_SQL_LEN 512 #define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_LOG_VAR_LEN 32 #define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_TYPE_LEN 16 @@ -503,6 +504,10 @@ typedef struct { int32_t primary; } SDiskCfg; +typedef struct { + char name[TSDB_LOG_VAR_LEN]; +} SLogVar; + #define TMQ_SEPARATOR ':' enum { diff --git a/include/util/tlog.h b/include/util/tlog.h index 6d393bfefb..11ac0e1fae 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -67,6 +67,7 @@ extern int32_t smaDebugFlag; extern int32_t idxDebugFlag; extern int32_t tdbDebugFlag; extern int32_t sndDebugFlag; +extern int32_t simDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d6d1e1810d..79d21955d4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -423,7 +423,7 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; @@ -497,7 +497,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; @@ -962,6 +962,7 @@ static void taosSetClientLogCfg(SConfig *pCfg) { rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; } static void taosSetServerLogCfg(SConfig *pCfg) { @@ -1278,7 +1279,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetServerLogCfg(pCfg); } - taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false); + taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1356,6 +1357,8 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); if (taosSetFileHandlesLimit() != 0) return -1; + taosSetAllDebugFlag(cfgGetItem(tsCfg, "debugFlag")->i32); + cfgDumpCfg(tsCfg, tsc, false); if (taosCheckGlobalCfg() != 0) { @@ -1399,7 +1402,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, *pVar = flag; if (isDebugflag) { - taosSetDebugFlag(pOptions[d].optionVar, optName, flag, true); + taosSetDebugFlag(pOptions[d].optionVar, optName, flag); } terrno = TSDB_CODE_SUCCESS; } break; @@ -1447,8 +1450,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { } if (strncasecmp(name, "debugFlag", 9) == 0) { - int32_t flag = pItem->i32; - taosSetAllDebugFlag(flag, true); + taosSetAllDebugFlag(pItem->i32); return 0; } @@ -1460,7 +1462,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag}, - {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, + {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, }; static OptionNameAndVar options[] = { @@ -1524,8 +1526,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { switch (lowcaseName[0]) { case 'd': { if (strcasecmp("debugFlag", name) == 0) { - int32_t flag = pItem->i32; - taosSetAllDebugFlag(flag, true); + taosSetAllDebugFlag(pItem->i32); matched = true; } break; @@ -1691,7 +1692,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { {"cDebugFlag", &cDebugFlag}, {"dDebugFlag", &dDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"jniDebugFlag", &jniDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, - {"uDebugFlag", &uDebugFlag}, + {"uDebugFlag", &uDebugFlag}, {"simDebugFlag", &simDebugFlag}, }; static OptionNameAndVar options[] = { @@ -1736,9 +1737,9 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer) { return taosCfgDynamicOptionsForClient(pCfg, name); } -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite) { +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal) { SConfigItem *pItem = cfgGetItem(tsCfg, flagName); - if (pItem != NULL && (rewrite || pItem->i32 == 0)) { + if (pItem != NULL) { pItem->i32 = flagVal; } if (pFlagPtr != NULL) { @@ -1746,33 +1747,58 @@ void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, } } -void taosSetAllDebugFlag(int32_t flag, bool rewrite) { +static int taosLogVarComp(void const *lp, void const *rp) { + SLogVar *lpVar = (SLogVar *)lp; + SLogVar *rpVar = (SLogVar *)rp; + return strcasecmp(lpVar->name, rpVar->name); +} + +static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag, SArray *noNeedToSetVars) { + if (noNeedToSetVars != NULL && taosArraySearch(noNeedToSetVars, name, taosLogVarComp, TD_EQ) != NULL) { + return; + } + taosSetDebugFlag(pFlagPtr, name, flag); +} + +void taosSetAllDebugFlag(int32_t flag) { if (flag <= 0) return; - taosSetDebugFlag(NULL, "debugFlag", flag, rewrite); - taosSetDebugFlag(NULL, "simDebugFlag", flag, rewrite); - taosSetDebugFlag(NULL, "tmrDebugFlag", flag, rewrite); - taosSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, rewrite); - taosSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, rewrite); - taosSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, rewrite); - taosSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, rewrite); - taosSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, rewrite); - taosSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, rewrite); - taosSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, rewrite); - taosSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, rewrite); - taosSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, rewrite); - taosSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, rewrite); - taosSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, rewrite); - taosSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, rewrite); - taosSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, rewrite); - taosSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, rewrite); - taosSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, rewrite); - taosSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, rewrite); - taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite); - taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite); - taosSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, rewrite); - taosSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, rewrite); + SArray *noNeedToSetVars = NULL; + SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); + if (pItem != NULL) { + pItem->i32 = flag; + noNeedToSetVars = pItem->array; + } + + taosCheckAndSetDebugFlag(&simDebugFlag, "simDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tmrDebugFlag, "tmrDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, noNeedToSetVars); + + taosCheckAndSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, noNeedToSetVars); + + taosCheckAndSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, noNeedToSetVars); + + taosArrayClear(noNeedToSetVars); // reset array + uInfo("all debug flag are set to %d", flag); + if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist } int8_t taosGranted() { return atomic_load_8(&tsGrant); } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 3c08714218..756ac8167e 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -68,7 +68,7 @@ static struct { int64_t startTime; } global = {0}; -static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143, true); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143); } static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } static void dmStopDnode(int signum, void *sigInfo, void *context) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index a78d930326..d656f0c14e 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -307,6 +307,34 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, return 0; } +static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) { + SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag"); + if (resetArray) { + // reset + if (pDebugFlagItem == NULL) return -1; + + // logflag names that should 'not' be set by 'debugFlag' + if (pDebugFlagItem->array == NULL) { + pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar)); + if (pDebugFlagItem->array == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + taosArrayClear(pDebugFlagItem->array); + return 0; + } + + // update + if (pDebugFlagItem == NULL) return -1; + if (pDebugFlagItem->array != NULL) { + SLogVar logVar = {0}; + strncpy(logVar.name, name, TSDB_LOG_VAR_LEN - 1); + taosArrayPush(pDebugFlagItem->array, &logVar); + } + return 0; +} + int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) { GRANT_CFG_SET; SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -661,7 +689,6 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; - if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); for (int32_t j = 0; j < CFG_SRC_PRINT_LEN; ++j) { if (src[j] == 0) src[j] = ' '; @@ -931,6 +958,14 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + + size_t len = strlen(name); + const char *debugFlagStr = "debugFlag"; + const size_t debugFlagLen = strlen(debugFlagStr); + if (len >= debugFlagLen && strcasecmp(name + len - debugFlagLen, debugFlagStr) == 0) { + code = cfgUpdateDebugFlagItem(pConfig, name, len == debugFlagLen); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } } taosCloseFile(&pFile); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e113a95fcc..bd6c37a7b5 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -111,6 +111,7 @@ int32_t udfDebugFlag = 131; int32_t smaDebugFlag = 131; int32_t idxDebugFlag = 131; int32_t sndDebugFlag = 131; +int32_t simDebugFlag = 131; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h index 02c7540cde..f2360277e0 100644 --- a/utils/tsim/inc/simInt.h +++ b/utils/tsim/inc/simInt.h @@ -184,7 +184,6 @@ extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; extern SCommand simCmdList[]; extern int32_t simScriptPos; extern int32_t simScriptSucceed; -extern int32_t simDebugFlag; extern char simScriptDir[]; extern bool abortExecution; extern bool useValgrind; diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index 98f9217fd6..dcf5d6ab12 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -21,7 +21,6 @@ SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; int32_t simScriptPos = -1; int32_t simScriptSucceed = 0; -int32_t simDebugFlag = 143; void simCloseTaosdConnect(SScript *script); char simScriptDir[PATH_MAX] = {0}; @@ -32,7 +31,6 @@ int32_t simInitCfg() { taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); SConfig *pCfg = taosGetCfg(); - simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX); return 0; } From 860146d425dcea53420439ee2f2f591d7fb56400 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 3 Jan 2024 23:59:48 +0800 Subject: [PATCH 56/93] initRemainGroups --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2e93dbd803..e3f634f406 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2164,7 +2164,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool initRemainGroups = false; if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { + if (tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { initRemainGroups = true; } } From f4ad4ea998d9e3b150172c39a7f1b809899c7823 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 10:01:05 +0800 Subject: [PATCH 57/93] enh: use learner in split vgroup --- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 20fb7a4d31..6eb5d7e592 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2707,8 +2707,16 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro if (newVg1.replica == 1) { if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER; + + newVg1.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER; + + newVg1.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER; + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; + + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; } else if (newVg1.replica == 3) { SVnodeGid del1 = {0}; if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER; From b85f36bcea67d833b65194333fec446413912ca3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 12:00:49 +0800 Subject: [PATCH 58/93] enh: allow to drop an empty dnode, no matter online or not --- source/dnode/mnode/impl/src/mndDnode.c | 29 +++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e224aceec2..6585e70533 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1094,6 +1094,32 @@ _OVER: return code; } +static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) { + bool isEmpty = false; + SMnodeObj *pMObj = NULL; + SQnodeObj *pQObj = NULL; + SSnodeObj *pSObj = NULL; + + pQObj = mndAcquireQnode(pMnode, dnodeId); + if (pQObj) goto _OVER; + + pSObj = mndAcquireSnode(pMnode, dnodeId); + if (pSObj) goto _OVER; + + pMObj = mndAcquireMnode(pMnode, dnodeId); + if (pMObj) goto _OVER; + + int32_t numOfVnodes = mndGetVnodesNum(pMnode, dnodeId); + if (numOfVnodes > 0) goto _OVER; + + isEmpty = true; +_OVER: + mndReleaseMnode(pMnode, pMObj); + mndReleaseQnode(pMnode, pQObj); + mndReleaseSnode(pMnode, pSObj); + return isEmpty; +} + static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -1155,7 +1181,8 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (!isonline && !force) { + bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id); + if (!isonline && !force && !isEmpty) { terrno = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); From bd64a78094f08aa5ad9bd3c0f1d03b7ab5e6acef Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 4 Jan 2024 18:53:37 +0800 Subject: [PATCH 59/93] fix:[TD-28155] set consumer lost if all current topic is 0 --- source/dnode/mnode/impl/src/mndConsumer.c | 24 ++++++++++------------- source/dnode/mnode/impl/src/mndTrans.c | 5 ++++- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 9d9473d883..a365a44d56 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -749,18 +749,6 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { return 0; } -static void updateConsumerStatus(SMqConsumerObj *pConsumer) { - int32_t status = pConsumer->status; - - if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS_REBALANCE) { - pConsumer->status = MQ_CONSUMER_STATUS_READY; - } else if (status == MQ_CONSUMER_STATUS_READY) { - pConsumer->status = MQ_CONSUMER_STATUS_LOST; - } - } -} - // remove from new topic static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); @@ -881,7 +869,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - updateConsumerStatus(pOldConsumer); + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && + taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && + taosArrayGetSize(pOldConsumer->currentTopics) != 0){ + pOldConsumer->status = MQ_CONSUMER_STATUS_READY; + } // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = taosGetTimestampMs(); @@ -905,7 +897,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - updateConsumerStatus(pOldConsumer); + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && + taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && + taosArrayGetSize(pOldConsumer->currentTopics) == 0){ + pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; + } pOldConsumer->rebalanceTime = taosGetTimestampMs(); atomic_add_fetch_32(&pOldConsumer->epoch, 1); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 99553fc57a..9075f0145c 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1119,7 +1119,10 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode, topHalf)) return TSDB_CODE_MND_TRANS_CTX_SWITCH; + if (mndCannotExecuteTransAction(pMnode, topHalf)) { + terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH; + return TSDB_CODE_MND_TRANS_CTX_SWITCH; + } int64_t signature = pTrans->id; signature = (signature << 32); From 4ea2cad0bb6824e315c68be890e52422b818fceb Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 4 Jan 2024 18:59:55 +0800 Subject: [PATCH 60/93] comment snode_restart_with_checkpoint.py --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1761189b19..ac21f0fbb2 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -33,7 +33,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py #,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 -,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py From a811a60ad497220fad7410afb779ba0e93cabbd5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 16:07:58 +0800 Subject: [PATCH 61/93] enh: restore sync log store from snap on fsm state incomplete --- source/libs/sync/src/syncMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 52557f7b9c..9352d5662e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -790,7 +790,7 @@ int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) { SyncIndex commitIndex = snapshot.lastApplyIndex; SyncIndex firstVer = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore); SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); - if (lastVer < commitIndex || firstVer > commitIndex + 1) { + if ((lastVer < commitIndex || firstVer > commitIndex + 1) || pNode->fsmState == SYNC_FSM_STATE_INCOMPLETE) { if (pNode->pLogStore->syncLogRestoreFromSnapshot(pNode->pLogStore, commitIndex)) { sError("vgId:%d, failed to restore log store from snapshot since %s. lastVer:%" PRId64 ", snapshotVer:%" PRId64, pNode->vgId, terrstr(), lastVer, commitIndex); From 81d8b9b98b3775848c6398e136c260125f43fa3b Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 5 Jan 2024 00:35:29 +0800 Subject: [PATCH 62/93] test: add test case for create index --- source/libs/parser/src/parTranslater.c | 11 ++++------- tests/system-test/0-others/tag_index_basic.py | 8 ++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 40b869d2b7..c51e96d1b6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6930,10 +6930,11 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde } static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + int32_t code = 0; SName name; STableMeta* pMeta = NULL; - int32_t code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + + code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -6955,10 +6956,6 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS taosMemoryFree(pMeta); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, ((SColumnNode*)pNode)->colName); } - if (IS_IDX_ON(pSchema)) { - code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; - goto _exit; - } } SCreateTagIndexReq createTagIdxReq = {0}; @@ -6966,7 +6963,7 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSCreateTagIdxReq, &createTagIdxReq); } -_exit: +_exit: taosMemoryFree(pMeta); return code; } diff --git a/tests/system-test/0-others/tag_index_basic.py b/tests/system-test/0-others/tag_index_basic.py index c1e1d521d2..a55508e229 100644 --- a/tests/system-test/0-others/tag_index_basic.py +++ b/tests/system-test/0-others/tag_index_basic.py @@ -88,6 +88,13 @@ class TDTestCase: tdLog.info(f" create {count} child tables ok.") + def create_tagidx_check(self, stbname): + err_dict = {"NULL","",",","\"","\"\"","undef","t1,t2","t12,t12"} + for errs in err_dict: + sql = (f'create index idx_err_check on {stbname} (%s)'% (errs)) + tdLog.info(f' sql={sql}') + tdSql.error(f'{sql}') + tdLog.info(f' create tagidx check ok.') # create stable and child tables def create_tagidx(self, stbname): @@ -231,6 +238,7 @@ class TDTestCase: count = 1000 # do self.create_table(stable, tbname, count) + self.create_tagidx_check(stable) self.create_tagidx(stable) self.insert_data(tbname) self.show_tagidx(stable) From d3aee612e0cd4418c18c90b2c99f4acf3a4a8169 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 5 Jan 2024 13:08:16 +0800 Subject: [PATCH 63/93] chore: code optimization --- source/libs/parser/src/parTranslater.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c51e96d1b6..ea1a5f7d1d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6939,10 +6939,6 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS taosMemoryFree(pMeta); return code; } - if (TSDB_SUPER_TABLE != pMeta->tableType) { - taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only super table is allowed"); - } if (LIST_LENGTH(pStmt->pCols) != 1) { taosMemoryFree(pMeta); From 9eced80e49303929c8e21c1b80a2aca8a39c07b8 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Fri, 5 Jan 2024 14:33:42 +0800 Subject: [PATCH 64/93] updated cases.task --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ac21f0fbb2..7eedf505a5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -542,6 +542,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py From 8ddff2eecbe476c84c1a7aba2364e7fb1dd0bc96 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 5 Jan 2024 07:02:16 +0000 Subject: [PATCH 65/93] fix/TD-28073 --- source/dnode/mnode/impl/src/mndCompact.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 6471fa817d..101022a44f 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -26,6 +26,7 @@ #include "mndTrans.h" #define MND_COMPACT_VER_NUMBER 1 +#define MND_COMPACT_ID_LEN 11 static int32_t mndProcessCompactTimer(SRpcMsg *pReq); @@ -451,7 +452,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){ code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[10] = {0}; + char obj[MND_COMPACT_ID_LEN] = {0}; sprintf(obj, "%d", pCompact->compactId); auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); From 0b9e3c9871d530fb9b19720284ed879f457a02c3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 5 Jan 2024 16:20:55 +0800 Subject: [PATCH 66/93] fix stream state buff issue --- source/libs/executor/src/executil.c | 4 +- source/libs/stream/src/streamSessionState.c | 2 +- tests/script/tsim/stream/state1.sim | 43 +++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c1ddd6e4f4..b713b9b112 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2274,10 +2274,10 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr, const char* taskIdStr) { if (!pBlock) { - qDebug("%s===stream===%s: Block is Null", taskIdStr, flag); + qDebug("%s===stream===%s %s: Block is Null", taskIdStr, flag, opStr); return; } else if (pBlock->info.rows == 0) { - qDebug("%s===stream===%s: Block is Empty. type:%d", taskIdStr, flag, pBlock->info.type); + qDebug("%s===stream===%s %s: Block is Empty. block type %d", taskIdStr, flag, opStr, pBlock->info.type); return; } if (qDebugFlag & DEBUG_DEBUG) { diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 765403b1aa..5f19b47e1f 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -557,7 +557,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch void* pFileStore = getStateFileStore(pFileState); void* p = NULL; int32_t code_file = streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); - if (code_file == TSDB_CODE_SUCCESS) { + if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim index 775528bd4e..e6f7a81851 100644 --- a/tests/script/tsim/stream/state1.sim +++ b/tests/script/tsim/stream/state1.sim @@ -152,6 +152,49 @@ endi print step 2 over +print step 3 + +sql create database test3 vgroups 1; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791213001,1,2,3,1.0); + +print create stream streams3 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into streamt3 as select _wstart, max(a), count(*) c1 from t1 state_window(a); +sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into streamt3 as select _wstart, max(a), count(*) c1 from t1 state_window(a); + +sleep 1000 + +sql insert into t1 values(1648791203000,2,2,3,1.0); + +sleep 500 + +sql insert into t1 values(1648791214000,1,2,3,1.0); + +$loop_count = 0 +loop6: + +sleep 1000 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print select * from streamt3 +sql select * from streamt3; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 + +if $rows != 2 then + print =====rows=$rows + goto loop6 +endi + +print step 3 over + print state1 end system sh/stop_dnodes.sh From db9655c54999f7830ab224ab397b014768a8e74f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 5 Jan 2024 17:07:24 +0800 Subject: [PATCH 67/93] test: save case.sql when case is failed and set default saved sqlfile True --- tests/parallel_test/run.sh | 4 ++++ tests/pytest/util/sql.py | 4 ++-- tests/system-test/1-insert/influxdb_line_taosc_insert.py | 2 +- tests/system-test/1-insert/insert_double.py | 2 +- tests/system-test/1-insert/insert_perf.py | 2 +- tests/system-test/1-insert/mutil_stage.py | 2 +- .../1-insert/opentsdb_telnet_line_taosc_insert.py | 2 +- tests/system-test/1-insert/time_range_wise.py | 2 +- tests/system-test/2-query/abs.py | 2 +- tests/system-test/2-query/and_or_for_byte.py | 2 +- tests/system-test/2-query/apercentile.py | 2 +- tests/system-test/2-query/case_when.py | 2 +- tests/system-test/2-query/check_tsdb.py | 2 +- tests/system-test/2-query/count.py | 2 +- tests/system-test/2-query/countAlwaysReturnValue.py | 2 +- tests/system-test/2-query/db.py | 2 +- tests/system-test/2-query/fill_with_group.py | 2 +- tests/system-test/2-query/func_to_char_timestamp.py | 2 +- tests/system-test/2-query/function_null.py | 2 +- tests/system-test/2-query/histogram.py | 2 +- tests/system-test/2-query/interval_limit_opt.py | 2 +- tests/system-test/2-query/interval_limit_opt_2.py | 2 +- tests/system-test/2-query/interval_unit.py | 2 +- tests/system-test/2-query/irate.py | 2 +- tests/system-test/2-query/join.py | 2 +- tests/system-test/2-query/join2.py | 2 +- tests/system-test/2-query/json_tag.py | 2 +- tests/system-test/2-query/json_tag_large_tables.py | 2 +- tests/system-test/2-query/last_cache_scan.py | 5 +++-- tests/system-test/2-query/like.py | 2 +- tests/system-test/2-query/limit.py | 2 +- tests/system-test/2-query/odbc.py | 2 +- tests/system-test/2-query/partition_by_col.py | 2 +- tests/system-test/2-query/partition_by_col_agg.py | 2 +- tests/system-test/2-query/qnodeCluster.py | 2 +- tests/system-test/2-query/query_cols_tags_and_or.py | 2 +- tests/system-test/2-query/state_window.py | 2 +- tests/system-test/2-query/substr.py | 2 +- tests/system-test/2-query/ts_3405_3398_3423.py | 2 +- tests/system-test/2-query/tsbsQuery.py | 2 +- tests/system-test/2-query/ttl_comment.py | 2 +- tests/system-test/2-query/varbinary.py | 2 +- tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py | 2 +- tests/system-test/7-tmq/dataFromTsdbNWal.py | 2 +- tests/system-test/7-tmq/stbTagFilter-1ctb.py | 2 +- tests/system-test/7-tmq/stbTagFilter-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqAutoCreateTbl.py | 2 +- tests/system-test/7-tmq/tmqClientConsLog.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py | 2 +- .../7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py | 2 +- .../7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1.py | 2 +- tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py | 2 +- tests/system-test/7-tmq/tmqDataPrecisionUnit.py | 2 +- tests/system-test/7-tmq/tmqDelete-1ctb.py | 2 +- tests/system-test/7-tmq/tmqDelete-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart1.py | 2 +- tests/system-test/7-tmq/tmqDropConsumer.py | 2 +- tests/system-test/7-tmq/tmqDropNtb-snapshot0.py | 2 +- tests/system-test/7-tmq/tmqDropNtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqDropStbCtb.py | 2 +- tests/system-test/7-tmq/tmqMaxGroupIds.py | 2 +- tests/system-test/7-tmq/tmqMaxTopic.py | 2 +- tests/system-test/7-tmq/tmqMultiConsumer.py | 2 +- tests/system-test/7-tmq/tmqOffset.py | 2 +- tests/system-test/7-tmq/tmqSeekAndCommit.py | 2 +- tests/system-test/7-tmq/tmqUpdate-1ctb.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqUpdateWithConsume.py | 2 +- tests/system-test/7-tmq/tmqVnodeReplicate.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-column.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-db.py | 2 +- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py | 2 +- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-stb.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform-db.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform.py | 2 +- tests/system-test/7-tmq/tmq_per.py | 2 +- tests/system-test/8-stream/snode_restart.py | 2 +- tests/system-test/8-stream/snode_restart_with_checkpoint.py | 2 +- tests/system-test/8-stream/vnode_restart.py | 2 +- tests/system-test/99-TDcase/TD-17255.py | 2 +- 95 files changed, 101 insertions(+), 96 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index f05e0dfc83..cf21349513 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -200,6 +200,7 @@ function run_thread() { if [ -z "$case_file" ]; then continue fi + case_sql_file="$exec_dir/${case_file}.sql" case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}" count=$(( count + 1 )) local case_path=`dirname "$case_file"` @@ -324,12 +325,15 @@ function run_thread() { cmd="$tarcmd sh -c \"cd $remote_sim_dir; tar -czf sim.tar.gz sim\"" $cmd local remote_sim_tar="${workdirs[index]}/tmp/thread_volume/$thread_no/sim.tar.gz" + local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file)" scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" if [ -z ${passwords[index]} ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" $cmd + cmd="$scpcmd:${remote_case_sql_file} $log_dir/${case_file}.sql" + $cmd # backup source code (disabled) source_tar_dir=$log_dir/TDengine_${hosts[index]} source_tar_file=TDengine.tar.gz diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index f8a49983e1..000040d958 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -48,9 +48,9 @@ class TDSql: self.queryCols = 0 self.affectedRows = 0 - def init(self, cursor, log=False): + def init(self, cursor, log=True): self.cursor = cursor - + print(f"sqllog is :{log}") if (log): caller = inspect.getframeinfo(inspect.stack()[1][0]) self.cursor.log(caller.filename + ".sql") diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index 3e9bad4efd..85a0afed48 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self._conn = conn def createDb(self, name="test", db_update_tag=0): diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index b47b22ab44..b7af7237db 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -15,7 +15,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) self.database = "db1" tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_db(self): tdSql.execute(f"drop database if exists {self.database}") diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py index d96a031458..dac2afb3b2 100644 --- a/tests/system-test/1-insert/insert_perf.py +++ b/tests/system-test/1-insert/insert_perf.py @@ -30,7 +30,7 @@ class TDTestCase: self.tb3 = "t3" self.once = 1000 tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_db(self): tdSql.execute(f"drop database if exists {self.database}") diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 970554c6c1..ebaa1e7dc3 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -60,7 +60,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.taos_cfg_path = tdDnodes.dnodes[0].cfgPath self.taos_data_dir = tdDnodes.dnodes[0].dataDir diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index d419aee12c..2d9e34c287 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -33,7 +33,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self._conn = conn self.smlChildTableName_value = "id" diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 14b717d1da..a3df0eb6b0 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -137,7 +137,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.precision = "ms" self.sma_count = 0 self.sma_created_index = [] diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d64d550bc4..0eabd91535 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -17,7 +17,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 15e9110b3b..5b2fb51998 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -17,7 +17,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/apercentile.py b/tests/system-test/2-query/apercentile.py index e190bf9374..ee6835bd10 100644 --- a/tests/system-test/2-query/apercentile.py +++ b/tests/system-test/2-query/apercentile.py @@ -21,7 +21,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) self.rowNum = 10 self.ts = 1537146000000 self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/case_when.py b/tests/system-test/2-query/case_when.py index eebbcc3fa3..1ccd2b5076 100755 --- a/tests/system-test/2-query/case_when.py +++ b/tests/system-test/2-query/case_when.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) + tdSql.init(conn.cursor()) self.testcasePath = os.path.split(__file__)[0] self.testcaseFilename = os.path.split(__file__)[-1] diff --git a/tests/system-test/2-query/check_tsdb.py b/tests/system-test/2-query/check_tsdb.py index b3ff259cc5..c49c4f1481 100644 --- a/tests/system-test/2-query/check_tsdb.py +++ b/tests/system-test/2-query/check_tsdb.py @@ -15,7 +15,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_datas(self, dbname="db"): tdSql.execute( diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 3ba651622a..c06ee28d02 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -6,7 +6,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) self.setsql = TDSetSql() self.rowNum = 10 self.ts = 1537146000000 diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index 244964e486..f22fa3e01e 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_data(self, dbname="db"): tdSql.execute( diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index a48b4cf3bb..07514a0950 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): tdSql.execute("create database if not exists dbms precision 'ms'") diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index fce504d0f7..05be2ec9d2 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/func_to_char_timestamp.py b/tests/system-test/2-query/func_to_char_timestamp.py index d955e00a82..034d025fcd 100644 --- a/tests/system-test/2-query/func_to_char_timestamp.py +++ b/tests/system-test/2-query/func_to_char_timestamp.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/function_null.py b/tests/system-test/2-query/function_null.py index 44b86ee543..e5056b7c56 100644 --- a/tests/system-test/2-query/function_null.py +++ b/tests/system-test/2-query/function_null.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/histogram.py b/tests/system-test/2-query/histogram.py index cc6e8fe7e9..a69fb505d1 100644 --- a/tests/system-test/2-query/histogram.py +++ b/tests/system-test/2-query/histogram.py @@ -147,7 +147,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __create_hsg(self, sma:Hsgschema): return f"""{sma.histogram_flag}({sma.col}, '{sma.bin_type}', '{sma.bin_desc}', {sma.normalized})""" diff --git a/tests/system-test/2-query/interval_limit_opt.py b/tests/system-test/2-query/interval_limit_opt.py index 492f453de5..aa1702fe3c 100644 --- a/tests/system-test/2-query/interval_limit_opt.py +++ b/tests/system-test/2-query/interval_limit_opt.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/interval_limit_opt_2.py b/tests/system-test/2-query/interval_limit_opt_2.py index cadb32b388..cfabe46dca 100644 --- a/tests/system-test/2-query/interval_limit_opt_2.py +++ b/tests/system-test/2-query/interval_limit_opt_2.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/interval_unit.py b/tests/system-test/2-query/interval_unit.py index 9430c74cff..03917bfaf7 100644 --- a/tests/system-test/2-query/interval_unit.py +++ b/tests/system-test/2-query/interval_unit.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d', precision: str='ms'): if dropFlag == 1: diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index d882c9b8fb..82841541f0 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 120cc2eb30..c5c8f6c730 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -66,7 +66,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 7ddc5e57ba..1006033eae 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 8db92f38a2..33209620e1 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): # tdSql.prepare() diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py index ea3963640a..86bd6c2d82 100644 --- a/tests/system-test/2-query/json_tag_large_tables.py +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -36,7 +36,7 @@ class TDTestCase: self.testcaseFilename = os.path.split(__file__)[-1] # os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): # tdSql.prepare() diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 23433277c2..9b6dca56fd 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -376,7 +376,7 @@ class TDTestCase: tdSql.checkCols(1) p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"]) p.check_returncode() - tdSql.query_success_failed("select ts, last_row(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2") + tdSql.query_success_failed("select ts, last(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2") tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) tdSql.checkRows(1) tdSql.checkCols(3) @@ -413,7 +413,8 @@ class TDTestCase: tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) - tdSql.query_success_failed('select last(c2), c2, c3 from meters', queryTimes=10, expectErrInfo="Invalid column name: c2") + + def run(self): self.prepareTestEnv() diff --git a/tests/system-test/2-query/like.py b/tests/system-test/2-query/like.py index 95e2daf6d3..65d0a36b24 100644 --- a/tests/system-test/2-query/like.py +++ b/tests/system-test/2-query/like.py @@ -13,7 +13,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def initDB(self): tdSql.execute("drop database if exists db") diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index 961cff5087..e6736704f7 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1): if dropFlag == 1: diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index 8fbad93995..7daf5fd730 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -13,7 +13,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def check_ins_cols(self): tdSql.execute("create database if not exists db") diff --git a/tests/system-test/2-query/partition_by_col.py b/tests/system-test/2-query/partition_by_col.py index 230b7582d9..feb8693e3e 100644 --- a/tests/system-test/2-query/partition_by_col.py +++ b/tests/system-test/2-query/partition_by_col.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/partition_by_col_agg.py b/tests/system-test/2-query/partition_by_col_agg.py index 1bc7a2414a..405cff9663 100644 --- a/tests/system-test/2-query/partition_by_col_agg.py +++ b/tests/system-test/2-query/partition_by_col_agg.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index d15ef89eb8..851b5bb1d1 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): tsql.execute("use %s" %dbName) diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index b8d4e3c314..d86b5d07e7 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -20,7 +20,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) ## add for TD-6672 tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def insertData(self, tb_name): insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1, 2, 3, 4)', diff --git a/tests/system-test/2-query/state_window.py b/tests/system-test/2-query/state_window.py index 2bf363f0dc..7dedeb88f1 100644 --- a/tests/system-test/2-query/state_window.py +++ b/tests/system-test/2-query/state_window.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index 302711b13d..ad5a2bba55 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) def __substr_condition(self): # sourcery skip: extract-method substr_condition = [] diff --git a/tests/system-test/2-query/ts_3405_3398_3423.py b/tests/system-test/2-query/ts_3405_3398_3423.py index dcd8fb5a85..814d9de394 100644 --- a/tests/system-test/2-query/ts_3405_3398_3423.py +++ b/tests/system-test/2-query/ts_3405_3398_3423.py @@ -11,7 +11,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): """This test case is used to verify the query performance for the merge scans process of diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 0e2975cdba..5a49f653ec 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_ctable(self,tsql=None, dbName='db',stbName='stb',ctbPrefix='ctb',ctbNum=1): tsql.execute("use %s" %dbName) diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py index 3ae602fa23..ba24579611 100644 --- a/tests/system-test/2-query/ttl_comment.py +++ b/tests/system-test/2-query/ttl_comment.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): dbname="db" diff --git a/tests/system-test/2-query/varbinary.py b/tests/system-test/2-query/varbinary.py index 8f8021c8ad..c2e2693762 100644 --- a/tests/system-test/2-query/varbinary.py +++ b/tests/system-test/2-query/varbinary.py @@ -10,7 +10,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def test(self): tdLog.info(" test") diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 808a4935e3..cf6aafc374 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 8386c22cd0..e0fbe89c36 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 1e6011d5db..15fd4ed875 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 5043c46f00..c7af55daae 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index 5d0af636b2..e3fe9ae0b6 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqClientConsLog.py b/tests/system-test/7-tmq/tmqClientConsLog.py index d708e6642c..ab0c5f6a72 100644 --- a/tests/system-test/7-tmq/tmqClientConsLog.py +++ b/tests/system-test/7-tmq/tmqClientConsLog.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index f372a2b742..2af61f4bbb 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index c7f95f6f41..d3087c0455 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 26dacf514d..275d80ca18 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index d6f100041b..0572499079 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 11fc7dbcc0..7b31019572 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -26,7 +26,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 8ed4a6df97..a05b6a23ec 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 117c3ce637..85adb50e88 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 2864240441..facbb6c228 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index a717c4966d..ede17130df 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index d8606efe58..fa7aad9dae 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index 05aa82c929..efc186c3bf 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index dcaa6ceb7c..f66605f5f4 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py b/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py index dd2420dcfb..d94f8df35c 100644 --- a/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py +++ b/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqDataPrecisionUnit.py b/tests/system-test/7-tmq/tmqDataPrecisionUnit.py index f050116a1b..13b0fcbe5d 100644 --- a/tests/system-test/7-tmq/tmqDataPrecisionUnit.py +++ b/tests/system-test/7-tmq/tmqDataPrecisionUnit.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.db_name = "tmq_db" self.topic_name = "tmq_topic" diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index aa9c8d25d0..117bd66340 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index 7a47cd6025..0dc2dc35f0 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 0ac8482163..f4a63de53e 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index 2bde32800b..b398178c61 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDropConsumer.py b/tests/system-test/7-tmq/tmqDropConsumer.py index e3e9906ecf..5208d14069 100644 --- a/tests/system-test/7-tmq/tmqDropConsumer.py +++ b/tests/system-test/7-tmq/tmqDropConsumer.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index c8bcdd6235..2c87e4083c 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) # drop some ntbs def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 3fc5a2fdc7..7a35867ce7 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) # drop some ntbs def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index c9e34136cc..eacacd913b 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqMaxGroupIds.py b/tests/system-test/7-tmq/tmqMaxGroupIds.py index 5049ee5bd7..eba15a4bef 100644 --- a/tests/system-test/7-tmq/tmqMaxGroupIds.py +++ b/tests/system-test/7-tmq/tmqMaxGroupIds.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index 05b699ca00..4f25c39627 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def modifyMaxTopics(self, tmqMaxTopicNum): # single dnode diff --git a/tests/system-test/7-tmq/tmqMultiConsumer.py b/tests/system-test/7-tmq/tmqMultiConsumer.py index cc217e0c4c..e9727042e2 100644 --- a/tests/system-test/7-tmq/tmqMultiConsumer.py +++ b/tests/system-test/7-tmq/tmqMultiConsumer.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqOffset.py b/tests/system-test/7-tmq/tmqOffset.py index 500c6f53e4..991d1185a9 100644 --- a/tests/system-test/7-tmq/tmqOffset.py +++ b/tests/system-test/7-tmq/tmqOffset.py @@ -31,7 +31,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqSeekAndCommit.py b/tests/system-test/7-tmq/tmqSeekAndCommit.py index 253edcd10d..65ee438a41 100644 --- a/tests/system-test/7-tmq/tmqSeekAndCommit.py +++ b/tests/system-test/7-tmq/tmqSeekAndCommit.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.db_name = "tmq_db" self.topic_name = "tmq_topic" diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index 920e8e77e4..216f67ffe4 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index 2f1d3e2631..e51ac9f029 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index 6b8c10de27..3988565154 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index 3975013e74..86e49d378a 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index 6a9bb0ae92..e88e16eec0 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqVnodeReplicate.py b/tests/system-test/7-tmq/tmqVnodeReplicate.py index 0ee11781ed..3e40cb13ad 100644 --- a/tests/system-test/7-tmq/tmqVnodeReplicate.py +++ b/tests/system-test/7-tmq/tmqVnodeReplicate.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 1fe2b5809a..74213c4536 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py index f66acf4fcd..b9eac4b692 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py index 68fb07b813..f6b523e5f4 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 6140e8a544..d4c76c4f61 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py index 18b80b7f8d..e79a0ca77d 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py index c203350322..a96d52ee2b 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db.py b/tests/system-test/7-tmq/tmqVnodeTransform-db.py index 5c61908d96..cef90ff65f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-db.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 64cdf6d153..95703c52ca 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index 3698297618..811b72c35f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmq_per.py b/tests/system-test/7-tmq/tmq_per.py index f3701dacab..bd319aa3a4 100644 --- a/tests/system-test/7-tmq/tmq_per.py +++ b/tests/system-test/7-tmq/tmq_per.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/8-stream/snode_restart.py b/tests/system-test/8-stream/snode_restart.py index 3657163ab0..e80ff54578 100644 --- a/tests/system-test/8-stream/snode_restart.py +++ b/tests/system-test/8-stream/snode_restart.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/8-stream/snode_restart_with_checkpoint.py b/tests/system-test/8-stream/snode_restart_with_checkpoint.py index d7bfd7b407..9eb8c09ca3 100644 --- a/tests/system-test/8-stream/snode_restart_with_checkpoint.py +++ b/tests/system-test/8-stream/snode_restart_with_checkpoint.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/8-stream/vnode_restart.py b/tests/system-test/8-stream/vnode_restart.py index a53432b77a..be0ee81787 100644 --- a/tests/system-test/8-stream/vnode_restart.py +++ b/tests/system-test/8-stream/vnode_restart.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 0f83468754..fa28372ae3 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") From f2a1d24ab1a5a6c7b19a1ffdbafef4ebc44846b0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 5 Jan 2024 19:28:40 +0800 Subject: [PATCH 68/93] fix:[TD-28155] set consumer lost if all current topic is 0 --- source/dnode/mnode/impl/src/mndConsumer.c | 24 +++++++++++++--------- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++-- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index a365a44d56..4db000287c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -749,6 +749,18 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { return 0; } +static void updateConsumerStatus(SMqConsumerObj *pConsumer) { + int32_t status = pConsumer->status; + + if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { + if (status == MQ_CONSUMER_STATUS_REBALANCE) { + pConsumer->status = MQ_CONSUMER_STATUS_READY; + } else if (status == MQ_CONSUMER_STATUS_READY && taosArrayGetSize(pConsumer->currentTopics) == 0) { + pConsumer->status = MQ_CONSUMER_STATUS_LOST; + } + } +} + // remove from new topic static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); @@ -869,11 +881,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && - taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && - taosArrayGetSize(pOldConsumer->currentTopics) != 0){ - pOldConsumer->status = MQ_CONSUMER_STATUS_READY; - } + updateConsumerStatus(pOldConsumer); // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = taosGetTimestampMs(); @@ -897,11 +905,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && - taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && - taosArrayGetSize(pOldConsumer->currentTopics) == 0){ - pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; - } + updateConsumerStatus(pOldConsumer); pOldConsumer->rebalanceTime = taosGetTimestampMs(); atomic_add_fetch_32(&pOldConsumer->epoch, 1); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 16ae7f2548..62b671a12f 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -737,8 +737,6 @@ static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { SMqConsumerObj *pConsumer; void *pIter = NULL; - mInfo("start to process mq timer"); - // iterate all consumers, find all modification while (1) { pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); @@ -852,6 +850,8 @@ void mndRebCntDec() { static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { int code = 0; + mInfo("start to process mq timer"); + if (!mndRebTryStart()) { mInfo("mq rebalance already in progress, do nothing"); return code; From 0aef021d885e313c1df2b51ac7acf47c0cbce44b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 6 Jan 2024 21:01:34 +0800 Subject: [PATCH 69/93] fix: backslash add G only valid in string end --- tools/shell/src/shellEngine.c | 21 ++++++++++++++++++++- tools/shell/src/shellWebsocket.c | 3 ++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 60b1dfab41..cec4a9683c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -195,6 +195,25 @@ int32_t shellRunCommand(char *command, bool recordHistory) { return shellRunSingleCommand(cmd); } + +char * strendG(const char* pstr) { + if(pstr == NULL) { + return NULL; + } + + size_t len = strlen(pstr); + if(len < 3) { + return NULL; + } + + char * p = pstr + len - 2 + if (strcmp(p, "\\G") == 0) { + return p; + } + + return NULL; +} + void shellRunSingleCommandImp(char *command) { int64_t st, et; char *sptr = NULL; @@ -213,7 +232,7 @@ void shellRunSingleCommandImp(char *command) { } } - if ((sptr = strstr(command, "\\G")) != NULL) { + if ((sptr = strendG(command)) != NULL) { *sptr = '\0'; printMode = true; // When output to a file, the switch does not work. } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index bff2ef7592..8c3a46aa05 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -238,6 +238,7 @@ static int shellDumpWebsocket(WS_RES *wres, char *fname, return numOfRows; } +char * strendG(const char* pstr); void shellRunSingleCommandWebsocketImp(char *command) { int64_t st, et; char *sptr = NULL; @@ -256,7 +257,7 @@ void shellRunSingleCommandWebsocketImp(char *command) { } } - if ((sptr = strstr(command, "\\G")) != NULL) { + if ((sptr = strendG(command)) != NULL) { *sptr = '\0'; printMode = true; // When output to a file, the switch does not work. } From 96947e24d702d9ae28c473a723d5a249f6286127 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 6 Jan 2024 23:24:36 +0800 Subject: [PATCH 70/93] fix: modify build error --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index cec4a9683c..563481ede7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,7 +206,7 @@ char * strendG(const char* pstr) { return NULL; } - char * p = pstr + len - 2 + char * p = (char *)pstr + len - 2; if (strcmp(p, "\\G") == 0) { return p; } From 772a644de0a4262b6a0b5a9da90b496e2cd8c137 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 6 Jan 2024 23:41:52 +0800 Subject: [PATCH 71/93] fix: end must G; --- tools/shell/src/shellEngine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 563481ede7..d3a2dfdf79 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -202,7 +202,7 @@ char * strendG(const char* pstr) { } size_t len = strlen(pstr); - if(len < 3) { + if(len < 4) { return NULL; } @@ -210,6 +210,11 @@ char * strendG(const char* pstr) { if (strcmp(p, "\\G") == 0) { return p; } + + p = (char *)pstr + len - 3; + if (strcmp(p, "\\G;") == 0 ){ + return p; + } return NULL; } From b38a8102655d8211977b2c753e8e1e0df2133338 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 6 Jan 2024 23:43:55 +0800 Subject: [PATCH 72/93] fix: end must G; fix --- tools/shell/src/shellEngine.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d3a2dfdf79..060e28282e 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,12 +206,7 @@ char * strendG(const char* pstr) { return NULL; } - char * p = (char *)pstr + len - 2; - if (strcmp(p, "\\G") == 0) { - return p; - } - - p = (char *)pstr + len - 3; + char * p = (char *)pstr + len - 3; if (strcmp(p, "\\G;") == 0 ){ return p; } From fbfa1c9abe901f6fefc3d9e180461ff53d11110f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 7 Jan 2024 15:10:57 +0800 Subject: [PATCH 73/93] fix: use \G have no ; end --- tools/shell/src/shellEngine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 563481ede7..02a3084d4f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -202,7 +202,7 @@ char * strendG(const char* pstr) { } size_t len = strlen(pstr); - if(len < 3) { + if(len < 4) { return NULL; } @@ -211,6 +211,11 @@ char * strendG(const char* pstr) { return p; } + p = (char *)pstr + len - 3; + if (strcmp(p, "\\G;") == 0) { + return p; + } + return NULL; } From d3b13bbb84d41c1c72bea877509da431c5c9a1f3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 7 Jan 2024 15:15:56 +0800 Subject: [PATCH 74/93] fix: git add G no ; --- tools/shell/src/shellEngine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9f077c7817..f6f834f3d7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,8 +206,8 @@ char * strendG(const char* pstr) { return NULL; } - char * p = (char *)pstr + len - 3; - if (strcmp(p, "\\G;") == 0 ){ + char * p = (char *)pstr + len - 2; + if (strcmp(p, "\\G") == 0 ){ return p; } From 96460914923c2263accf2c2c774cfd00b52929db Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 7 Jan 2024 16:18:24 +0800 Subject: [PATCH 75/93] fix: sort error due to out of space --- source/libs/executor/src/tsort.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 205cd7d3ef..ee1d831a24 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -878,6 +878,9 @@ static int32_t blockCompareTsFn(const void* pLeft, const void* pRight, void* par static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk, SArray* aPgId) { int32_t pageId = -1; void* pPage = getNewBufPage(pHandle->pBuf, &pageId); + if (pPage == NULL) { + return terrno; + } taosArrayPush(aPgId, &pageId); int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t); @@ -944,16 +947,15 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO totalRows += blk->info.rows; } - SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); - SMultiwayMergeTreeInfo* pTree = NULL; code = tMergeTreeCreate(&pTree, taosArrayGetSize(aBlk), &sup, blockCompareTsFn); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(sup.aRowIdx); taosMemoryFree(sup.aTs); - return code; } + + SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); int32_t nRows = 0; int32_t nMergedRows = 0; bool mergeLimitReached = false; @@ -969,7 +971,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) { SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; - appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pTree); + taosArrayDestroy(aPgId); + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); + return code; + } nMergedRows += pHandle->pDataBlock->info.rows; blockDataCleanup(pHandle->pDataBlock); blkPgSz = pgHeaderSz; @@ -1001,7 +1010,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO if (!mergeLimitReached) { SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; - appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(aPgId); + taosMemoryFree(pTree); + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); + return code; + } nMergedRows += pHandle->pDataBlock->info.rows; if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { mergeLimitReached = true; From 1a02bb4232a36f12bf13f3b88f1066c1673d4a7f Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 7 Jan 2024 18:25:30 +0800 Subject: [PATCH 76/93] doc: fix insert stb syntax --- docs/en/12-taos-sql/05-insert.md | 2 +- docs/zh/12-taos-sql/05-insert.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/05-insert.md b/docs/en/12-taos-sql/05-insert.md index 57e15746cc..462e7fc0ae 100644 --- a/docs/en/12-taos-sql/05-insert.md +++ b/docs/en/12-taos-sql/05-insert.md @@ -157,7 +157,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c Automatically creating table and the table name is specified through the `tbname` column ```sql -INSERT INTO meters(tbname, location, groupId, ts, current, phase) +INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index 1f31698d88..fa330cabae 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -157,7 +157,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c 自动建表, 表名通过tbname列指定 ```sql -INSERT INTO meters(tbname, location, groupId, ts, current, phase) +INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) From c5932c9d1c6226863b0e960ec7174a284044bf87 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 8 Jan 2024 01:13:53 +0800 Subject: [PATCH 77/93] fix: wildcard character in both target and pattern --- source/util/src/tcompare.c | 22 ++++++++++------ tests/system-test/2-query/like.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 2baa21f5d9..53fdc96c37 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1102,10 +1102,13 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t c1 = str[j++]; ++nMatchChar; - if (c == '\\' && pattern[i] == c1 && - (c1 == '_' || c1 == '%')) { - i++; - continue; + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + if (c1 != pattern[i]) { + return TSDB_PATTERN_NOMATCH; + } else { + i++; + continue; + } } if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { @@ -1176,10 +1179,13 @@ int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, c1 = str[j++]; nMatchChar++; - if (c == '\\' && pattern[i] == c1 && - (c1 == '_' || c1 == '%')) { - i++; - continue; + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + if (c1 != pattern[i]) { + return TSDB_PATTERN_NOMATCH; + } else { + i++; + continue; + } } if (c == c1 || towlower(c) == towlower(c1) || (c == pInfo->umatchOne && c1 != 0)) { diff --git a/tests/system-test/2-query/like.py b/tests/system-test/2-query/like.py index 95e2daf6d3..88c1e3b8dd 100644 --- a/tests/system-test/2-query/like.py +++ b/tests/system-test/2-query/like.py @@ -126,6 +126,47 @@ class TDTestCase: tdSql.query("select * from db.t4x where c1 like '\%\_%\%%'") tdSql.checkRows(1) + def like_wildcard_test2(self): + tdSql.execute("create table db.t5x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t5x values(now(), 'a\%c')") + tdSql.execute("insert into db.t5x values(now+1s, 'a\%bbbc')") + tdSql.execute("insert into db.t5x values(now()+2s, 'a%c')") + + tdSql.query("select * from db.t5x where c1 like 'a\%c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a%c") + + tdSql.execute("create table db.t6x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t6x values(now(), '\%c')") + tdSql.execute("insert into db.t6x values(now+1s, '\%bbbc')") + tdSql.execute("insert into db.t6x values(now()+2s, '%c')") + + tdSql.query("select * from db.t6x where c1 like '\%c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "%c") + + tdSql.execute("create table db.t7x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t7x values(now(), 'a\_c')") + tdSql.execute("insert into db.t7x values(now+1s, 'a\_bbbc')") + tdSql.execute("insert into db.t7x values(now()+2s, 'a_c')") + + tdSql.query("select * from db.t7x where c1 like 'a\_c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a_c") + + tdSql.execute("create table db.t8x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t8x values(now(), '\_c')") + tdSql.execute("insert into db.t8x values(now+1s, '\_bbbc')") + tdSql.execute("insert into db.t8x values(now()+2s, '_c')") + + tdSql.query("select * from db.t8x where c1 like '\_c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "_c") + def run(self): tdLog.printNoPrefix("==========start like_wildcard_test run ...............") tdSql.prepare(replica = self.replicaVar) @@ -135,6 +176,7 @@ class TDTestCase: self.like_wildcard_test() self.like_cnc_wildcard_test() self.like_multi_wildcard_test() + self.like_wildcard_test2() tdLog.printNoPrefix("==========end like_wildcard_test run ...............") self.stopTest() From 6b38e5b8fb083b8e784ae9cee778e00e40e732cb Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 8 Jan 2024 10:44:19 +0800 Subject: [PATCH 78/93] feat: websocket support bi with appending &conn_mode=1 string to dsn --- tools/shell/src/shellEngine.c | 4 +--- tools/shell/src/shellWebsocket.c | 11 ++++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index f6f834f3d7..23424cea98 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1318,9 +1318,7 @@ int32_t shellExecute() { if(shell.args.is_bi_mode) { // need set bi mode printf("Set BI mode is true.\n"); -#ifdef WEBSOCKET - //ws_taos_set_conn_mode(shell.ws_conn, TAOS_CONN_MODE_BI, 1); -#else +#ifndef WEBSOCKET taos_set_conn_mode(shell.conn, TAOS_CONN_MODE_BI, 1); #endif } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 8c3a46aa05..2d3aa27a3b 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -30,7 +30,16 @@ int shell_conn_ws_server(bool first) { fprintf(stdout, "trying to connect %s****** ", cuttedDsn); fflush(stdout); for (int i = 0; i < shell.args.timeout; i++) { - shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + if(shell.args.is_bi_mode) { + size_t len = strlen(shell.args.dsn); + char * dsn = taosMemoryMalloc(len + 32); + sprintf(dsn, "%s&conn_mode=1", shell.args.dsn); + shell.ws_conn = ws_connect_with_dsn(dsn); + taosMemoryFree(dsn); + } else { + shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + } + if (NULL == shell.ws_conn) { int errNo = ws_errno(NULL); if (0xE001 == errNo) { From b4f869d051d62d6b68755e80ce8193691ca9b8c1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 8 Jan 2024 11:19:54 +0800 Subject: [PATCH 79/93] case: s3_basic.py except arm64 --- tests/army/enterprise/s3/s3_basic.py | 3 +++ tests/army/frame/eos.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 389b8a52c7..112cb65b15 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -84,6 +84,9 @@ class TDTestCase(TBase): # run def run(self): tdLog.debug(f"start to excute {__file__}") + if eos.isArm64Cpu(): + tdLog.success(f"{__file__} arm64 ignore executed") + return # insert data self.insertData() diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index b24be74393..1c00f94380 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -22,10 +22,31 @@ import datetime import platform import subprocess +# +# platform +# + # if windows platform return True def isWin(): return platform.system().lower() == 'windows' +def isArm64Cpu(): + system = platform.system() + + if system == 'Linux': + machine = platform.machine().lower() + + # Check for ARM64 architecture on Linux systems + return machine in ['aarch64', 'armv8l'] + elif system == 'Darwin' or system == 'Windows': + processor = platform.processor().lower() + + # Check for ARM64 architecture on macOS and Windows systems + return processor in ['arm64', 'aarch64'] + else: + print("Unsupported operating system") + return False + # # execute programe # From 8cdd28131f642a5d087b04e6d3817db8c4377a0e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 12:29:09 +0800 Subject: [PATCH 80/93] test: save case.sql when case is failed and set default saved sqlfile True --- tests/parallel_test/run.sh | 330 +++++++++++++++++++----------------- tests/system-test/pytest.sh | 12 +- 2 files changed, 176 insertions(+), 166 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index cf21349513..ef9413a9d5 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -15,61 +15,61 @@ function usage() { ent=0 while getopts "m:t:b:l:o:w:eh" opt; do case $opt in - m) - config_file=$OPTARG - ;; - t) - t_file=$OPTARG - ;; - b) - branch=$OPTARG - ;; - l) - log_dir=$OPTARG - ;; - e) - ent=1 - ;; - o) - timeout_param="-o $OPTARG" - ;; - w) - web_server=$OPTARG - ;; - h) - usage - exit 0 - ;; - \?) - echo "Invalid option: -$OPTARG" - usage - exit 0 - ;; + m) + config_file=$OPTARG + ;; + t) + t_file=$OPTARG + ;; + b) + branch=$OPTARG + ;; + l) + log_dir=$OPTARG + ;; + e) + ent=1 + ;; + o) + timeout_param="-o $OPTARG" + ;; + w) + web_server=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; esac done #config_file=$1 -if [ -z $config_file ]; then +if [ -z "$config_file" ]; then usage exit 1 fi -if [ ! -f $config_file ]; then +if [ ! -f "$config_file" ]; then echo "$config_file not found" usage exit 1 fi #t_file=$2 -if [ -z $t_file ]; then +if [ -z "$t_file" ]; then usage exit 1 fi -if [ ! -f $t_file ]; then +if [ ! -f "$t_file" ]; then echo "$t_file not found" usage exit 1 fi -date_tag=`date +%Y%m%d-%H%M%S` +date_tag=$(date +%Y%m%d-%H%M%S) test_log_dir=${branch}_${date_tag} -if [ -z $log_dir ]; then +if [ -z "$log_dir" ]; then log_dir="log/${test_log_dir}" else log_dir="$log_dir/${test_log_dir}" @@ -82,42 +82,41 @@ workdirs=() threads=() i=0 -while [ 1 ]; do - host=`jq .[$i].host $config_file` +while true; do + host=$(jq .[$i].host "$config_file") if [ "$host" = "null" ]; then break fi - username=`jq .[$i].username $config_file` + username=$(jq .[$i].username "$config_file") if [ "$username" = "null" ]; then break fi - password=`jq .[$i].password $config_file` + password=$(jq .[$i].password "$config_file") if [ "$password" = "null" ]; then password="" fi - workdir=`jq .[$i].workdir $config_file` + workdir=$(jq .[$i].workdir "$config_file") if [ "$workdir" = "null" ]; then break fi - thread=`jq .[$i].thread $config_file` + thread=$(jq .[$i].thread "$config_file") if [ "$thread" = "null" ]; then break fi - hosts[i]=`echo $host|sed 's/\"$//'|sed 's/^\"//'` - usernames[i]=`echo $username|sed 's/\"$//'|sed 's/^\"//'` - passwords[i]=`echo $password|sed 's/\"$//'|sed 's/^\"//'` - workdirs[i]=`echo $workdir|sed 's/\"$//'|sed 's/^\"//'` + hosts[i]=$(echo "$host" | sed 's/\"$//' | sed 's/^\"//') + usernames[i]=$(echo "$username" | sed 's/\"$//' | sed 's/^\"//') + passwords[i]=$(echo "$password" | sed 's/\"$//' | sed 's/^\"//') + workdirs[i]=$(echo "$workdir" | sed 's/\"$//' | sed 's/^\"//') threads[i]=$thread - i=$(( i + 1 )) + i=$((i + 1)) done - function prepare_cases() { - cat $t_file >>$task_file + cat "$t_file" >>"$task_file" local i=0 - while [ $i -lt $1 ]; do - echo "%%FINISHED%%" >>$task_file - i=$(( i + 1 )) + while [ $i -lt "$1" ]; do + echo "%%FINISHED%%" >>"$task_file" + i=$((i + 1)) done } @@ -125,7 +124,7 @@ function clean_tmp() { # clean tmp dir local index=$1 local ssh_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then ssh_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi local cmd="${ssh_script} rm -rf ${workdirs[index]}/tmp" @@ -136,7 +135,7 @@ function run_thread() { local index=$1 local thread_no=$2 local runcase_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then runcase_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi local count=0 @@ -147,8 +146,9 @@ function run_thread() { local cmd="${runcase_script} ${script}" # script="echo" - while [ 1 ]; do - local line=`flock -x $lock_file -c "head -n1 $task_file;sed -i \"1d\" $task_file"` + while true; do + local line + line=$(flock -x "$lock_file" -c "head -n1 $task_file;sed -i \"1d\" $task_file") if [ "x$line" = "x%%FINISHED%%" ]; then # echo "$index . $thread_no EXIT" break @@ -156,15 +156,17 @@ function run_thread() { if [ -z "$line" ]; then continue fi - echo "$line"|grep -q "^#" - if [ $? -eq 0 ]; then + + if echo "$line" | grep -q "^#"; then continue fi - local case_redo_time=`echo "$line"|cut -d, -f2` + local case_redo_time + case_redo_time=$(echo "$line" | cut -d, -f2) if [ -z "$case_redo_time" ]; then case_redo_time=${DEFAULT_RETRY_TIME:-2} fi - local case_build_san=`echo "$line"|cut -d, -f3` + local case_build_san + case_build_san=$(echo "$line" | cut -d, -f3) if [ "${case_build_san}" == "y" ]; then case_build_san="y" DEBUGPATH="debugSan" @@ -175,140 +177,150 @@ function run_thread() { usage exit 1 fi - local exec_dir=`echo "$line"|cut -d, -f4` - local case_cmd=`echo "$line"|cut -d, -f5` + local exec_dir + exec_dir=$(echo "$line" | cut -d, -f4) + local case_cmd + case_cmd=$(echo "$line" | cut -d, -f5) local case_file="" - echo "$case_cmd"|grep -q "\.sh" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.sh"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "\.sh"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.sh" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "^python3" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "^python3"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.py" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "^./pytest.sh" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "^./pytest.sh"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.py" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "\.sim" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.sim"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "\.sim"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.sim" | awk '{print $NF}') fi if [ -z "$case_file" ]; then - case_file=`echo "$case_cmd"|awk '{print $NF}'` + case_file=$(echo "$case_cmd" | awk '{print $NF}') fi if [ -z "$case_file" ]; then continue fi case_sql_file="$exec_dir/${case_file}.sql" case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}" - count=$(( count + 1 )) - local case_path=`dirname "$case_file"` - if [ ! -z "$case_path" ]; then - mkdir -p $log_dir/$case_path + count=$((count + 1)) + local case_path + case_path=$(dirname "$case_file") + if [ -n "$case_path" ]; then + mkdir -p "$log_dir"/"$case_path" fi cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} -s ${case_build_san} ${timeout_param}" # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 local case_log_file=$log_dir/${case_file}.txt - start_time=`date +%s` - local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` - case_index=`printf "%5d" $case_index` - local case_info=`echo "$line"|cut -d, -f 3,4,5` + start_time=$(date +%s) + local case_index + case_index=$(flock -x "$lock_file" -c "sh -c \"echo \$(( \$( cat $index_file ) + 1 )) | tee $index_file\"") + case_index=$(printf "%5d" "$case_index") + local case_info + case_info=$(echo "$line" | cut -d, -f 3,4,5) while [ ${redo_count} -lt 6 ]; do - if [ -f $case_log_file ]; then - cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt + if [ -f "$case_log_file" ]; then + cp "$case_log_file" "$log_dir"/"$case_file".${redo_count}.redotxt fi - echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$case_log_file - local current_time=`date "+%Y-%m-%d %H:%M:%S"` + echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >"$case_log_file" + local current_time + current_time=$(date "+%Y-%m-%d %H:%M:%S") echo -e "$case_index \e[33m START >>>>> \e[0m ${case_info} \e[33m[$current_time]\e[0m" - echo "$current_time" >>$case_log_file - local real_start_time=`date +%s` + echo "$current_time" >>"$case_log_file" + local real_start_time + real_start_time=$(date +%s) # $cmd 2>&1 | tee -a $case_log_file # ret=${PIPESTATUS[0]} - $cmd >>$case_log_file 2>&1 + $cmd >>"$case_log_file" 2>&1 ret=$? - local real_end_time=`date +%s` - local time_elapsed=$(( real_end_time - real_start_time )) - echo "execute time: ${time_elapsed}s" >>$case_log_file - current_time=`date "+%Y-%m-%d %H:%M:%S"` - echo "${hosts[index]} $current_time exit code:${ret}" >>$case_log_file + local real_end_time + real_end_time=$(date +%s) + local time_elapsed + time_elapsed=$((real_end_time - real_start_time)) + echo "execute time: ${time_elapsed}s" >>"$case_log_file" + current_time=$(date "+%Y-%m-%d %H:%M:%S") + echo "${hosts[index]} $current_time exit code:${ret}" >>"$case_log_file" if [ $ret -eq 0 ]; then break fi redo=0 - grep -q "wait too long for taosd start" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "wait too long for taosd start" "$case_log_file"; then redo=1 fi - grep -q "kex_exchange_identification: Connection closed by remote host" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "kex_exchange_identification: Connection closed by remote host" "$case_log_file"; then redo=1 fi - grep -q "ssh_exchange_identification: Connection closed by remote host" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "ssh_exchange_identification: Connection closed by remote host" "$case_log_file"; then redo=1 fi - grep -q "kex_exchange_identification: read: Connection reset by peer" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "kex_exchange_identification: read: Connection reset by peer" "$case_log_file"; then redo=1 fi - grep -q "Database not ready" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "Database not ready" "$case_log_file"; then redo=1 fi - grep -q "Unable to establish connection" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "Unable to establish connection" "$case_log_file"; then redo=1 fi - if [ $redo_count -lt $case_redo_time ]; then + if [ $redo_count -lt "$case_redo_time" ]; then redo=1 fi if [ $redo -eq 0 ]; then break fi - redo_count=$(( redo_count + 1 )) + redo_count=$((redo_count + 1)) done - end_time=`date +%s` - echo >>$case_log_file - total_time=$(( end_time - start_time )) - echo "${hosts[index]} total time: ${total_time}s" >>$case_log_file + end_time=$(date +%s) + echo >>"$case_log_file" + total_time=$((end_time - start_time)) + echo "${hosts[index]} total time: ${total_time}s" >>"$case_log_file" # echo "$thread_no ${line} DONE" if [ $ret -eq 0 ]; then echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[32m success\e[0m" - flock -x $lock_file -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" + flock -x "$lock_file" -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" else - if [ ! -z ${web_server} ]; then - flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" + if [ -n "${web_server}" ]; then + flock -x "$lock_file" -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" else - flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" + flock -x "$lock_file" -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" fi - mkdir -p $log_dir/${case_file}.coredump + mkdir -p "${log_dir}"/"${case_file}".coredump local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump" local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" $cmd # 2>/dev/null - local corefile=`ls $log_dir/${case_file}.coredump/` + local corefile + corefile=$(ls "$log_dir/${case_file}.coredump/") echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[31m failed\e[0m" echo "=========================log============================" - cat $case_log_file + cat "$case_log_file" echo "=====================================================" echo -e "\e[34m log file: $case_log_file \e[0m" - if [ ! -z "${web_server}" ]; then + if [ -n "${web_server}" ]; then echo "${web_server}/$test_log_dir/${case_file}.txt" fi - if [ ! -z "$corefile" ]; then + if [ -n "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" # if [ $ent -ne 0 ]; then # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" # fi - mkdir $build_dir 2>/dev/null + mkdir "$build_dir" 2>/dev/null if [ $? -eq 0 ]; then # scp build binary cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" @@ -319,15 +331,15 @@ function run_thread() { # get remote sim dir local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no" local tarcmd="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then tarcmd="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi cmd="$tarcmd sh -c \"cd $remote_sim_dir; tar -czf sim.tar.gz sim\"" $cmd local remote_sim_tar="${workdirs[index]}/tmp/thread_volume/$thread_no/sim.tar.gz" - local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file)" + local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file}" scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" @@ -341,11 +353,9 @@ function run_thread() { source_tar_dir=$log_dir/TDinternal_${hosts[index]} source_tar_file=TDinternal.tar.gz fi - mkdir $source_tar_dir 2>/dev/null + mkdir "$source_tar_dir" 2>/dev/null if [ $? -eq 0 ]; then cmd="$scpcmd:${workdirs[index]}/$source_tar_file $source_tar_dir" - # echo "$cmd" - # $cmd fi fi done @@ -361,12 +371,12 @@ function run_thread() { i=0 while [ $i -lt ${#hosts[*]} ]; do clean_tmp $i & - i=$(( i + 1 )) + i=$((i + 1)) done wait -mkdir -p $log_dir -rm -rf $log_dir/* +mkdir -p "$log_dir" +rm -rf "${log_dir:?}"/* task_file=$log_dir/$$.task lock_file=$log_dir/$$.lock index_file=$log_dir/case_index.txt @@ -374,71 +384,71 @@ stat_file=$log_dir/stat.txt failed_case_file=$log_dir/failed.txt success_case_file=$log_dir/success.txt -echo "0" >$index_file +echo "0" >"$index_file" i=0 j=0 while [ $i -lt ${#hosts[*]} ]; do - j=$(( j + threads[i] )) - i=$(( i + 1 )) + j=$((j + threads[i])) + i=$((i + 1)) done prepare_cases $j i=0 while [ $i -lt ${#hosts[*]} ]; do j=0 - while [ $j -lt ${threads[i]} ]; do + while [ $j -lt "${threads[i]}" ]; do run_thread $i $j & - j=$(( j + 1 )) + j=$((j + 1)) done - i=$(( i + 1 )) + i=$((i + 1)) done wait -rm -f $lock_file -rm -f $task_file +rm -f "$lock_file" +rm -f "$task_file" # docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f echo "=====================================================================" echo "log dir: $log_dir" -total_cases=`cat $index_file` +total_cases=$(cat "$index_file") failed_cases=0 -if [ -f $failed_case_file ]; then - if [ ! -z "$web_server" ]; then - failed_cases=`grep -v "$web_server" $failed_case_file|wc -l` +if [ -f "$failed_case_file" ]; then + if [ -n "$web_server" ]; then + failed_cases=$(grep -c -v "$web_server" "$failed_case_file") else - failed_cases=`grep -v "log file:" $failed_case_file|wc -l` + failed_cases=$(grep -c -v "log file:" "$failed_case_file") fi fi -success_cases=$(( total_cases - failed_cases )) -echo "Total Cases: $total_cases" >$stat_file -echo "Successful: $success_cases" >>$stat_file -echo "Failed: $failed_cases" >>$stat_file -cat $stat_file +success_cases=$((total_cases - failed_cases)) +echo "Total Cases: $total_cases" >"$stat_file" +echo "Successful: $success_cases" >>"$stat_file" +echo "Failed: $failed_cases" >>"$stat_file" +cat "$stat_file" RET=0 i=1 if [ -f "${failed_case_file}" ]; then echo "=====================================================" - while read line; do - if [ ! -z "${web_server}" ]; then - echo "$line"|grep -q "${web_server}" - if [ $? -eq 0 ]; then + while read -r line; do + if [ -n "${web_server}" ]; then + + if echo "$line" | grep -q "${web_server}"; then echo " $line" continue fi else - echo "$line"|grep -q "log file:" - if [ $? -eq 0 ]; then + + if echo "$line" | grep -q "log file:"; then echo " $line" continue fi fi - line=`echo "$line"|cut -d, -f 3,4,5` + line=$(echo "$line" | cut -d, -f 3,4,5) echo -e "$i. $line \e[31m failed\e[0m" >&2 - i=$(( i + 1 )) - done <${failed_case_file} + i=$((i + 1)) + done <"${failed_case_file}" RET=1 fi diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index bae0fdf278..2837c817be 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -60,16 +60,16 @@ echo "ASAN_DIR : $ASAN_DIR" # prevent delete / folder or /usr/bin if [ ${#SIM_DIR} -lt 10 ]; then - echo "len(SIM_DIR) < 10 , danger so exit. SIM_DIR=$SIM_DIR" - exit 1 + echo "len(SIM_DIR) < 10 , danger so exit. SIM_DIR=$SIM_DIR" + exit 1 fi -rm -rf $SIM_DIR/* +rm -rf "${SIM_DIR:?}"/* mkdir -p $PRG_DIR mkdir -p $ASAN_DIR -cd $CODE_DIR +cd "$CODE_DIR" || exit ulimit -n 600000 ulimit -c unlimited @@ -78,10 +78,10 @@ ulimit -c unlimited echo "ExcuteCmd:" $* if [[ "$TD_OS" == "Alpine" ]]; then - $* + "$@" else AsanFile=$ASAN_DIR/psim.info - echo "AsanFile:" $AsanFile + echo "AsanFile:" "$AsanFile" unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5 From 826c098aee032c9388bd63716c4188b11536775f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 14:46:04 +0800 Subject: [PATCH 81/93] test: test --- tests/system-test/2-query/last_cache_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 9b6dca56fd..8e8ac837e7 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -394,7 +394,7 @@ class TDTestCase: tdSql.query('select last(c2), c2, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) tdSql.checkRows(1) - tdSql.checkCols(3) + tdSql.checkCols(5) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) From 031bcaec8ff80f84d5ae47b9ef6e10f8d38ffac9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 14:47:25 +0800 Subject: [PATCH 82/93] test: test failed case --- tests/system-test/2-query/last_cache_scan.py | 2 +- tests/system-test/8-stream/stream_basic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 8e8ac837e7..9b6dca56fd 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -394,7 +394,7 @@ class TDTestCase: tdSql.query('select last(c2), c2, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) tdSql.checkRows(1) - tdSql.checkCols(5) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index 3ebc255114..04e635ea24 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -101,7 +101,7 @@ class TDTestCase: # check ts interval is correct sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;" tdSql.query(sql) - tdSql.checkRows(0) + tdSql.checkRows(1) # stop def stop(self): From 1eee7b44e08ce34693dc1d8783047bf98d2536f7 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 07:31:35 +0000 Subject: [PATCH 83/93] fix crash found on 3.1 --- include/libs/transport/trpc.h | 4 +- include/util/taoserror.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 9 ++- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 62 +++++++++--------- source/dnode/mnode/impl/src/mndProfile.c | 43 +++++++------ source/libs/executor/src/exchangeoperator.c | 70 +++++++++++---------- source/libs/executor/src/sysscanoperator.c | 15 +++-- source/libs/transport/inc/transComm.h | 5 +- source/libs/transport/src/trans.c | 12 +++- source/libs/transport/src/transCli.c | 12 +++- source/util/src/terror.c | 1 + 11 files changed, 140 insertions(+), 94 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index accb7e6f24..e7b0b25653 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -163,7 +163,8 @@ int rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc in // These functions will not be called in the child process int rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); int rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -int rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int32_t timeoutMs); +int rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated, + int32_t timeoutMs); int rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); void *rpcAllocHandle(); void rpcSetIpWhite(void *thandl, void *arg); @@ -171,6 +172,7 @@ void rpcSetIpWhite(void *thandl, void *arg); int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); +int32_t rpcCvtErrCode(int32_t code); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c4d61157b6..b41078dfbf 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -68,6 +68,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED TAOS_DEF_ERROR_CODE(0, 0x0020) // "Vgroup could not be connected" #define TSDB_CODE_RPC_SOMENODE_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0021) // #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) // +#define TSDB_CODE_RPC_NETWORK_ERROR TAOS_DEF_ERROR_CODE(0, 0x0023) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index ac94390619..a95ec42bd0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -159,13 +159,18 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq); SEpSet epSet = {0}; + int8_t epUpdated = 0; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, 5000); + rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, 5000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); - char tbuf[256]; + char tbuf[512]; dmEpSetToStr(tbuf, sizeof(tbuf), &epSet); dError("failed to send status req since %s, epSet:%s, inUse:%d", tstrerror(rpcRsp.code), tbuf, epSet.inUse); + } else { + if (epUpdated == 1) { + dmSetMnodeEpSet(pMgmt->pData, &epSet); + } } dmProcessStatusRsp(pMgmt, &rpcRsp); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index af43804db4..b79b3f9e11 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -22,10 +22,8 @@ static void *dmStatusThreadFp(void *param) { int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-status"); - const static int16_t TRIM_FREQ = 30; - int32_t trimCount = 0; - int32_t upTimeCount = 0; - int64_t upTime = 0; + int32_t upTimeCount = 0; + int64_t upTime = 0; while (1) { taosMsleep(200); @@ -38,11 +36,6 @@ static void *dmStatusThreadFp(void *param) { dmSendStatusReq(pMgmt); lastTime = curTime; - trimCount = (trimCount + 1) % TRIM_FREQ; - if (trimCount == 0) { - taosMemoryTrim(0); - } - if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { upTime = taosGetOsUptime() - tsDndStartOsUptime; tsDndUpTime = TMAX(tsDndUpTime, upTime); @@ -55,27 +48,27 @@ static void *dmStatusThreadFp(void *param) { SDmNotifyHandle dmNotifyHdl = {.state = 0}; static void *dmNotifyThreadFp(void *param) { - SDnodeMgmt *pMgmt = param; - setThreadName("dnode-notify"); + SDnodeMgmt *pMgmt = param; + setThreadName("dnode-notify"); - if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { - return NULL; + if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { + return NULL; } - bool wait = true; - while (1) { - if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; + bool wait = true; + while (1) { + if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; if (wait) tsem_wait(&dmNotifyHdl.sem); atomic_store_8(&dmNotifyHdl.state, 1); - dmSendNotifyReq(pMgmt); - if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { - wait = true; - continue; + dmSendNotifyReq(pMgmt); + if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { + wait = true; + continue; } - wait = false; + wait = false; } - return NULL; + return NULL; } static void *dmMonitorThreadFp(void *param) { @@ -83,6 +76,9 @@ static void *dmMonitorThreadFp(void *param) { int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-monitor"); + static int32_t TRIM_FREQ = 20; + int32_t trimCount = 0; + while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; @@ -93,6 +89,11 @@ static void *dmMonitorThreadFp(void *param) { if (interval >= tsMonitorInterval) { (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; + + trimCount = (trimCount + 1) % TRIM_FREQ; + if (trimCount == 0) { + taosMemoryTrim(0); + } } } @@ -126,14 +127,15 @@ static void *dmCrashReportThreadFp(void *param) { setThreadName("dnode-crashReport"); char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s%s.taosdCrashLog", tsLogDir, TD_DIRSEP); - char *pMsg = NULL; - int64_t msgLen = 0; + char *pMsg = NULL; + int64_t msgLen = 0; TdFilePtr pFile = NULL; - bool truncateFile = false; - int32_t sleepTime = 200; - int32_t reportPeriodNum = 3600 * 1000 / sleepTime;; + bool truncateFile = false; + int32_t sleepTime = 200; + int32_t reportPeriodNum = 3600 * 1000 / sleepTime; + ; int32_t loopTimes = reportPeriodNum; - + while (1) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; if (loopTimes++ < reportPeriodNum) { @@ -167,13 +169,13 @@ static void *dmCrashReportThreadFp(void *param) { pMsg = NULL; continue; } - + if (pFile) { taosReleaseCrashLogFile(pFile, truncateFile); pFile = NULL; truncateFile = false; } - + taosMsleep(sleepTime); loopTimes = 0; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 34ee18a5cc..e0559b4c48 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" +#include "audit.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -26,7 +27,6 @@ #include "mndView.h" #include "tglobal.h" #include "tversion.h" -#include "audit.h" typedef struct { uint32_t id; @@ -530,23 +530,24 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } #ifdef TD_ENTERPRISE - bool needCheck = true; - int32_t key = HEARTBEAT_KEY_DYN_VIEW; - SDynViewVersion* pDynViewVer = NULL; - SKv* pKv = taosHashGet(pHbReq->info, &key, sizeof(key)); + bool needCheck = true; + int32_t key = HEARTBEAT_KEY_DYN_VIEW; + SDynViewVersion *pDynViewVer = NULL; + SKv *pKv = taosHashGet(pHbReq->info, &key, sizeof(key)); if (NULL != pKv) { pDynViewVer = pKv->value; mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer); - SDynViewVersion* pRspVer = NULL; + SDynViewVersion *pRspVer = NULL; if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) { return -1; } - + if (needCheck) { SKv kv1 = {.key = HEARTBEAT_KEY_DYN_VIEW, .valueLen = sizeof(*pDynViewVer), .value = pRspVer}; taosArrayPush(hbRsp.info, &kv1); - mTrace("need to check view ver, lastest bootTs:%" PRId64 ", ver:%" PRIu64, pRspVer->svrBootTs, pRspVer->dynViewVer); + mTrace("need to check view ver, lastest bootTs:%" PRId64 ", ver:%" PRIu64, pRspVer->svrBootTs, + pRspVer->dynViewVer); } } #endif @@ -594,7 +595,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb if (!needCheck) { break; } - + void *rspMsg = NULL; int32_t rspLen = 0; mndValidateViewInfo(pMnode, kv->value, kv->valueLen / sizeof(SViewVersion), &rspMsg, &rspLen); @@ -814,8 +815,9 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl * @param offset skip [offset] queries in pConn * @param rowsToPack at most rows to pack * @return rows packed -*/ -static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBlock* pBlock, uint32_t offset, uint32_t rowsToPack) { + */ +static int32_t packQueriesIntoBlock(SShowObj *pShow, SConnObj *pConn, SSDataBlock *pBlock, uint32_t offset, + uint32_t rowsToPack) { int32_t cols = 0; taosRLockLatch(&pConn->queryLock); int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); @@ -826,7 +828,7 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc int32_t i = offset; for (; i < numOfQueries && (i - offset) < rowsToPack; ++i) { - int32_t curRowIndex = pBlock->info.rows; + int32_t curRowIndex = pBlock->info.rows; SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i); cols = 0; @@ -877,14 +879,19 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->subPlanNum, false); char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; + int64_t reserve = 64; int32_t strSize = sizeof(subStatus); int32_t offset = VARSTR_HEADER_SIZE; - for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) { + for (int32_t i = 0; i < pQuery->subPlanNum && offset + reserve < strSize; ++i) { if (i) { - offset += snprintf(subStatus + offset, strSize - offset - 1, ","); + offset += sprintf(subStatus + offset, ","); + } + if (offset + reserve < strSize) { + SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); + offset += sprintf(subStatus + offset, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); + } else { + break; } - SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); - offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); } varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -903,8 +910,8 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc } static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode * pMnode = pReq->info.node; - SSdb * pSdb = pMnode->pSdb; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SConnObj *pConn = NULL; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index ef8bf9674a..2797cb2d82 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -16,16 +16,14 @@ #include "executorInt.h" #include "filter.h" #include "function.h" -#include "index.h" #include "operator.h" -#include "os.h" #include "query.h" #include "querytask.h" #include "tdatablock.h" #include "thash.h" #include "tmsg.h" -#include "tname.h" #include "tref.h" +#include "trpc.h" typedef struct SFetchRspHandleWrapper { uint32_t exchangeId; @@ -131,14 +129,14 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, try next %d/%" PRIzu, + " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, totalSources); } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb", + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%" PRId64 + ", totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } @@ -232,7 +230,7 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { if (blockDataGetNumOfRows(pBlock) == 0) { continue; } - + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); @@ -261,7 +259,7 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const if (pInfo->dynamicOp) { return TSDB_CODE_SUCCESS; } - + for (int32_t i = 0; i < numOfSources; ++i) { SSourceDataInfo dataInfo = {0}; dataInfo.status = EX_SOURCE_DATA_NOT_READY; @@ -343,8 +341,8 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode goto _error; } - pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); return pOperator; _error: @@ -383,7 +381,7 @@ void doDestroyExchangeOperatorInfo(void* param) { blockDataDestroy(pExInfo->pDummyBlock); tSimpleHashCleanup(pExInfo->pHashSources); - + tsem_destroy(&pExInfo->ready); taosMemoryFreeClear(param); } @@ -411,13 +409,18 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, + pRsp->numOfBlocks, pRsp->numOfRows, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); - pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), - pExchangeInfo); + pSourceDataInfo->code = rpcCvtErrCode(code); + if (pSourceDataInfo->code != code) { + qError("%s fetch rsp received, index:%d, error:%s, cvted error: %s, %p", pSourceDataInfo->taskId, index, + tstrerror(code), tstrerror(pSourceDataInfo->code), pExchangeInfo); + } else { + qError("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), + pExchangeInfo); + } } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -450,7 +453,7 @@ int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, in return TSDB_CODE_OUT_OF_MEMORY; } pScan->tableSeq = tableSeq; - + (*ppRes)->opType = srcOpType; (*ppRes)->downstreamIdx = 0; (*ppRes)->value = pScan; @@ -459,9 +462,8 @@ int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, in return TSDB_CODE_SUCCESS; } - int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); if (EX_SOURCE_DATA_NOT_READY != pDataInfo->status) { return TSDB_CODE_SUCCESS; } @@ -490,7 +492,8 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas req.queryId = pTaskInfo->id.queryId; req.execId = pSource->execId; if (pDataInfo->pSrcUidList) { - int32_t code = buildTableScanOperatorParam(&req.pOpParam, pDataInfo->pSrcUidList, pDataInfo->srcOpType, pDataInfo->tableSeq); + int32_t code = + buildTableScanOperatorParam(&req.pOpParam, pDataInfo->pSrcUidList, pDataInfo->srcOpType, pDataInfo->tableSeq); taosArrayDestroy(pDataInfo->pSrcUidList); pDataInfo->pSrcUidList = NULL; if (TSDB_CODE_SUCCESS != code) { @@ -499,7 +502,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas return pTaskInfo->code; } } - + int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; @@ -741,8 +744,8 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 + ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); @@ -769,7 +772,7 @@ _error: } int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasicParam* pBasicParam) { - SExchangeInfo* pExchangeInfo = pOperator->info; + SExchangeInfo* pExchangeInfo = pOperator->info; SExchangeSrcIndex* pIdx = tSimpleHashGet(pExchangeInfo->pHashSources, &pBasicParam->vgId, sizeof(pBasicParam->vgId)); if (NULL == pIdx) { qError("No exchange source for vgId: %d", pBasicParam->vgId); @@ -784,7 +787,7 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic dataInfo.pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL); dataInfo.srcOpType = pBasicParam->srcOpType; dataInfo.tableSeq = pBasicParam->tableSeq; - + taosArrayPush(pExchangeInfo->pSourceDataInfo, &dataInfo); pIdx->inUseIdx = taosArrayGetSize(pExchangeInfo->pSourceDataInfo) - 1; } else { @@ -800,15 +803,14 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic return TSDB_CODE_SUCCESS; } - int32_t addDynamicExchangeSource(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - int32_t code = TSDB_CODE_SUCCESS; + SExchangeInfo* pExchangeInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; SExchangeOperatorBasicParam* pBasicParam = NULL; - SExchangeOperatorParam* pParam = (SExchangeOperatorParam*)pOperator->pOperatorGetParam->value; + SExchangeOperatorParam* pParam = (SExchangeOperatorParam*)pOperator->pOperatorGetParam->value; if (pParam->multiParams) { SExchangeOperatorBatchParam* pBatch = (SExchangeOperatorBatchParam*)pOperator->pOperatorGetParam->value; - int32_t iter = 0; + int32_t iter = 0; while (NULL != (pBasicParam = tSimpleHashIterate(pBatch->pBatchs, pBasicParam, &iter))) { code = addSingleExchangeSource(pOperator, pBasicParam); if (code) { @@ -826,11 +828,11 @@ int32_t addDynamicExchangeSource(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } - int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; - int32_t code = TSDB_CODE_SUCCESS; - if ((OPTR_IS_OPENED(pOperator) && !pExchangeInfo->dynamicOp) || (pExchangeInfo->dynamicOp && NULL == pOperator->pOperatorGetParam)) { + int32_t code = TSDB_CODE_SUCCESS; + if ((OPTR_IS_OPENED(pOperator) && !pExchangeInfo->dynamicOp) || + (pExchangeInfo->dynamicOp && NULL == pOperator->pOperatorGetParam)) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 76de204cab..0c421bf354 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -15,9 +15,7 @@ #include "executorInt.h" #include "filter.h" -#include "function.h" #include "functionMgt.h" -#include "os.h" #include "querynodes.h" #include "systable.h" #include "tname.h" @@ -32,6 +30,7 @@ #include "storageapi.h" #include "tcompare.h" #include "thash.h" +#include "trpc.h" #include "ttypes.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); @@ -1789,8 +1788,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); return pOperator; _error: @@ -1867,7 +1866,13 @@ int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->handle = htobe64(pRsp->handle); pRsp->compLen = htonl(pRsp->compLen); } else { - operator->pTaskInfo->code = code; + operator->pTaskInfo->code = rpcCvtErrCode(code); + if (operator->pTaskInfo->code != code) { + qError("load systable rsp received, error:%s, cvted error:%s", tstrerror(code), + tstrerror(operator->pTaskInfo->code)); + } else { + qError("load systable rsp received, error:%s", tstrerror(code)); + } } tsem_post(&pScanResInfo->ready); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 19b2dfc300..7483588593 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -122,6 +122,8 @@ typedef struct SExHandle { typedef struct { STransMsg* pRsp; + SEpSet epSet; + int8_t hasEpSet; tsem_t* pSem; int8_t inited; SRWLatch latch; @@ -317,7 +319,8 @@ int transReleaseSrvHandle(void* handle); int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); -int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int32_t timeoutMs); +int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs); int transSendResponse(const STransMsg* msg); int transRegisterMsg(const STransMsg* msg); int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index b23d229931..64302a78fc 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -169,8 +169,9 @@ int rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, in int rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { return transSendRecv(shandle, pEpSet, pMsg, pRsp); } -int rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int32_t timeoutMs) { - return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, timeoutMs); +int rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); } int rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } @@ -197,6 +198,13 @@ int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { return transUtilSWhiteListToStr(pWhiteList, ppBuf); } +int32_t rpcCvtErrCode(int32_t code) { + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return TSDB_CODE_RPC_NETWORK_ERROR; + } + return code; +} + int32_t rpcInit() { transInit(); return 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 936d11c151..4b3000b47e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2425,6 +2425,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STransSyncMsg* pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); if (pSyncMsg != NULL) { memcpy(pSyncMsg->pRsp, (char*)pResp, sizeof(*pResp)); + if (cliIsEpsetUpdated(pResp->code, pCtx)) { + pSyncMsg->hasEpSet = 1; + epsetAssign(&pSyncMsg->epSet, &pCtx->epSet); + } tsem_post(pSyncMsg->pSem); taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); } else { @@ -2640,10 +2644,12 @@ int64_t transCreateSyncMsg(STransMsg* pTransMsg) { pSyncMsg->inited = 0; pSyncMsg->pRsp = pTransMsg; pSyncMsg->pSem = sem; + pSyncMsg->hasEpSet = 0; return taosAddRef(transGetSyncMsgMgt(), pSyncMsg); } -int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int32_t timeoutMs) { +int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); STransMsg* pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransInst == NULL) { @@ -2695,6 +2701,10 @@ int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pRe ret = TSDB_CODE_TIMEOUT_ERROR; } else { memcpy(pRsp, pSyncMsg->pRsp, sizeof(STransMsg)); + if (pSyncMsg->hasEpSet == 1) { + epsetAssign(pEpSet, &pSyncMsg->epSet); + *epUpdated = 1; + } ret = 0; } _RETURN: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c294270e4a..360689cef3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -53,6 +53,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_BROKEN_LINK, "Conn is broken") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED, "some vnode/qnode/mnode(s) out of service") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, "rpc open too many session") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_ERROR, "rpc network error") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") From 1c28f535255f91b0430e1560719cf56de0cce02d Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 08:28:00 +0000 Subject: [PATCH 84/93] refactor stream code --- source/libs/stream/inc/streamBackendRocksdb.h | 3 - source/libs/stream/src/streamBackendRocksdb.c | 128 +++++++++--------- 2 files changed, 63 insertions(+), 68 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 9bfec5577c..b89664a6c1 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -19,10 +19,7 @@ #include "rocksdb/c.h" //#include "streamInt.h" #include "streamState.h" -#include "tcoding.h" #include "tcommon.h" -#include "tcompare.h" -#include "ttimer.h" typedef struct SCfComparator { rocksdb_comparator_t** comp; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index a5a4bd4dc1..4b2a9e4a65 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -14,8 +14,6 @@ */ #include "streamBackendRocksdb.h" -#include "executor.h" -#include "query.h" #include "streamInt.h" #include "tcommon.h" #include "tref.h" @@ -1059,14 +1057,14 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) { rocksdb_column_family_handle_t** ppCf = NULL; int32_t nCf = chkpGetAllDbCfHandle2(pTaskDb, &ppCf); - qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); + stDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); if ((code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf)) == 0) { if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) { stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir); } else { - qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, - taosGetTimestampMs() - st); + stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, + taosGetTimestampMs() - st); } } else { stError("stream backend:%p failed to flush db at:%s", pTaskDb, pChkpIdDir); @@ -1797,7 +1795,7 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { cfNames = NULL; } - qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); + stDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); return pTaskDb; _EXIT: @@ -1826,7 +1824,7 @@ void taskDbDestroy(void* pDb, bool flush) { streamMetaRemoveDB(wrapper->pMeta, wrapper->idstr); - qDebug("succ to destroy stream backend:%p", wrapper); + stDebug("succ to destroy stream backend:%p", wrapper); int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); @@ -2323,7 +2321,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { stError("failed to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); taosMemoryFree(err); } else { - qDebug("succ to open cf, %p %s_%s", pState, wrapper->idstr, funcName); + stDebug("succ to open cf, %p %s_%s", pState, wrapper->idstr, funcName); wrapper->pCf[idx] = cf; } } @@ -2365,14 +2363,14 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe char* err = NULL; \ int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ + stWarn("streamState failed to get cf name: %s", funcname); \ code = -1; \ break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ wrapper->dataWritten += 1; \ char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ @@ -2385,54 +2383,54 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe taosMemoryFree(err); \ code = -1; \ } else { \ - qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \ - ttlVLen, wrapper); \ + stTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \ + ttlVLen, wrapper); \ } \ taosMemoryFree(ttlV); \ } while (0); -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->db; \ - rocksdb_readoptions_t* opts = wrapper->readOpt; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL || len == 0) { \ - if (err == NULL) { \ - qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ - } else { \ - stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ - taosMemoryFreeClear(err); \ - } \ - code = -1; \ - } else { \ - char* p = NULL; \ - int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (tlen <= 0) { \ - stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ - funcname); \ - code = -1; \ - } else { \ - qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, \ - wrapper); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = tlen; \ - } \ +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + stWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->db; \ + rocksdb_readoptions_t* opts = wrapper->readOpt; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL || len == 0) { \ + if (err == NULL) { \ + stTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ + } else { \ + stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFreeClear(err); \ + } \ + code = -1; \ + } else { \ + char* p = NULL; \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ + stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ + funcname); \ + code = -1; \ + } else { \ + stTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, \ + tlen, wrapper); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = tlen; \ + } \ } while (0); #define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ @@ -2442,14 +2440,14 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe char* err = NULL; \ int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ - qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ + stWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ code = -1; \ break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ wrapper->dataWritten += 1; \ char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_t* db = wrapper->db; \ @@ -2460,7 +2458,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe taosMemoryFree(err); \ code = -1; \ } else { \ - qTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ + stTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ } \ } while (0); @@ -2681,7 +2679,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { } #ifdef BUILD_NO_CALL SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateGetCur_rocksdb"); + stDebug("streamStateGetCur_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); @@ -2775,7 +2773,7 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k } SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) { - qDebug("streamStateSessionSeekToLast_rocksdb"); + stDebug("streamStateSessionSeekToLast_rocksdb"); int32_t code = 0; @@ -2812,7 +2810,7 @@ SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) { } int32_t streamStateSessionCurPrev_rocksdb(SStreamStateCur* pCur) { - qDebug("streamStateCurPrev_rocksdb"); + stDebug("streamStateCurPrev_rocksdb"); if (!pCur) return -1; rocksdb_iter_prev(pCur->iter); @@ -2862,7 +2860,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta return pCur; } SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { - qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); + stDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { @@ -2900,7 +2898,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta } SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { - qDebug("streamStateSessionSeekKeyNext_rocksdb"); + stDebug("streamStateSessionSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { @@ -3004,7 +3002,7 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { } SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillGetCur_rocksdb"); + stDebug("streamStateFillGetCur_rocksdb"); SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -3064,7 +3062,7 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, } SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillSeekKeyNext_rocksdb"); + stDebug("streamStateFillSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (!pCur) { @@ -3102,7 +3100,7 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const return NULL; } SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillSeekKeyPrev_rocksdb"); + stDebug("streamStateFillSeekKeyPrev_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { From 4c5441da7ee711ac919a302306cf857c22b27f56 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 08:38:16 +0000 Subject: [PATCH 85/93] refactor stream code --- source/libs/stream/src/streamMeta.c | 6 +- source/libs/stream/src/streamSessionState.c | 126 ++++++++++---------- source/libs/stream/src/streamSnapshot.c | 14 +-- source/libs/stream/src/streamUpdate.c | 6 +- 4 files changed, 77 insertions(+), 75 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9c94799237..7ea80ad8d8 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -201,7 +201,7 @@ int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) { void* key = taosHashGetKey(pIter, NULL); code = streamStateCvtDataFormat(pMeta->path, key, *(void**)pIter); if (code != 0) { - qError("failed to cvt data"); + stError("failed to cvt data"); goto _EXIT; } @@ -225,11 +225,11 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { if (compatible == STREAM_STATA_COMPATIBLE) { return 0; } else if (compatible == STREAM_STATA_NEED_CONVERT) { - qInfo("stream state need covert backend format"); + stInfo("stream state need covert backend format"); return streamMetaCvtDbFormat(pMeta); } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { - qError( + stError( "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " "manually", tsDataDir); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 5f19b47e1f..2cb77c60dc 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -17,17 +17,14 @@ #include "query.h" #include "streamBackendRocksdb.h" -#include "taos.h" #include "tcommon.h" -#include "thash.h" #include "tsimplehash.h" - -typedef int (*__session_compare_fn_t) (const SSessionKey* pWin, const void* pDatas, int pos); +typedef int (*__session_compare_fn_t)(const SSessionKey* pWin, const void* pDatas, int pos); int sessionStateKeyCompare(const SSessionKey* pWin1, const void* pDatas, int pos) { SRowBuffPos* pPos2 = taosArrayGetP(pDatas, pos); - SSessionKey* pWin2 = (SSessionKey*) pPos2->pKey; + SSessionKey* pWin2 = (SSessionKey*)pPos2->pKey; return sessionWinKeyCmpr(pWin1, pWin2); } @@ -61,12 +58,12 @@ int32_t binarySearch(void* keyList, int num, const void* key, __session_compare_ int64_t getSessionWindowEndkey(void* data, int32_t index) { SArray* pWinInfos = (SArray*)data; SRowBuffPos** ppos = taosArrayGet(pWinInfos, index); - SSessionKey* pWin = (SSessionKey*)((*ppos)->pKey); + SSessionKey* pWin = (SSessionKey*)((*ppos)->pKey); return pWin->win.ekey; } bool inSessionWindow(SSessionKey* pKey, TSKEY ts, int64_t gap) { - if (ts + gap >= pKey->win.skey && ts - gap <= pKey->win.ekey) { + if (ts + gap >= pKey->win.skey && ts - gap <= pKey->win.ekey) { return true; } return false; @@ -80,7 +77,8 @@ static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pW return pNewPos; } -static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, int32_t index) { +static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, + int32_t index) { SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); @@ -97,11 +95,12 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe return pNewPos; } -int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen) { - int32_t code = TSDB_CODE_SUCCESS; +int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, + int32_t* pVLen) { + int32_t code = TSDB_CODE_SUCCESS; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -146,7 +145,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, if (index + 1 < size) { pPos = taosArrayGetP(pWinStates, index + 1); - if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) ) { + if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap))) { (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; @@ -157,9 +156,9 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, if (index + 1 == 0) { if (!isDeteled(pFileState, endTs) && isFlushedState(pFileState, endTs, gap)) { - void* p = NULL; - void* pFileStore = getStateFileStore(pFileState); - int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); + void* p = NULL; + void* pFileStore = getStateFileStore(pFileState); + int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); code = code_file; @@ -184,10 +183,10 @@ _end: } int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSessionKey* pKey = pPos->pKey; - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -202,7 +201,7 @@ int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) } // find the first position which is smaller than the pKey - int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); + int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); if (index >= 0) { taosArrayInsert(pWinStates, index, &pPos); } else { @@ -218,7 +217,7 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; - void* pBuff = NULL; + void* pBuff = NULL; int32_t code = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); if (code != TSDB_CODE_SUCCESS) { return code; @@ -229,16 +228,16 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v return TSDB_CODE_SUCCESS; } -int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen) { - SSHashObj* pSessionBuff = (SSHashObj*) pBuff; - SSessionKey* pWinKey = (SSessionKey*) key; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); +int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen) { + SSHashObj* pSessionBuff = (SSHashObj*)pBuff; + SSessionKey* pWinKey = (SSessionKey*)key; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (!ppBuff) { return TSDB_CODE_SUCCESS; } SArray* pWinStates = (SArray*)(*ppBuff); int32_t size = taosArrayGetSize(pWinStates); - TSKEY gap = 0; + TSKEY gap = 0; int32_t index = binarySearch(pWinStates, size, pWinKey, sessionStateKeyCompare); if (index >= 0) { SRowBuffPos* pPos = taosArrayGetP(pWinStates, index); @@ -251,15 +250,15 @@ int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen) } int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SSessionKey* pWinKey = (SSessionKey*) pPos->pKey; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SSessionKey* pWinKey = (SSessionKey*)pPos->pKey; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (!ppBuff) { return TSDB_CODE_SUCCESS; } SArray* pWinStates = (SArray*)(*ppBuff); int32_t size = taosArrayGetSize(pWinStates); - TSKEY gap = 0; + TSKEY gap = 0; int32_t index = binarySearch(pWinStates, size, pWinKey, sessionStateKeyCompare); if (index >= 0) { SRowBuffPos* pItemPos = taosArrayGetP(pWinStates, index); @@ -270,11 +269,12 @@ int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffP return TSDB_CODE_SUCCESS; } -int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { +int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, + const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { SRowBuffPos* pNewPos = NULL; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); - SArray* pWinStates = NULL; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; if (!ppBuff) { pWinStates = taosArrayInit(16, POINTER_BYTES); tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); @@ -297,10 +297,10 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream } else { if (size > 0) { SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0); - if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >=0) { + if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >= 0) { // pCur is invalid SSessionKey pTmpKey = *pWinKey; - int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); + int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); ASSERT(code == TSDB_CODE_FAILED); goto _end; } @@ -320,7 +320,7 @@ void sessionWinStateClear(SStreamFileState* pFileState) { void* pIte = NULL; size_t keyLen = 0; int32_t iter = 0; - void* pBuff = getRowStateBuff(pFileState); + void* pBuff = getRowStateBuff(pFileState); while ((pIte = tSimpleHashIterate(pBuff, pIte, &iter)) != NULL) { SArray* pWinStates = *((void**)pIte); int32_t size = taosArrayGetSize(pWinStates); @@ -336,7 +336,7 @@ void sessionWinStateCleanup(void* pBuff) { size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pBuff, pIte, &iter)) != NULL) { - SArray* pWinStates = (SArray*) (*(void**)pIte); + SArray* pWinStates = (SArray*)(*(void**)pIte); taosArrayDestroy(pWinStates); } tSimpleHashCleanup(pBuff); @@ -395,11 +395,13 @@ static void transformCursor(SStreamFileState* pFileState, SStreamStateCur* pCur) pCur->pStreamFileState = pFileState; } -static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t groupId, SArray* pWinStates, SStreamStateCur** ppCur) { +static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t groupId, SArray* pWinStates, + SStreamStateCur** ppCur) { SSessionKey key = {.groupId = groupId}; - int32_t code = streamStateSessionGetKVByCur_rocksdb(*ppCur, &key, NULL, NULL); - if (taosArrayGetSize(pWinStates) > 0 && (code == TSDB_CODE_FAILED || sessionStateKeyCompare(&key, pWinStates, 0) >= 0)) { - if ( !(*ppCur) ) { + int32_t code = streamStateSessionGetKVByCur_rocksdb(*ppCur, &key, NULL, NULL); + if (taosArrayGetSize(pWinStates) > 0 && + (code == TSDB_CODE_FAILED || sessionStateKeyCompare(&key, pWinStates, 0) >= 0)) { + if (!(*ppCur)) { (*ppCur) = createStreamStateCursor(); } transformCursor(pFileState, *ppCur); @@ -410,8 +412,8 @@ static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t } SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey) { - SArray* pWinStates = NULL; - int32_t index = -1; + SArray* pWinStates = NULL; + int32_t index = -1; SStreamStateCur* pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index); if (pCur) { if (sessionStateKeyCompare(pWinKey, pWinStates, index) > 0) { @@ -427,8 +429,8 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, } SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey) { - SArray* pWinStates = NULL; - int32_t index = -1; + SArray* pWinStates = NULL; + int32_t index = -1; SStreamStateCur* pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index); if (pCur) { sessionWinStateMoveToNext(pCur); @@ -449,7 +451,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void SSHashObj* pSessionBuff = getRowStateBuff(pCur->pStreamFileState); void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); - SArray* pWinStates = NULL; + SArray* pWinStates = NULL; if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } @@ -467,7 +469,8 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void } else { void* pData = NULL; code = streamStateSessionGetKVByCur_rocksdb(pCur, pKey, &pData, pVLen); - if (taosArrayGetSize(pWinStates) > 0 && (code == TSDB_CODE_FAILED || sessionStateKeyCompare(pKey, pWinStates, 0) >= 0)) { + if (taosArrayGetSize(pWinStates) > 0 && + (code == TSDB_CODE_FAILED || sessionStateKeyCompare(pKey, pWinStates, 0) >= 0)) { transformCursor(pCur->pStreamFileState, pCur); SRowBuffPos* pPos = taosArrayGetP(pWinStates, pCur->buffIndex); if (pVal) { @@ -498,9 +501,9 @@ int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) { int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey) { SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key); - SSessionKey tmpKey = *key; - int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); - bool hasCurrentPrev = true; + SSessionKey tmpKey = *key; + int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); + bool hasCurrentPrev = true; if (code == TSDB_CODE_FAILED) { streamStateFreeCur(pCur); pCur = sessionWinStateSeekKeyNext(pFileState, key); @@ -535,13 +538,13 @@ _end: } int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { SSessionKey* pWinKey = key; - TSKEY gap = 0; - int32_t code = TSDB_CODE_SUCCESS; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + TSKEY gap = 0; + int32_t code = TSDB_CODE_SUCCESS; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -560,7 +563,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; - qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); } else { (*pVal) = addNewSessionWindow(pFileState, pWinStates, key); code = TSDB_CODE_FAILED; @@ -589,7 +593,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (index + 1 < size) { pPos = taosArrayGetP(pWinStates, index + 1); void* stateKey = (char*)(pPos->pRowBuff) + (valSize - keyDataLen); - if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) || fn(pKeyData, stateKey) == true) { + if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) || + fn(pKeyData, stateKey) == true) { (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; @@ -607,7 +612,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; - qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); goto _end; } else { taosMemoryFree(p); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index e29f2ba7de..a69cb75dad 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -15,10 +15,8 @@ #include "streamSnapshot.h" #include "query.h" -#include "rocksdb/c.h" #include "streamBackendRocksdb.h" #include "streamInt.h" -#include "tcommon.h" enum SBackendFileType { ROCKSDB_OPTIONS_TYPE = 1, @@ -158,7 +156,7 @@ void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { } sprintf(buf + strlen(buf) - 1, "]"); - qInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, + stInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, pSnapFile->snapInfo.taskId, buf); taosMemoryFree(buf); } @@ -203,7 +201,7 @@ int32_t snapFileGenMeta(SBackendSnapFile2* pSnapFile) { int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { TdDirPtr pDir = taosOpenDir(pSnapFile->path); if (NULL == pDir) { - qError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path); + stError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path); return -1; } @@ -397,7 +395,7 @@ _NEXT: } item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); - qDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 + stDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d, total set:%d, current set idx: %d", STREAM_STATE_TRANSFER, item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx, (int)taosArrayGetSize(pHandle->pDbSnapSet), pHandle->currIdx); @@ -515,7 +513,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); return code; } else { - qInfo("succ to write data %s", pItem->name); + stInfo("succ to write data %s", pItem->name); } pSnapFile->offset += bytes; } else { @@ -538,7 +536,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t } taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); - qInfo("succ to write data %s", pItem->name); + stInfo("succ to write data %s", pItem->name); pSnapFile->offset += pHdr->size; } code = 0; @@ -563,7 +561,7 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa "checkpoint", snapInfo.chkpId); if (!taosIsDir(path)) { code = taosMulMkDir(path); - qInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); + stInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); ASSERT(code == 0); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index f78f6f4df1..d607f26de3 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#include "query.h" -#include "tdatablock.h" #include "tencode.h" #include "tstreamUpdate.h" #include "ttime.h" @@ -387,8 +385,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); - bool res = true; - if ( pMapMaxTs && ts < *pMapMaxTs ) { + bool res = true; + if (pMapMaxTs && ts < *pMapMaxTs) { res = false; } else { taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), &ts, sizeof(TSKEY)); From 5e45ecd3fda3e3ea5530d800f672760f871184ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Jan 2024 16:46:07 +0800 Subject: [PATCH 86/93] fix: possible race condition --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbMemTable.c | 3 ++ source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 48 +++++++++++++--------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 75f90af378..9d8d5013fa 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -397,6 +397,7 @@ struct STbData { tb_uid_t uid; TSKEY minKey; TSKEY maxKey; + SRWLatch lock; SDelData *pHead; SDelData *pTail; SMemSkipList sl; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index cc77474e79..4657b60f62 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -181,6 +181,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pDelData->sKey = sKey; pDelData->eKey = eKey; pDelData->pNext = NULL; + taosWLockLatch(&pTbData->lock); if (pTbData->pHead == NULL) { ASSERT(pTbData->pTail == NULL); pTbData->pHead = pTbData->pTail = pDelData; @@ -188,6 +189,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pTbData->pTail->pNext = pDelData; pTbData->pTail = pDelData; } + taosWUnLockLatch(&pTbData->lock); pMemTable->nDel++; pMemTable->minVer = TMIN(pMemTable->minVer, version); @@ -401,6 +403,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SL_NODE_BACKWARD(pTbData->sl.pHead, iLevel) = NULL; SL_NODE_FORWARD(pTbData->sl.pTail, iLevel) = NULL; } + taosInitRWLatch(&pTbData->lock); taosWLockLatch(&pMemTable->latch); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index a223a2dc2d..33b7667168 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -22,9 +22,14 @@ #include "tsdbUtil2.h" #include "tsimplehash.h" -#define INIT_TIMEWINDOW(_w) do { (_w)->skey = INT64_MAX; (_w)->ekey = INT64_MIN;} while(0); +#define INIT_TIMEWINDOW(_w) \ + do { \ + (_w)->skey = INT64_MAX; \ + (_w)->ekey = INT64_MIN; \ + } while (0); -static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order); +static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, + int32_t order); static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { int32_t num = numOfTables / pBuf->numPerBucket; @@ -449,7 +454,7 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); taosArrayPush(pBlockIter->blockList, &blockInfo); - STableBlockScanInfo *pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; + STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; if (pTableScanInfo->pBlockIdxList == NULL) { size_t szTableDataBlocks = taosArrayGetSize(pTableScanInfo->pBlockList); pTableScanInfo->pBlockIdxList = taosArrayInit(szTableDataBlocks, sizeof(STableDataBlockIdx)); @@ -504,7 +509,7 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ int32_t code = 0; STombRecord record = {0}; - uint64_t uid = pReader->status.uidList.tableUidList[*j]; + uint64_t uid = pReader->status.uidList.tableUidList[*j]; STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); @@ -595,12 +600,12 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs return code; } -// uint64_t uid = pReader->status.uidList.tableUidList[j]; + // uint64_t uid = pReader->status.uidList.tableUidList[j]; -// STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); -// if (pScanInfo->pFileDelData == NULL) { -// pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); -// } + // STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + // if (pScanInfo->pFileDelData == NULL) { + // pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); + // } ETombBlkCheckEnum ret = 0; code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret); @@ -650,6 +655,7 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT SDelData* p = NULL; if (pMemTbData != NULL) { + taosRLockLatch(&pMemTbData->lock); p = pMemTbData->pHead; while (p) { if (p->version <= ver) { @@ -658,6 +664,7 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT p = p->pNext; } + taosRUnLockLatch(&pMemTbData->lock); } if (piMemTbData != NULL) { @@ -681,7 +688,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo } int32_t i = 0; - while((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) { + while ((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) { ++i; } @@ -689,7 +696,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo return 0; } - SStatisBlk *p = &pStatisBlkArray->data[i]; + SStatisBlk* p = &pStatisBlkArray->data[i]; STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock)); tStatisBlockInit(pStatisBlock); @@ -759,14 +766,14 @@ void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { if (size < numOfFileObj) { int32_t inc = numOfFileObj - size; for (int32_t k = 0; k < inc; ++k) { - SLDataIter *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + SLDataIter* pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); taosArrayPush(pLDIterList, &pIter); } } else if (size > numOfFileObj) { // remove unused LDataIter int32_t inc = size - numOfFileObj; for (int i = 0; i < inc; ++i) { - SLDataIter *pIter = taosArrayPop(pLDIterList); + SLDataIter* pIter = taosArrayPop(pLDIterList); destroyLDataIter(pIter); } } @@ -781,9 +788,9 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) taosArrayPush(pSttFileBlockIterArray, &pList); } - for(int32_t j = 0; j < numOfLevels; ++j) { + for (int32_t j = 0; j < numOfLevels; ++j) { SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; - SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); + SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr)); } @@ -829,8 +836,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra } // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file - TStatisBlkArray *pStatisBlkArray = NULL; - int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pStatisBlkArray); + TStatisBlkArray* pStatisBlkArray = NULL; + int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); continue; @@ -899,11 +906,12 @@ static int32_t sortUidComparFn(const void* p1, const void* p2) { if (px1->skey == px2->skey) { return 0; } else { - return px1->skey < px2->skey? -1:1; + return px1->skey < px2->skey ? -1 : 1; } } -bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo *pScanInfo, int32_t order) { +bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, + int32_t order) { // check if it overlap with del skyline taosArraySort(pTimewindowList, sortUidComparFn); @@ -935,7 +943,7 @@ bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableB } static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, - int32_t startIndex) { + int32_t startIndex) { size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); for (int32_t i = startIndex; i < num; i += 1) { From 767feba65a24ec22b78b9aa48294667966cac19d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 16:49:02 +0800 Subject: [PATCH 87/93] test: revert failed case --- tests/system-test/8-stream/stream_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index 04e635ea24..3ebc255114 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -101,7 +101,7 @@ class TDTestCase: # check ts interval is correct sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;" tdSql.query(sql) - tdSql.checkRows(1) + tdSql.checkRows(0) # stop def stop(self): From 75c25a194fd78626e3020afe080aa0dd90da3a5b Mon Sep 17 00:00:00 2001 From: zk66214 Date: Mon, 8 Jan 2024 17:19:48 +0800 Subject: [PATCH 88/93] add test cases for TD-28164 --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/backslash_g.py | 72 ++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 tests/system-test/2-query/backslash_g.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3db6eccffc..336f58afa3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -546,6 +546,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/backslash_g.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py diff --git a/tests/system-test/2-query/backslash_g.py b/tests/system-test/2-query/backslash_g.py new file mode 100644 index 0000000000..69b06289c3 --- /dev/null +++ b/tests/system-test/2-query/backslash_g.py @@ -0,0 +1,72 @@ + +import taos + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def checksql(self, sql): + result = os.popen(f"taos -s \"{sql}\" ") + res = result.read() + print(res) + if ("Query OK" in res): + tdLog.info(f"checkEqual success") + else : + tdLog.exit(f"checkEqual error") + + def td_28164(self): + tdSql.execute("drop database if exists td_28164;") + tdSql.execute("create database td_28164;") + tdSql.execute("create table td_28164.test (ts timestamp, name varchar(10));") + tdSql.execute("insert into td_28164.test values (now(), 'ac\\\\G') (now() + 1s, 'ac\\\\G') (now()+2s, 'ac\\G') (now()+3s, 'acG') (now()+4s, 'acK') ;") + + tdSql.query(f"select * from td_28164.test;") + tdSql.checkRows(5) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\\\\\G';") + tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\\\G';") + tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\G';") + tdSql.checkRows(2) + + # tdSql.query(f"select * from td_28164.test where name like 'ac\\\g';") + # tdSql.checkRows(0) + # + # tdSql.query(f"select * from td_28164.test where name like 'ac\\g';") + # tdSql.checkRows(0) + + self.checksql(f'select * from td_28164.test where name like \'ac\\G\'\G;') + # tdSql.checkRows(2) + + self.checksql(f"select * from td_28164.test where name like \'ac\\G\' \G;") + # tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac/\\G';") + tdSql.checkRows(0) + + tdSql.query(f"select * from td_28164.test where name like 'ac/G';") + tdSql.checkRows(0) + + def run(self): + # tdSql.prepare() + self.td_28164() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 5bd29a413b30df61024d4c6dfa5d1efbc669a02b Mon Sep 17 00:00:00 2001 From: fullhonest Date: Mon, 8 Jan 2024 15:04:04 +0800 Subject: [PATCH 89/93] Fix: not use last row scan when select tbname and last --- source/libs/planner/src/planOptimizer.c | 2 +- tests/system-test/2-query/last.py | 81 ++++++++++++++++++++ tests/system-test/2-query/last_cache_scan.py | 8 +- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index aa3181e166..035377d22e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2529,7 +2529,7 @@ static bool lastRowScanOptCheckFuncList(SLogicNode* pNode, bool* hasOtherFunc) { SNode* pParam = nodesListGetNode(pAggFunc->pParameterList, 0); if (QUERY_NODE_COLUMN == nodeType(pParam)) { SColumnNode* pCol = (SColumnNode*)pParam; - if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { + if (COLUMN_TYPE_COLUMN == pCol->colType && PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { if (selectNonPKColId != pCol->colId) { selectNonPKColId = pCol->colId; selectNonPKColNum++; diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index f4a1ab1790..a160aeab11 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -289,6 +289,82 @@ class TDTestCase: tdSql.query(f"select * from {stbname}") tdSql.checkRows(tables * rows) + def check_explain_res_has_row(self, plan_str_expect: str, rows, sql): + plan_found = False + for row in rows: + if str(row).find(plan_str_expect) >= 0: + tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row))) + plan_found = True + break + if not plan_found: + tdLog.exit("plan: %s not found in res: [%s] in sql: %s" % (plan_str_expect, str(rows), sql)) + + def check_explain_res_no_row(self, plan_str_not_expect: str, res, sql): + for row in res: + if str(row).find(plan_str_not_expect) >= 0: + tdLog.exit('plan: [%s] found in: [%s] for sql: %s' % (plan_str_not_expect, str(row), sql)) + + def explain_sql(self, sql: str): + sql = "explain " + sql + tdSql.query(sql, queryTimes=1) + return tdSql.queryResult + + def last_check_scan_type(self, cacheModel): + tdSql.execute("create database test_last_tbname cachemodel '%s';" % cacheModel) + tdSql.execute("use test_last_tbname;") + tdSql.execute("create stable test_last_tbname.st(ts timestamp, id int) tags(tid int);") + tdSql.execute("create table test_last_tbname.test_t1 using test_last_tbname.st tags(1);") + + maxRange = 100 + # 2023-11-13 00:00:00.000 + startTs = 1699804800000 + for i in range(maxRange): + insertSqlString = "insert into test_last_tbname.test_t1 values(%d, %d);" % (startTs + i, i) + tdSql.execute(insertSqlString) + + last_ts = startTs + maxRange + tdSql.execute("insert into test_last_tbname.test_t1 (ts) values(%d)" % (last_ts)) + sql = f'select tbname, last(ts) from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, "test_t1") + tdSql.checkData(0, 1, last_ts) + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + + sql = f'select last(ts), tbname from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, last_ts) + tdSql.checkData(0, 1, "test_t1") + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + sql = f'select tbname, last(ts), tbname from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, "test_t1") + tdSql.checkData(0, 1, last_ts) + tdSql.checkData(0, 2, "test_t1") + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + tdSql.execute("drop table if exists test_last_tbname.test_t1 ;") + tdSql.execute("drop stable if exists test_last_tbname.st;") + tdSql.execute("drop database if exists test_last_tbname;") def run(self): self.last_check_stb_tb_base() @@ -296,6 +372,11 @@ class TDTestCase: self.last_check_stb_distribute() self.last_file_check() + self.last_check_scan_type("none") + self.last_check_scan_type("last_row") + self.last_check_scan_type("last_value") + self.last_check_scan_type("both") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 23433277c2..1269bd4fde 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -181,7 +181,7 @@ class TDTestCase: select_items = [ "last(ts), ts", "last(ts), c1", "last(ts), c2", "last(ts), c3",\ "last(ts), c4", "last(ts), tbname", "last(ts), t1", "last(ts), ts, ts"] - has_last_row_scan_res = [1, 0, 0, 0, 0, 0, 0, 1] + has_last_row_scan_res = [1, 0, 0, 0, 0, 1, 1, 1] res_expect = [ ["2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"], None, None, None, None, None, None, @@ -193,7 +193,7 @@ class TDTestCase: select_items = ["last(c1),ts", "last(c1), c1", "last(c1), c2", "last(c1), c3",\ "last(c1), c4", "last(c1), tbname", "last(c1), t1", "last(c1), ts, ts", "last(c1), c1, c1"] - has_last_row_scan_res = [1, 1, 0, 0, 0, 0, 0, 1, 1] + has_last_row_scan_res = [1, 1, 0, 0, 0, 1, 1, 1, 1] res_expect = [ [999, "2018-11-25 19:30:00.000"], [999, 999], None, None, None, None, None, @@ -207,7 +207,7 @@ class TDTestCase: sql_template = 'select %s from t1' select_items = ["last(c4),ts", "last(c4), c1", "last(c4), c2", "last(c4), c3",\ "last(c4), c4", "last(c4), tbname", "last(c4), t1"] - has_last_row_scan_res = [1, 0, 0, 0, 1, 0, 0] + has_last_row_scan_res = [1, 0, 0, 0, 1, 1, 1] res_expect = [ [4999.000000000000000, "2018-11-25 19:30:00.000"], None,None,None, @@ -220,7 +220,7 @@ class TDTestCase: sql_template = 'select %s from meters' select_items = ["last(c8), ts", "last(c8), c1", "last(c8), c8", "last(c8), tbname", \ "last(c8), t1", "last(c8), c8, c8", "last(c8), ts, ts"] - has_last_row_scan_res = [1, 0, 1, 0, 0, 1, 1] + has_last_row_scan_res = [1, 0, 1, 1, 1, 1, 1] res_expect = [ ["binary9999", "2018-11-25 19:30:00.000"], None, From d93b4f88b38c7aa5616d9c58f0f097741bcc713a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Jan 2024 15:23:15 +0800 Subject: [PATCH 90/93] fix(tsdb/cache): destory brin block with fs loading --- source/dnode/vnode/src/tsdb/tsdbCache.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index a39c74a679..ab504acea7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1955,6 +1955,7 @@ typedef struct SFSNextRowIter { SArray *pIndexList; int32_t iBrinIndex; SBrinBlock brinBlock; + SBrinBlock *pBrinBlock; int32_t iBrinRecord; SBrinRecord brinRecord; SBlockData blockData; @@ -2143,6 +2144,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie pBrinBlk = taosArrayGet(state->pIndexList, state->iBrinIndex); } + if (!state->pBrinBlock) { + state->pBrinBlock = &state->brinBlock; + } else { + tBrinBlockClear(&state->brinBlock); + } code = tsdbDataFileReadBrinBlock(state->pr->pFileReader, pBrinBlk, &state->brinBlock); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -2420,6 +2426,16 @@ int32_t clearNextRowFromFS(void *iter) { state->pBlockData = NULL; } + if (state->pBrinBlock) { + tBrinBlockDestroy(state->pBrinBlock); + state->pBrinBlock = NULL; + } + + if (state->pIndexList) { + taosArrayDestroy(state->pIndexList); + state->pIndexList = NULL; + } + if (state->pTSRow) { taosMemoryFree(state->pTSRow); state->pTSRow = NULL; From 404e17e67797f99f1c491171348ac463e0f99261 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 9 Jan 2024 16:18:55 +0800 Subject: [PATCH 91/93] fix: get expired user for each cluster --- source/client/src/clientHb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1de5941ac3..b3d9a9ced5 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -902,6 +902,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } + taosHashPut(clientHbMgr.appHbHash, &hbParam->clusterId, sizeof(uint64_t), NULL, 0); } // invoke after hbGetExpiredUserInfo @@ -1121,7 +1122,6 @@ static void *hbThreadFunc(void *param) { asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); - taosHashPut(clientHbMgr.appHbHash, &pAppHbMgr->pAppInstInfo->clusterId, sizeof(uint64_t), NULL, 0); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } From 6444af84e47f5ed90f1af99e7e549a2be968f97b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 9 Jan 2024 16:28:58 +0800 Subject: [PATCH 92/93] opti:[TD-28142]memory size if block num is too large --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 109 ++++++++++++--------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 25 ++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 19 +++- source/util/src/tarray.c | 11 ++- 4 files changed, 110 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index b57d2dfb45..255d5d4105 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -882,6 +882,21 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo } } +static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo){ + record->uid = pBlockInfo->uid; + record->firstKey = pBlockInfo->firstKey; + record->lastKey = pBlockInfo->lastKey; + record->minVer = pBlockInfo->minVer; + record->maxVer = pBlockInfo->maxVer; + record->blockOffset = pBlockInfo->blockOffset; + record->smaOffset = pBlockInfo->smaOffset; + record->blockSize = pBlockInfo->blockSize; + record->blockKeySize = pBlockInfo->blockKeySize; + record->smaSize = pBlockInfo->smaSize; + record->numRow = pBlockInfo->numRow; + record->count = pBlockInfo->count; +} + static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -899,7 +914,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; - SBrinRecord* pRecord = &pBlockInfo->record; + SBrinRecord tmp; + blockInfoToRecord(&tmp, pBlockInfo); + SBrinRecord* pRecord = &tmp; // no data exists, return directly. if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) { @@ -1092,14 +1109,16 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBrinRecord* pRecord = &pBlockInfo->record; + SBrinRecord tmp; + blockInfoToRecord(&tmp, pBlockInfo); + SBrinRecord* pRecord = &tmp; code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1], pSup->numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, code:%s %s", - pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.firstKey, - pBlockInfo->record.lastKey, pBlockInfo->record.numRow, tstrerror(code), pReader->idStr); + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->firstKey, + pBlockInfo->lastKey, pBlockInfo->numRow, tstrerror(code), pReader->idStr); return code; } @@ -1120,10 +1139,10 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI * This is an two rectangles overlap cases. */ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SFileDataBlockInfo* pBlock) { - return (pWindow->ekey < pBlock->record.lastKey && pWindow->ekey >= pBlock->record.firstKey) || - (pWindow->skey > pBlock->record.firstKey && pWindow->skey <= pBlock->record.lastKey) || - (pVerRange->minVer > pBlock->record.minVer && pVerRange->minVer <= pBlock->record.maxVer) || - (pVerRange->maxVer < pBlock->record.maxVer && pVerRange->maxVer >= pBlock->record.minVer); + return (pWindow->ekey < pBlock->lastKey && pWindow->ekey >= pBlock->firstKey) || + (pWindow->skey > pBlock->firstKey && pWindow->skey <= pBlock->lastKey) || + (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || + (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, @@ -1142,7 +1161,7 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); - memcpy(pRecord, &p->record, sizeof(SBrinRecord)); + blockInfoToRecord(pRecord, p); *nextIndex = pBlockInfo->tbBlockIdx + step; return true; @@ -1211,9 +1230,9 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order) { // it is the last block in current file, no chance to overlap with neighbor blocks. if (ASCENDING_TRAVERSE(order)) { - return pBlock->record.lastKey == pRec->firstKey; + return pBlock->lastKey == pRec->firstKey; } else { - return pBlock->record.firstKey == pRec->lastKey; + return pBlock->firstKey == pRec->lastKey; } } @@ -1223,9 +1242,9 @@ static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanI int64_t key = 0; if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; - key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt); + key = ascScan ? TMIN(pBlock->firstKey, keyInStt) : TMAX(pBlock->lastKey, keyInStt); } else { - key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey; + key = ascScan ? pBlock->firstKey : pBlock->lastKey; } return key; @@ -1241,8 +1260,8 @@ static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBloc } static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) { - return (key.ts >= pBlock->record.firstKey && key.ts <= pBlock->record.lastKey) && - (pBlock->record.maxVer >= pVerRange->minVer) && (pBlock->record.minVer <= pVerRange->maxVer); + return (key.ts >= pBlock->firstKey && key.ts <= pBlock->lastKey) && + (pBlock->maxVer >= pVerRange->minVer) && (pBlock->minVer <= pVerRange->maxVer); } static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, @@ -1258,18 +1277,20 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order); } + SBrinRecord pRecord; + blockInfoToRecord(&pRecord, pBlockInfo); // has duplicated ts of different version in this block - pInfo->hasDupTs = (pBlockInfo->record.numRow > pBlockInfo->record.count) || (pBlockInfo->record.count <= 0); - pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pBlockInfo->record, pReader->info.order); + pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0); + pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, pReader->info.order); ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey; pInfo->overlapWithSttBlock = - !(pBlockInfo->record.lastKey < nextProcKeyInStt || pBlockInfo->record.firstKey > nextProcKeyInStt); + !(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt); } - pInfo->moreThanCapcity = pBlockInfo->record.numRow > pReader->resBlockInfo.capacity; + pInfo->moreThanCapcity = pBlockInfo->numRow > pReader->resBlockInfo.capacity; pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo); pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange); } @@ -2361,17 +2382,14 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBrinRecord* pRecord = NULL; STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo == NULL) { return 0; } - pRecord = &pBlockInfo->record; - if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; } @@ -2380,13 +2398,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { goto _end; } - pRecord = &pBlockInfo->record; TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly int64_t cap = pReader->resBlockInfo.capacity; - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pRecord->numRow <= cap)) { - if (((asc && (pRecord->firstKey < keyInBuf.ts)) || (!asc && (pRecord->lastKey > keyInBuf.ts))) && + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pBlockInfo->numRow <= cap)) { + if (((asc && (pBlockInfo->firstKey < keyInBuf.ts)) || (!asc && (pBlockInfo->lastKey > keyInBuf.ts))) && (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA)) { code = copyBlockDataToSDataBlock(pReader); if (code) { @@ -2394,7 +2411,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } // record the last key value - pBlockScanInfo->lastProcKey = asc ? pRecord->lastKey : pRecord->firstKey; + pBlockScanInfo->lastProcKey = asc ? pBlockInfo->lastKey : pBlockInfo->firstKey; goto _end; } } @@ -2422,7 +2439,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { bool loadNeighbor = false; code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor); if ((!loadNeighbor) || (code != 0)) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); break; } } @@ -2441,7 +2458,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); break; } @@ -2801,7 +2818,7 @@ static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockSc return true; } else { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; - return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt); + return (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt); } } @@ -2815,7 +2832,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; } @@ -2850,20 +2867,20 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (notOverlapWithSttFiles(pBlockInfo, pScanInfo, asc)) { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; - pInfo->rows = pBlockInfo->record.numRow; + pInfo->rows = pBlockInfo->numRow; pInfo->id.uid = pScanInfo->uid; pInfo->dataLoad = 0; - pInfo->window = (STimeWindow){.skey = pBlockInfo->record.firstKey, .ekey = pBlockInfo->record.lastKey}; + pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey}; setComposedBlockFlag(pReader, false); - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); // update the last key for the corresponding table pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey; tsdbDebug("%p uid:%" PRIu64 " clean file block retrieved from file, global index:%d, " "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", - pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.numRow, - pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, + pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); } else { SBlockData* pBData = &pReader->status.fileBlockData; tBlockDataReset(pBData); @@ -2892,10 +2909,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // data in stt now overlaps with current active file data block, need to composed with file data block. int64_t lastKeyInStt = getCurrentKeyInSttBlock(pSttBlockReader); - if ((lastKeyInStt >= pBlockInfo->record.firstKey && asc) || - (lastKeyInStt <= pBlockInfo->record.lastKey && (!asc))) { + if ((lastKeyInStt >= pBlockInfo->firstKey && asc) || + (lastKeyInStt <= pBlockInfo->lastKey && (!asc))) { tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader, - lastKeyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + lastKeyInStt, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); break; } } @@ -3012,8 +3029,8 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) lastKey = pScanInfo->lastProcKey; } - pDumpInfo->totalRows = pBlockInfo->record.numRow; - pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->record.numRow - 1; + pDumpInfo->totalRows = pBlockInfo->numRow; + pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { pDumpInfo->totalRows = 0; pDumpInfo->rowIndex = 0; @@ -4632,7 +4649,9 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, // int64_t st = taosGetTimestampUs(); TARRAY2_CLEAR(&pSup->colAggArray, 0); - code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pFBlock->record, &pSup->colAggArray); + SBrinRecord pRecord; + blockInfoToRecord(&pRecord, pFBlock); + code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pRecord, &pSup->colAggArray); if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), pReader->idStr); @@ -4663,7 +4682,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pFBlock->record.numRow, numOfCols, pTsAgg); + doFillNullColSMA(pSup, pFBlock->numRow, numOfCols, pTsAgg); size_t size = pSup->colAggArray.size; @@ -4884,7 +4903,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT while (true) { if (hasNext) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); - int32_t numOfRows = pBlockInfo->record.numRow; + int32_t numOfRows = pBlockInfo->numRow; pTableBlockInfo->totalRows += numOfRows; @@ -4896,7 +4915,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT pTableBlockInfo->minRows = numOfRows; } - pTableBlockInfo->totalSize += pBlockInfo->record.blockSize; + pTableBlockInfo->totalSize += pBlockInfo->blockSize; int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets); pTableBlockInfo->blockRowsHisto[bucketIndex]++; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 33b7667168..b4b92c0008 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -359,6 +359,21 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v return pLeftBlock->offset > pRightBlock->offset ? 1 : -1; } +static void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record){ + pBlockInfo->uid = record->uid; + pBlockInfo->firstKey = record->firstKey; + pBlockInfo->lastKey = record->lastKey; + pBlockInfo->minVer = record->minVer; + pBlockInfo->maxVer = record->maxVer; + pBlockInfo->blockOffset = record->blockOffset; + pBlockInfo->smaOffset = record->smaOffset; + pBlockInfo->blockSize = record->blockSize; + pBlockInfo->blockKeySize = record->blockKeySize; + pBlockInfo->smaSize = record->smaSize; + pBlockInfo->numRow = record->numRow; + pBlockInfo->count = record->count; +} + int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->info.order); @@ -416,8 +431,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 pTableScanInfo->pBlockIdxList = taosArrayInit(numOfBlocks, sizeof(STableDataBlockIdx)); } for (int32_t i = 0; i < numOfBlocks; ++i) { - SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; - blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); + SFileDataBlockInfo blockInfo = {.tbBlockIdx = i}; + SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); + recordToBlockInfo(&blockInfo, record); taosArrayPush(pBlockIter->blockList, &blockInfo); STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i}; @@ -450,8 +466,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 int32_t pos = tMergeTreeGetChosenIndex(pTree); int32_t index = sup.indexPerTable[pos]++; - SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[pos][index].uid, .tbBlockIdx = index}; - blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + SFileDataBlockInfo blockInfo = {.tbBlockIdx = index}; + SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + recordToBlockInfo(&blockInfo, record); taosArrayPush(pBlockIter->blockList, &blockInfo); STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 39e65f22b1..cd9942a0a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -175,9 +175,22 @@ typedef struct SFilesetIter { typedef struct SFileDataBlockInfo { // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it - uint64_t uid; - int32_t tbBlockIdx; - SBrinRecord record; + // int64_t suid; + int64_t uid; + int64_t firstKey; +// int64_t firstKeyVer; + int64_t lastKey; +// int64_t lastKeyVer; + int64_t minVer; + int64_t maxVer; + int64_t blockOffset; + int64_t smaOffset; + int32_t blockSize; + int32_t blockKeySize; + int32_t smaSize; + int32_t numRow; + int32_t count; + int32_t tbBlockIdx; } SFileDataBlockInfo; typedef struct SDataBlockIter { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index bc98461592..88eb51d500 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -18,6 +18,9 @@ #include "tcoding.h" // todo refactor API +#define BOUNDARY_SIZE 1024*1024*1024 // 1G +#define BOUNDARY_SMALL_FACTOR 1.2 +#define BOUNDARY_BIG_FACTOR 2 SArray* taosArrayInit(size_t size, size_t elemSize) { if (elemSize == 0) { @@ -85,9 +88,13 @@ static int32_t taosArrayResize(SArray* pArray) { int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { if (newCap > pArray->capacity) { - size_t tsize = (pArray->capacity << 1u); + float factor = BOUNDARY_BIG_FACTOR; + if(newCap * pArray->elemSize > BOUNDARY_SIZE){ + factor = BOUNDARY_SMALL_FACTOR; + } + size_t tsize = (pArray->capacity * factor); while (newCap > tsize) { - tsize = (tsize << 1u); + tsize = (tsize * factor); } pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); From 220cef4a61c1c2a8d857c812549b2ab378e6dd47 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Tue, 9 Jan 2024 11:02:10 +0000 Subject: [PATCH 93/93] opt msg on mnode --- source/dnode/mnode/impl/src/mndMain.c | 164 +++++++++++++++++--------- 1 file changed, 109 insertions(+), 55 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 38a92b43e7..c0fd93c65d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" #include "mndCluster.h" +#include "mndCompact.h" +#include "mndCompactDetail.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -42,8 +44,6 @@ #include "mndUser.h" #include "mndVgroup.h" #include "mndView.h" -#include "mndCompact.h" -#include "mndCompactDetail.h" static inline int32_t mndAcquireRpc(SMnode *pMnode) { int32_t code = 0; @@ -158,16 +158,16 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { } } -static void mndStreamCheckpointRemain(SMnode* pMnode) { +static void mndStreamCheckpointRemain(SMnode *pMnode) { int32_t contLen = 0; - void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); + void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } } -static void mndStreamCheckNode(SMnode* pMnode) { +static void mndStreamCheckNode(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -275,6 +275,100 @@ static void mndCheckDnodeOffline(SMnode *pMnode) { mndReleaseRpc(pMnode); } +static bool mnodeIsNotLeader(SMnode *pMnode) { + terrno = 0; + taosThreadRwlockRdlock(&pMnode->lock); + SSyncState state = syncGetState(pMnode->syncMgmt.sync); + if (terrno != 0) { + taosThreadRwlockUnlock(&pMnode->lock); + return true; + } + + if (state.state != TAOS_SYNC_STATE_LEADER) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_NOT_LEADER; + return true; + } + if (!state.restored || !pMnode->restored) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_RESTORING; + return true; + } + taosThreadRwlockUnlock(&pMnode->lock); + return false; +} + +static int32_t minCronTime() { + int64_t min = INT64_MAX; + min = TMIN(min, tsTtlPushIntervalSec); + min = TMIN(min, tsTrimVDbIntervalSec); + min = TMIN(min, tsTransPullupInterval); + min = TMIN(min, tsCompactPullupInterval); + min = TMIN(min, tsMqRebalanceInterval); + min = TMIN(min, tsStreamCheckpointInterval); + min = TMIN(min, 5); // checkpointRemain + min = TMIN(min, tsStreamNodeCheckInterval); + + int64_t telemInt = TMIN(60, (tsTelemInterval - 1)); + min = TMIN(min, telemInt); + min = TMIN(min, tsGrantHBInterval); + min = TMIN(min, tsUptimeInterval); + + return min <= 1 ? 2 : min; +} +void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { + if (sec % tsTtlPushIntervalSec == 0) { + mndPullupTtl(pMnode); + } + + if (sec % tsTrimVDbIntervalSec == 0) { + mndPullupTrimDb(pMnode); + } + + if (sec % tsTransPullupInterval == 0) { + mndPullupTrans(pMnode); + } + + if (sec % tsCompactPullupInterval == 0) { + mndPullupCompacts(pMnode); + } + + if (sec % tsMqRebalanceInterval == 0) { + mndCalMqRebalance(pMnode); + } + + if (sec % tsStreamCheckpointInterval == 0) { + mndStreamCheckpointTick(pMnode, sec); + } + + if (sec % 5 == 0) { + mndStreamCheckpointRemain(pMnode); + } + + if (sec % tsStreamNodeCheckInterval == 0) { + mndStreamCheckNode(pMnode); + } + + if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) { + mndPullupTelem(pMnode); + } + + if (sec % tsGrantHBInterval == 0) { + mndPullupGrant(pMnode); + } + + if (sec % tsUptimeInterval == 0) { + mndIncreaseUpTime(pMnode); + } +} +void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) { + if (sec % (tsStatusInterval * 5) == 0) { + mndCheckDnodeOffline(pMnode); + } + if (sec % (MNODE_TIMEOUT_SEC / 2) == 0) { + mndSyncCheckTimeout(pMnode); + } +} static void *mndThreadFp(void *param) { SMnode *pMnode = param; int64_t lastTime = 0; @@ -287,57 +381,15 @@ static void *mndThreadFp(void *param) { if (lastTime % 10 != 0) continue; int64_t sec = lastTime / 10; - if (sec % tsTtlPushIntervalSec == 0) { - mndPullupTtl(pMnode); - } + mndDoTimerCheckTask(pMnode, sec); - if (sec % tsTrimVDbIntervalSec == 0) { - mndPullupTrimDb(pMnode); - } - - if (sec % tsTransPullupInterval == 0) { - mndPullupTrans(pMnode); - } - - if (sec % tsCompactPullupInterval == 0) { - mndPullupCompacts(pMnode); - } - - if (sec % tsMqRebalanceInterval == 0) { - mndCalMqRebalance(pMnode); - } - - if (sec % tsStreamCheckpointInterval == 0) { - mndStreamCheckpointTick(pMnode, sec); - } - - if (sec % 5 == 0) { - mndStreamCheckpointRemain(pMnode); - } - - if (sec % tsStreamNodeCheckInterval == 0) { - mndStreamCheckNode(pMnode); - } - - if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) { - mndPullupTelem(pMnode); - } - - if (sec % tsGrantHBInterval == 0) { - mndPullupGrant(pMnode); - } - - if (sec % tsUptimeInterval == 0) { - mndIncreaseUpTime(pMnode); - } - - if (sec % (tsStatusInterval * 5) == 0) { - mndCheckDnodeOffline(pMnode); - } - - if (sec % (MNODE_TIMEOUT_SEC / 2) == 0) { - mndSyncCheckTimeout(pMnode); + int64_t minCron = minCronTime(); + if (sec % minCron == 0 && mnodeIsNotLeader(pMnode)) { + // not leader, do nothing + mTrace("timer not process since mnode is not leader, reason: %s", tstrerror(terrno)) terrno = 0; + continue; } + mndDoTimerPullupTask(pMnode, sec); } return NULL; @@ -712,7 +764,9 @@ _OVER: if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER || - pMsg->msgType == TDMT_MND_COMPACT_TIMER) { + pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER || + pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE || + pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, pMnode->stopped, state.restored, syncStr(state.state)); return -1;