From e3098accc4c7dc222670311e887d01437d63907c Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 13 Dec 2023 23:25:37 +0800 Subject: [PATCH 001/198] support count empty table --- .gitignore | 4 +- include/libs/nodes/plannodes.h | 1 + include/libs/nodes/querynodes.h | 1 + source/dnode/vnode/src/sma/smaRollup.c | 1 + source/libs/executor/inc/executorInt.h | 4 +- source/libs/executor/src/scanoperator.c | 102 +++++++++- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 9 +- source/libs/nodes/src/nodesMsgFuncs.c | 7 +- source/libs/parser/src/parTranslater.c | 1 + source/libs/planner/src/planPhysiCreater.c | 22 +++ tests/parallel_test/cases.task | 1 + tests/system-test/2-query/count.py | 33 ++-- tests/system-test/2-query/group_partition.py | 183 ++++++++++++++++++ .../2-query/nestedQueryInterval.py | 26 +-- 15 files changed, 359 insertions(+), 37 deletions(-) create mode 100644 tests/system-test/2-query/group_partition.py diff --git a/.gitignore b/.gitignore index 08e3d57717..f8b42f9176 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -build/ +*build/ compile_commands.json CMakeSettings.json .cache @@ -132,3 +132,5 @@ tools/taos-tools tools/taosws-rs tags .clangd +*CMakeCache* +*CMakeFiles* diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b99a97a194..4b3c846389 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -434,6 +434,7 @@ typedef struct STableScanPhysiNode { bool assignBlockUid; int8_t igCheckUpdate; bool filesetDelimited; + bool needCountEmptyTable; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5c5172b9cd..fdf598153f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -363,6 +363,7 @@ typedef struct SSelectStmt { bool hasLastRowFunc; bool hasLastFunc; bool hasTimeLineFunc; + bool hasCountFunc; bool hasUdaf; bool hasStateKey; bool onlyHasKeepOrderFunc; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index abe4c3f2fc..5b33451be5 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -750,6 +750,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); + pReq = NULL; TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e3e504cdbc..2523b87cfb 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -262,6 +262,7 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; + SHashObj* pValuedTables; // non empty table uids SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; @@ -269,8 +270,9 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; - // TsdReader readerAPI; bool filesetDelimited; + bool needCountEmptyTable; + bool processingEmptyTable; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..1889492aa0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,6 +655,50 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } + +// record processed (non empty) table +static int32_t insertTableToProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { + if (!pTableScanInfo->needCountEmptyTable) { + return TSDB_CODE_SUCCESS; + } + if (NULL == pTableScanInfo->pValuedTables) { + int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); + pTableScanInfo->pValuedTables = + taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (NULL == pTableScanInfo->pValuedTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + taosHashPut(pTableScanInfo->pValuedTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, + sizeof(pTableScanInfo->scanTimes)); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { + STableScanInfo* pTableScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSDataBlock* pBlock = pTableScanInfo->pResBlock; + + blockDataEmpty(pBlock); + pBlock->info.rows = 1; + pBlock->info.id.uid = tbInfo->uid; + pBlock->info.id.groupId = pOperator->dynamicTask ? tbInfo->uid : tbInfo->groupId; + + // only one row: set all col data to null & hasNull + int32_t col_num = blockDataGetNumOfCols(pBlock); + for (int32_t i = 0; i < col_num; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + colDataSetNULL(pColInfoData, 0); + } + + // set tag/tbname + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); + + pOperator->resultInfo.totalRows++; + return pBlock; +} + static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -722,7 +766,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { return NULL; } -static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { +static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKeyInfo* pList, int32_t num) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -736,6 +780,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { + insertTableToProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -764,6 +809,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { + insertTableToProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -780,6 +826,39 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } } + if (pTableScanInfo->needCountEmptyTable) { + if (num == 0 && 0 == taosHashGetSize(pTableScanInfo->pValuedTables)) { + // table by table, num is 0 + if (!pTableScanInfo->processingEmptyTable) { + pTableScanInfo->processingEmptyTable = true; + // current table is empty, fill result block info & return + const STableKeyInfo* info = tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->currentTable); + return getBlockForEmptyTable(pOperator, info); + } + + } else if (num > taosHashGetSize(pTableScanInfo->pValuedTables)) { + // group by group, num >= 1 + if (!pTableScanInfo->processingEmptyTable) { + pTableScanInfo->processingEmptyTable = true; + pTableScanInfo->currentTable = 0; + } + if (pTableScanInfo->currentTable < num) { + // loop: get empty table uid & process + while (pTableScanInfo->currentTable < num) { + const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; + if (pTableScanInfo->pValuedTables && + NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { + } else { + return getBlockForEmptyTable(pOperator, info); + } + } + } + } + + pTableScanInfo->processingEmptyTable = false; + } + taosHashClear(pTableScanInfo->pValuedTables); + return NULL; } @@ -861,7 +940,7 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -876,15 +955,16 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int32_t num = 0; + STableKeyInfo* pList = NULL; if (pInfo->currentGroupId == -1) { - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo) || numOfTables == 0) { setOperatorCompleted(pOperator); return NULL; } - int32_t num = 0; - STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); @@ -899,9 +979,11 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; } + } else { + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); } - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -923,7 +1005,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - + if (pOperator->pOperatorGetParam) { pOperator->dynamicTask = true; int32_t code = createTableListInfoFromParam(pOperator); @@ -952,7 +1034,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo tInfo = {0}; while (1) { - SSDataBlock* result = doGroupedTableScan(pOperator); + SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) { return result; } @@ -1012,6 +1094,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); + taosHashCleanup(pTableScanInfo->pValuedTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1075,6 +1158,9 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; + pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; + pInfo->processingEmptyTable = false; + pInfo->base.pTableListInfo = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 97438b84a6..8a154dcf00 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -652,6 +652,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(igExpired); COPY_SCALAR_FIELD(filesetDelimited); + COPY_SCALAR_FIELD(needCountEmptyTable); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c445af61cc..402a6c6e3d 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1841,6 +1841,7 @@ static const char* jkTableScanPhysiPlanSubtable = "Subtable"; static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate"; static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited"; +static const char* jkTableScanPhysiPlanNeedCountEmptyTable = "NeedCountEmptyTable"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1912,7 +1913,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanFilesetDelimited, pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, pNode->needCountEmptyTable); + } return code; } @@ -1986,7 +1989,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanFilesetDelimited, &pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanNeedCountEmptyTable, &pNode->needCountEmptyTable); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index d6eb3360aa..b36e2695f6 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2170,6 +2170,9 @@ static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEnc if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueBool(pEncoder, pNode->filesetDelimited); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->needCountEmptyTable); + } return code; } @@ -2251,7 +2254,9 @@ static int32_t msgToPhysiTableScanNodeInline(STlvDecoder* pDecoder, void* pObj) if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueBool(pDecoder, &pNode->filesetDelimited); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->needCountEmptyTable); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..98799d3d1d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1902,6 +1902,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { if (NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt)) { SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt; pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); + pSelect->hasCountFunc = pSelect->hasCountFunc ? true : (FUNCTION_TYPE_COUNT == pFunc->funcType); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); if (fmIsIndefiniteRowsFunc(pFunc->funcId)) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index d1fbd0681d..901927b1d1 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,6 +587,27 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } +static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + // refuse interval + if (pScanLogicNode->interval > 0) { + return false; + } + SNode* pRoot = pCxt->pPlanCxt->pAstRoot; + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + SSelectStmt* pSelect = (SSelectStmt*)pRoot; + // select & count + if (pSelect->hasCountFunc) { + // key only accept tag/tbname + if (NULL != pSelect->pGroupByList) { + return !keysHasCol(pSelect->pGroupByList); + } else if (NULL != pSelect->pPartitionByList) { + return !keysHasCol(pSelect->pPartitionByList); + } + } + } + return false; +} + static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, @@ -623,6 +644,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; + pTableScan->needCountEmptyTable = calcNeedCountEmpty(pCxt, pScanLogicNode); int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..e63f86743b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -347,6 +347,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 40d9b3ff8b..7e99f9e4dd 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -57,8 +57,11 @@ class TDTestCase: tdSql.query(f'select count(*) from {self.stbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + rows = [2, 0] function_names = ['count', 'hyperloglog'] - for function_name in function_names: + for i in range(2): + function_name = function_names[i] + row = rows[i] tdSql.query(f'select {function_name}(tbname) from {self.stbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -93,17 +96,17 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.stbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by t0') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by t0') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.stbname} group by c1)') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -113,17 +116,24 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1 interval(1s)') tdSql.checkRows(0) - tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname)') + tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + if 'count' == function_name: + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + elif 'hyperloglog' == function_name: + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 0, 0) def query_empty_ntb(self): tdSql.query(f'select count(*) from {self.ntbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + rows = [1, 0] function_names = ['count', 'hyperloglog'] - for function_name in function_names: + for i in range(2): + function_name = function_names[i] + row = rows[i] tdSql.query(f'select {function_name}(tbname) from {self.ntbname}') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -158,7 +168,7 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.ntbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by tbname') - tdSql.checkRows(0) + tdSql.checkRows(row) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.ntbname} group by c1)') @@ -170,10 +180,11 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by c1 interval(1s)') tdSql.checkRows(0) - tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) tdSql.checkData(0, 1, None) + def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') tdSql.checkEqual(tdSql.queryResult[0][0],tbnum*rownum) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py new file mode 100644 index 0000000000..eb5f069b3d --- /dev/null +++ b/tests/system-test/2-query/group_partition.py @@ -0,0 +1,183 @@ +# author : bobliu +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.row_nums = 10 + self.tb_nums = 10 + self.ts = 1537146000000 + self.dbname = "db" + self.stable = "stb" + + def prepare_db(self): + tdSql.execute(f" use {self.dbname} ") + tdSql.execute(f" create stable {self.dbname}.{self.stable} (ts timestamp , c1 int , c2 bigint , c3 float , c4 double , c5 smallint , c6 tinyint , c7 bool , c8 binary(36) , c9 nchar(36) , uc1 int unsigned,\ + uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags(t1 timestamp , t2 int , t3 bigint , t4 float , t5 double , t6 smallint , t7 tinyint , t8 bool , t9 binary(36)\ + , t10 nchar(36) , t11 int unsigned , t12 bigint unsigned ,t13 smallint unsigned , t14 tinyint unsigned ) ") + + for i in range(self.tb_nums): + tbname = f"{self.dbname}.sub_{self.stable}_{i}" + ts = self.ts + i*10000 + tdSql.execute(f"create table {tbname} using {self.dbname}.{self.stable} tags ({ts} , {i} , {i}*10 ,{i}*1.0,{i}*1.0 , 1 , 2, 'true', 'binary_{i}' ,'nchar_{i}',{i},{i},10,20 )") + + def insert_db(self, tb_nums, row_nums): + for i in range(tb_nums): + tbname = f"{self.dbname}.sub_{self.stable}_{i}" + ts_base = self.ts + i*10000 + for row in range(row_nums): + ts = ts_base + row*1000 + tdSql.execute(f"insert into {tbname} values({ts} , {row} , {row} , {row} , {row} , 1 , 2 , 'true' , 'binary_{row}' , 'nchar_{row}' , {row} , {row} , 1 ,2 )") + + + def test_groupby(self, check_num, real_num): + # tbname + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + # having filter out empty + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # tag + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by t2 ") + tdSql.checkRows(check_num) + + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") + tdSql.checkRows(check_num) + + # having + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # col where filter nothing + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + # tdSql.checkRows(check_num) + + ############### same with old ############### + # col where filter all + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + # tdSql.checkRows(0) + + # col where filter part + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") + tdSql.checkRows(real_num) + + # col + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(real_num) + + # count + sum(col) + tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(real_num) + + tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") + num = 0 + if real_num > 0: + num = self.row_nums + tdSql.checkRows(num) + + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} group by ts ") + tdSql.checkRows(real_num * self.row_nums) + + # col + tag + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") + tdSql.checkRows(real_num * self.row_nums) + + + def test_partitionby(self, check_num, real_num): + tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + # having filter out empty + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + #tag + tdSql.query(f"select count(*) from {self.dbname}.{self.stable} partition by t2 ") + tdSql.checkRows(check_num) + + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") + tdSql.checkRows(check_num) + + # having + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") + tdSql.checkRows(check_num - real_num) + + # col where filter nothing + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") + # tdSql.checkRows(check_num) + + ############### same with old ############### + # col where filter all + # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") + # tdSql.checkRows(0) + + # col where filter part + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") + tdSql.checkRows(real_num) + + #col + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(real_num) + + tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") + num = 0 + if real_num > 0: + num = self.row_nums + tdSql.checkRows(num) + + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} partition by ts ") + tdSql.checkRows(real_num * self.row_nums) + + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") + tdSql.checkRows(real_num * self.row_nums) + + def test_error(self): + tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") + tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") + + + def run(self): + tdSql.prepare() + self.prepare_db() + check_num = self.tb_nums + self.test_groupby(check_num, 0) + self.test_partitionby(check_num, 0) + # insert into half of tables + real_num = 5 + self.insert_db(real_num, self.row_nums) + self.test_groupby(check_num, real_num) + self.test_partitionby(check_num, real_num) + + # test old version before changed + # self.test_groupby(0, 0) + # self.test_partitionby(0, 0) + # self.insert_db(5, self.row_nums) + # self.test_groupby(5, 5) + # self.test_partitionby(5, 5) + + self.test_error() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index c16fc03c27..b6ef50dcda 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -1112,13 +1112,13 @@ class TDTestCase: def TS_3932(self): tdLog.debug("test insert data into stable") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 100) tdSql.checkData(1, 1, 200) tdSql.query(f"insert into nested.stable_1 (ts,tbname) values(now,'stable_1_1');") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 101) tdSql.checkData(1, 1, 200) @@ -1127,7 +1127,7 @@ class TDTestCase: coulmn_name = qlist[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;",queryTimes=5) - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 111) tdSql.checkData(1, 1, 200) @@ -1136,7 +1136,7 @@ class TDTestCase: coulmn_name = q_null_list[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;",queryTimes=5) - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 121) tdSql.checkData(1, 1, 200) @@ -1184,7 +1184,7 @@ class TDTestCase: def TS_3932_flushdb(self): tdLog.debug("test flush db and insert data into stable") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 121) tdSql.checkData(1, 1, 200) @@ -1192,7 +1192,7 @@ class TDTestCase: q_null_list = ['q_int_null', 'q_bigint_null', 'q_smallint_null', 'q_tinyint_null', 'q_float_null', 'q_double_null', 'q_bool_null', 'q_binary_null', 'q_nchar_null', 'q_ts_null'] tdSql.query(f"insert into nested.stable_1 (ts,tbname) values(now,'stable_1_1');") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 122) tdSql.checkData(1, 1, 200) @@ -1200,7 +1200,7 @@ class TDTestCase: coulmn_name = qlist[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 132) tdSql.checkData(1, 1, 200) @@ -1208,7 +1208,7 @@ class TDTestCase: coulmn_name = q_null_list[i] tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 142) tdSql.checkData(1, 1, 200) @@ -1223,7 +1223,7 @@ class TDTestCase: nested.stable_1 (ts,tbname,q_nchar) values(now+8a,'stable_1_1',1)\ nested.stable_1 (ts,tbname,q_ts) values(now+9a,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 152); tdSql.checkData(1, 1, 200); @@ -1330,7 +1330,7 @@ class TDTestCase: nested.stable_null_childtable (ts,tbname,q_ts) values(now+9a,'stable_null_childtable_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(2) + tdSql.checkRows(6) tdSql.checkData(0, 1, 162); tdSql.checkData(1, 1, 200); @@ -1349,7 +1349,7 @@ class TDTestCase: nested.stable_null_childtable (ts,tbname,q_int) values(now,'$^%$%^&',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); @@ -1387,7 +1387,7 @@ class TDTestCase: nested.stable_null_childtable(tbname,ts,q_int,q_binary) file '{self.testcasePath}/stable_null_childtable.csv';") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); @@ -1423,7 +1423,7 @@ class TDTestCase: tdSql.query(f"insert into nested.stable_null_childtable(tbname,ts,q_int,q_binary) file '{self.testcasePath}/stable_null_childtable.csv';") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(3) + tdSql.checkRows(7) tdSql.checkData(0, 1, 1); tdSql.checkData(1, 1, 162); tdSql.checkData(2, 1, 200); From 3bfc7b34862cbea233e33b9b5617f154bf28f295 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 00:47:15 +0800 Subject: [PATCH 002/198] adjust case --- tests/system-test/2-query/group_partition.py | 44 +++++++++++++------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index eb5f069b3d..58be8563fa 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -46,6 +46,12 @@ class TDTestCase: tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname order by count(*)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname having count(*)>=0") + tdSql.checkRows(check_num) + # having filter out empty tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -62,26 +68,26 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # col where filter nothing - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") - # tdSql.checkRows(check_num) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + tdSql.checkRows(check_num) - ############### same with old ############### # col where filter all - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") - # tdSql.checkRows(0) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + tdSql.checkRows(check_num) # col where filter part tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) # col tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) # count + sum(col) tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") num = 0 if real_num > 0: @@ -103,6 +109,12 @@ class TDTestCase: tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname having count(*)>=0") + tdSql.checkRows(check_num) + # having filter out empty tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -119,22 +131,25 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # col where filter nothing - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") - # tdSql.checkRows(check_num) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") + tdSql.checkRows(check_num) - ############### same with old ############### # col where filter all - # tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") - # tdSql.checkRows(0) + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") + tdSql.checkRows(check_num) # col where filter part tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") - tdSql.checkRows(real_num) + tdSql.checkRows(check_num) #col tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") tdSql.checkRows(real_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") num = 0 if real_num > 0: @@ -150,6 +165,7 @@ class TDTestCase: def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") + tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 interval(1d)") def run(self): From 3b0d1480f7783d05b4cab34a4548fd95c436c476 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 11:28:26 +0800 Subject: [PATCH 003/198] adjust case --- tests/script/tsim/field/3.sim | 14 ++++---- tests/script/tsim/field/4.sim | 18 +++++----- tests/script/tsim/field/6.sim | 38 ++++++++++---------- tests/script/tsim/field/bigint.sim | 4 +-- tests/script/tsim/field/double.sim | 4 +-- tests/script/tsim/field/smallint.sim | 4 +-- tests/script/tsim/field/unsigined_bigint.sim | 4 +-- tests/script/tsim/parser/mixed_blocks.sim | 3 +- tests/system-test/2-query/count_partition.py | 4 +-- tests/system-test/2-query/last_row.py | 2 +- 10 files changed, 48 insertions(+), 47 deletions(-) diff --git a/tests/script/tsim/field/3.sim b/tests/script/tsim/field/3.sim index 8b428febcd..4d4801c54a 100644 --- a/tests/script/tsim/field/3.sim +++ b/tests/script/tsim/field/3.sim @@ -447,44 +447,44 @@ if $data00 != 100 then endi print =============== step17 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step18 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/4.sim b/tests/script/tsim/field/4.sim index 361ca4c326..d83de81c88 100644 --- a/tests/script/tsim/field/4.sim +++ b/tests/script/tsim/field/4.sim @@ -619,56 +619,56 @@ if $data00 != 100 then endi print =============== step22 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step23 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/6.sim b/tests/script/tsim/field/6.sim index 52fd0b3780..2d2b5623c7 100644 --- a/tests/script/tsim/field/6.sim +++ b/tests/script/tsim/field/6.sim @@ -824,117 +824,117 @@ if $data00 != 25 then endi print =============== step28 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol3 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol3 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol4 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol4 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol5 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol5 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol6 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt group by tgcol6 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step29 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol1 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol1 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step30 -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 endi -sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol2 +sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where ts < 1626739440001 and tbcol1 = 1 and tbcol2 = 1 and tbcol3 = 1 and tbcol4 = 1 and tbcol5 = 1 and tbcol6 = 1 group by tgcol2 order by count(tbcol1) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/bigint.sim b/tests/script/tsim/field/bigint.sim index ce35cacd84..a791d6a1cb 100644 --- a/tests/script/tsim/field/bigint.sim +++ b/tests/script/tsim/field/bigint.sim @@ -127,7 +127,7 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 print $data10 $data11 $data12 $data13 $data14 $data15 $data16 if $data00 != 100 then @@ -136,7 +136,7 @@ if $data00 != 100 then endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/double.sim b/tests/script/tsim/field/double.sim index 1f0cea4be8..2daa78eb7a 100644 --- a/tests/script/tsim/field/double.sim +++ b/tests/script/tsim/field/double.sim @@ -127,14 +127,14 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/smallint.sim b/tests/script/tsim/field/smallint.sim index 66bfee5838..980d69297a 100644 --- a/tests/script/tsim/field/smallint.sim +++ b/tests/script/tsim/field/smallint.sim @@ -128,14 +128,14 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 100 then return -1 endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/field/unsigined_bigint.sim b/tests/script/tsim/field/unsigined_bigint.sim index baa57ce1f6..de8098c297 100644 --- a/tests/script/tsim/field/unsigined_bigint.sim +++ b/tests/script/tsim/field/unsigined_bigint.sim @@ -131,7 +131,7 @@ if $data00 != 100 then endi print =============== step6 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 print $data10 $data11 $data12 $data13 $data14 $data15 $data16 if $data00 != 100 then @@ -140,7 +140,7 @@ if $data00 != 100 then endi print =============== step7 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = 1 group by tgcol order by count(tbcol) desc print $data00 $data01 $data02 $data03 $data04 $data05 $data06 if $data00 != 25 then return -1 diff --git a/tests/script/tsim/parser/mixed_blocks.sim b/tests/script/tsim/parser/mixed_blocks.sim index 04e9df3ff4..8716192858 100644 --- a/tests/script/tsim/parser/mixed_blocks.sim +++ b/tests/script/tsim/parser/mixed_blocks.sim @@ -100,7 +100,8 @@ if $data05 != 1 then endi sql select max(c1), min(c1), sum(c1), avg(c1), count(c1), t1 from $stb where c1 > 0 group by t1 -if $rows != 1 then +if $rows != 2 then + print === rows $rows return -1 endi if $data00 != 319 then diff --git a/tests/system-test/2-query/count_partition.py b/tests/system-test/2-query/count_partition.py index f59376a979..e970b00cec 100644 --- a/tests/system-test/2-query/count_partition.py +++ b/tests/system-test/2-query/count_partition.py @@ -66,9 +66,9 @@ class TDTestCase: tdSql.checkRows(self.row_nums+1) tdSql.query(f"select count(c1) , count(t2) from {dbname}.stb where abs(c1+t2)=1 partition by tbname") - tdSql.checkRows(2) + tdSql.checkRows(10) tdSql.query(f"select count(c1) from {dbname}.stb where abs(c1+t2)=1 partition by tbname") - tdSql.checkRows(2) + tdSql.checkRows(10) tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname order by tbname") tdSql.checkRows(self.tb_nums) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index a6bcc2c5f1..5b989eb456 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -436,7 +436,7 @@ class TDTestCase: tdSql.checkData(1,1,None) tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 ") - tdSql.checkRows(2) + tdSql.checkRows(4) # filter by tbname tdSql.query(f"select last_row(c1) from {dbname}.stb1 where tbname = 'ct1' ") From ef5b5786b38e3d14465ae16780b47ed4f5935265 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 16:03:09 +0800 Subject: [PATCH 004/198] adjust --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1889492aa0..6df46ef4d0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -683,7 +683,7 @@ static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STable blockDataEmpty(pBlock); pBlock->info.rows = 1; pBlock->info.id.uid = tbInfo->uid; - pBlock->info.id.groupId = pOperator->dynamicTask ? tbInfo->uid : tbInfo->groupId; + pBlock->info.id.groupId = tbInfo->groupId; // only one row: set all col data to null & hasNull int32_t col_num = blockDataGetNumOfCols(pBlock); From 6e4703e5b43af47d83aca6acf4e5d8fe1dc16d7d Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 22 Dec 2023 22:58:40 +0800 Subject: [PATCH 005/198] adjust ts --- .../2-query/nestedQueryInterval.py | 100 ++++++++++-------- 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index b6ef50dcda..185848f6e0 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -5,6 +5,8 @@ import socket import os import threading +from datetime import timezone, datetime + from util.log import * from util.sql import * from util.cases import * @@ -1196,9 +1198,14 @@ class TDTestCase: tdSql.checkData(0, 1, 122) tdSql.checkData(1, 1, 200) + pd = datetime.datetime.now() + ts = int(datetime.datetime.timestamp(pd)*1000 - 10000) + print(f"start time {ts}") + for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values({ts+i},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 132) @@ -1206,92 +1213,101 @@ class TDTestCase: for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values({ts+i},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 142) tdSql.checkData(1, 1, 200) - tdSql.execute(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'stable_1_1',1) \ - nested.stable_1 (ts,tbname,q_bigint) values(now+1a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_smallint) values(now+2a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_tinyint) values(now+3a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_float) values(now+4a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_double) values(now+5a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_bool) values(now+6a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_binary) values(now+7a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_nchar) values(now+8a,'stable_1_1',1)\ - nested.stable_1 (ts,tbname,q_ts) values(now+9a,'stable_1_1',1);") + tdSql.execute(f"insert into nested.stable_1 (ts,tbname,q_int) values({ts},'stable_1_1',1) \ + nested.stable_1 (ts,tbname,q_bigint) values({ts+1},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_smallint) values({ts+2},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_tinyint) values({ts+3},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_float) values({ts+4},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_double) values({ts+5},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_bool) values({ts+6},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_binary) values({ts+7},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_nchar) values({ts+8},'stable_1_1',1)\ + nested.stable_1 (ts,tbname,q_ts) values({ts+9},'stable_1_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(6) tdSql.checkData(0, 1, 152); tdSql.checkData(1, 1, 200); - tdSql.query(f"insert into nested.stable_null_data (ts,tbname) values(now,'stable_null_data_1');") + tdSql.query(f"insert into nested.stable_null_data (ts,tbname) values({ts},'stable_null_data_1');") + ts = ts + 1 tdSql.query(f"select tbname,count(*) from nested.stable_null_data_1 group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 22); for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 32) for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") - + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 42) - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values(now,'stable_null_data_1',1) \ - nested.stable_null_data (ts,tbname,q_bigint) values(now+1a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_smallint) values(now+2a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_tinyint) values(now+3a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_float) values(now+4a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_double) values(now+5a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_bool) values(now+6a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_binary) values(now+7a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_nchar) values(now+8a,'stable_null_data_1',1)\ - nested.stable_null_data (ts,tbname,q_ts) values(now+9a,'stable_null_data_1',1);") + tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values({ts},'stable_null_data_1',1) \ + nested.stable_null_data (ts,tbname,q_bigint) values({ts+1},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_smallint) values({ts+2},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_tinyint) values({ts+3},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_float) values({ts+4},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_double) values({ts+5},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_bool) values({ts+6},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_binary) values({ts+7},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_nchar) values({ts+8},'stable_null_data_1',1)\ + nested.stable_null_data (ts,tbname,q_ts) values({ts+9},'stable_null_data_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 52); - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname) values(now,'stable_null_childtable_1');") + tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname) values({ts},'stable_null_childtable_1');") + ts = ts + 1 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 22) for i in range(10): coulmn_name = qlist[i] - tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 32) for i in range(10): coulmn_name = q_null_list[i] - tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values({ts+i},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 42); - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int) values(now,'stable_null_childtable_1',1) \ - nested.stable_null_childtable (ts,tbname,q_bigint) values(now+1a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_smallint) values(now+2a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_tinyint) values(now+3a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_float) values(now+4a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_double) values(now+5a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_bool) values(now+6a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_binary) values(now+7a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_nchar) values(now+8a,'stable_null_childtable_1',1)\ - nested.stable_null_childtable (ts,tbname,q_ts) values(now+9a,'stable_null_childtable_1',1);") + tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int) values({ts},'stable_null_childtable_1',1) \ + nested.stable_null_childtable (ts,tbname,q_bigint) values({ts+1},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_smallint) values({ts+2},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_tinyint) values({ts+3},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_float) values({ts+4},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_double) values({ts+5},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_bool) values({ts+6},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_binary) values({ts+7},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_nchar) values({ts+8},'stable_null_childtable_1',1)\ + nested.stable_null_childtable (ts,tbname,q_ts) values({ts+9},'stable_null_childtable_1',1);") + ts = ts + 20 tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 52); @@ -1344,9 +1360,9 @@ class TDTestCase: #test special character - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'!@!@$$^$',1) \ - nested.stable_null_data (ts,tbname,q_int) values(now,'%^$^&^&',1) \ - nested.stable_null_childtable (ts,tbname,q_int) values(now,'$^%$%^&',1);") + tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now+10a,'!@!@$$^$',1) \ + nested.stable_null_data (ts,tbname,q_int) values(now+10a,'%^$^&^&',1) \ + nested.stable_null_childtable (ts,tbname,q_int) values(now+10a,'$^%$%^&',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(7) From c9b69316c1f8f71281d1506ed681989b5f4e4990 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 01:19:16 +0800 Subject: [PATCH 006/198] calcNeedCountEmpty --- include/libs/nodes/plannodes.h | 1 + source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/planner/src/planLogicCreater.c | 9 +++++++++ source/libs/planner/src/planOptimizer.c | 1 + source/libs/planner/src/planPhysiCreater.c | 22 +++++++++++----------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 4b3c846389..55ac3b86bf 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -147,6 +147,7 @@ typedef struct SAggLogicNode { bool isGroupTb; bool isPartTb; // true if partition keys has tbname bool hasGroup; + bool isCountByTag; // true if hasCountFunc & part by tag/tbname } SAggLogicNode; typedef struct SProjectLogicNode { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 8a154dcf00..2af31a8c7a 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -453,6 +453,7 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { COPY_SCALAR_FIELD(isGroupTb); COPY_SCALAR_FIELD(isPartTb); COPY_SCALAR_FIELD(hasGroup); + COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2adc5b3072..17b1b272ba 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -748,6 +748,15 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; + bool isCountByTag = false; + if (pSelect->hasCountFunc) { + if (pSelect->pGroupByList) { + isCountByTag = !keysHasCol(pSelect->pGroupByList); + } else if (pSelect->pPartitionByList) { + isCountByTag = !keysHasCol(pSelect->pPartitionByList); + } + } + pAgg->isCountByTag = isCountByTag; if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index aa3181e166..9ecbd3ae9f 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2783,6 +2783,7 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, pNew->isGroupTb = pAgg->isGroupTb; pNew->isPartTb = pAgg->isPartTb; pNew->hasGroup = pAgg->hasGroup; + pNew->isCountByTag = pAgg->isCountByTag; pNew->node.pChildren = nodesCloneList(pAgg->node.pChildren); int32_t code = 0; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 901927b1d1..9c4e9f97ab 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -37,6 +37,7 @@ typedef struct SPhysiPlanContext { SArray* pLocationHelper; bool hasScan; bool hasSysScan; + SLogicNode* pNode; // tmp record LogicSubplan->pNode } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey, int32_t keyBufSize) { @@ -592,17 +593,13 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog if (pScanLogicNode->interval > 0) { return false; } + // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { - SSelectStmt* pSelect = (SSelectStmt*)pRoot; - // select & count - if (pSelect->hasCountFunc) { - // key only accept tag/tbname - if (NULL != pSelect->pGroupByList) { - return !keysHasCol(pSelect->pGroupByList); - } else if (NULL != pSelect->pPartitionByList) { - return !keysHasCol(pSelect->pPartitionByList); - } + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) + && pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { + SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; + if (pNode->isCountByTag) { + return true; } } return false; @@ -2302,7 +2299,9 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } + pCxt->pNode = pLogicSubplan->pNode; code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); + pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); } @@ -2454,7 +2453,8 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), .hasScan = false, - .hasSysScan = false}; + .hasSysScan = false, + .pNode = NULL}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } From 651b681d4c95879c5a20668c8b85c45583383e98 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 18:45:51 +0800 Subject: [PATCH 007/198] adjust case --- tests/system-test/2-query/count.py | 4 ++-- tests/system-test/2-query/last_row.py | 4 ++-- tests/system-test/2-query/nestedQueryInterval.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 7e99f9e4dd..0e1d332d89 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -119,8 +119,8 @@ class TDTestCase: tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname order by tbname)') tdSql.checkRows(1) if 'count' == function_name: - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 2) elif 'hyperloglog' == function_name: tdSql.checkData(0, 0, 0) tdSql.checkData(0, 0, 0) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 5b989eb456..6469b3ea54 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -435,8 +435,8 @@ class TDTestCase: tdSql.checkData(1,0,'ct4') tdSql.checkData(1,1,None) - tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 ") - tdSql.checkRows(4) + tdSql.query(f"select t1 ,count(c1) from {dbname}.stb1 partition by t1 having count(c1)>0") + tdSql.checkRows(2) # filter by tbname tdSql.query(f"select last_row(c1) from {dbname}.stb1 where tbname = 'ct1' ") diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index 185848f6e0..07b5519432 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -1113,8 +1113,8 @@ class TDTestCase: def TS_3932(self): tdLog.debug("test insert data into stable") - tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") - tdSql.checkRows(6) + tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname having count(*)>0 order by tbname;") + tdSql.checkRows(2) tdSql.checkData(0, 1, 100) tdSql.checkData(1, 1, 200) From 52c244d9ec0a9ce286a25379b3516159f0380222 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 23:48:37 +0800 Subject: [PATCH 008/198] adjust --- source/libs/planner/src/planPhysiCreater.c | 22 +++++++++++++++----- tests/system-test/2-query/count.py | 2 +- tests/system-test/2-query/group_partition.py | 10 ++++++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 9c4e9f97ab..5c78ff4432 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -595,13 +595,25 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog } // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) - && pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { - SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; - if (pNode->isCountByTag) { - return true; + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + // case1 root select + SSelectStmt* pSelect = (SSelectStmt*)pRoot; + if (pSelect->hasCountFunc) { + if (NULL != pSelect->pGroupByList) { + return !keysHasCol(pSelect->pGroupByList); + } else if (NULL != pSelect->pPartitionByList) { + return !keysHasCol(pSelect->pPartitionByList); + } + } + // case2 inner agg + if (pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { + SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; + if (pNode->isCountByTag) { + return true; + } } } + return false; } diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 0e1d332d89..a80c6f07e9 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -123,7 +123,7 @@ class TDTestCase: tdSql.checkData(0, 1, 2) elif 'hyperloglog' == function_name: tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) def query_empty_ntb(self): tdSql.query(f'select count(*) from {self.ntbname}') diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 58be8563fa..c08ea98ef6 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -83,6 +83,10 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + #inner select + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") + tdSql.checkRows(check_num) + # count + sum(col) tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) @@ -146,6 +150,10 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + #inner select + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") + tdSql.checkRows(check_num) + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") tdSql.checkRows(real_num) @@ -196,4 +204,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From da0ecc18b95af94abff05cece0acd002dfd38bd0 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 02:51:52 +0800 Subject: [PATCH 009/198] test --- source/libs/planner/src/planPhysiCreater.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 5c78ff4432..397258d6ca 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2311,9 +2311,11 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - pCxt->pNode = pLogicSubplan->pNode; + + if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pLogicSubplan->pNode)) { + pCxt->pNode = pLogicSubplan->pNode; + } code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); - pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); } @@ -2414,6 +2416,7 @@ static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogi break; } } + pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code) { *pPhysiPlan = pPlan; From 15694df001da02ab54ced68b014d6e48fe4a1cff Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 11:54:10 +0800 Subject: [PATCH 010/198] adjust --- source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/planner/src/planLogicCreater.c | 20 ++++++++-------- source/libs/planner/src/planOptimizer.c | 1 - source/libs/planner/src/planPhysiCreater.c | 27 +++------------------- tests/parallel_test/cases.task | 4 ++++ 5 files changed, 19 insertions(+), 35 deletions(-) diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 2af31a8c7a..c68fd81d22 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -424,6 +424,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(groupOrderScan); COPY_SCALAR_FIELD(onlyMetaCtbIdx); COPY_SCALAR_FIELD(filesetDelimited); + COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } @@ -453,7 +454,6 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { COPY_SCALAR_FIELD(isGroupTb); COPY_SCALAR_FIELD(isPartTb); COPY_SCALAR_FIELD(hasGroup); - COPY_SCALAR_FIELD(isCountByTag); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 17b1b272ba..f55689c472 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -486,6 +486,16 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = tagScanSetExecutionMode(pScan); } + bool isCountByTag = false; + if (pSelect->hasCountFunc) { + if (pSelect->pGroupByList) { + isCountByTag = !keysHasCol(pSelect->pGroupByList); + } else if (pSelect->pPartitionByList) { + isCountByTag = !keysHasCol(pSelect->pPartitionByList); + } + } + pScan->isCountByTag = isCountByTag; + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pScan; } else { @@ -748,15 +758,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; - bool isCountByTag = false; - if (pSelect->hasCountFunc) { - if (pSelect->pGroupByList) { - isCountByTag = !keysHasCol(pSelect->pGroupByList); - } else if (pSelect->pPartitionByList) { - isCountByTag = !keysHasCol(pSelect->pPartitionByList); - } - } - pAgg->isCountByTag = isCountByTag; + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 9ecbd3ae9f..aa3181e166 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2783,7 +2783,6 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, pNew->isGroupTb = pAgg->isGroupTb; pNew->isPartTb = pAgg->isPartTb; pNew->hasGroup = pAgg->hasGroup; - pNew->isCountByTag = pAgg->isCountByTag; pNew->node.pChildren = nodesCloneList(pAgg->node.pChildren); int32_t code = 0; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 397258d6ca..0db0fd77fb 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -37,7 +37,6 @@ typedef struct SPhysiPlanContext { SArray* pLocationHelper; bool hasScan; bool hasSysScan; - SLogicNode* pNode; // tmp record LogicSubplan->pNode } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey, int32_t keyBufSize) { @@ -595,23 +594,8 @@ static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLog } // limit: root node is select SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { - // case1 root select - SSelectStmt* pSelect = (SSelectStmt*)pRoot; - if (pSelect->hasCountFunc) { - if (NULL != pSelect->pGroupByList) { - return !keysHasCol(pSelect->pGroupByList); - } else if (NULL != pSelect->pPartitionByList) { - return !keysHasCol(pSelect->pPartitionByList); - } - } - // case2 inner agg - if (pCxt->pNode && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pCxt->pNode)) { - SAggLogicNode* pNode = (SAggLogicNode*)pCxt->pNode; - if (pNode->isCountByTag) { - return true; - } - } + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) && pScanLogicNode->isCountByTag) { + return true; } return false; @@ -2312,9 +2296,6 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pLogicSubplan->pNode)) { - pCxt->pNode = pLogicSubplan->pNode; - } code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); @@ -2416,7 +2397,6 @@ static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogi break; } } - pCxt->pNode = NULL; if (TSDB_CODE_SUCCESS == code) { *pPhysiPlan = pPlan; @@ -2468,8 +2448,7 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), .hasScan = false, - .hasSysScan = false, - .pNode = NULL}; + .hasSysScan = false}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e63f86743b..5e7ef407f0 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -348,6 +348,10 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/group_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py From baee536cb06976332fe4d22305305edc0ee1d013 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 11:54:23 +0800 Subject: [PATCH 011/198] adjust --- include/libs/nodes/plannodes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 55ac3b86bf..f0383e8168 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -119,6 +119,7 @@ typedef struct SScanLogicNode { bool groupOrderScan; bool onlyMetaCtbIdx; // for tag scan with no tbname bool filesetDelimited; // returned blocks delimited by fileset + bool isCountByTag; // true if selectstmt hasCountFunc & part by tag/tbname } SScanLogicNode; typedef struct SJoinLogicNode { @@ -147,7 +148,6 @@ typedef struct SAggLogicNode { bool isGroupTb; bool isPartTb; // true if partition keys has tbname bool hasGroup; - bool isCountByTag; // true if hasCountFunc & part by tag/tbname } SAggLogicNode; typedef struct SProjectLogicNode { From 31c54e5edabc71924d72befc3194942b8b40fb73 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 12:42:25 +0800 Subject: [PATCH 012/198] adjust --- tests/system-test/2-query/count.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index a80c6f07e9..9bf9a4b9cc 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -182,8 +182,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 2) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') From f672df239e176a5256e76f9937d9e19a8da5b2cc Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 13:33:45 +0800 Subject: [PATCH 013/198] adjust --- tests/system-test/2-query/count.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 9bf9a4b9cc..3ba651622a 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -182,8 +182,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname order by tbname)') tdSql.checkRows(1) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') From 62cfcefbb0e26c29c06ff1d22f414a71018274c8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 00:34:04 +0800 Subject: [PATCH 014/198] refactor: remove stream-scan-history event for stream task. --- include/libs/stream/tstream.h | 19 +++-- source/dnode/mnode/impl/src/mndStream.c | 88 +++++++++++----------- source/dnode/vnode/src/tq/tq.c | 27 +++---- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- source/libs/parser/src/parTranslater.c | 2 +- source/libs/stream/src/streamExec.c | 2 +- source/libs/stream/src/streamMeta.c | 2 +- source/libs/stream/src/streamStart.c | 20 +++-- source/libs/stream/src/streamTaskSm.c | 32 ++++---- 9 files changed, 99 insertions(+), 95 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2d2db5c1dc..0c09e0e1ea 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -81,7 +81,7 @@ typedef enum ETaskStatus { TASK_STATUS__HALT, // pause, but not be manipulated by user command TASK_STATUS__PAUSE, // pause TASK_STATUS__CK, // stream task is in checkpoint status, no data are allowed to put into inputQ anymore - TASK_STATUS__STREAM_SCAN_HISTORY, +// TASK_STATUS__STREAM_SCAN_HISTORY, } ETaskStatus; enum { @@ -138,15 +138,14 @@ enum { typedef enum EStreamTaskEvent { TASK_EVENT_INIT = 0x1, TASK_EVENT_INIT_SCANHIST = 0x2, - TASK_EVENT_INIT_STREAM_SCANHIST = 0x3, - TASK_EVENT_SCANHIST_DONE = 0x4, - TASK_EVENT_STOP = 0x5, - TASK_EVENT_GEN_CHECKPOINT = 0x6, - TASK_EVENT_CHECKPOINT_DONE = 0x7, - TASK_EVENT_PAUSE = 0x8, - TASK_EVENT_RESUME = 0x9, - TASK_EVENT_HALT = 0xA, - TASK_EVENT_DROPPING = 0xB, + TASK_EVENT_SCANHIST_DONE = 0x3, + TASK_EVENT_STOP = 0x4, + TASK_EVENT_GEN_CHECKPOINT = 0x5, + TASK_EVENT_CHECKPOINT_DONE = 0x6, + TASK_EVENT_PAUSE = 0x7, + TASK_EVENT_RESUME = 0x8, + TASK_EVENT_HALT = 0x9, + TASK_EVENT_DROPPING = 0xA, } EStreamTaskEvent; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 25f51b85df..2464a9207e 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2886,38 +2886,38 @@ static SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { return NULL; } -static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStreamExecInfo *pExecNode) { - if (pTaskEntry->status == TASK_STATUS__STREAM_SCAN_HISTORY && pTaskEntry->statusLastDuration >= 10) { - if (!pTaskEntry->inputQChanging && pTaskEntry->inputQUnchangeCounter > 10) { - int32_t numOfReady = 0; - int32_t numOfTotal = 0; - for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { - STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); - if (pTaskEntry->id.streamId == pId->streamId) { - numOfTotal++; - - if (pTaskEntry->id.taskId != pId->taskId) { - STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); - if (pEntry->status == TASK_STATUS__READY) { - numOfReady++; - } - } - } - } - - if (numOfReady > 0) { - mDebug("stream:0x%" PRIx64 - " %d tasks are ready, %d tasks in stream-scan-history for more than 50s, drop related fill-history task", - pTaskEntry->id.streamId, numOfReady, numOfTotal - numOfReady); - return true; - } else { - return false; - } - } - } - - return false; -} +//static bool needDropRelatedFillhistoryTask(STaskStatusEntry *pTaskEntry, SStreamExecInfo *pExecNode) { +// if (pTaskEntry->status == TASK_STATUS__STREAM_SCAN_HISTORY && pTaskEntry->statusLastDuration >= 10) { +// if (!pTaskEntry->inputQChanging && pTaskEntry->inputQUnchangeCounter > 10) { +// int32_t numOfReady = 0; +// int32_t numOfTotal = 0; +// for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { +// STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); +// if (pTaskEntry->id.streamId == pId->streamId) { +// numOfTotal++; +// +// if (pTaskEntry->id.taskId != pId->taskId) { +// STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); +// if (pEntry->status == TASK_STATUS__READY) { +// numOfReady++; +// } +// } +// } +// } +// +// if (numOfReady > 0) { +// mDebug("stream:0x%" PRIx64 +// " %d tasks are ready, %d tasks in stream-scan-history for more than 50s, drop related fill-history task", +// pTaskEntry->id.streamId, numOfReady, numOfTotal - numOfReady); +// return true; +// } else { +// return false; +// } +// } +// } +// +// return false; +//} // currently only handle the sink task // 1. sink task, drop related fill-history task msg is missing @@ -3091,18 +3091,18 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (p->status != TASK_STATUS__READY) { mDebug("received s-task:0x%" PRIx64 " not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status)); - if (p->status == TASK_STATUS__STREAM_SCAN_HISTORY) { - bool drop = needDropRelatedFillhistoryTask(pTaskEntry, &execInfo); - if (drop) { - SStreamObj *pStreamObj = mndGetStreamObj(pMnode, pTaskEntry->id.streamId); - if (pStreamObj == NULL) { - mError("failed to acquire the streamObj:0x%" PRIx64 " it may have been dropped", pStreamObj->uid); - } else { - mndDropRelatedFillhistoryTask(pMnode, pTaskEntry, pStreamObj); - mndReleaseStream(pMnode, pStreamObj); - } - } - } +// if (p->status == TASK_STATUS__STREAM_SCAN_HISTORY) { +// bool drop = needDropRelatedFillhistoryTask(pTaskEntry, &execInfo); +// if (drop) { +// SStreamObj *pStreamObj = mndGetStreamObj(pMnode, pTaskEntry->id.streamId); +// if (pStreamObj == NULL) { +// mError("failed to acquire the streamObj:0x%" PRIx64 " it may have been dropped", pStreamObj->uid); +// } else { +// mndDropRelatedFillhistoryTask(pMnode, pTaskEntry, pStreamObj); +// mndReleaseStream(pMnode, pStreamObj); +// } +// } +// } } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c8cc4f9d54..a8b5842b93 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1037,15 +1037,16 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pMeta, pStreamTask); } else { - STimeWindow* pWindow = &pTask->dataRange.window; - ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) || streamTaskShouldStop(pTask)); - - // Not update the fill-history time window until the state transfer is completed. - tqDebug("s-task:%s scan-history in stream time window completed, start to handle data from WAL, startVer:%" PRId64 - ", window:%" PRId64 " - %" PRId64, - id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); - - code = streamTaskScanHistoryDataComplete(pTask); + ASSERT(0); +// STimeWindow* pWindow = &pTask->dataRange.window; +// ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) || streamTaskShouldStop(pTask)); +// +// // Not update the fill-history time window until the state transfer is completed. +// tqDebug("s-task:%s scan-history in stream time window completed, start to handle data from WAL, startVer:%" PRId64 +// ", window:%" PRId64 " - %" PRId64, +// id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); +// +// code = streamTaskScanHistoryDataComplete(pTask); } atomic_store_32(&pTask->status.inScanHistorySentinel, 0); @@ -1170,7 +1171,7 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, } else if (status == TASK_STATUS__UNINIT) { // todo: fill-history task init ? if (pTask->info.fillHistory == 0) { - EStreamTaskEvent event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + EStreamTaskEvent event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; streamTaskHandleEvent(pTask->status.pSM, event); } } @@ -1363,9 +1364,9 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { taosThreadMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask, NULL); - if (status == TASK_STATUS__STREAM_SCAN_HISTORY) { - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); - } +// if (status == TASK_STATUS__STREAM_SCAN_HISTORY) { +// streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); +// } SStreamTaskId id = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId}; streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &id); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index f576345f64..2991e3cef5 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -545,7 +545,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + EStreamTaskEvent event = /*(HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; streamTaskHandleEvent(p->status.pSM, event); } else if (!restored) { tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..6f3915c4d7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7989,7 +7989,7 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void } if (TSDB_CODE_SUCCESS == code) { if (interval.interval > 0) { - pStmt->pReq->lastTs = taosTimeTruncate(lastTs, &interval); + pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, interval.precision); } else { pStmt->pReq->lastTs = lastTs; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 25f32195be..7d2a572cda 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -646,7 +646,7 @@ bool streamTaskIsIdle(const SStreamTask* pTask) { bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { ETaskStatus st = streamTaskGetStatus(pTask, NULL); - return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__STREAM_SCAN_HISTORY || + return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY/* || st == TASK_STATUS__STREAM_SCAN_HISTORY*/ || st == TASK_STATUS__CK); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d09d370a36..f2dcc3bcc6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1484,7 +1484,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { continue; } - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + EStreamTaskEvent event = /*(HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); if (ret != TSDB_CODE_SUCCESS) { stError("vgId:%d failed to handle event:%d", pMeta->vgId, event); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index ea5e2edc09..70f83e27a5 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -52,7 +52,7 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); ETaskStatus status = streamTaskGetStatus(pTask, &p); - if ((status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY) && + if ((status == TASK_STATUS__SCAN_HISTORY/* || status == TASK_STATUS__STREAM_SCAN_HISTORY*/) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->upstreamInfo.pList); stDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", @@ -158,7 +158,7 @@ int32_t streamTaskStartScanHistory(SStreamTask* pTask) { ETaskStatus status = streamTaskGetStatus(pTask, NULL); ASSERT(pTask->status.downstreamReady == 1 && - ((status == TASK_STATUS__SCAN_HISTORY) || (status == TASK_STATUS__STREAM_SCAN_HISTORY))); + ((status == TASK_STATUS__SCAN_HISTORY)/* || (status == TASK_STATUS__STREAM_SCAN_HISTORY)*/)); if (level == TASK_LEVEL__SOURCE) { return doStartScanHistoryTask(pTask); @@ -374,10 +374,14 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { char* p = NULL; ETaskStatus status = streamTaskGetStatus(pTask, &p); - ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY); + ASSERT(status == TASK_STATUS__SCAN_HISTORY/* || status == TASK_STATUS__STREAM_SCAN_HISTORY*/); - stDebug("s-task:%s enter into scan-history data stage, status:%s", id, p); - streamTaskStartScanHistory(pTask); + if (pTask->info.fillHistory == 1) { + stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", id, p); + streamTaskStartScanHistory(pTask); + } else { + stDebug("s-task:%s scan wal data, status:%s", id, p); + } // NOTE: there will be an deadlock if launch fill history here. // // start the related fill-history task, when current task is ready @@ -391,7 +395,7 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { void doProcessDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event; if (pTask->info.fillHistory == 0) { - event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; } else { event = TASK_EVENT_INIT_SCANHIST; } @@ -631,7 +635,7 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory char* p = NULL; ETaskStatus status = streamTaskGetStatus(pTask, &p); - if (status != TASK_STATUS__SCAN_HISTORY && status != TASK_STATUS__STREAM_SCAN_HISTORY) { + if (status != TASK_STATUS__SCAN_HISTORY /*&& status != TASK_STATUS__STREAM_SCAN_HISTORY*/) { stError("s-task:%s not in scan-history status, status:%s return upstream:0x%x scan-history finish directly", id, p, pReq->upstreamTaskId); @@ -693,7 +697,7 @@ int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) { return TSDB_CODE_INVALID_MSG; } - ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY); + ASSERT(status == TASK_STATUS__SCAN_HISTORY/* || status == TASK_STATUS__STREAM_SCAN_HISTORY*/); SStreamMeta* pMeta = pTask->pMeta; // execute in the scan history complete call back msg, ready to process data from inputQ diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 51e6ad23fe..4bd6483f7f 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -31,14 +31,14 @@ SStreamTaskState StreamTaskStatusList[9] = { {.state = TASK_STATUS__HALT, .name = "halt"}, {.state = TASK_STATUS__PAUSE, .name = "paused"}, {.state = TASK_STATUS__CK, .name = "checkpoint"}, - {.state = TASK_STATUS__STREAM_SCAN_HISTORY, .name = "stream-scan-history"}, +// {.state = TASK_STATUS__STREAM_SCAN_HISTORY, .name = "stream-scan-history"}, }; SStreamEventInfo StreamTaskEventList[12] = { {.event = 0, .name = ""}, // dummy event, place holder {.event = TASK_EVENT_INIT, .name = "initialize"}, {.event = TASK_EVENT_INIT_SCANHIST, .name = "scan-history-init"}, - {.event = TASK_EVENT_INIT_STREAM_SCANHIST, .name = "stream-scan-history-init"}, +// {.event = TASK_EVENT_INIT_STREAM_SCANHIST, .name = "stream-scan-history-init"}, {.event = TASK_EVENT_SCANHIST_DONE, .name = "scan-history-completed"}, {.event = TASK_EVENT_STOP, .name = "stopping"}, {.event = TASK_EVENT_GEN_CHECKPOINT, .name = "checkpoint"}, @@ -110,12 +110,12 @@ int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) { // todo check rsp code for handle Event:TASK_EVENT_SCANHIST_DONE static bool isInvalidStateTransfer(ETaskStatus state, const EStreamTaskEvent event) { - if (event == TASK_EVENT_INIT_STREAM_SCANHIST || event == TASK_EVENT_INIT || event == TASK_EVENT_INIT_SCANHIST) { + if (/*event == TASK_EVENT_INIT_STREAM_SCANHIST || */event == TASK_EVENT_INIT || event == TASK_EVENT_INIT_SCANHIST) { return (state != TASK_STATUS__UNINIT); } if (event == TASK_EVENT_SCANHIST_DONE) { - return (state != TASK_STATUS__SCAN_HISTORY && state != TASK_STATUS__STREAM_SCAN_HISTORY); + return (state != TASK_STATUS__SCAN_HISTORY/* && state != TASK_STATUS__STREAM_SCAN_HISTORY*/); } if (event == TASK_EVENT_GEN_CHECKPOINT) { @@ -482,14 +482,14 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); +// taosArrayPush(streamTaskSMTrans, &trans); // scan-history related event trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); +// taosArrayPush(streamTaskSMTrans, &trans); // halt stream task, from other task status trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); @@ -499,8 +499,8 @@ void doInitStateTransferTable(void) { SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); +// taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); @@ -519,8 +519,8 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); +// taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); @@ -554,8 +554,8 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); +// taosArrayPush(streamTaskSMTrans, &trans); // dropping related event trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); @@ -574,7 +574,7 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); - taosArrayPush(streamTaskSMTrans, &trans); +// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); +// taosArrayPush(streamTaskSMTrans, &trans); } //clang-format on \ No newline at end of file From 2c9fa56e9c3ea6a0bbca237800da86995aa98cee Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 26 Dec 2023 10:24:50 +0800 Subject: [PATCH 015/198] calcNeedCountEmpty --- source/libs/planner/src/planPhysiCreater.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0db0fd77fb..444bda84cd 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,14 +587,12 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static bool calcNeedCountEmpty(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - // refuse interval +static bool calcNeedCountEmpty(SScanLogicNode* pScanLogicNode) { if (pScanLogicNode->interval > 0) { return false; } - // limit: root node is select - SNode* pRoot = pCxt->pPlanCxt->pAstRoot; - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot) && pScanLogicNode->isCountByTag) { + + if (pScanLogicNode->isCountByTag) { return true; } @@ -637,7 +635,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; - pTableScan->needCountEmptyTable = calcNeedCountEmpty(pCxt, pScanLogicNode); + pTableScan->needCountEmptyTable = calcNeedCountEmpty(pScanLogicNode); int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { From b74e98b25c11d40a290d03988dd848d6a75319c2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 14:17:46 +0800 Subject: [PATCH 016/198] fix(stream): set the correct split timestamp for session/state window. --- source/dnode/mnode/impl/src/mndScheduler.c | 6 +++--- source/libs/parser/src/parTranslater.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 3ef4c9a4d2..39121cecd9 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -251,7 +251,7 @@ int32_t doAddSinkTask(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, in static int32_t addSourceTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SArray* pSinkTaskList, SStreamObj* pStream, SSubplan* plan, uint64_t uid, SEpSet* pEpset, bool fillHistory, - bool hasExtraSink, int64_t firstWindowSkey, bool hasFillHistory) { + bool hasExtraSink, int64_t nextWindowSkey, bool hasFillHistory) { SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, fillHistory, pStream->conf.triggerParam, pTaskList, hasFillHistory); if (pTask == NULL) { @@ -262,7 +262,7 @@ static int32_t addSourceTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, STimeWindow* pWindow = &pTask->dataRange.window; pWindow->skey = INT64_MIN; - pWindow->ekey = firstWindowSkey - 1; + pWindow->ekey = nextWindowSkey - 1; mDebug("add source task 0x%x window:%" PRId64 " - %" PRId64, pTask->id.taskId, pWindow->skey, pWindow->ekey); // sink or dispatch @@ -382,7 +382,7 @@ static int32_t doAddSourceTask(SArray* pTaskList, bool isFillhistory, int64_t ui epsetAssign(&(pTask)->info.mnodeEpset, pEpset); - // todo set the correct ts, which should be last key of queried table. + // set the correct ts, which is the last key of queried table. STimeWindow* pWindow = &pTask->dataRange.window; pWindow->skey = INT64_MIN; pWindow->ekey = nextWindowSkey - 1; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6f3915c4d7..cf6db9f549 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7991,7 +7991,7 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void if (interval.interval > 0) { pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, interval.precision); } else { - pStmt->pReq->lastTs = lastTs; + pStmt->pReq->lastTs = lastTs + 1; // start key of the next time window } code = buildCmdMsg(&cxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, pStmt->pReq); } From 2cd61804bcfd3b70b10d85464f6334a557c97e26 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 14:40:15 +0800 Subject: [PATCH 017/198] test:update the test case. --- tests/system-test/8-stream/stream_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index e838950bb8..3ebc255114 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -89,7 +89,7 @@ class TDTestCase: sql = "select count(*) from sta" # loop wait max 60s to check count is ok tdLog.info("loop wait result ...") - tdSql.checkDataLoop(0, 0, 99999, sql, loopCount=120, waitTime=0.5) + tdSql.checkDataLoop(0, 0, 100000, sql, loopCount=120, waitTime=0.5) time.sleep(5) From e2564a61899ca75df168f62d8ad1c0dc4a3a241c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 26 Dec 2023 18:35:49 +0800 Subject: [PATCH 018/198] fix issue --- source/libs/executor/src/scanoperator.c | 4 ++- .../executor/src/streameventwindowoperator.c | 1 + source/libs/stream/src/streamSessionState.c | 8 ++--- tests/script/tsim/stream/event2.sim | 33 +++++++++++++++++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..cf5ea95088 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1961,8 +1961,10 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { } static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) { - if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { + if (pInfo->pUpdateInfo) { pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); + } + if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { checkUpdateData(pInfo, true, pBlock, true); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); if (pInfo->pUpdateDataRes->info.rows > 0) { diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 6adab74344..bd247eba07 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -622,6 +622,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { } setEventWindowFlag(pAggSup, &curInfo); if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) { + saveSessionOutputBuf(pAggSup, &curInfo.winInfo); continue; } diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 45d656c456..c71edccb99 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -449,13 +449,13 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void SSHashObj* pSessionBuff = getRowStateBuff(pCur->pStreamFileState); void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); - if (!ppBuff) { - return TSDB_CODE_FAILED; + SArray* pWinStates = NULL; + if (ppBuff) { + pWinStates = (SArray*)(*ppBuff); } - SArray* pWinStates = (SArray*)(*ppBuff); - int32_t size = taosArrayGetSize(pWinStates); if (pCur->buffIndex >= 0) { + int32_t size = taosArrayGetSize(pWinStates); if (pCur->buffIndex >= size) { return TSDB_CODE_FAILED; } diff --git a/tests/script/tsim/stream/event2.sim b/tests/script/tsim/stream/event2.sim index eb9fca46e6..9fc7615fb8 100644 --- a/tests/script/tsim/stream/event2.sim +++ b/tests/script/tsim/stream/event2.sim @@ -31,6 +31,8 @@ sql insert into t4 values(1648791223000,1,1,10,3.0); sql insert into t4 values(1648791233000,0,2,11,1.0); sql insert into t4 values(1648791243000,1,9,12,2.0); +sleep 1000 + sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 1 into streamt0 as select _wstart as s, count(*) c1, sum(b), max(c), _wend as e from st partition by tbname event_window start with a = 0 end with b = 9; sleep 1000 @@ -79,6 +81,37 @@ if $data21 != 2 then goto loop0 endi +sql insert into t3 values(1648791222000,0,1,7,3.0); + +$loop_count = 0 +loop1: + +sleep 300 +print 2 sql select * from streamt0 order by 1, 2, 3, 4; +sql select * from streamt0 order by 1, 2, 3, 4; + +print +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +print $data20 $data21 $data22 $data23 $data24 +print $data30 $data31 $data32 $data33 $data34 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then + print ======rows=$rows + goto loop1 +endi + +if $data01 != 5 then + print ======data01=$data01 + goto loop0 +endi + print event1 end system sh/exec.sh -n dnode1 -s stop -x SIGINT From 5592e11235cb04824e46782eb5a172a6324af0bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 19:02:52 +0800 Subject: [PATCH 019/198] fix(stream): handle error when checkpoint is interrupted by nodeUpdate. --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamExec.c | 25 ++++++++++++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a8b5842b93..1ee5256775 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1295,8 +1295,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (status == TASK_STATUS__CK) { ASSERT(pTask->chkInfo.checkpointingId == req.checkpointId); tqWarn("s-task:%s recv checkpoint-source msg again checkpointId:%" PRId64 - " already received, ignore this msg and continue process checkpoint", - pTask->id.idStr, pTask->chkInfo.checkpointingId); + " transId:%d already received, ignore this msg and continue process checkpoint", + pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index bfa269b0d5..00a8940b6a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -746,7 +746,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { taosArrayClear(pTask->pReadyMsgList); stDebug("s-task:%s level:%d source checkpoint completed msg sent to mnode", pTask->id.idStr, pTask->info.taskLevel); } else { - stDebug("s-task:%s level:%d already send rsp to mnode", pTask->id.idStr, pTask->info.taskLevel); + stDebug("s-task:%s level:%d already send rsp checkpoint success to mnode", pTask->id.idStr, pTask->info.taskLevel); } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7d2a572cda..eafa5adea8 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -625,10 +625,29 @@ int32_t doStreamExecTask(SStreamTask* pTask) { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. if (type == STREAM_INPUT__CHECKPOINT) { + + // todo add lock char* p = NULL; - streamTaskGetStatus(pTask, &p); - stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); - streamTaskBuildCheckpoint(pTask); + ETaskStatus s = streamTaskGetStatus(pTask, &p); + if (s == TASK_STATUS__CK) { + stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); + streamTaskBuildCheckpoint(pTask); + } else { + // todo refactor + int32_t code = 0; + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + code = streamTaskSendCheckpointSourceRsp(pTask); + } else { + code = streamTaskSendCheckpointReadyMsg(pTask); + } + + if (code != TSDB_CODE_SUCCESS) { + // todo: let's retry send rsp to upstream/mnode + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + 0, tstrerror(code)); + } + } + return 0; } } From e553b84904309b7ea51edecd039820ebce965a50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 19:16:49 +0800 Subject: [PATCH 020/198] fix(stream): fix syntax error. --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index eafa5adea8..1f7bb56ec1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -643,7 +643,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", pTask->id.idStr, 0, tstrerror(code)); } } From 3faf1514f43d1e03cb6557349a644ffcc598f93b Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 26 Dec 2023 20:47:15 +0800 Subject: [PATCH 021/198] doGroupedTableScan --- source/libs/executor/src/scanoperator.c | 45 +++++++++++----------- source/libs/planner/src/planLogicCreater.c | 1 - source/libs/planner/src/planPhysiCreater.c | 1 - 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6df46ef4d0..0f9b2f1eac 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -657,7 +657,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, // record processed (non empty) table -static int32_t insertTableToProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { +static int32_t markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (!pTableScanInfo->needCountEmptyTable) { return TSDB_CODE_SUCCESS; } @@ -770,6 +770,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; + bool outputAll = pTableScanInfo->base.pTableListInfo->oneTableForEachGroup; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { @@ -780,7 +781,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - insertTableToProcessed(pTableScanInfo, p->info.id.uid); + markTableProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -809,7 +810,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - insertTableToProcessed(pTableScanInfo, p->info.id.uid); + markTableProcessed(pTableScanInfo, p->info.id.uid); return p; } @@ -827,30 +828,30 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->needCountEmptyTable) { - if (num == 0 && 0 == taosHashGetSize(pTableScanInfo->pValuedTables)) { - // table by table, num is 0 - if (!pTableScanInfo->processingEmptyTable) { - pTableScanInfo->processingEmptyTable = true; - // current table is empty, fill result block info & return - const STableKeyInfo* info = tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->currentTable); - return getBlockForEmptyTable(pOperator, info); - } - - } else if (num > taosHashGetSize(pTableScanInfo->pValuedTables)) { - // group by group, num >= 1 + // pList is NULL in mode TABLE_SCAN__TABLE_ORDER for streamscan, no need to process + // pList not NULL, group by group, num >= 1 + int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pValuedTables); + if (pList && num > tb_cnt) { if (!pTableScanInfo->processingEmptyTable) { pTableScanInfo->processingEmptyTable = true; pTableScanInfo->currentTable = 0; } if (pTableScanInfo->currentTable < num) { - // loop: get empty table uid & process - while (pTableScanInfo->currentTable < num) { - const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; - if (pTableScanInfo->pValuedTables && - NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { - } else { - return getBlockForEmptyTable(pOperator, info); + if (outputAll) { + // loop: get empty table uid & process + while (pTableScanInfo->currentTable < num) { + const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; + if (pTableScanInfo->pValuedTables && + NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { + } else { + return getBlockForEmptyTable(pOperator, info); + } } + } else if (tb_cnt == 0) { + // only need one & all empty table in this group + // output first one + pTableScanInfo->currentTable = num; + return getBlockForEmptyTable(pOperator, pList); } } } @@ -1005,7 +1006,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - + if (pOperator->pOperatorGetParam) { pOperator->dynamicTask = true; int32_t code = createTableListInfoFromParam(pOperator); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f55689c472..e642b5bd5f 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -758,7 +758,6 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; - if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 444bda84cd..96205904e3 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2293,7 +2293,6 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic } else { pSubplan->msgType = TDMT_SCH_MERGE_QUERY; } - code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); From 42f9d54a70b67170278adf3dac701ec10742d4a9 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 00:28:42 +0800 Subject: [PATCH 022/198] adjust --- source/libs/executor/src/scanoperator.c | 3 +- tests/system-test/2-query/group_partition.py | 42 +++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0f9b2f1eac..c69d963c79 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -770,7 +770,8 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - bool outputAll = pTableScanInfo->base.pTableListInfo->oneTableForEachGroup; + // Only when all tables are scanned can you determine how many groups the tag has + bool outputAll = true; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index c08ea98ef6..3d9874bfe2 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -40,9 +40,6 @@ class TDTestCase: tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) @@ -57,12 +54,13 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) # tag - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by t2 ") - tdSql.checkRows(check_num) - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") tdSql.checkRows(check_num) + # multi tag + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") + tdSql.checkRows(check_num) + # having tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -87,10 +85,16 @@ class TDTestCase: tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") tdSql.checkRows(check_num) - # count + sum(col) + # multi agg tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") tdSql.checkRows(check_num) + tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(real_num) + ############### same with old ############### tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") num = 0 @@ -105,14 +109,20 @@ class TDTestCase: tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} group by t2, t3, c1 ") + tdSql.checkRows(real_num * self.row_nums) + def test_partitionby(self, check_num, real_num): tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + tdSql.query(f" select count(c5), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(real_num) + tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") tdSql.checkRows(check_num) @@ -124,12 +134,12 @@ class TDTestCase: tdSql.checkRows(check_num - real_num) #tag - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} partition by t2 ") - tdSql.checkRows(check_num) - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") tdSql.checkRows(check_num) + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} partition by t2, t3, tbname") + tdSql.checkRows(check_num) + # having tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") tdSql.checkRows(check_num - real_num) @@ -150,6 +160,13 @@ class TDTestCase: tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") tdSql.checkRows(check_num) + #multi agg + tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(check_num) + + tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(real_num) + #inner select tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) @@ -170,6 +187,9 @@ class TDTestCase: tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} partition by t2, t3, c1 ") + tdSql.checkRows(real_num * self.row_nums) + def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") From 86351d5487ae55010b6922d58e058c8a58420173 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 01:05:04 +0800 Subject: [PATCH 023/198] check count & window --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e642b5bd5f..ea8bb666b1 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -487,7 +487,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } bool isCountByTag = false; - if (pSelect->hasCountFunc) { + if (pSelect->hasCountFunc && !pSelect->pWindow) { if (pSelect->pGroupByList) { isCountByTag = !keysHasCol(pSelect->pGroupByList); } else if (pSelect->pPartitionByList) { From 158c5996dfb98aadb767847fce4d63555d63f957 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 15:51:36 +0800 Subject: [PATCH 024/198] adjust case --- source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/planner/src/planPhysiCreater.c | 14 +- tests/system-test/2-query/group_partition.py | 219 +++++++++---------- 3 files changed, 101 insertions(+), 134 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ea8bb666b1..12b7360165 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -487,7 +487,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } bool isCountByTag = false; - if (pSelect->hasCountFunc && !pSelect->pWindow) { + if (pSelect->hasCountFunc && NULL == pSelect->pWindow) { if (pSelect->pGroupByList) { isCountByTag = !keysHasCol(pSelect->pGroupByList); } else if (pSelect->pPartitionByList) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 96205904e3..e266c55425 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -587,18 +587,6 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static bool calcNeedCountEmpty(SScanLogicNode* pScanLogicNode) { - if (pScanLogicNode->interval > 0) { - return false; - } - - if (pScanLogicNode->isCountByTag) { - return true; - } - - return false; -} - static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, @@ -635,7 +623,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; - pTableScan->needCountEmptyTable = calcNeedCountEmpty(pScanLogicNode); + pTableScan->needCountEmptyTable = pScanLogicNode->isCountByTag; int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 3d9874bfe2..0ac1be8482 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -35,161 +35,132 @@ class TDTestCase: tdSql.execute(f"insert into {tbname} values({ts} , {row} , {row} , {row} , {row} , 1 , 2 , 'true' , 'binary_{row}' , 'nchar_{row}' , {row} , {row} , 1 ,2 )") - def test_groupby(self, check_num, real_num): - # tbname - tdSql.query(f"select count(*) from {self.dbname}.{self.stable} group by tbname ") + def test_groupby(self, keyword, check_num, nonempty_tb_num): + ####### by tbname + tdSql.query(f"select count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname ") + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} {keyword} by tbname ") tdSql.checkRows(check_num) - tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname order by count(*)") + tdSql.query(f"select tbname from {self.dbname}.{self.stable} {keyword} by tbname order by count(*)") tdSql.checkRows(check_num) - tdSql.query(f"select tbname from {self.dbname}.{self.stable} group by tbname having count(*)>=0") + # last + tdSql.query(f"select last(ts), count(*) from {self.dbname}.{self.stable} {keyword} by tbname order by last(ts)") + tdSql.checkRows(check_num) + + tdSql.query(f"select tbname from {self.dbname}.{self.stable} {keyword} by tbname having count(*)>=0") tdSql.checkRows(check_num) # having filter out empty - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} group by tbname having count(*) <= 0") - tdSql.checkRows(check_num - real_num) + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} {keyword} by tbname having count(*) <= 0") + tdSql.checkRows(check_num - nonempty_tb_num) - # tag - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 ") + ####### by tag + tdSql.query(f"select t2, count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by t2 ") tdSql.checkRows(check_num) - # multi tag - tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} {keyword} by t2 having count(*) <= 0") + tdSql.checkRows(check_num - nonempty_tb_num) + + # where + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now {keyword} by t2 ") tdSql.checkRows(check_num) - # having - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 having count(*) <= 0") - tdSql.checkRows(check_num - real_num) - - # col where filter nothing - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now group by t2 ") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 {keyword} by t2 ") tdSql.checkRows(check_num) - # col where filter all - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 group by t2 ") + tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 {keyword} by t2 ") tdSql.checkRows(check_num) - # col where filter part - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 group by t2 ") - tdSql.checkRows(check_num) - - # col - tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - #inner select - tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") - tdSql.checkRows(check_num) - - # multi agg - tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} group by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") - tdSql.checkRows(real_num) - - ############### same with old ############### - tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} group by c1 ") + ####### by col + tdSql.query(f"select c1, count(*), count(1), count(c1) from {self.dbname}.{self.stable} {keyword} by c1 ") num = 0 - if real_num > 0: + if nonempty_tb_num > 0: num = self.row_nums tdSql.checkRows(num) - tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} group by ts ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} {keyword} by ts ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) # col + tag - tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} group by t2, c1 ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} {keyword} by t2, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} group by t2, t3, c1 ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} {keyword} by t2, t3, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - - def test_partitionby(self, check_num, real_num): - tdSql.query(f"select tbname , count(*) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) - - tdSql.query(f"select count(*), sum(1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) - tdSql.query(f" select count(c5), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") - tdSql.checkRows(real_num) - - tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname order by count(*)") - tdSql.checkRows(check_num) - - tdSql.query(f"select tbname from {self.dbname}.{self.stable} partition by tbname having count(*)>=0") - tdSql.checkRows(check_num) - - # having filter out empty - tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname having count(*) <= 0") - tdSql.checkRows(check_num - real_num) - - #tag - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 ") + def test_multi_group_key(self, check_num, nonempty_tb_num): + # multi tag/tbname + tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} group by t2, t3, tbname") tdSql.checkRows(check_num) tdSql.query(f"select t2, t3, tbname, count(*) from {self.dbname}.{self.stable} partition by t2, t3, tbname") tdSql.checkRows(check_num) - # having - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} partition by t2 having count(*) <= 0") - tdSql.checkRows(check_num - real_num) + # multi tag + col + tdSql.query(f"select t1, t2, c1, count(*) from {self.dbname}.{self.stable} partition by t1, t2, c1 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - # col where filter nothing - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts < now partition by t2 ") - tdSql.checkRows(check_num) + # tag + multi col + tdSql.query(f"select t2, c1, c2, count(*) from {self.dbname}.{self.stable} partition by t2, c1, c2 ") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - # col where filter all - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where ts > 1737146000000 partition by t2 ") - tdSql.checkRows(check_num) - # col where filter part - tdSql.query(f"select t2, count(*) from {self.dbname}.{self.stable} where c1 = 1 partition by t2 ") - tdSql.checkRows(check_num) + def test_multi_agg(self, all_tb_num, nonempty_tb_num): + tdSql.query(f"select count(*), sum(c1) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(all_tb_num) - #col - tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) + tdSql.query(f"select count(1), sum(1), avg(c1), apercentile(c1, 50), spread(ts) from {self.dbname}.{self.stable} group by tbname ") + tdSql.checkRows(all_tb_num) - #multi agg - tdSql.query(f"select count(1), sum(1) from {self.dbname}.{self.stable} partition by tbname ") - tdSql.checkRows(check_num) + tdSql.query(f"select count(c1), sum(c1), min(c1), mode(c1), stddev(c1), spread(c1) from {self.dbname}.{self.stable} partition by tbname ") + tdSql.checkRows(all_tb_num) + + # elapsed: continuous duration in a statistical period, table merge scan + tdSql.query(f" select count(c1), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(nonempty_tb_num) tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") - tdSql.checkRows(real_num) + tdSql.checkRows(nonempty_tb_num) + + + def test_innerSelect(self, check_num): + tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") + tdSql.checkRows(check_num) - #inner select tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) + tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.stable} partition by t1)") + tdSql.checkRows(check_num) + + + def test_window(self, nonempty_tb_num): + # empty group optimization condition is not met + # time window tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname interval(1d)") - tdSql.checkRows(real_num) + tdSql.checkRows(nonempty_tb_num) - ############### same with old ############### - tdSql.query(f"select c1, count(*) from {self.dbname}.{self.stable} partition by c1 ") - num = 0 - if real_num > 0: - num = self.row_nums - tdSql.checkRows(num) - - tdSql.query(f"select ts, count(*) from {self.dbname}.{self.stable} partition by ts ") - tdSql.checkRows(real_num * self.row_nums) + tdSql.query(f"select _wstart, _wend, count(c1), max(c1), apercentile(c1, 50) from {self.dbname}.{self.stable} partition by tbname interval(1d)") + tdSql.checkRows(nonempty_tb_num) - tdSql.query(f"select t2, c1, count(*) from {self.dbname}.{self.stable} partition by t2, c1 ") - tdSql.checkRows(real_num * self.row_nums) + # state window + tdSql.query(f"select tbname, count(*), c1 from {self.dbname}.{self.stable} partition by tbname state_window(c1)") + tdSql.checkRows(nonempty_tb_num * self.row_nums) - tdSql.query(f"select t2, t3, c1, count(*) from {self.dbname}.{self.stable} partition by t2, t3, c1 ") - tdSql.checkRows(real_num * self.row_nums) + # session window + tdSql.query(f"select count(c1) from {self.dbname}.{self.stable} partition by tbname session(ts, 5s)") + tdSql.checkRows(nonempty_tb_num) + # event window + tdSql.query(f"select tbname, count(*) from {self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") + tdSql.checkRows(nonempty_tb_num) + + + def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") @@ -199,21 +170,29 @@ class TDTestCase: def run(self): tdSql.prepare() self.prepare_db() - check_num = self.tb_nums - self.test_groupby(check_num, 0) - self.test_partitionby(check_num, 0) - # insert into half of tables - real_num = 5 - self.insert_db(real_num, self.row_nums) - self.test_groupby(check_num, real_num) - self.test_partitionby(check_num, real_num) + # empty table only + self.test_groupby('group', self.tb_nums, 0) + self.test_groupby('partition', self.tb_nums, 0) + self.test_innerSelect(self.tb_nums) + self.test_multi_group_key(self.tb_nums, 0) + self.test_multi_agg(self.tb_nums, 0) + self.test_window(0) - # test old version before changed - # self.test_groupby(0, 0) - # self.test_partitionby(0, 0) + # insert data to 5 tables + nonempty_tb_num = 5 + self.insert_db(nonempty_tb_num, self.row_nums) + + self.test_groupby('group', self.tb_nums, nonempty_tb_num) + self.test_groupby('partition', self.tb_nums, nonempty_tb_num) + self.test_innerSelect(self.tb_nums) + self.test_multi_group_key(self.tb_nums, nonempty_tb_num) + self.test_multi_agg(self.tb_nums, nonempty_tb_num) + self.test_window(nonempty_tb_num) + + ## test old version before changed + # self.test_groupby('group', 0, 0) # self.insert_db(5, self.row_nums) - # self.test_groupby(5, 5) - # self.test_partitionby(5, 5) + # self.test_groupby('group', 5, 5) self.test_error() From f6e55d358b375e86c5d82aca6ed089303d4b9347 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 16:12:50 +0800 Subject: [PATCH 025/198] =?UTF-8?q?=C2=A0adjust=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/system-test/2-query/group_partition.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 0ac1be8482..8b06f0d6fd 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -134,7 +134,7 @@ class TDTestCase: tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} partition by tbname) ") tdSql.checkRows(check_num) - tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.stable} partition by t1)") + tdSql.query(f"select t1, c from (select t1, sum(c1) as s, count(*) as c from {self.dbname}.{self.stable} partition by t1)") tdSql.checkRows(check_num) @@ -156,7 +156,7 @@ class TDTestCase: tdSql.checkRows(nonempty_tb_num) # event window - tdSql.query(f"select tbname, count(*) from {self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") tdSql.checkRows(nonempty_tb_num) From 10e86fc733a53b57c056c1a4e7bc61a3624cb2df Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Dec 2023 19:25:08 +0800 Subject: [PATCH 026/198] fix:[TS-4391] rebalance cnt always 1 if msg lost --- source/dnode/mnode/impl/inc/mndConsumer.h | 1 + source/dnode/mnode/impl/src/mndConsumer.c | 28 +++++++++++++++------- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 94c937e8f4..f075510428 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -47,6 +47,7 @@ int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj int32_t mndSetConsumerDropLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); bool mndRebTryStart(); +bool mndRebCanStart(); void mndRebEnd(); void mndRebCntInc(); void mndRebCntDec(); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index c9ee66d3a0..11c929c898 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -101,6 +101,16 @@ bool mndRebTryStart() { return old == 0; } +bool mndRebCanStart() { + int32_t val = atomic_load_32(&mqRebInExecCnt); + if (val < 0) { + mError("rebalance trans end, rebalance counter:%d should not be less equalled than 0, ignore counter desc", val); + return false; + } + mInfo("tq timer, rebalance counter val:%d", val); + return val == 0; +} + void mndRebEnd() { mndRebCntDec(); } void mndRebCntInc() { @@ -119,7 +129,7 @@ void mndRebCntDec() { int32_t newVal = val - 1; int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal); if (oldVal == val) { - mDebug("rebalance trans end, rebalance counter:%d", newVal); + mInfo("rebalance trans end, rebalance counter:%d", newVal); break; } } @@ -284,10 +294,10 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqConsumerObj *pConsumer; void *pIter = NULL; - mDebug("start to process mq timer"); + mInfo("start to process mq timer"); // rebalance cannot be parallel - if (!mndRebTryStart()) { + if (!mndRebCanStart()) { mInfo("mq rebalance already in progress, do nothing"); return 0; } @@ -295,7 +305,6 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); if (pRebMsg == NULL) { mError("failed to create the rebalance msg, size:%d, quit mq timer", (int32_t)sizeof(SMqDoRebalanceMsg)); - mndRebEnd(); return TSDB_CODE_OUT_OF_MEMORY; } @@ -303,7 +312,6 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { if (pRebMsg->rebSubHash == NULL) { mError("failed to create rebalance hashmap"); rpcFreeCont(pRebMsg); - mndRebEnd(); return TSDB_CODE_OUT_OF_MEMORY; } @@ -390,6 +398,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } + + if (newTopicNum == 0 && removedTopicNum == 0 && taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); + } + taosRUnLockLatch(&pConsumer->lock); } @@ -397,7 +410,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { - mInfo("mq rebalance will be triggered"); + mInfo("mq send msg to rebalance"); SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_DO_REBALANCE, .pCont = pRebMsg, @@ -407,8 +420,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); - mDebug("mq timer finished, no need to re-balance"); - mndRebEnd(); + mInfo("mq timer finished, no need to re-balance"); } return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 01d4d1029c..8cd7f6bc9c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -732,6 +732,10 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMqDoRebalanceMsg *pReq = pMsg->pCont; void *pIter = NULL; // bool rebalanceOnce = false; // to ensure only once. + if (!mndRebTryStart()) { + mInfo("mq rebalance already in progress, do nothing"); + return 0; + } mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); From d795798648ac0d1945e90bbbf39d445ed0f88679 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 21:37:00 +0800 Subject: [PATCH 027/198] count state --- source/libs/executor/inc/executorInt.h | 11 ++- source/libs/executor/src/scanoperator.c | 94 ++++++++++++++----------- 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 2523b87cfb..8b0bef5fa6 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -217,6 +217,12 @@ enum { TABLE_SCAN__BLOCK_ORDER = 2, }; +typedef enum ETableCountState { + TABLE_COUNT_STATE_NONE = 0, // before start scan + TABLE_COUNT_STATE_SCAN = 1, // scanning + TABLE_COUNT_STATE_END = 2, // finish or noneed to process +} ETableCountState; + typedef struct SAggSupporter { SSHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer @@ -262,17 +268,18 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; - SHashObj* pValuedTables; // non empty table uids + SHashObj* pRemainTables; // remain table to process SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; int8_t scanMode; int8_t assignBlockUid; + uint8_t countState; // empty table count state + bool isOneGroup; // whether or not only one group in this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; bool needCountEmptyTable; - bool processingEmptyTable; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c69d963c79..31fe31007c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,26 +655,44 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } - -// record processed (non empty) table -static int32_t markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { +static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { return TSDB_CODE_SUCCESS; } - if (NULL == pTableScanInfo->pValuedTables) { + pTableScanInfo->isOneGroup = true; + if (NULL == pTableScanInfo->pRemainTables) { int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); - pTableScanInfo->pValuedTables = + pTableScanInfo->pRemainTables = taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (NULL == pTableScanInfo->pValuedTables) { + if (NULL == pTableScanInfo->pRemainTables) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_OUT_OF_MEMORY; } } - - taosHashPut(pTableScanInfo->pValuedTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, - sizeof(pTableScanInfo->scanTimes)); + uint64_t groupId = 0; + for (int32_t i = 0; i < num; i++) { + const STableKeyInfo* pInfo = pList + i; + if (pTableScanInfo->isOneGroup) { + if (i == 0) { + groupId = pInfo->groupId; + } else if (groupId != pInfo->groupId) { + pTableScanInfo->isOneGroup = false; + } + } + taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); + } return TSDB_CODE_SUCCESS; } +static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { + // case0 group scanning, mark + // case1 stream scan: no need to mark + if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { + return; + } + taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); +} + static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -770,14 +788,17 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - // Only when all tables are scanned can you determine how many groups the tag has - bool outputAll = true; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; } + if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { + initRemainTable(pTableScanInfo, pList, num); + pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; + } + // do the ascending order traverse in the first place. while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); @@ -829,37 +850,28 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->needCountEmptyTable) { - // pList is NULL in mode TABLE_SCAN__TABLE_ORDER for streamscan, no need to process - // pList not NULL, group by group, num >= 1 - int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pValuedTables); - if (pList && num > tb_cnt) { - if (!pTableScanInfo->processingEmptyTable) { - pTableScanInfo->processingEmptyTable = true; - pTableScanInfo->currentTable = 0; - } - if (pTableScanInfo->currentTable < num) { - if (outputAll) { - // loop: get empty table uid & process - while (pTableScanInfo->currentTable < num) { - const STableKeyInfo* info = pList + pTableScanInfo->currentTable++; - if (pTableScanInfo->pValuedTables && - NULL != taosHashGet(pTableScanInfo->pValuedTables, &info->uid, sizeof(info->uid))) { - } else { - return getBlockForEmptyTable(pOperator, info); - } - } - } else if (tb_cnt == 0) { - // only need one & all empty table in this group - // output first one - pTableScanInfo->currentTable = num; - return getBlockForEmptyTable(pOperator, pList); + int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); + if (tb_cnt) { + if (!pTableScanInfo->isOneGroup) { + // get first empty table uid, mark processed & rm from hash + void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); + if (pIte != NULL) { + size_t keySize = 0; + uint64_t* pUid = taosHashGetKey(pIte, &keySize); + STableKeyInfo info = {.uid = *pUid, .groupId = *(uint64_t*)pIte}; + taosHashCancelIterate(pTableScanInfo->pRemainTables, pIte); + markTableProcessed(pTableScanInfo, *pUid); + return getBlockForEmptyTable(pOperator, &info); } + } else { + // output one table for this group + taosHashClear(pTableScanInfo->pRemainTables); + return getBlockForEmptyTable(pOperator, pList); } } - pTableScanInfo->processingEmptyTable = false; + pTableScanInfo->countState = TABLE_COUNT_STATE_END; } - taosHashClear(pTableScanInfo->pValuedTables); return NULL; } @@ -937,7 +949,8 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - + pInfo->countState = TABLE_COUNT_STATE_NONE; + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; @@ -968,6 +981,7 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { } tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); + pInfo->countState = TABLE_COUNT_STATE_NONE; ASSERT(pInfo->base.dataReader == NULL); int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, @@ -1034,6 +1048,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { int32_t numOfTables = 0; // tableListGetSize(pTaskInfo->pTableListInfo); STableKeyInfo tInfo = {0}; + pInfo->countState = TABLE_COUNT_STATE_END; while (1) { SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); @@ -1096,7 +1111,7 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - taosHashCleanup(pTableScanInfo->pValuedTables); + taosHashCleanup(pTableScanInfo->pRemainTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1161,7 +1176,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->exprSupp.numOfExprs = numOfCols; pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; - pInfo->processingEmptyTable = false; pInfo->base.pTableListInfo = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); From abed781bd324dda7c636e162a25646292214004a Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 27 Dec 2023 23:15:11 +0800 Subject: [PATCH 028/198] opt --- source/libs/executor/src/scanoperator.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 31fe31007c..0714019352 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -657,6 +657,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_SUCCESS; } pTableScanInfo->isOneGroup = true; @@ -681,6 +682,7 @@ static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyIn } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); } + pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; return TSDB_CODE_SUCCESS; } @@ -690,6 +692,11 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { return; } + // case2 only one group, uid ready + if (pTableScanInfo->isOneGroup) { + pTableScanInfo->countState = TABLE_COUNT_STATE_END; + return; + } taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); } @@ -796,7 +803,6 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { initRemainTable(pTableScanInfo, pList, num); - pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; } // do the ascending order traverse in the first place. @@ -849,7 +855,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } } - if (pTableScanInfo->needCountEmptyTable) { + if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); if (tb_cnt) { if (!pTableScanInfo->isOneGroup) { @@ -865,14 +871,14 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } } else { // output one table for this group - taosHashClear(pTableScanInfo->pRemainTables); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return getBlockForEmptyTable(pOperator, pList); } } - pTableScanInfo->countState = TABLE_COUNT_STATE_END; } + taosHashClear(pTableScanInfo->pRemainTables); return NULL; } From a5f88a86e9b18ac7b3a3a61592659fa72a40c9a1 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 09:45:01 +0800 Subject: [PATCH 029/198] adjust --- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/scanoperator.c | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 8b0bef5fa6..b87dee475d 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -275,7 +275,7 @@ typedef struct STableScanInfo { int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state - bool isOneGroup; // whether or not only one group in this scan + bool isSameGroup; // whether all tables are in the same group this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0714019352..dd6dff0301 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,12 +655,12 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } -static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { +static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { if (!pTableScanInfo->needCountEmptyTable) { pTableScanInfo->countState = TABLE_COUNT_STATE_END; return TSDB_CODE_SUCCESS; } - pTableScanInfo->isOneGroup = true; + pTableScanInfo->isSameGroup = true; if (NULL == pTableScanInfo->pRemainTables) { int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); pTableScanInfo->pRemainTables = @@ -673,11 +673,11 @@ static int32_t initRemainTable(STableScanInfo* pTableScanInfo, const STableKeyIn uint64_t groupId = 0; for (int32_t i = 0; i < num; i++) { const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isOneGroup) { + if (pTableScanInfo->isSameGroup) { if (i == 0) { groupId = pInfo->groupId; } else if (groupId != pInfo->groupId) { - pTableScanInfo->isOneGroup = false; + pTableScanInfo->isSameGroup = false; } } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); @@ -692,8 +692,8 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { return; } - // case2 only one group, uid ready - if (pTableScanInfo->isOneGroup) { + // case2 if all table in same group, process only once + if (pTableScanInfo->isSameGroup) { pTableScanInfo->countState = TABLE_COUNT_STATE_END; return; } @@ -802,7 +802,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { - initRemainTable(pTableScanInfo, pList, num); + initTableCountEnv(pTableScanInfo, pList, num); } // do the ascending order traverse in the first place. @@ -858,7 +858,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); if (tb_cnt) { - if (!pTableScanInfo->isOneGroup) { + if (!pTableScanInfo->isSameGroup) { // get first empty table uid, mark processed & rm from hash void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); if (pIte != NULL) { From 424ab1bbe32672570d4b8549b9b8f375e55bcbb5 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 16:18:38 +0800 Subject: [PATCH 030/198] support table merge scan --- source/libs/executor/inc/executorInt.h | 3 ++- source/libs/executor/src/scanoperator.c | 27 ++++++++++++++------ tests/system-test/2-query/group_partition.py | 17 +++++++++--- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index b87dee475d..ae38d4940c 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -307,7 +307,8 @@ typedef struct STableMergeScanInfo { SHashObj* mSkipTables; int64_t mergeLimit; SSortExecInfo sortExecInfo; - + bool needCountEmptyTable; + bool bGroupProcessed; // the group return data means processed bool filesetDelimited; bool bNewFilesetEvent; bool bNextDurationBlockEvent; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index dd6dff0301..a1f9baa082 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -700,17 +700,14 @@ static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); } -static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { - STableScanInfo* pTableScanInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - +static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, + const STableKeyInfo* tbInfo) { blockDataEmpty(pBlock); pBlock->info.rows = 1; pBlock->info.id.uid = tbInfo->uid; pBlock->info.id.groupId = tbInfo->groupId; - // only one row: set all col data to null & hasNull + // only one row: set all col data to null & hasNull int32_t col_num = blockDataGetNumOfCols(pBlock); for (int32_t i = 0; i < col_num; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); @@ -718,7 +715,14 @@ static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STable } // set tag/tbname - doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); + doSetTagColumnData(pBase, pBlock, pTaskInfo, pBlock->info.rows); + return pBlock; +} + +static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) { + STableScanInfo* pTableScanInfo = pOperator->info; + SSDataBlock* pBlock = + getOneRowResultBlock(pOperator->pTaskInfo, &pTableScanInfo->base, pTableScanInfo->pResBlock, tbInfo); pOperator->resultInfo.totalRows++; return pBlock; @@ -3585,7 +3589,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { } pInfo->tableEndIndex = i - 1; } - + pInfo->bGroupProcessed = false; int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; @@ -3707,9 +3711,14 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity, pOperator); + if (pBlock == NULL && !pInfo->bGroupProcessed && pInfo->needCountEmptyTable) { + STableKeyInfo* tbInfo = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + pBlock = getOneRowResultBlock(pTaskInfo, &pInfo->base, pInfo->pResBlock, tbInfo); + } if (pBlock != NULL) { pBlock->info.id.groupId = pInfo->groupId; pOperator->resultInfo.totalRows += pBlock->info.rows; + pInfo->bGroupProcessed = true; return pBlock; } else { if (pInfo->bNewFilesetEvent) { @@ -3864,6 +3873,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN } else { pInfo->filesetDelimited = pTableScanNode->filesetDelimited; } + pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable; + setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 8b06f0d6fd..e228351f0e 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -120,12 +120,21 @@ class TDTestCase: tdSql.checkRows(all_tb_num) # elapsed: continuous duration in a statistical period, table merge scan - tdSql.query(f" select count(c1), max(c5), avg(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") - tdSql.checkRows(nonempty_tb_num) - - tdSql.query(f" select count(c1), max(c1), avg(c1), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.query(f" select count(c1), max(c5), last_row(c5), elapsed(ts), spread(c1) from {self.dbname}.{self.stable} group by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select count(c1), min(c1), avg(c1), elapsed(ts), mode(c1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select count(c1), elapsed(ts), twa(c1), irate(c1), leastsquares(c1,1,1) from {self.dbname}.{self.stable} partition by tbname") + tdSql.checkRows(all_tb_num) + + tdSql.query(f" select avg(c1), elapsed(ts), twa(c1), irate(c1) from {self.dbname}.{self.stable} partition by tbname") tdSql.checkRows(nonempty_tb_num) + # if nonempty_tb_num > 0: + # tdSql.query(f" select avg(c1), percentile(c1, 50) from {self.dbname}.sub_{self.stable}_1") + # tdSql.checkRows(1) def test_innerSelect(self, check_num): tdSql.query(f"select * from (select count(c1) from {self.dbname}.{self.stable} group by tbname) ") From 53ce344b362b10d0e7afd5f49d06893ee9b10f15 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 28 Dec 2023 17:09:52 +0800 Subject: [PATCH 031/198] fix:set error msg if build consumer error --- source/client/src/clientTmq.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index d05cdc0156..15c8903978 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1028,11 +1028,16 @@ static void tmqMgmtInit(void) { } } +#define SET_ERROR_MSG(MSG) if(errstr!=NULL)snprintf(errstr,errstrLen,MSG); tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - if(conf == NULL) return NULL; + if(conf == NULL) { + SET_ERROR_MSG("configure is null") + return NULL; + } taosThreadOnce(&tmqInit, tmqMgmtInit); if (tmqInitRes != 0) { terrno = tmqInitRes; + SET_ERROR_MSG("tmq timer init error") return NULL; } @@ -1040,6 +1045,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to create consumer, groupId:%s, code:%s", conf->groupId, terrstr()); + SET_ERROR_MSG("malloc tmq failed") return NULL; } @@ -1055,6 +1061,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG("malloc tmq element failed or group is empty") goto _failed; } @@ -1086,6 +1093,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (tsem_init(&pTmq->rspSem, 0, 0) != 0) { tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId); + SET_ERROR_MSG("init t_sem failed") goto _failed; } @@ -1094,11 +1102,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq->pTscObj == NULL) { tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); tsem_destroy(&pTmq->rspSem); + SET_ERROR_MSG("init tscObj failed") goto _failed; } pTmq->refId = taosAddRef(tmqMgmt.rsetId, pTmq); if (pTmq->refId < 0) { + SET_ERROR_MSG("add tscObj ref failed") goto _failed; } From 2526df745ee62db8786adcd732e8b0cd4646a723 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:15:09 +0800 Subject: [PATCH 032/198] enh: adjust threshold of snap replication timeout --- include/libs/sync/sync.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e54237fe8b..cb053d2548 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -46,8 +46,8 @@ extern "C" { #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 -#define SYNC_SNAP_RESEND_MS 1000 * 300 -#define SYNC_SNAP_TIMEOUT_MS 1000 * 1800 +#define SYNC_SNAP_RESEND_MS 1000 * 60 +#define SYNC_SNAP_TIMEOUT_MS 1000 * 300 #define SYNC_VND_COMMIT_MIN_MS 3000 From d83f85bf2642982d8fdcb5bd53b58011cc8f81e1 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:26:46 +0800 Subject: [PATCH 033/198] enh: send rsp msg on rejecting snap replication due to smaller term --- source/libs/sync/src/syncSnapshot.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 10a8734617..f0e457ef8d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1001,6 +1001,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { sRError(pReceiver, "reject snap replication with smaller term. msg term:%" PRId64 ", seq:%d", pMsg->term, pMsg->seq); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, terrno); return -1; } From baa7f9c895faf0ed12d59c5ff02460ae8eb3f1da Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 28 Dec 2023 17:29:49 +0800 Subject: [PATCH 034/198] enh: adjust error msg as warn for not ready to propose etc --- source/dnode/vnode/src/vnd/vnodeSync.c | 8 ++++---- source/libs/sync/src/syncMain.c | 6 +++--- source/libs/sync/src/syncPipeline.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 048092131d..0f491e9a58 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -220,8 +220,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) isWeak, isBlock, msg, numOfMsgs, arrayPos, pMsg->info.handle); if (!pVnode->restored) { - vGError("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, - TMSG_INFO(pMsg->msgType)); + vGWarn("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, + TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_SYN_RESTORING; vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); @@ -284,8 +284,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle); if (!pVnode->restored) { - vGError("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, - TMSG_INFO(pMsg->msgType)); + vGWarn("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, + TMSG_INFO(pMsg->msgType)); vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 6f3b3fdf98..52557f7b9c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -662,14 +662,14 @@ ESyncRole syncGetRole(int64_t rid) { int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { terrno = TSDB_CODE_SYN_NOT_LEADER; - sNError(pSyncNode, "sync propose not leader, type:%s", TMSG_INFO(pMsg->msgType)); + sNWarn(pSyncNode, "sync propose not leader, type:%s", TMSG_INFO(pMsg->msgType)); return -1; } if (!pSyncNode->restoreFinish) { terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; - sNError(pSyncNode, "failed to sync propose since not ready, type:%s, last:%" PRId64 ", cmt:%" PRId64, - TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); + sNWarn(pSyncNode, "failed to sync propose since not ready, type:%s, last:%" PRId64 ", cmt:%" PRId64, + TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); return -1; } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 28ee5ba841..70bdd4a837 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -206,7 +206,7 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { } if (pLogStore->syncLogGetEntry(pLogStore, index, &pEntry) < 0) { - sError("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); + sWarn("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); break; } @@ -1237,7 +1237,7 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, } else { *pInBuf = false; if (pNode->pLogStore->syncLogGetEntry(pNode->pLogStore, index, &pEntry) < 0) { - sError("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); + sWarn("vgId:%d, failed to get log entry since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); } } return pEntry; @@ -1253,7 +1253,7 @@ int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex ind pEntry = syncLogBufferGetOneEntry(pBuf, pNode, index, &inBuf); if (pEntry == NULL) { - sError("vgId:%d, failed to get raft entry for index:%" PRId64 "", pNode->vgId, index); + sWarn("vgId:%d, failed to get raft entry for index:%" PRId64 "", pNode->vgId, index); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); if (pMgr) { From ef22830ca39a70be588db086c6d769d49159e444 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 18:01:30 +0800 Subject: [PATCH 035/198] adjust --- tests/system-test/2-query/irate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index d976edb49c..d882c9b8fb 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -209,7 +209,7 @@ class TDTestCase: tdSql.error(f"select irate(c1), abs(c1) from {dbname}.ct4 ") # agg functions mix with agg functions - tdSql.query(f"select irate(c1), count(c5) from {dbname}.stb1 partition by tbname order by tbname") + tdSql.query(f"select irate(c1), count(c5) from {dbname}.stb1 partition by tbname having count(c5)>0 order by tbname") tdSql.checkData(0, 0, 0.000000000) tdSql.checkData(1, 0, 0.000000000) tdSql.checkData(0, 1, 13) From 9052ca6eded8c879f3c6a9728ce308e3bd2eac15 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 28 Dec 2023 18:05:28 +0800 Subject: [PATCH 036/198] add test case for ts-4403 --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/test_ts4403.py | 77 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 tests/system-test/2-query/test_ts4403.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e7c26b9833..a04a9f6d0a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -519,6 +519,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py diff --git a/tests/system-test/2-query/test_ts4403.py b/tests/system-test/2-query/test_ts4403.py new file mode 100644 index 0000000000..7f0561d084 --- /dev/null +++ b/tests/system-test/2-query/test_ts4403.py @@ -0,0 +1,77 @@ +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TS-4403 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db' + self.stbname = 'st' + + def prepareData(self): + # db + tdSql.execute("create database {};".format(self.dbname)) + tdSql.execute("use {};".format(self.dbname)) + tdLog.debug("Create database %s" % self.dbname) + + # super table + tdSql.execute("create table {} (ts timestamp, col1 int) tags (t1 int, t2 binary(16), t3 nchar(16));".format(self.stbname)) + tdLog.debug("Create super table %s" % self.stbname) + + # create index for all tags + tdSql.execute("create index t2_st on {} (t2);".format(self.stbname)) + tdSql.execute("create index t3_st on {} (t3);".format(self.stbname)) + + def run(self): + self.prepareData() + # check index number + tdSql.query("show indexes from {};".format(self.stbname)) + assert(3 == len(tdSql.queryResult)) + tdLog.debug("All tags of super table have index successfully") + + # drop default first tag index + tdSql.execute("drop index t1_st;") + tdSql.query("show indexes from {};".format(self.stbname)) + assert(2 == len(tdSql.queryResult)) + tdLog.debug("Delete the default index of first tag successfully") + + # create index for first tag + tdSql.execute("create index t1_st on {} (t1);".format(self.stbname)) + tdSql.query("show indexes from {};".format(self.stbname)) + assert(3 == len(tdSql.queryResult)) + tdLog.debug("Create index for first tag successfully") + + # null as index value to create child table + tdSql.execute("create table ct1 using {} tags(null, null, null);".format(self.stbname)) + tdSql.query("show tables;") + assert(1 == len(tdSql.queryResult)) + tdLog.debug("Create child table with tags value as 'null' successfully") + + # redundant index with different index name for some tag + tdSql.error("create index t2_ct1 on st (t2);") + tdLog.debug("Verify redundant index with different index name for some tag successfully") + + # same index name for different tag + tdSql.error("create index t2_st on st (t3);") + tdLog.debug("Verify same index name for some tag successfully") + + # add index for multiple tags(TD-28078) + tdSql.error("create index tt on {} (t2, t3);".format(self.stbname)) + tdLog.debug("Verify add index for multiple tags successfully") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 9f8f69a791f9544e11475af4f0f7c0fa04663b56 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 28 Dec 2023 21:54:16 +0800 Subject: [PATCH 037/198] adjust --- source/libs/executor/src/scanoperator.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a1f9baa082..7b2df4e206 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -670,15 +670,11 @@ static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKey return TSDB_CODE_OUT_OF_MEMORY; } } - uint64_t groupId = 0; + uint64_t groupId = pList->groupId; for (int32_t i = 0; i < num; i++) { const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isSameGroup) { - if (i == 0) { - groupId = pInfo->groupId; - } else if (groupId != pInfo->groupId) { - pTableScanInfo->isSameGroup = false; - } + if (pTableScanInfo->isSameGroup && groupId != pInfo->groupId) { + pTableScanInfo->isSameGroup = false; } taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); } From e2b61a55fc13aee349cb63abb3c7d003c1de783f Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 29 Dec 2023 01:40:17 +0800 Subject: [PATCH 038/198] refactor --- source/libs/executor/inc/executorInt.h | 6 +- source/libs/executor/src/executor.c | 12 +-- source/libs/executor/src/scanoperator.c | 129 ++++++++---------------- 3 files changed, 52 insertions(+), 95 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index ae38d4940c..9acef69f9c 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -268,14 +268,12 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; - SHashObj* pRemainTables; // remain table to process SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; - int32_t currentTable; + int32_t tableStartIndex; // current group scan start + int32_t tableEndIndex; // current group scan end int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state - bool isSameGroup; // whether all tables are in the same group this scan bool hasGroupByTag; bool countOnly; bool filesetDelimited; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6cee79bff2..fb39de484f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1209,7 +1209,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0); uid = pTableInfo->uid; ts = INT64_MIN; - pScanInfo->currentTable = 0; + pScanInfo->tableEndIndex = 0; } else { taosRUnLockLatch(&pTaskInfo->lock); qError("no table in table list, %s", id); @@ -1223,16 +1223,16 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pInfo->pTableScanOp->resultInfo.totalRows = 0; // start from current accessed position - // we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start + // we cannot start from the pScanInfo->tableEndIndex, since the commit offset may cause the rollback of the start // position, let's find it from the beginning. index = tableListFind(pTableListInfo, uid, 0); taosRUnLockLatch(&pTaskInfo->lock); if (index >= 0) { - pScanInfo->currentTable = index; + pScanInfo->tableEndIndex = index; } else { qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, - numOfTables, pScanInfo->currentTable, id); + numOfTables, pScanInfo->tableEndIndex, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1255,12 +1255,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", - uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id); } else { pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1); pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", - uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id); } // restore the key value diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7b2df4e206..e990d3d975 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -655,45 +655,29 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, colDataDestroy(&infoData); } -static int32_t initTableCountEnv(STableScanInfo* pTableScanInfo, const STableKeyInfo* pList, int32_t num) { - if (!pTableScanInfo->needCountEmptyTable) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return TSDB_CODE_SUCCESS; - } - pTableScanInfo->isSameGroup = true; - if (NULL == pTableScanInfo->pRemainTables) { - int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); - pTableScanInfo->pRemainTables = - taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (NULL == pTableScanInfo->pRemainTables) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return TSDB_CODE_OUT_OF_MEMORY; - } - } - uint64_t groupId = pList->groupId; - for (int32_t i = 0; i < num; i++) { - const STableKeyInfo* pInfo = pList + i; - if (pTableScanInfo->isSameGroup && groupId != pInfo->groupId) { - pTableScanInfo->isSameGroup = false; - } - taosHashPut(pTableScanInfo->pRemainTables, &(pInfo->uid), sizeof(pInfo->uid), &(pInfo->groupId), sizeof(pInfo->groupId)); - } - pTableScanInfo->countState = TABLE_COUNT_STATE_SCAN; - return TSDB_CODE_SUCCESS; -} -static void markTableProcessed(STableScanInfo* pTableScanInfo, uint64_t uid) { - // case0 group scanning, mark - // case1 stream scan: no need to mark - if (pTableScanInfo->countState > TABLE_COUNT_STATE_SCAN) { - return; +static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) { + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + int32_t i = pInfo->tableStartIndex + 1; + for (; i < numOfTables; ++i) { + STableKeyInfo* pCur = tableListGetInfo(pInfo->base.pTableListInfo, i); + if (pCur->groupId != pStart->groupId) { + break; + } } - // case2 if all table in same group, process only once - if (pTableScanInfo->isSameGroup) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return; + + pInfo->tableEndIndex = i - 1; + if (!pInfo->needCountEmptyTable) { + pInfo->countState = TABLE_COUNT_STATE_END; + } else { + pInfo->countState = TABLE_COUNT_STATE_SCAN; } - taosHashRemove(pTableScanInfo->pRemainTables, &uid, sizeof(uid)); + + *pKeyInfo = pStart; + *size = i - pInfo->tableStartIndex; } static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, @@ -791,7 +775,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { return NULL; } -static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKeyInfo* pList, int32_t num) { +static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -801,15 +785,11 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey return NULL; } - if (TABLE_COUNT_STATE_NONE == pTableScanInfo->countState) { - initTableCountEnv(pTableScanInfo, pList, num); - } - // do the ascending order traverse in the first place. while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - markTableProcessed(pTableScanInfo, p->info.id.uid); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return p; } @@ -838,7 +818,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - markTableProcessed(pTableScanInfo, p->info.id.uid); + pTableScanInfo->countState = TABLE_COUNT_STATE_END; return p; } @@ -856,29 +836,13 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator, const STableKey } if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { - int32_t tb_cnt = taosHashGetSize(pTableScanInfo->pRemainTables); - if (tb_cnt) { - if (!pTableScanInfo->isSameGroup) { - // get first empty table uid, mark processed & rm from hash - void *pIte = taosHashIterate(pTableScanInfo->pRemainTables, NULL); - if (pIte != NULL) { - size_t keySize = 0; - uint64_t* pUid = taosHashGetKey(pIte, &keySize); - STableKeyInfo info = {.uid = *pUid, .groupId = *(uint64_t*)pIte}; - taosHashCancelIterate(pTableScanInfo->pRemainTables, pIte); - markTableProcessed(pTableScanInfo, *pUid); - return getBlockForEmptyTable(pOperator, &info); - } - } else { - // output one table for this group - pTableScanInfo->countState = TABLE_COUNT_STATE_END; - return getBlockForEmptyTable(pOperator, pList); - } - } + // output once for this group pTableScanInfo->countState = TABLE_COUNT_STATE_END; + STableKeyInfo* pStart = + (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); + return getBlockForEmptyTable(pOperator, pStart); } - taosHashClear(pTableScanInfo->pRemainTables); return NULL; } @@ -938,8 +902,8 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { + int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + if (pInfo->tableEndIndex + 1 >= numOfTables) { setOperatorCompleted(pOperator); if (pOperator->dynamicTask) { taosArrayClear(pInfo->base.pTableListInfo->pTableList); @@ -954,14 +918,13 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pInfo->countState = TABLE_COUNT_STATE_NONE; + initNextGroupScan(pInfo, &pList, &num); pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; - SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -979,15 +942,14 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - if (pInfo->currentGroupId == -1) { + if (pInfo->tableEndIndex == -1) { int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo) || numOfTables == 0) { + if (pInfo->tableEndIndex + 1 == numOfTables) { setOperatorCompleted(pOperator); return NULL; } - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - pInfo->countState = TABLE_COUNT_STATE_NONE; + initNextGroupScan(pInfo, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, @@ -1001,11 +963,9 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; } - } else { - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); } - SSDataBlock* result = doGroupedTableScan(pOperator, pList, num); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; @@ -1038,7 +998,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } if (pOperator->status == OP_EXEC_DONE) { - pInfo->currentGroupId = -1; + pInfo->tableEndIndex = -1; pOperator->status = OP_OPENED; SSDataBlock* result = NULL; while (true) { @@ -1057,29 +1017,29 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->countState = TABLE_COUNT_STATE_END; while (1) { - SSDataBlock* result = doGroupedTableScan(pOperator, NULL, 0); + SSDataBlock* result = doGroupedTableScan(pOperator); if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) { return result; } // if no data, switch to next table and continue scan - pInfo->currentTable++; + pInfo->tableEndIndex++; taosRLockLatch(&pTaskInfo->lock); numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if (pInfo->currentTable >= numOfTables) { + if (pInfo->tableEndIndex >= numOfTables) { qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); taosRUnLockLatch(&pTaskInfo->lock); return NULL; } - tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); + tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableEndIndex); taosRUnLockLatch(&pTaskInfo->lock); pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, - pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); + pInfo->tableEndIndex, numOfTables, GET_TASKID(pTaskInfo)); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; @@ -1117,7 +1077,6 @@ static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); taosHashCleanup(pTableScanInfo->pIgnoreTables); - taosHashCleanup(pTableScanInfo->pRemainTables); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -1173,7 +1132,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } - pInfo->currentGroupId = -1; + pInfo->tableEndIndex = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; @@ -1268,7 +1227,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.startVersion = 0; pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; - pTableScanInfo->currentGroupId = -1; + pTableScanInfo->tableEndIndex = -1; pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -2170,7 +2129,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->pTableScanOp->status = OP_OPENED; pTSInfo->scanTimes = 0; - pTSInfo->currentGroupId = -1; + pTSInfo->tableEndIndex = -1; } if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { From 0a0f1a9d22986c5140a5adda57475455b69a43df Mon Sep 17 00:00:00 2001 From: fullhonest Date: Tue, 26 Dec 2023 19:51:14 +0800 Subject: [PATCH 039/198] Fix TD-27989 --- source/libs/function/inc/tpercentile.h | 4 +-- source/libs/function/src/tpercentile.c | 46 +++++++++++++------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 65b7b38a05..5351594b2f 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -26,12 +26,12 @@ extern "C" { typedef struct MinMaxEntry { union { double dMinVal; - int64_t i64MinVal; + //double i64MinVal; uint64_t u64MinVal; }; union { double dMaxVal; - int64_t i64MaxVal; + //double i64MaxVal; int64_t u64MaxVal; }; } MinMaxEntry; diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 93008b565a..c671e7717c 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -63,8 +63,8 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) static void resetBoundingBox(MinMaxEntry *range, int32_t type) { if (IS_SIGNED_NUMERIC_TYPE(type)) { - range->i64MaxVal = INT64_MIN; - range->i64MinVal = INT64_MAX; + range->dMaxVal = INT64_MIN; + range->dMinVal = INT64_MAX; } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { range->u64MaxVal = 0; range->u64MinVal = UINT64_MAX; @@ -80,8 +80,8 @@ static int32_t setBoundingBox(MinMaxEntry *range, int16_t type, double minval, d } if (IS_SIGNED_NUMERIC_TYPE(type)) { - range->i64MinVal = (int64_t)minval; - range->i64MaxVal = (int64_t)maxval; + range->dMinVal = (int64_t)minval; + range->dMaxVal = (int64_t)maxval; } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { range->u64MinVal = (uint64_t)minval; range->u64MaxVal = (uint64_t)maxval; @@ -137,21 +137,21 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { int32_t index = -1; - if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) { + if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { return index; } // divide the value range into 1024 buckets - uint64_t span = (uint64_t)(pBucket->range.i64MaxVal - pBucket->range.i64MinVal); + uint64_t span = pBucket->range.dMaxVal - pBucket->range.dMinVal; if (span < pBucket->numOfSlots) { - int64_t delta = v - pBucket->range.i64MinVal; + int64_t delta = v - pBucket->range.dMinVal; index = (delta % pBucket->numOfSlots); } else { double slotSpan = ((double)span) / pBucket->numOfSlots; - uint64_t delta = (uint64_t)(v - pBucket->range.i64MinVal); + uint64_t delta = (uint64_t)(v - pBucket->range.dMinVal); - index = (int32_t)(delta / slotSpan); - if (v == pBucket->range.i64MaxVal || index == pBucket->numOfSlots) { + index = delta / slotSpan; + if (v == pBucket->range.dMaxVal || index == pBucket->numOfSlots) { index -= 1; } } @@ -318,23 +318,23 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT int64_t v = 0; GET_TYPED_DATA(v, int64_t, dataType, data); - if (r->i64MinVal > v) { - r->i64MinVal = v; + if (r->dMinVal > v) { + r->dMinVal = v; } - if (r->i64MaxVal < v) { - r->i64MaxVal = v; + if (r->dMaxVal < v) { + r->dMaxVal = v; } } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { uint64_t v = 0; GET_TYPED_DATA(v, uint64_t, dataType, data); - if (r->i64MinVal > v) { - r->i64MinVal = v; + if (r->u64MinVal > v) { + r->u64MinVal = v; } - if (r->i64MaxVal < v) { - r->i64MaxVal = v; + if (r->u64MaxVal < v) { + r->u64MaxVal = v; } } else if (IS_FLOAT_TYPE(dataType)) { double v = 0; @@ -438,7 +438,7 @@ static double getIdenticalDataVal(tMemBucket *pMemBucket, int32_t slotIndex) { double finalResult = 0.0; if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - finalResult = (double)pSlot->range.i64MinVal; + finalResult = (double)pSlot->range.dMinVal; } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { finalResult = (double)pSlot->range.u64MinVal; } else { @@ -469,8 +469,8 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction double maxOfThisSlot = 0; double minOfNextSlot = 0; if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - maxOfThisSlot = (double)pSlot->range.i64MaxVal; - minOfNextSlot = (double)next.i64MinVal; + maxOfThisSlot = (double)pSlot->range.dMaxVal; + minOfNextSlot = (double)next.dMinVal; } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { maxOfThisSlot = (double)pSlot->range.u64MaxVal; minOfNextSlot = (double)next.u64MinVal; @@ -577,7 +577,7 @@ int32_t getPercentile(tMemBucket *pMemBucket, double percent, double *result) { MinMaxEntry *pRange = &pMemBucket->range; if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - *result = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); + *result = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->dMaxVal : pRange->dMinVal); } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { *result = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal); } else { @@ -603,6 +603,6 @@ bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) { if (IS_FLOAT_TYPE(pMemBucket->type)) { return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON; } else { - return pSeg->range.i64MinVal == pSeg->range.i64MaxVal; + return pSeg->range.dMinVal == pSeg->range.dMaxVal; } } From b58a23df49b034490521d28faa51548caa1b2aba Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Dec 2023 13:57:13 +0800 Subject: [PATCH 040/198] fix:[TS-4391] rebalance cnt always 1 if msg lost --- include/common/tmsg.h | 6 - source/dnode/mnode/impl/inc/mndConsumer.h | 5 +- source/dnode/mnode/impl/src/mndConsumer.c | 184 +-------------------- source/dnode/mnode/impl/src/mndSubscribe.c | 170 +++++++++++++++---- 4 files changed, 151 insertions(+), 214 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bafe7583b7..c314d82036 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2544,12 +2544,6 @@ _err: return NULL; } -// this message is sent from mnode to mnode(read thread to write thread), -// so there is no need for serialization or deserialization -typedef struct { - SHashObj* rebSubHash; // SHashObj -} SMqDoRebalanceMsg; - typedef struct { int64_t streamId; int64_t checkpointId; diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index f075510428..59a22b76cd 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -46,9 +46,10 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); int32_t mndSetConsumerDropLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); -bool mndRebTryStart(); +const char *mndConsumerStatusName(int status); + bool mndRebCanStart(); -void mndRebEnd(); +bool mndRebTryStart(); void mndRebCntInc(); void mndRebCntDec(); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 11c929c898..5987f0ec34 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -29,13 +29,9 @@ #define MND_CONSUMER_RESERVE_SIZE 64 #define MND_MAX_GROUP_PER_TOPIC 100 -#define MND_CONSUMER_LOST_HB_CNT 6 -#define MND_CONSUMER_LOST_CLEAR_THRESHOLD 43200 static int32_t mqRebInExecCnt = 0; -static const char *mndConsumerStatusName(int status); - static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer); @@ -45,7 +41,6 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg); static int32_t mndProcessAskEpReq(SRpcMsg *pMsg); static int32_t mndProcessMqHbReq(SRpcMsg *pMsg); -static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg); @@ -63,7 +58,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TMQ_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); - mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); +// mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg); @@ -97,42 +92,18 @@ void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo* bool mndRebTryStart() { int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); - mDebug("tq timer, rebalance counter old val:%d", old); + mInfo("rebalance counter old val:%d", old); return old == 0; } -bool mndRebCanStart() { - int32_t val = atomic_load_32(&mqRebInExecCnt); - if (val < 0) { - mError("rebalance trans end, rebalance counter:%d should not be less equalled than 0, ignore counter desc", val); - return false; - } - mInfo("tq timer, rebalance counter val:%d", val); - return val == 0; -} - -void mndRebEnd() { mndRebCntDec(); } - void mndRebCntInc() { int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1); - mInfo("rebalance trans start, rebalance counter:%d", val); + mInfo("rebalance cnt inc, value:%d", val); } void mndRebCntDec() { - while (1) { - int32_t val = atomic_load_32(&mqRebInExecCnt); - if (val <= 0) { - mError("rebalance trans end, rebalance counter:%d should not be less equalled than 0, ignore counter desc", val); - break; - } - - int32_t newVal = val - 1; - int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal); - if (oldVal == val) { - mInfo("rebalance trans end, rebalance counter:%d", newVal); - break; - } - } + int32_t val = atomic_sub_fetch_32(&mqRebInExecCnt, 1); + mInfo("rebalance cnt sub, value:%d", val); } static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { @@ -282,149 +253,6 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebInfo; } -static void freeRebalanceItem(void *param) { - SMqRebInfo *pInfo = param; - taosArrayDestroy(pInfo->newConsumers); - taosArrayDestroy(pInfo->removedConsumers); -} - -static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SSdb *pSdb = pMnode->pSdb; - SMqConsumerObj *pConsumer; - void *pIter = NULL; - - mInfo("start to process mq timer"); - - // rebalance cannot be parallel - if (!mndRebCanStart()) { - mInfo("mq rebalance already in progress, do nothing"); - return 0; - } - - SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); - if (pRebMsg == NULL) { - mError("failed to create the rebalance msg, size:%d, quit mq timer", (int32_t)sizeof(SMqDoRebalanceMsg)); - return TSDB_CODE_OUT_OF_MEMORY; - } - - pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); - if (pRebMsg->rebSubHash == NULL) { - mError("failed to create rebalance hashmap"); - rpcFreeCont(pRebMsg); - return TSDB_CODE_OUT_OF_MEMORY; - } - - taosHashSetFreeFp(pRebMsg->rebSubHash, freeRebalanceItem); - - // iterate all consumers, find all modification - while (1) { - pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); - if (pIter == NULL) { - break; - } - - int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); - int32_t status = atomic_load_32(&pConsumer->status); - - mInfo("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", - pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, - hbStatus); - - if (status == MQ_CONSUMER_STATUS_READY) { - if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close - mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); - } else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { - taosRLockLatch(&pConsumer->lock); - int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); - for (int32_t i = 0; i < topicNum; i++) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); - mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); - } - taosRUnLockLatch(&pConsumer->lock); - }else{ - int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics); - for (int32_t i = 0; i < newTopicNum; i++) { - char * topic = taosArrayGetP(pConsumer->currentTopics, i); - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); - if (pSub == NULL) { - continue; - } - taosRLockLatch(&pSub->lock); - - // 2.2 iterate all vg assigned to the consumer of that topic - SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); - int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); - - for (int32_t j = 0; j < vgNum; j++) { - SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); - SVgObj * pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); - if (!pVgroup) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, pConsumer->cgroup, topic); - mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); - } - mndReleaseVgroup(pMnode, pVgroup); - } - taosRUnLockLatch(&pSub->lock); - mndReleaseSubscribe(pMnode, pSub); - } - } - } else if (status == MQ_CONSUMER_STATUS_LOST) { - if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { // clear consumer if lost a day - mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); - } - } else { // MQ_CONSUMER_STATUS_REBALANCE - taosRLockLatch(&pConsumer->lock); - - int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); - for (int32_t i = 0; i < newTopicNum; i++) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i); - mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic); - SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); - } - - int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics); - for (int32_t i = 0; i < removedTopicNum; i++) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i); - mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); - } - - if (newTopicNum == 0 && removedTopicNum == 0 && taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close - mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); - } - - taosRUnLockLatch(&pConsumer->lock); - } - - mndReleaseConsumer(pMnode, pConsumer); - } - - if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { - mInfo("mq send msg to rebalance"); - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_DO_REBALANCE, - .pCont = pRebMsg, - .contLen = sizeof(SMqDoRebalanceMsg), - }; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - } else { - taosHashCleanup(pRebMsg->rebSubHash); - rpcFreeCont(pRebMsg); - mInfo("mq timer finished, no need to re-balance"); - } - return 0; -} - static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t code = 0; SMnode *pMnode = pMsg->info.node; @@ -1263,7 +1091,7 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -static const char *mndConsumerStatusName(int status) { +const char *mndConsumerStatusName(int status) { switch (status) { case MQ_CONSUMER_STATUS_READY: return "ready"; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8cd7f6bc9c..b2370374b7 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -27,7 +27,8 @@ #define MND_SUBSCRIBE_VER_NUMBER 2 #define MND_SUBSCRIBE_RESERVE_SIZE 64 -#define MND_SUBSCRIBE_REBALANCE_CNT 3 +#define MND_CONSUMER_LOST_HB_CNT 6 +#define MND_CONSUMER_LOST_CLEAR_THRESHOLD 43200 static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); @@ -38,14 +39,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg); static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); - -static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { - SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; - return 0; -} +static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* hash); static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); @@ -68,7 +62,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_TMQ_SUBSCRIBE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_TMQ_DELETE_SUB_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DO_REBALANCE, mndProcessRebalanceReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessRebalanceReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_CGROUP, mndProcessDropCgroupReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_CGROUP_RSP, mndTransProcessRsp); @@ -727,21 +721,140 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu return 0; } -static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SMqDoRebalanceMsg *pReq = pMsg->pCont; - void *pIter = NULL; -// bool rebalanceOnce = false; // to ensure only once. - if (!mndRebTryStart()) { - mInfo("mq rebalance already in progress, do nothing"); - return 0; +static void freeRebalanceItem(void *param) { + SMqRebInfo *pInfo = param; + taosArrayDestroy(pInfo->newConsumers); + taosArrayDestroy(pInfo->removedConsumers); +} + +static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { + SMnode *pMnode = pMsg->info.node; + SSdb *pSdb = pMnode->pSdb; + SMqConsumerObj *pConsumer; + void *pIter = NULL; + + mInfo("start to process mq timer"); + + // iterate all consumers, find all modification + while (1) { + pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) { + break; + } + + int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); + int32_t status = atomic_load_32(&pConsumer->status); + + mInfo("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", + pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, + hbStatus); + + if (status == MQ_CONSUMER_STATUS_READY) { + if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); + } else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { + taosRLockLatch(&pConsumer->lock); + int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < topicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } + taosRUnLockLatch(&pConsumer->lock); + }else{ + int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < newTopicNum; i++) { + char * topic = taosArrayGetP(pConsumer->currentTopics, i); + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); + if (pSub == NULL) { + continue; + } + taosRLockLatch(&pSub->lock); + + // 2.2 iterate all vg assigned to the consumer of that topic + SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t)); + int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); + + for (int32_t j = 0; j < vgNum; j++) { + SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); + SVgObj * pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); + if (!pVgroup) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + mndMakeSubscribeKey(key, pConsumer->cgroup, topic); + mndGetOrCreateRebSub(rebSubHash, key); + mInfo("vnode splitted, vgId:%d rebalance will be triggered", pVgEp->vgId); + } + mndReleaseVgroup(pMnode, pVgroup); + } + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + } + } + } else if (status == MQ_CONSUMER_STATUS_LOST) { + if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { // clear consumer if lost a day + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); + } + } else { + taosRLockLatch(&pConsumer->lock); + + int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); + for (int32_t i = 0; i < newTopicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); + taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); + } + + int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics); + for (int32_t i = 0; i < removedTopicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } + + if (newTopicNum == 0 && removedTopicNum == 0 && taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId, &pMsg->info); + } + + taosRUnLockLatch(&pConsumer->lock); + } + + mndReleaseConsumer(pMnode, pConsumer); } - mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); + return 0; +} + +static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { + int code = 0; + if (!mndRebTryStart()) { + mInfo("mq rebalance already in progress, do nothing"); + return code; + } + + SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + if (rebSubHash == NULL) { + mError("failed to create rebalance hashmap"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; + goto END; + } + + taosHashSetFreeFp(rebSubHash, freeRebalanceItem); + + mndCheckConsumer(pMsg, rebSubHash); + mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(rebSubHash)); // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction. + void *pIter = NULL; + SMnode *pMnode = pMsg->info.node; while (1) { - pIter = taosHashIterate(pReq->rebSubHash, pIter); + pIter = taosHashIterate(rebSubHash, pIter); if (pIter == NULL) { break; } @@ -760,12 +873,11 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { taosArrayDestroy(rebOutput.modifyConsumers); taosArrayDestroy(rebOutput.rebVgs); - taosHashCancelIterate(pReq->rebSubHash, pIter); + taosHashCancelIterate(rebSubHash, pIter); terrno = TSDB_CODE_OUT_OF_MEMORY; - mInfo("mq re-balance failed, due to out of memory"); - taosHashCleanup(pReq->rebSubHash); - mndRebEnd(); - return -1; + mError("mq re-balance failed, due to out of memory"); + code = -1; + goto END; } SMqRebInfo *pRebInfo = (SMqRebInfo *)pIter; @@ -833,10 +945,12 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { // reset flag mInfo("mq re-balance completed successfully"); - taosHashCleanup(pReq->rebSubHash); - mndRebEnd(); - return 0; +END: + taosHashCleanup(rebSubHash); + mndRebCntDec(); + + return code; } static int32_t sendDeleteSubToVnode(SMqSubscribeObj *pSub, STrans *pTrans){ From c83989754f448500aa91e4c486454a57f46c2dd9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Dec 2023 14:09:49 +0800 Subject: [PATCH 041/198] fix:[TS-4391] rebalance cnt always 1 if msg lost --- source/dnode/mnode/impl/inc/mndConsumer.h | 5 ----- source/dnode/mnode/impl/inc/mndSubscribe.h | 9 ++++----- source/dnode/mnode/impl/src/mndConsumer.c | 18 ------------------ source/dnode/mnode/impl/src/mndSubscribe.c | 18 ++++++++++++++++++ source/dnode/mnode/impl/src/mndTrans.c | 2 +- 5 files changed, 23 insertions(+), 29 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 59a22b76cd..8c89ddc825 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -48,11 +48,6 @@ int32_t mndSetConsumerDropLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *p const char *mndConsumerStatusName(int status); -bool mndRebCanStart(); -bool mndRebTryStart(); -void mndRebCntInc(); -void mndRebCntDec(); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h index 10864da5fb..23b3a7d1fe 100644 --- a/source/dnode/mnode/impl/inc/mndSubscribe.h +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -32,14 +32,13 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); -//static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { -// return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); -//} - -//int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic); int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub); +bool mndRebTryStart(); +void mndRebCntInc(); +void mndRebCntDec(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 5987f0ec34..cf8b9e019a 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -30,8 +30,6 @@ #define MND_MAX_GROUP_PER_TOPIC 100 -static int32_t mqRebInExecCnt = 0; - static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer); @@ -90,22 +88,6 @@ void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo* return; } -bool mndRebTryStart() { - int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); - mInfo("rebalance counter old val:%d", old); - return old == 0; -} - -void mndRebCntInc() { - int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1); - mInfo("rebalance cnt inc, value:%d", val); -} - -void mndRebCntDec() { - int32_t val = atomic_sub_fetch_32(&mqRebInExecCnt, 1); - mInfo("rebalance cnt sub, value:%d", val); -} - static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { SMqTopicObj *pTopic = NULL; int32_t code = 0; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index b2370374b7..b7958c1484 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -30,6 +30,8 @@ #define MND_CONSUMER_LOST_HB_CNT 6 #define MND_CONSUMER_LOST_CLEAR_THRESHOLD 43200 +static int32_t mqRebInExecCnt = 0; + static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); @@ -830,6 +832,22 @@ static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { return 0; } +bool mndRebTryStart() { + int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); + mInfo("rebalance counter old val:%d", old); + return old == 0; +} + +void mndRebCntInc() { + int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1); + mInfo("rebalance cnt inc, value:%d", val); +} + +void mndRebCntDec() { + int32_t val = atomic_sub_fetch_32(&mqRebInExecCnt, 1); + mInfo("rebalance cnt sub, value:%d", val); +} + static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { int code = 0; if (!mndRebTryStart()) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9e478f3aa5..99553fc57a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndConsumer.h" +#include "mndSubscribe.h" #include "mndDb.h" #include "mndPrivilege.h" #include "mndShow.h" From 90a8fe558196d58198221a46b1c08f0e27203bf3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 29 Dec 2023 15:08:56 +0800 Subject: [PATCH 042/198] fix: fill prev not working when desc fill --- source/libs/executor/src/tfill.c | 13 +++-- tests/system-test/2-query/fill_with_group.py | 51 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 6c537d7b98..7b63bc8720 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -169,13 +169,14 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* setNotFillColumn(pFillInfo, pDstCol, index, i); } } else { - SGroupKeys* pKey = taosArrayGet(pFillInfo->prev.pRowVal, i); + SRowVal* pRVal = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; + SGroupKeys* pKey = taosArrayGet(pRVal->pRowVal, i); if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { colDataSetNULL(pDstCol, index); continue; } - SGroupKeys* pKey1 = taosArrayGet(pFillInfo->prev.pRowVal, pFillInfo->tsSlotId); + SGroupKeys* pKey1 = taosArrayGet(pRVal->pRowVal, pFillInfo->tsSlotId); int64_t prevTs = *(int64_t*)pKey1->pData; int32_t srcSlotId = GET_DEST_SLOT_ID(pCol); @@ -346,9 +347,10 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t char* src = colDataGetData(pSrc, pFillInfo->index); if (!colDataIsNull_s(pSrc, pFillInfo->index)) { colDataSetVal(pDst, index, src, false); - saveColData(pFillInfo->prev.pRowVal, i, src, false); + SRowVal* pRVal = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; + saveColData(pRVal->pRowVal, i, src, false); if (pFillInfo->srcTsSlotId == dstSlotId) { - pFillInfo->prev.key = *(int64_t*)src; + pRVal->key = *(int64_t*)src; } } else { // the value is null if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { @@ -361,7 +363,8 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t } else if (pFillInfo->type == TSDB_FILL_LINEAR) { bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); colDataSetVal(pDst, index, src, isNull); - saveColData(pFillInfo->prev.pRowVal, i, src, isNull); // todo: + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; + saveColData(p, i, src, isNull); // todo: } else if (pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { colDataSetNULL(pDst, index); } else if (pFillInfo->type == TSDB_FILL_NEXT) { diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index b442647ff4..fce504d0f7 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -144,10 +144,61 @@ class TDTestCase: tdSql.query(sql) tdSql.checkRows(6) + def test_fill_with_order_by2(self): + ## window size: 5 minutes, with 6 rows in meters every 10 minutes + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart asc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, 10) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(prev) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + + sql = "select _wstart, count(*) from meters where ts >= '2018-09-20 00:00:00.000' and ts < '2018-09-20 01:00:00.000' interval(5m) fill(linear) order by _wstart desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(12) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, 10) + tdSql.checkData(2, 1, 10) + tdSql.checkData(3, 1, 10) + tdSql.checkData(4, 1, 10) + tdSql.checkData(5, 1, 10) + tdSql.checkData(6, 1, 10) + tdSql.checkData(7, 1, 10) + tdSql.checkData(8, 1, 10) + tdSql.checkData(9, 1, 10) + tdSql.checkData(10, 1, 10) + tdSql.checkData(11, 1, 10) + def run(self): self.prepareTestEnv() self.test_partition_by_with_interval_fill_prev_new_group_fill_error() self.test_fill_with_order_by() + self.test_fill_with_order_by2() def stop(self): tdSql.close() From 7cb12db8ce4b4eaa3e1446fdb0eca21c74376c8c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 29 Dec 2023 15:20:27 +0800 Subject: [PATCH 043/198] remove more useless code --- source/dnode/vnode/src/inc/tsdb.h | 18 - .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 796 ------------------ 2 files changed, 814 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ebaf0851e6..75f90af378 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -258,18 +258,6 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback); int32_t tsdbFSClose(STsdb *pTsdb); void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); // tsdbReaderWriter.c ============================================================================================== -// SDataFWriter -int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); -int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); -int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); -int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx); -int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast); -int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo); - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -281,12 +269,6 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *p int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); -// SDelFWriter -int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); -int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx); -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx); -int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e6f419362c..babf8c75fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -433,635 +433,6 @@ _exit: return code; } -// SDataFWriter ==================================================== -int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { - int32_t code = 0; - int32_t flag; - int64_t n; - int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; - SDataFWriter *pWriter = NULL; - char fname[TSDB_FILENAME_LEN]; - char hdr[TSDB_FHDR_SIZE] = {0}; - - // alloc - pWriter = taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pWriter->pTsdb = pTsdb; - pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, - .fid = pSet->fid, - .pHeadF = &pWriter->fHead, - .pDataF = &pWriter->fData, - .pSmaF = &pWriter->fSma, - .nSttF = pSet->nSttF}; - pWriter->fHead = *pSet->pHeadF; - pWriter->fData = *pSet->pDataF; - pWriter->fSma = *pSet->pSmaF; - for (int8_t iStt = 0; iStt < pSet->nSttF; iStt++) { - pWriter->wSet.aSttF[iStt] = &pWriter->fStt[iStt]; - pWriter->fStt[iStt] = *pSet->aSttF[iStt]; - } - - // head - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pHeadFD); - if (code) goto _err; - - code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fHead.size += TSDB_FHDR_SIZE; - - // data - if (pWriter->fData.size == 0) { - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_READ | TD_FILE_WRITE; - } - tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pDataFD); - if (code) goto _err; - if (pWriter->fData.size == 0) { - code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fData.size += TSDB_FHDR_SIZE; - } - - // sma - if (pWriter->fSma.size == 0) { - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_READ | TD_FILE_WRITE; - } - tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pSmaFD); - if (code) goto _err; - if (pWriter->fSma.size == 0) { - code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - pWriter->fSma.size += TSDB_FHDR_SIZE; - } - - // stt - ASSERT(pWriter->fStt[pSet->nSttF - 1].size == 0); - flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbSttFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fStt[pSet->nSttF - 1], fname); - code = tsdbOpenFile(fname, pTsdb, flag, &pWriter->pSttFD); - if (code) goto _err; - code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - pWriter->fStt[pWriter->wSet.nSttF - 1].size += TSDB_FHDR_SIZE; - - *ppWriter = pWriter; - return code; - -_err: - tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; - return code; -} - -int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - STsdb *pTsdb = NULL; - - if (*ppWriter == NULL) goto _exit; - - pTsdb = (*ppWriter)->pTsdb; - if (sync) { - code = tsdbFsyncFile((*ppWriter)->pHeadFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pDataFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pSmaFD); - if (code) goto _err; - - code = tsdbFsyncFile((*ppWriter)->pSttFD); - if (code) goto _err; - } - - tsdbCloseFile(&(*ppWriter)->pHeadFD); - tsdbCloseFile(&(*ppWriter)->pDataFD); - tsdbCloseFile(&(*ppWriter)->pSmaFD); - tsdbCloseFile(&(*ppWriter)->pSttFD); - - for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree((*ppWriter)->aBuf[iBuf]); - } - taosMemoryFree(*ppWriter); -_exit: - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { - int32_t code = 0; - int64_t n; - char hdr[TSDB_FHDR_SIZE]; - - // head ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutHeadFile(hdr, &pWriter->fHead); - code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // data ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutDataFile(hdr, &pWriter->fData); - code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // sma ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSmaFile(hdr, &pWriter->fSma); - code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - // stt ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSttFile(hdr, &pWriter->fStt[pWriter->wSet.nSttF - 1]); - code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; - int64_t n; - - // check - if (taosArrayGetSize(aBlockIdx) == 0) { - pHeadFile->offset = pHeadFile->size; - goto _exit; - } - - // prepare - size = 0; - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pHeadFile->offset = pHeadFile->size; - pHeadFile->size += size; - -_exit: - // tsdbTrace("vgId:%d, write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", - // TD_VID(pWriter->pTsdb->pVnode), - // pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); - return code; - -_err: - tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; - int64_t n; - - ASSERT(mDataBlk->nItem > 0); - - // alloc - size = tPutMapData(NULL, mDataBlk); - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = tPutMapData(pWriter->aBuf[0], mDataBlk); - - // write - code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pBlockIdx->offset = pHeadFile->size; - pBlockIdx->size = size; - pHeadFile->size += size; - - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%" PRId64 " suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 - " size:%" PRId64 " nItem:%d", - TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, - pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); - return code; - -_err: - tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { - int32_t code = 0; - SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; - int64_t size = 0; - int64_t n; - - // check - if (taosArrayGetSize(aSttBlk) == 0) { - pSttFile->offset = pSttFile->size; - goto _exit; - } - - // size - size = 0; - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { - size += tPutSttBlk(NULL, taosArrayGet(aSttBlk, iBlockL)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // encode - n = 0; - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { - n += tPutSttBlk(pWriter->aBuf[0] + n, taosArrayGet(aSttBlk, iBlockL)); - } - - // write - code = tsdbWriteFile(pWriter->pSttFD, pSttFile->size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pSttFile->offset = pSttFile->size; - pSttFile->size += size; - -_exit: - tsdbTrace("vgId:%d, tsdb write stt block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pSttFile->offset, size); - return code; - -_err: - tsdbError("vgId:%d, tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { - int32_t code = 0; - - pSmaInfo->offset = 0; - pSmaInfo->size = 0; - - // encode - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; - - SColumnDataAgg sma = {.colId = pColData->cid}; - tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); - - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); - if (code) goto _err; - pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); - } - - // write - if (pSmaInfo->size) { - code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); - if (code) goto _err; - - pSmaInfo->offset = pWriter->fSma.size; - pWriter->fSma.size += pSmaInfo->size; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast) { - int32_t code = 0; - - ASSERT(pBlockData->nRow > 0); - - if (toLast) { - pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; - } else { - pBlkInfo->offset = pWriter->fData.size; - } - pBlkInfo->szBlock = 0; - pBlkInfo->szKey = 0; - - int32_t aBufN[4] = {0}; - code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); - if (code) goto _err; - - // write ================= - STsdbFD *pFD = toLast ? pWriter->pSttFD : pWriter->pDataFD; - - pBlkInfo->szKey = aBufN[3] + aBufN[2]; - pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - - int64_t offset = pBlkInfo->offset; - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[3], aBufN[3]); - if (code) goto _err; - offset += aBufN[3]; - - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[2], aBufN[2]); - if (code) goto _err; - offset += aBufN[2]; - - if (aBufN[1]) { - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[1], aBufN[1]); - if (code) goto _err; - offset += aBufN[1]; - } - - if (aBufN[0]) { - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], aBufN[0]); - if (code) goto _err; - } - - // update info - if (toLast) { - pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; - } else { - pWriter->fData.size += pBlkInfo->szBlock; - } - - // ================= SMA ==================== - if (pSmaInfo) { - code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); - if (code) goto _err; - } - -_exit: - tsdbTrace("vgId:%d, tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", - TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, - pBlkInfo->szBlock); - return code; - -_err: - tsdbError("vgId:%d, tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) { - int32_t code = 0; - int32_t lino = 0; - - STsdbFD *pFD = NULL; - if (pSmaInfo) { - pFD = pWriter->pDataFD; - pBlkInfo->offset = pWriter->fData.size; - } else { - pFD = pWriter->pSttFD; - pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; - } - pBlkInfo->szBlock = 0; - pBlkInfo->szKey = 0; - - // hdr - int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr); - code = tRealloc(&pWriter->aBuf[0], n); - TSDB_CHECK_CODE(code, lino, _exit); - - tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr); - - code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += n; - pBlkInfo->szBlock += n; - - // uid + ver + key - if (pDiskData->pUid) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szUid; - pBlkInfo->szBlock += pDiskData->hdr.szUid; - } - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szVer; - pBlkInfo->szBlock += pDiskData->hdr.szVer; - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey); - TSDB_CHECK_CODE(code, lino, _exit); - pBlkInfo->szKey += pDiskData->hdr.szKey; - pBlkInfo->szBlock += pDiskData->hdr.szKey; - - // aBlockCol - if (pDiskData->hdr.szBlkCol) { - code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol); - TSDB_CHECK_CODE(code, lino, _exit); - - n = 0; - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol); - } - ASSERT(n == pDiskData->hdr.szBlkCol); - - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskData->hdr.szBlkCol; - } - - // aDiskCol - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - - if (pDiskCol->pBit) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szBitmap; - } - - if (pDiskCol->pOff) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szOffset; - } - - if (pDiskCol->pVal) { - code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue); - TSDB_CHECK_CODE(code, lino, _exit); - - pBlkInfo->szBlock += pDiskCol->bCol.szValue; - } - } - - if (pSmaInfo) { - pWriter->fData.size += pBlkInfo->szBlock; - } else { - pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; - goto _exit; - } - - pSmaInfo->offset = 0; - pSmaInfo->size = 0; - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - - if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue; - if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue; - if (!pDiskCol->bCol.smaOn) continue; - - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg)); - TSDB_CHECK_CODE(code, lino, _exit); - pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg); - } - - if (pSmaInfo->size) { - pSmaInfo->offset = pWriter->fSma.size; - - code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->fSma.size += pSmaInfo->size; - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { - int32_t code = 0; - int64_t n; - int64_t size; - TdFilePtr pOutFD = NULL; - TdFilePtr PInFD = NULL; - int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; - char fNameFrom[TSDB_FILENAME_LEN]; - char fNameTo[TSDB_FILENAME_LEN]; - - // head - tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); - tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // data - tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); - tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // sma - tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); - tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // stt - for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { - tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); - tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - // SDataFReader ==================================================== int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; @@ -1478,173 +849,6 @@ _exit: return code; } -// SDelFWriter ==================================================== -int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { - int32_t code = 0; - int32_t lino = 0; - char fname[TSDB_FILENAME_LEN]; - uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter = NULL; - int64_t n; - - // alloc - pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); - if (pDelFWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pDelFWriter->pTsdb = pTsdb; - pDelFWriter->fDel = *pFile; - - tsdbDelFileName(pTsdb, pFile, fname); - code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); - TSDB_CHECK_CODE(code, lino, _exit); - - // update header - code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); - TSDB_CHECK_CODE(code, lino, _exit); - - pDelFWriter->fDel.size = TSDB_FHDR_SIZE; - pDelFWriter->fDel.offset = 0; - - *ppWriter = pDelFWriter; - -_exit: - if (code) { - if (pDelFWriter) { - tsdbCloseFile(&pDelFWriter->pWriteH); - taosMemoryFree(pDelFWriter); - } - *ppWriter = NULL; - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); - } else { - *ppWriter = pDelFWriter; - } - return code; -} - -int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - SDelFWriter *pWriter = *ppWriter; - STsdb *pTsdb = pWriter->pTsdb; - - // sync - if (sync) { - code = tsdbFsyncFile(pWriter->pWriteH); - if (code) goto _err; - } - - // close - tsdbCloseFile(&pWriter->pWriteH); - - for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree(pWriter->aBuf[iBuf]); - } - taosMemoryFree(pWriter); - - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { - int32_t code = 0; - int64_t size; - int64_t n; - - // prepare - size = 0; - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pDelIdx->offset = pWriter->fDel.size; - pDelIdx->size = size; - pWriter->fDel.size += size; - - return code; - -_err: - tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { - int32_t code = 0; - int64_t size; - int64_t n; - SDelIdx *pDelIdx; - - // prepare - size = 0; - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); - } - - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // build - n = 0; - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); - } - ASSERT(n == size); - - // write - code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); - if (code) goto _err; - - // update - pWriter->fDel.offset = pWriter->fDel.size; - pWriter->fDel.size += size; - - return code; - -_err: - tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) { - int32_t code = 0; - char hdr[TSDB_FHDR_SIZE] = {0}; - int64_t size = TSDB_FHDR_SIZE; - int64_t n; - - // build - tPutDelFile(hdr, &pWriter->fDel); - - // write - code = tsdbWriteFile(pWriter->pWriteH, 0, hdr, size); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} // SDelFReader ==================================================== struct SDelFReader { STsdb *pTsdb; From 7e1d59c565e408dc952b26ecdcd349c070cd3f9f Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 29 Dec 2023 16:22:41 +0800 Subject: [PATCH 044/198] sort --- source/libs/executor/inc/executorInt.h | 5 ++-- source/libs/executor/src/executil.c | 4 +++- source/libs/executor/src/operator.c | 2 +- source/libs/executor/src/scanoperator.c | 32 ++++++++++++++++++------- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 9acef69f9c..c09f615735 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -269,8 +269,9 @@ typedef struct STableScanInfo { SSDataBlock* pResBlock; SHashObj* pIgnoreTables; SSampleExecInfo sample; // sample execution info - int32_t tableStartIndex; // current group scan start - int32_t tableEndIndex; // current group scan end + int32_t tableStartIndex; // current group scan start + int32_t tableEndIndex; // current group scan end + int32_t currentGroupIndex; // current group index of groupOffset int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 377de99fc0..4b3b1ffead 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2122,6 +2122,9 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool groupByTbname = groupbyTbname(group); size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); + if (!numOfTables) { + return code; + } if (group == NULL || groupByTbname) { for (int32_t i = 0; i < numOfTables; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); @@ -2143,7 +2146,6 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* if (code != TSDB_CODE_SUCCESS) { return code; } - if (pScanNode->groupOrderScan) pTableListInfo->numOfOuputGroups = taosArrayGetSize(pTableListInfo->pTableList); if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 69a8acb3d7..0b5e7f51ed 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -302,7 +302,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pTableListInfo->idInfo.suid = pTableScanNode->scan.suid; pTableListInfo->idInfo.tableType = pTableScanNode->scan.tableType; } else { - code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, + code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e990d3d975..27c9a76757 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -659,17 +659,30 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) { pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); - int32_t i = pInfo->tableStartIndex + 1; - for (; i < numOfTables; ++i) { - STableKeyInfo* pCur = tableListGetInfo(pInfo->base.pTableListInfo, i); - if (pCur->groupId != pStart->groupId) { - break; + STableListInfo* pTableListInfo = pInfo->base.pTableListInfo; + int32_t numOfTables = tableListGetSize(pTableListInfo); + STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pTableListInfo, pInfo->tableStartIndex); + + if (pTableListInfo->oneTableForEachGroup) { + pInfo->tableEndIndex = pInfo->tableStartIndex; + } else if (pTableListInfo->groupOffset) { + pInfo->currentGroupIndex++; + if (pInfo->currentGroupIndex + 1 < pTableListInfo->numOfOuputGroups) { + pInfo->tableEndIndex = pTableListInfo->groupOffset[pInfo->currentGroupIndex + 1] - 1; + } else { + pInfo->tableEndIndex = numOfTables - 1; } + } else { + int32_t i = pInfo->tableStartIndex + 1; + for (; i < numOfTables; ++i) { + STableKeyInfo* pCur = tableListGetInfo(pTableListInfo, i); + if (pCur->groupId != pStart->groupId) { + break; + } + } + pInfo->tableEndIndex = i - 1; } - pInfo->tableEndIndex = i - 1; if (!pInfo->needCountEmptyTable) { pInfo->countState = TABLE_COUNT_STATE_END; } else { @@ -677,7 +690,7 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i } *pKeyInfo = pStart; - *size = i - pInfo->tableStartIndex; + *size = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; } static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, @@ -1133,6 +1146,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->tableEndIndex = -1; + pInfo->currentGroupIndex = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; From d158ca74d841b0cf373725fed96010b8a7872f4a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Dec 2023 17:22:27 +0800 Subject: [PATCH 045/198] fix:[TS-4391] rebalance cnt always 1 if msg lost --- source/dnode/mnode/impl/src/mndConsumer.c | 15 --------------- source/dnode/mnode/impl/src/mndSubscribe.c | 14 ++++++++------ 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index cf8b9e019a..9d9473d883 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -220,21 +220,6 @@ FAIL: return -1; } -static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); - if (pRebInfo == NULL) { - pRebInfo = tNewSMqRebSubscribe(key); - if (pRebInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - taosHashPut(pHash, key, strlen(key) + 1, pRebInfo, sizeof(SMqRebInfo)); - taosMemoryFree(pRebInfo); - pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); - } - return pRebInfo; -} - static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t code = 0; SMnode *pMnode = pMsg->info.node; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index b7958c1484..16ae7f2548 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -209,16 +209,18 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, } static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebInfo *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); - if (pRebSub == NULL) { - pRebSub = tNewSMqRebSubscribe(key); - if (pRebSub == NULL) { + SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); + if (pRebInfo == NULL) { + pRebInfo = tNewSMqRebSubscribe(key); + if (pRebInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebInfo)); + taosHashPut(pHash, key, strlen(key) + 1, pRebInfo, sizeof(SMqRebInfo)); + taosMemoryFree(pRebInfo); + pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); } - return pRebSub; + return pRebInfo; } static void doRemoveLostConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { From 8ac02adc0b94aae0e013c7914ec6182b24413a22 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 29 Dec 2023 17:42:23 +0800 Subject: [PATCH 046/198] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tqRead.c | 3 ++- source/libs/wal/src/walRead.c | 5 ----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 41b1aa7bd1..bfa8cfdb53 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -195,7 +195,8 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t int64_t committedVer = walGetCommittedVer(pHandle->pWalReader->pWal); int64_t appliedVer = walGetAppliedVer(pHandle->pWalReader->pWal); - wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64", 0x%"PRIx64, + wDebug("vgId:%d, start to fetch wal, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64 + ", 0x%" PRIx64, vgId, offset, lastVer, committedVer, appliedVer, id); while (offset <= appliedVer) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c0435ca774..1660a0ecf0 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -259,11 +259,6 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { int64_t contLen; bool seeked = false; - wDebug("vgId:%d, try to fetch ver %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64", 0x%"PRIx64, - pRead->pWal->cfg.vgId, ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, - pRead->pWal->vers.appliedVer, pRead->readerId); - // TODO: valid ver if (ver > pRead->pWal->vers.commitVer) { return -1; From 87b7a9a980233880deda02ef05a0f3915560379c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 29 Dec 2023 17:51:35 +0800 Subject: [PATCH 047/198] fix: stream test case --- tests/script/tsim/stream/basic0.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 917a0a73ab..a525594861 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -33,6 +33,8 @@ if $rows != 3 then return -1 endi +sleep 1000 + sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) sql show stables From 673d3bc376ad8955bd4b3b9163853eebb983b915 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 31 Dec 2023 13:15:05 +0800 Subject: [PATCH 048/198] arrangeTableGroup --- source/libs/executor/src/executil.c | 70 +++++++++++++++++++++++++++++ source/libs/executor/src/operator.c | 2 +- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 4b3b1ffead..f3e2e99332 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2116,6 +2116,70 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TSDB_CODE_SUCCESS; } +static int32_t arrangeTableGroup(STableListInfo* pTableListInfo) { + int32_t code = TSDB_CODE_SUCCESS; + size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); + SArray* pDup = NULL; + SHashObj* pHashObj = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (!pHashObj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t num = 1; + // first: get the number of tables per group + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, i); + int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); + // update each group's table count + if (!pVal) { + taosHashPut(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId), &num, sizeof(num)); + } else { + (*pVal)++; + } + } + pTableListInfo->numOfOuputGroups = taosHashGetSize(pHashObj); + pTableListInfo->oneTableForEachGroup = (pTableListInfo->numOfOuputGroups == numOfTables); + + if (pTableListInfo->numOfOuputGroups > 1 && pTableListInfo->numOfOuputGroups < numOfTables) { + pDup = taosArrayDup(pTableListInfo->pTableList, NULL); + pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups); + if (pDup == NULL || pTableListInfo->groupOffset == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + int32_t idx = 0, sum = 0; + + void* pIter = taosHashIterate(pHashObj, NULL); + while (pIter != NULL) { + // record each group's offset + pTableListInfo->groupOffset[idx] = sum; + int32_t* pData = (int32_t*)pIter; + sum += *pData; + // change value to record group item's first position + *pData = pTableListInfo->groupOffset[idx++]; + pIter = taosHashIterate(pHashObj, pIter); + } + + // second: arrange the tables and put the items with same groupId together + STableKeyInfo* pStart = taosArrayGet(pTableListInfo->pTableList, 0); + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pInfo = taosArrayGet(pDup, i); + int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); + if (*pVal != i) { + pStart[*pVal] = *pInfo; + } + (*pVal)++; // update to next item's position + } + } + +_exit: + taosHashCleanup(pHashObj); + taosArrayDestroy(pDup); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pTableListInfo->groupOffset); + } + return code; +} + int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, bool groupSort, uint8_t* digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; @@ -2149,6 +2213,12 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; + if (pTableScanNode->needCountEmptyTable) { + // only put together tables with the same groupid + arrangeTableGroup(pTableListInfo); + } } } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 0b5e7f51ed..69a8acb3d7 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -302,7 +302,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pTableListInfo->idInfo.suid = pTableScanNode->scan.suid; pTableListInfo->idInfo.tableType = pTableScanNode->scan.tableType; } else { - code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, + code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; From 40205541a67f8db225254d111aef6390abd9a650 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:19:23 +0800 Subject: [PATCH 049/198] fix: check in client when create index --- source/libs/parser/src/parTranslater.c | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bdee4d54ac..42aebb3806 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6929,11 +6929,45 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde } static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + SName name; + STableMeta* pMeta = NULL; + int32_t code = + getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + if (code) { + taosMemoryFree(pMeta); + return code; + } + if (TSDB_SUPER_TABLE != pMeta->tableType) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + } + + if (LIST_LENGTH(pStmt->pCols) != 1) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only 1 tag is allowed"); + } + + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pCols) { + SColumnNode* p = (SColumnNode*)pNode; + const SSchema* pSchema = getTagSchema(pMeta, p->colName); + if (!pSchema) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, p->colName); + } + if (IS_IDX_ON(pSchema)) { + code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + goto _exit; + } + } + SCreateTagIndexReq createTagIdxReq = {0}; - int32_t code = buildCreateTagIndexReq(pCxt, pStmt, &createTagIdxReq); + code = buildCreateTagIndexReq(pCxt, pStmt, &createTagIdxReq); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSCreateTagIdxReq, &createTagIdxReq); } +_exit: + taosMemoryFree(pMeta); return code; } From 0d26c186f0d3f11818264498e9e5b40b97626881 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:23:19 +0800 Subject: [PATCH 050/198] chore: code optimization --- source/libs/parser/src/parTranslater.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42aebb3806..d980c0f5ae 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6949,11 +6949,10 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS SNode* pNode = NULL; FOREACH(pNode, pStmt->pCols) { - SColumnNode* p = (SColumnNode*)pNode; - const SSchema* pSchema = getTagSchema(pMeta, p->colName); + const SSchema* pSchema = getTagSchema(pMeta, ((SColumnNode*)pNode)->colName); if (!pSchema) { taosMemoryFree(pMeta); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, p->colName); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, ((SColumnNode*)pNode)->colName); } if (IS_IDX_ON(pSchema)) { code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; From ecf3a7b6d350f5e3f40ae9736f84e1a6f2b8c9d3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 2 Jan 2024 00:28:30 +0800 Subject: [PATCH 051/198] fix: code optimization --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d980c0f5ae..c3b9f9fc85 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6939,12 +6939,12 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS } if (TSDB_SUPER_TABLE != pMeta->tableType) { taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only super table is allowed"); } if (LIST_LENGTH(pStmt->pCols) != 1) { taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only 1 tag is allowed"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM, "Only one tag is allowed"); } SNode* pNode = NULL; From e79516ade302ddf508230fe950d7b48fd6dbba6d Mon Sep 17 00:00:00 2001 From: fullhonest Date: Tue, 2 Jan 2024 10:02:30 +0800 Subject: [PATCH 052/198] Fix TD-27939: error for sub query is empty when where 1=2 --- source/libs/parser/src/parCalcConst.c | 7 ++++++ tests/system-test/2-query/select_null.py | 31 +++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 441f4da3b1..af24624067 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -334,6 +334,13 @@ static int32_t calcConstSelectWithoutFrom(SCalcConstContext* pCxt, SSelectStmt* static int32_t calcConstSelectFrom(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { int32_t code = calcConstFromTable(pCxt, pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && + ((STempTableNode*)pSelect->pFromTable)->pSubquery != NULL && + QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery) && + ((SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery)->isEmptyResult){ + pSelect->isEmptyResult = true; + return code; + } if (TSDB_CODE_SUCCESS == code) { code = calcConstProjections(pCxt, pSelect, subquery); } diff --git a/tests/system-test/2-query/select_null.py b/tests/system-test/2-query/select_null.py index e20f6131c9..8411a33a1f 100755 --- a/tests/system-test/2-query/select_null.py +++ b/tests/system-test/2-query/select_null.py @@ -401,7 +401,29 @@ class TDTestCase: tdSql.execute(sql) sql = "select * from %s.`12345` order by `567` desc limit 2;"%(database) tdSql.error(sql) - + + def td_27939(self,database): + sql = "create table %s.`test1eq2` (`ts` timestamp, id int);"%(database) + tdSql.execute(sql) + + sql = "insert into %s.test1eq2 values (now,1);"%(database) + tdSql.execute(sql) + + sql = "insert into %s.`test1eq2` values (now,2);"%(database) + tdSql.execute(sql) + + sql = "select * from %s.`test1eq2` where 1=2;"%(database) + tdSql.query(sql) + tdSql.checkRows(0) + + sql = "select * from (select * from %s.`test1eq2` where 1=2);"%(database) + tdSql.query(sql) + tdSql.checkRows(0) + + sql = "drop table %s.`test1eq2` ;"%(database) + tdSql.execute(sql) + + def run(self): startTime = time.time() @@ -418,6 +440,8 @@ class TDTestCase: self.ts_3110("%s" %self.db) self.ts_23505("%s" %self.db) self.ts_3036("%s" %self.db) + + self.td_27939("%s" %self.db) tdSql.query("flush database %s" %self.db) @@ -430,8 +454,9 @@ class TDTestCase: self.ts_3110("%s" %self.db) self.ts_23505("%s" %self.db) self.ts_3036("%s" %self.db) - - + + self.td_27939("%s" %self.db) + self.test_select_as_chinese_characters(); endTime = time.time() print("total time %ds" % (endTime - startTime)) From c787535693f3d2d411f2c92e28c59d3c421018c2 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 2 Jan 2024 10:11:22 +0800 Subject: [PATCH 053/198] fix session state issue --- source/libs/stream/src/streamSessionState.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index c71edccb99..765403b1aa 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -117,7 +117,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* pFileStore = getStateFileStore(pFileState); void* p = NULL; int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); - if (code_file == TSDB_CODE_SUCCESS) { + if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); code = code_file; qDebug("===stream===0 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); From be0d35a5af93f21e1ee774b440d0a093577432f3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 2 Jan 2024 10:20:04 +0800 Subject: [PATCH 054/198] fix: last table scan reported invalid param when col was dropped which should report invalid col --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- tests/system-test/2-query/last_cache_scan.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 1f54f13592..4ca7881814 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -120,7 +120,7 @@ static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInf } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing i += 1; } else { - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; } } diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 39271318ba..23433277c2 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -413,8 +413,7 @@ class TDTestCase: tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) - - + tdSql.query_success_failed('select last(c2), c2, c3 from meters', queryTimes=10, expectErrInfo="Invalid column name: c2") def run(self): self.prepareTestEnv() From d21741269170f92deebd51684db588154ab1d7bb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 10:33:25 +0800 Subject: [PATCH 055/198] fix(stream): avoid race condition in starting threads. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 2991e3cef5..f576345f64 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -545,7 +545,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = /*(HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; streamTaskHandleEvent(p->status.pSM, event); } else if (!restored) { tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 048092131d..c41aea255d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -571,7 +571,20 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta); - tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); + + { + streamMetaWLock(pMeta); + if (pMeta->startInfo.taskStarting == 1) { + pMeta->startInfo.restartCount += 1; + tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, + pMeta->startInfo.restartCount); + streamMetaWUnLock(pMeta); + } else { + pMeta->startInfo.taskStarting = 1; + streamMetaWUnLock(pMeta); + tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); + } + } } } else { vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); From b3d8f4e9d6914c6d8746da174a4bcd6dc2859104 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 10:39:01 +0800 Subject: [PATCH 056/198] fix(stream): fix a typo. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index f576345f64..a3d860fd78 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -545,7 +545,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + EStreamTaskEvent event = TASK_EVENT_INIT; streamTaskHandleEvent(p->status.pSM, event); } else if (!restored) { tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); From 4368bfb20a0fb4737b91011a0772dfe4ab841879 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 11:40:29 +0800 Subject: [PATCH 057/198] fix(stream): fix dead lock. --- source/dnode/vnode/src/vnd/vnodeSync.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index c41aea255d..84ef36df74 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -587,10 +587,9 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) } } } else { + streamMetaWUnLock(pMeta); vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); } - - streamMetaWUnLock(pMeta); } static void vnodeBecomeFollower(const SSyncFSM *pFsm) { From afc2a346e5719a9529a038b7d6a36d31461a623d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 2 Jan 2024 14:16:34 +0800 Subject: [PATCH 058/198] reset update info --- source/libs/executor/inc/executorInt.h | 1 + .../libs/executor/src/streameventwindowoperator.c | 1 + .../libs/executor/src/streamtimewindowoperator.c | 15 +++++++++++++++ 3 files changed, 17 insertions(+) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e3e504cdbc..a2ced573f7 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -845,6 +845,7 @@ void resetWinRange(STimeWindow* winRange); bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts); int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval); void resetUnCloseSessionWinInfo(SSHashObj* winMap); +void reloadAggSupFromDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup); int32_t encodeSSessionKey(void** buf, SSessionKey* key); void* decodeSSessionKey(void* buf, SSessionKey* key); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index bd247eba07..85f15cc368 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -654,6 +654,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) { if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } + reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); } SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9490e2c55..946b02f2d4 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2759,6 +2759,18 @@ void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, } } +void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup) { + SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; + + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + reloadAggSupFromDownStream(downstream->pDownstream[0], pAggSup); + return; + } + + SStreamScanInfo* pScanInfo = downstream->info; + pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; +} + void streamSessionSemiReloadState(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -2790,6 +2802,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } + reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); } void streamSessionReloadState(SOperatorInfo* pOperator) { @@ -2842,6 +2855,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } + reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); } SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -3724,6 +3738,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } + reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); } SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, From 85ba9a4a5fe2332dd3fc889ee71e8f4e0b96fc20 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Tue, 2 Jan 2024 15:45:03 +0800 Subject: [PATCH 059/198] remainGroups --- source/libs/executor/inc/executil.h | 1 + source/libs/executor/inc/executorInt.h | 7 +- source/libs/executor/src/executil.c | 103 +++++++----------------- source/libs/executor/src/scanoperator.c | 51 ++++++++---- 4 files changed, 72 insertions(+), 90 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 946d3311a4..640ed2f2f9 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -105,6 +105,7 @@ typedef struct STableListInfo { int32_t* groupOffset; // keep the offset value for each group in the tableList SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid + SHashObj* remainGroups; // remaining group has not yet processed the empty group STableListIdInfo idInfo; // this maybe the super table or ordinary table } STableListInfo; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index c09f615735..bee378effc 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -218,9 +218,10 @@ enum { }; typedef enum ETableCountState { - TABLE_COUNT_STATE_NONE = 0, // before start scan - TABLE_COUNT_STATE_SCAN = 1, // scanning - TABLE_COUNT_STATE_END = 2, // finish or noneed to process + TABLE_COUNT_STATE_NONE = 0, // before start scan + TABLE_COUNT_STATE_SCAN = 1, // cur group scanning + TABLE_COUNT_STATE_PROCESSED = 2, // cur group processed + TABLE_COUNT_STATE_END = 3, // finish or noneed to process } ETableCountState; typedef struct SAggSupporter { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f3e2e99332..fe3528cce6 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -456,7 +456,7 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C } int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest, - SStorageAPI* pAPI) { + SStorageAPI* pAPI, bool initRemainGroups) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -590,6 +590,15 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf goto end; } + if (initRemainGroups) { + pTableListInfo->remainGroups = + taosHashInit(rows, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pTableListInfo->remainGroups == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + } + for (int i = 0; i < rows; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); @@ -631,6 +640,14 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf int32_t len = (int32_t)(pStart - (char*)keyBuf); info->groupId = calcGroupId(keyBuf, len); + if (initRemainGroups) { + // groupId ~ table uid + taosHashPut(pTableListInfo->remainGroups, &(info->groupId), sizeof(info->groupId), &(info->uid), sizeof(info->uid)); + } + } + + if (initRemainGroups) { + pTableListInfo->numOfOuputGroups = taosHashGetSize(pTableListInfo->remainGroups); } if (tsTagFilterCache) { @@ -2025,6 +2042,7 @@ STableListInfo* tableListCreate() { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pListInfo->remainGroups = NULL; pListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); if (pListInfo->pTableList == NULL) { @@ -2054,7 +2072,7 @@ void* tableListDestroy(STableListInfo* pTableListInfo) { taosMemoryFreeClear(pTableListInfo->groupOffset); taosHashCleanup(pTableListInfo->map); - + taosHashCleanup(pTableListInfo->remainGroups); pTableListInfo->pTableList = NULL; pTableListInfo->map = NULL; taosMemoryFree(pTableListInfo); @@ -2068,6 +2086,7 @@ void tableListClear(STableListInfo* pTableListInfo) { taosArrayClear(pTableListInfo->pTableList); taosHashClear(pTableListInfo->map); + taosHashClear(pTableListInfo->remainGroups); taosMemoryFree(pTableListInfo->groupOffset); pTableListInfo->numOfOuputGroups = 1; pTableListInfo->oneTableForEachGroup = false; @@ -2116,70 +2135,6 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TSDB_CODE_SUCCESS; } -static int32_t arrangeTableGroup(STableListInfo* pTableListInfo) { - int32_t code = TSDB_CODE_SUCCESS; - size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); - SArray* pDup = NULL; - SHashObj* pHashObj = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (!pHashObj) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int32_t num = 1; - // first: get the number of tables per group - for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, i); - int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); - // update each group's table count - if (!pVal) { - taosHashPut(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId), &num, sizeof(num)); - } else { - (*pVal)++; - } - } - pTableListInfo->numOfOuputGroups = taosHashGetSize(pHashObj); - pTableListInfo->oneTableForEachGroup = (pTableListInfo->numOfOuputGroups == numOfTables); - - if (pTableListInfo->numOfOuputGroups > 1 && pTableListInfo->numOfOuputGroups < numOfTables) { - pDup = taosArrayDup(pTableListInfo->pTableList, NULL); - pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups); - if (pDup == NULL || pTableListInfo->groupOffset == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - int32_t idx = 0, sum = 0; - - void* pIter = taosHashIterate(pHashObj, NULL); - while (pIter != NULL) { - // record each group's offset - pTableListInfo->groupOffset[idx] = sum; - int32_t* pData = (int32_t*)pIter; - sum += *pData; - // change value to record group item's first position - *pData = pTableListInfo->groupOffset[idx++]; - pIter = taosHashIterate(pHashObj, pIter); - } - - // second: arrange the tables and put the items with same groupId together - STableKeyInfo* pStart = taosArrayGet(pTableListInfo->pTableList, 0); - for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pInfo = taosArrayGet(pDup, i); - int32_t* pVal = taosHashGet(pHashObj, &pInfo->groupId, sizeof(pInfo->groupId)); - if (*pVal != i) { - pStart[*pVal] = *pInfo; - } - (*pVal)++; // update to next item's position - } - } - -_exit: - taosHashCleanup(pHashObj); - taosArrayDestroy(pDup); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pTableListInfo->groupOffset); - } - return code; -} - int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, bool groupSort, uint8_t* digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; @@ -2206,19 +2161,21 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI); + bool initRemainGroups = false; + if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; + if (pTableScanNode->needCountEmptyTable) { + initRemainGroups = true; + } + } + + code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI, initRemainGroups); if (code != TSDB_CODE_SUCCESS) { return code; } if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { - STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable) { - // only put together tables with the same groupid - arrangeTableGroup(pTableListInfo); - } } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 27c9a76757..8999340ddc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -673,14 +673,7 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i pInfo->tableEndIndex = numOfTables - 1; } } else { - int32_t i = pInfo->tableStartIndex + 1; - for (; i < numOfTables; ++i) { - STableKeyInfo* pCur = tableListGetInfo(pTableListInfo, i); - if (pCur->groupId != pStart->groupId) { - break; - } - } - pInfo->tableEndIndex = i - 1; + pInfo->tableEndIndex = numOfTables - 1; } if (!pInfo->needCountEmptyTable) { @@ -693,6 +686,17 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i *size = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; } +void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { + if (pInfo->countState == TABLE_COUNT_STATE_END) { + return; + } + if (pInfo->base.pTableListInfo->oneTableForEachGroup) { + pInfo->countState = TABLE_COUNT_STATE_PROCESSED; + } else { + taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId)); + } +} + static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock, const STableKeyInfo* tbInfo) { blockDataEmpty(pBlock); @@ -802,7 +806,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; + markGroupProcessed(pTableScanInfo, p->info.id.groupId); return p; } @@ -831,7 +835,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { - pTableScanInfo->countState = TABLE_COUNT_STATE_END; + markGroupProcessed(pTableScanInfo, p->info.id.groupId); return p; } @@ -849,11 +853,30 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { - // output once for this group + STableListInfo* pTableListInfo = pTableScanInfo->base.pTableListInfo; + if (pTableListInfo->oneTableForEachGroup) { // group by tbname + if (pTableScanInfo->countState < TABLE_COUNT_STATE_PROCESSED) { + pTableScanInfo->countState = TABLE_COUNT_STATE_PROCESSED; + STableKeyInfo* pStart = + (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); + return getBlockForEmptyTable(pOperator, pStart); + } + } else { // group by tag + int32_t numOfTables = tableListGetSize(pTableListInfo); + if (pTableScanInfo->tableEndIndex + 1 >= numOfTables) { + // get empty group, mark processed & rm from hash + void* pIte = taosHashIterate(pTableListInfo->remainGroups, NULL); + if (pIte != NULL) { + size_t keySize = 0; + uint64_t* pGroupId = taosHashGetKey(pIte, &keySize); + STableKeyInfo info = {.uid = *(uint64_t*)pIte, .groupId = *pGroupId}; + taosHashCancelIterate(pTableListInfo->remainGroups, pIte); + markGroupProcessed(pTableScanInfo, *pGroupId); + return getBlockForEmptyTable(pOperator, &info); + } + } + } pTableScanInfo->countState = TABLE_COUNT_STATE_END; - STableKeyInfo* pStart = - (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); - return getBlockForEmptyTable(pOperator, pStart); } return NULL; From 1a265a2e9dfbf1c38f4262b9f8e5b839bca71b6e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:26:46 +0800 Subject: [PATCH 060/198] test: modify testcase of hot refresh config --- tests/pytest/util/common.py | 16 ++++++++++++++++ .../test_hot_refresh_configurations.py | 18 ++++++++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 9c45c09715..2e7bd2a89c 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -506,7 +506,23 @@ class TDCom: if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] break + if platform.system().lower() == 'windows': + win_sep = "\\" + buildPath = buildPath.replace(win_sep,'/') + return buildPath + + def getTaosdPath(self, dnodeID="dnode1"): + buildPath = self.getBuildPath() + print("123456",buildPath) + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + taosdPath = buildPath + "/../sim/" + dnodeID + tdLog.info("taosdPath: %s" % taosdPath) + return taosdPath + def getClientCfgPath(self): buildPath = self.getBuildPath() diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 09e634974b..d281b9c1f7 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -201,6 +201,17 @@ class TDTestCase: tdSql.error(f'alter {dnode} "{name} {v}";') def run(self): + + # reset log + taosdPath = tdCom.getTaosdPath() + print(taosdPath,"123") + taosdLogAbsoluteFilename = taosdPath + "/log/" + "taosdlog*" + print(taosdLogAbsoluteFilename) + tdSql.execute("alter all dnodes 'resetlog';") + r = subprocess.Popen("cat {} | grep 'reset log file'".format(taosdLogAbsoluteFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = r.communicate() + assert('reset log file' in stdout.decode()) + for key in self.configration_dic: if "cli" == key: for item in self.configration_dic[key]: @@ -214,12 +225,7 @@ class TDTestCase: self.svr_check(item["name"], item["alias"], item["except_values"], True) else: raise Exception(f"unknown key: {key}") - # reset log - path = os.sep.join([tdDnodes.getDnodesRootDir(), "dnode1", "log", "taosdlog.*"]) - tdSql.execute("alter all dnodes 'resetlog';") - r = subprocess.Popen("cat {} | grep 'reset log file'".format(path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = r.communicate() - assert('reset log file' in stdout.decode()) + def stop(self): tdSql.close() From 480f571c2bda07f02a7b77cd3b4d2beff8763b29 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:29:01 +0800 Subject: [PATCH 061/198] test: modify testcase of hot refresh config --- tests/pytest/util/common.py | 1 - .../system-test/0-others/test_hot_refresh_configurations.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 2e7bd2a89c..010f45a573 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -514,7 +514,6 @@ class TDCom: def getTaosdPath(self, dnodeID="dnode1"): buildPath = self.getBuildPath() - print("123456",buildPath) if (buildPath == ""): tdLog.exit("taosd not found!") else: diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index d281b9c1f7..71f6290469 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -203,10 +203,7 @@ class TDTestCase: def run(self): # reset log - taosdPath = tdCom.getTaosdPath() - print(taosdPath,"123") - taosdLogAbsoluteFilename = taosdPath + "/log/" + "taosdlog*" - print(taosdLogAbsoluteFilename) + taosdLogAbsoluteFilename = tdCom.getTaosdPath() + "/log/" + "taosdlog*" tdSql.execute("alter all dnodes 'resetlog';") r = subprocess.Popen("cat {} | grep 'reset log file'".format(taosdLogAbsoluteFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = r.communicate() From 7333fe54052ac97ac306c727499cf3a3162353bd Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 16:35:01 +0800 Subject: [PATCH 062/198] test: add script loop.bat --- tests/system-test/loop.bat | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/system-test/loop.bat diff --git a/tests/system-test/loop.bat b/tests/system-test/loop.bat new file mode 100644 index 0000000000..41e3901b3f --- /dev/null +++ b/tests/system-test/loop.bat @@ -0,0 +1,13 @@ +@echo off +set LoopNumner=%1 +set cmd=%2 + +echo "Loop Number: %LoopNumner%" +echo "command : %cmd:"=%" + +For /l %%i in (1,1,%LoopNumner% ) do ( + echo "execute times:"%%i + %cmd:"=% +) + +% case loop.bat 100 "python3 .\test.py -f 2-query\query_cols_tags_and_or.py" % \ No newline at end of file From 2387d5bade30ab3630059ea237bc6197437fe14e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 17:10:12 +0800 Subject: [PATCH 063/198] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 1 + source/libs/stream/inc/streamsm.h | 2 ++ source/libs/stream/src/streamExec.c | 13 ++++++++++--- source/libs/stream/src/streamTaskSm.c | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 0c09e0e1ea..a6dac7f5ba 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -792,6 +792,7 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); int32_t streamTaskClearHTaskAttr(SStreamTask* pTask); int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, void* pFn); int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event); void streamTaskRestoreStatus(SStreamTask* pTask); diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index 7be655fbed..ea0522bd5a 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -34,6 +34,8 @@ typedef int32_t (*__state_trans_succ_fn)(SStreamTask*); typedef struct SAttachedEventInfo { ETaskStatus status; // required status that this event can be handled EStreamTaskEvent event; // the delayed handled event + void* pParam; + void* pFn; } SAttachedEventInfo; typedef struct STaskStateTrans { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 1f7bb56ec1..b9839dfc0c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -356,9 +356,16 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); } else { - ASSERT(status == TASK_STATUS__READY|| status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); - streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); - stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, id); + ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); + int32_t code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id, + pStreamTask->id.idStr, tstrerror(code)); + streamMetaReleaseTask(pMeta, pStreamTask); + return code; + } else { + stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, id); + } } // wait for the stream task to handle all in the inputQ, and to be idle diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 4bd6483f7f..d785932109 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -278,10 +278,10 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt ETaskStatus s = streamTaskGetStatus(pTask, NULL); taosThreadMutexUnlock(&pTask->lock); - if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) { + if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event)); return TSDB_CODE_SUCCESS; - } else if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__STOP) { // this event has been handled already + } else if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT) { stDebug("s-task:%s not handle event:%s yet, wait for 100ms and recheck", id, GET_EVT_NAME(event)); taosMsleep(100); } else { From 93fe9558289052437385d5b24fe339593dcfd944 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 17:48:05 +0800 Subject: [PATCH 064/198] test: modify testcase of windows --- tests/script/win-test-file | 2 +- tests/system-test/win-test-file | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/script/win-test-file b/tests/script/win-test-file index d9ff09f468..ada8574165 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -15,7 +15,6 @@ ./test.sh -f tsim/dnode/balance2.sim ./test.sh -f tsim/vnode/replica3_repeat.sim ./test.sh -f tsim/parser/col_arithmetic_operation.sim -./test.sh -f tsim/trans/create_db.sim ./test.sh -f tsim/dnode/balance3.sim ./test.sh -f tsim/vnode/replica3_many.sim ./test.sh -f tsim/stable/metrics_idx.sim @@ -201,6 +200,7 @@ ./test.sh -f tsim/query/show_db_table_kind.sim ./test.sh -f tsim/query/bi_star_table.sim ./test.sh -f tsim/query/bi_tag_scan.sim +./test.sh -f tsim/query/bi_tbname_col.sim ./test.sh -f tsim/query/tag_scan.sim ./test.sh -f tsim/query/nullColSma.sim ./test.sh -f tsim/query/bug3398.sim diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index aefdb1e824..0db83e71a3 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -149,7 +149,7 @@ python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -python3 ./test.py -f 7-tmq/tmq_taosx.py +# python3 ./test.py -f 7-tmq/tmq_taosx.py python3 ./test.py -f 7-tmq/tmq_replay.py python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py python3 ./test.py -f 7-tmq/tmq_offset.py @@ -163,6 +163,7 @@ python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 @@ -196,6 +197,7 @@ python3 ./test.py -f 0-others/ttlChangeOnWrite.py python3 ./test.py -f 0-others/compress_tsz1.py python3 ./test.py -f 0-others/compress_tsz2.py python3 ./test.py -f 0-others/view/non_marterial_view/test_view.py +python3 ./test.py -f 0-others/test_show_table_distributed.py python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 0-others/tag_index_basic.py python3 ./test.py -N 3 -f 0-others/walRetention.py @@ -204,6 +206,7 @@ python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 python3 ./test.py -f 0-others/timeRangeWise.py -N 3 python3 ./test.py -f 0-others/delete_check.py python3 ./test.py -f 0-others/test_hot_refresh_configurations.py +python3 ./test.py -f 1-insert/insert_double.py python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/alter_replica.py -N 3 python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py @@ -252,6 +255,10 @@ python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 python3 ./test.py -f 1-insert/precisionUS.py python3 ./test.py -f 1-insert/precisionNS.py python3 ./test.py -f 1-insert/test_ts4219.py +python3 ./test.py -f 1-insert/ts-4272.py +python3 ./test.py -f 1-insert/test_ts4295.py +python3 ./test.py -f 1-insert/test_td27388.py +python3 ./test.py -f 1-insert/insert_timestamp.py python3 ./test.py -f 0-others/show.py python3 ./test.py -f 0-others/show_tag_index.py python3 ./test.py -f 0-others/information_schema.py @@ -795,6 +802,7 @@ python3 ./test.py -f 2-query/blockSMA.py -Q 4 python3 ./test.py -f 2-query/projectionDesc.py -Q 4 python3 ./test.py -f 2-query/odbc.py python3 ./test.py -f 2-query/fill_with_group.py +python3 ./test.py -f 2-query/state_window.py -Q 3 python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 python3 ./test.py -f 99-TDcase/TD-20582.py python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 From 5bbf12c2c9240df4677fa3ba00325ce01b8d4b38 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Jan 2024 18:35:17 +0800 Subject: [PATCH 065/198] fix(tsdb): check if stt blocks are clean before building composed data blocks. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 1f54f13592..0aef1efc7b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1713,6 +1713,9 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* } else if ((!dataInDataFile) && dataInSttFile) { // no data ile block exists return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + } else if (pBlockScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) { + // opt model for count data in stt file, which is not overlap with data blocks in files. + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else { // row in both stt file blocks and data file blocks TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); From 866fac624fa9985bd3b0d90c371cae19bdc83bc4 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Tue, 2 Jan 2024 19:26:59 +0800 Subject: [PATCH 066/198] Add test cases for TS-4348 and TD-27939 --- tests/system-test/2-query/ts-4348-td-27939.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/system-test/2-query/ts-4348-td-27939.py diff --git a/tests/system-test/2-query/ts-4348-td-27939.py b/tests/system-test/2-query/ts-4348-td-27939.py new file mode 100644 index 0000000000..2a861899d5 --- /dev/null +++ b/tests/system-test/2-query/ts-4348-td-27939.py @@ -0,0 +1,46 @@ +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + tdSql.execute("drop database if exists ts_4338;") + tdSql.execute("create database ts_4338;") + tdSql.execute("drop table if exists ts_4338.t;") + tdSql.execute("create database if not exists ts_4338;") + tdSql.execute("create table ts_4338.t (ts timestamp, i8 tinyint);") + tdSql.execute("insert into ts_4338.t (ts, i8) values (now(), 1) (now()+1s, 2);") + + def run(self): + # TS-4348 + tdSql.query(f'select i8 from ts_4338.t;') + tdSql.checkRows(2) + + tdSql.query(f'select i8 from ts_4338.t where 1 = 1;') + tdSql.checkRows(2) + + tdSql.query(f'select i8 from ts_4338.t where i8 = 1;') + tdSql.checkRows(1) + + tdSql.query(f'select * from (select * from ts_4338.t where i8 = 3);') + tdSql.checkRows(0) + + # TD-27939 + tdSql.query(f'select * from (select * from ts_4338.t where 1 = 100);') + tdSql.checkRows(0) + + tdSql.query(f'select * from (select * from (select * from ts_4338.t where 1 = 200));') + tdSql.checkRows(0) + + tdSql.execute("drop database if exists ts_4338;") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From b48df60fdd157ac058f1d4e4e2731304210c5872 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 20:18:10 +0800 Subject: [PATCH 067/198] test: add testcase that merge same ver stt file after split db --- tests/system-test/0-others/splitVGroupRep1.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupRep1.py index f799c9dbd8..1745b34cc4 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupRep1.py @@ -164,7 +164,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 1 wal_retention_size 1" + sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 0 wal_retention_size 1" self.exeDouble(sql) # create super talbe st @@ -221,14 +221,14 @@ class TDTestCase: sql1 = sql.replace('@db_name', self.db1) tdLog.info(sql1) start1 = time.time() - rows1 = tdSql.query(sql1) + rows1 = tdSql.query(sql1,queryTimes=2) spend1 = time.time() - start1 res1 = copy.deepcopy(tdSql.queryResult) sql2 = sql.replace('@db_name', self.db2) tdLog.info(sql2) start2 = time.time() - tdSql.query(sql2) + tdSql.query(sql2,queryTimes=2) spend2 = time.time() - start2 res2 = tdSql.queryResult @@ -269,7 +269,7 @@ class TDTestCase: def checkResult(self): # check vgroupid sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{self.db2}'" - tdSql.query(sql) + tdSql.query(sql,queryTimes=2) tdSql.checkRows(self.vgroups2) # check child table count same @@ -412,8 +412,16 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - + tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver3.stt to db2 and db1 ") + for dbname in [self.db2, self.db1]: + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') tdLog.info("check db1 and db2 same after creating ...") + self.checkResult() for i in range(3): @@ -423,6 +431,15 @@ class TDTestCase: end = time.time() self.vgroups2 += 1 + # insert the same data per tables into splited vgroups + tdLog.info("insert the same data per tables into splited vgroups(3,4)") + for dbname in [self.db2, self.db1]: + for tableid in range(self.childCnt): + tdSql.execute(f'insert into {dbname}.t{tableid} values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.ta values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999",0);') + tdSql.execute(f'flush database {dbname}') + # check two db query result same self.checkResult() spend = "%.3f"%(end-start) From 7b805d2b002f49496d591bb2a2276f05442cc465 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 2 Jan 2024 20:19:09 +0800 Subject: [PATCH 068/198] test: add testcase that merge same ver stt file after split db --- tests/system-test/0-others/splitVGroupRep1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupRep1.py index 1745b34cc4..f7dab4fcf6 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupRep1.py @@ -412,7 +412,7 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver3.stt to db2 and db1 ") + tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver4.stt to db2 and db1 ") for dbname in [self.db2, self.db1]: tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') tdSql.execute(f'flush database {dbname}') From 453d3e8a3069f860b9aa0072986a0aa039084dd3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 10:50:38 +0800 Subject: [PATCH 069/198] fix(stream): fix dead lock on windows. --- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 84ef36df74..34ae6623a7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -567,13 +567,13 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) if (vnodeIsRoleLeader(pVnode)) { // start to restore all stream tasks if (tsDisableStream) { + streamMetaWUnLock(pMeta); vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta); { - streamMetaWLock(pMeta); if (pMeta->startInfo.taskStarting == 1) { pMeta->startInfo.restartCount += 1; tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, From 284ef7d0857d9420c447cffc9724ece4cf293fb6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 14:44:04 +0800 Subject: [PATCH 070/198] fix(stream): remove the related checkpoint trans when trying to drop stream. --- source/dnode/mnode/impl/inc/mndStream.h | 3 +- source/dnode/mnode/impl/src/mndStream.c | 48 ++++++++++++-------- source/dnode/mnode/impl/src/mndStreamTrans.c | 26 ++++++++++- 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 4d06b16297..7b71ad873b 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -71,7 +71,8 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamUid); int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId); -bool streamTransConflictOtherTrans(SMnode *pMnode, int64_t streamUid, const char *pTransName, bool lock); +bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamUid, const char *pTransName, bool lock); +int32_t mndStreamGetRelCheckpointTrans(SMnode *pMnode, int64_t streamUid); // for sma // TODO refactor diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index a656137770..3c700665d3 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -80,7 +80,9 @@ static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *p static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); static void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); -static int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); +static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); +static void killCheckpointTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); + static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); static void freeCheckpointCandEntry(void *); @@ -1047,7 +1049,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre return -1; } - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, true); + bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, true); if (conflict) { mndAddtoCheckpointWaitingList(pStream, checkpointId); mWarn("checkpoint conflict with other trans in %s, ignore the checkpoint for stream:%s %" PRIx64, pStream->sourceDb, @@ -1344,6 +1346,7 @@ static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq) { if (ps == NULL) { continue; } + mDebug("start to launch checkpoint for stream:%s %" PRIx64 " in candidate list", pEntry->pName, pEntry->streamId); code = mndProcessStreamCheckpointTrans(pMnode, ps, pEntry->checkpointId); @@ -1399,7 +1402,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_DROP_NAME, true); + bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_DROP_NAME, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); tFreeMDropStreamReq(&dropReq); @@ -1451,6 +1454,12 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return -1; } + // kill the related checkpoint trans + int32_t transId = mndStreamGetRelCheckpointTrans(pMnode, pStream->uid); + if (transId != 0) { + killCheckpointTransImpl(pMnode, transId, pStream->sourceDb); + } + removeStreamTasksInBuf(pStream, &execInfo); SName name = {0}; @@ -1894,7 +1903,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_PAUSE_NAME, true); + bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_PAUSE_NAME, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -2037,7 +2046,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { } // check if it is conflict with other trans in both sourceDb and targetDb. - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_RESUME_NAME, true); + bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_RESUME_NAME, true); if (conflict) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -2343,7 +2352,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange break; } - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_UPDATE_NAME, false); + bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_TASK_UPDATE_NAME, false); sdbRelease(pSdb, pStream); if (conflict) { @@ -2567,7 +2576,7 @@ static void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInf size_t len = 0; void * pKey = taosHashGetKey(pDb, &len); - killActiveCheckpointTrans(pMnode, pKey, len); + doKillCheckpointTrans(pMnode, pKey, len); } } @@ -2812,7 +2821,16 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return TSDB_CODE_ACTION_IN_PROGRESS; } -int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { +void killCheckpointTransImpl(SMnode* pMnode, int32_t transId, const char* pDbName) { + STrans *pTrans = mndAcquireTrans(pMnode, transId); + if (pTrans != NULL) { + mInfo("kill checkpoint transId:%d in Db:%s", transId, pDbName); + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + } +} + +int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { // data in the hash table will be removed automatically, no need to remove it here. SStreamTransInfo *pTransInfo = taosHashGet(execInfo.transMgmt.pDBTrans, pDBName, len); if (pTransInfo == NULL) { @@ -2825,15 +2843,9 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t le return TSDB_CODE_SUCCESS; } - STrans *pTrans = mndAcquireTrans(pMnode, pTransInfo->transId); - if (pTrans != NULL) { - char* pDupDBName = strndup(pDBName, len); - mInfo("kill checkpoint transId:%d in Db:%s", pTransInfo->transId, pDupDBName); - taosMemoryFree(pDupDBName); - - mndKillTrans(pMnode, pTrans); - mndReleaseTrans(pMnode, pTrans); - } + char* pDupDBName = strndup(pDBName, len); + killCheckpointTransImpl(pMnode, pTransInfo->transId, pDupDBName); + taosMemoryFree(pDupDBName); return TSDB_CODE_SUCCESS; } @@ -2855,7 +2867,7 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, in code = TSDB_CODE_STREAM_TASK_NOT_EXIST; mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid); } else { - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_RESET_NAME, false); + bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_TASK_RESET_NAME, false); if (conflict) { mError("stream:%s other trans exists in DB:%s, dstTable:%s failed to start reset-status trans", pStream->name, pStream->sourceDb, pStream->targetSTbName); diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 8f94843500..f5047acc49 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -65,7 +65,7 @@ int32_t clearFinishedTrans(SMnode* pMnode) { return 0; } -bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char* pTransName, bool lock) { +bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamUid, const char* pTransName, bool lock) { if (lock) { taosThreadMutexLock(&execInfo.lock); } @@ -113,6 +113,30 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char return false; } +int32_t mndStreamGetRelCheckpointTrans(SMnode* pMnode, int64_t streamUid) { + taosThreadMutexLock(&execInfo.lock); + int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); + if (num <= 0) { + taosThreadMutexUnlock(&execInfo.lock); + return 0; + } + + clearFinishedTrans(pMnode); + SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid)); + if (pEntry != NULL) { + SStreamTransInfo tInfo = *pEntry; + taosThreadMutexUnlock(&execInfo.lock); + + if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) { + return tInfo.transId; + } + } else { + taosThreadMutexUnlock(&execInfo.lock); + } + + return 0; +} + int32_t mndAddtoCheckpointWaitingList(SStreamObj* pStream, int64_t checkpointId) { SCheckpointCandEntry* pEntry = taosHashGet(execInfo.transMgmt.pWaitingList, &pStream->uid, sizeof(pStream->uid)); if (pEntry == NULL) { From c0186a214d9dc11faf56fedd2b483ce228ff91b5 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 3 Jan 2024 14:50:05 +0800 Subject: [PATCH 071/198] test: add testcase for TD-28106 of copy file in snapshot --- tests/parallel_test/cases.task | 6 +- tests/script/win-test-file | 1 + tests/system-test/0-others/splitVGroup.py | 464 ++++++++++++++++++ .../{splitVGroupRep1.py => splitVGroupWal.py} | 8 +- tests/system-test/win-test-file | 8 +- 5 files changed, 479 insertions(+), 8 deletions(-) create mode 100644 tests/system-test/0-others/splitVGroup.py rename tests/system-test/0-others/{splitVGroupRep1.py => splitVGroupWal.py} (99%) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 093fec78ab..edf28b6518 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -260,8 +260,10 @@ e ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 3 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py diff --git a/tests/script/win-test-file b/tests/script/win-test-file index ada8574165..744415c89f 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -189,6 +189,7 @@ ./test.sh -f tsim/query/unionall_as_table.sim ./test.sh -f tsim/query/multi_order_by.sim ./test.sh -f tsim/query/sys_tbname.sim +./test.sh -f tsim/query/sort-pre-cols.sim ./test.sh -f tsim/query/groupby.sim ./test.sh -f tsim/query/groupby_distinct.sim ./test.sh -f tsim/query/event.sim diff --git a/tests/system-test/0-others/splitVGroup.py b/tests/system-test/0-others/splitVGroup.py new file mode 100644 index 0000000000..c49713fc6d --- /dev/null +++ b/tests/system-test/0-others/splitVGroup.py @@ -0,0 +1,464 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time +import copy +import string + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # random string + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into @db_name.t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + + # insert nomal talbe + for i in range(20): + self.ts += 1000 + name = self.random_string(20) + sql = f"insert into @db_name.ta values({self.ts}, {i}, {self.ts%100000}, '{name}', false)" + self.exeDouble(sql) + + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + def exeDouble(self, sql): + # dbname replace + sql1 = sql.replace("@db_name", self.db1) + + if len(sql1) > 100: + tdLog.info(sql1[:100]) + else: + tdLog.info(sql1) + tdSql.execute(sql1) + + sql2 = sql.replace("@db_name", self.db2) + if len(sql2) > 100: + tdLog.info(sql2[:100]) + else: + tdLog.info(sql2) + tdSql.execute(sql2) + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000 + self.childCnt = 4 + self.childRow = 10000 + self.batchSize = 50000 + self.vgroups1 = 1 + self.vgroups2 = 1 + self.db1 = "db1" + self.db2 = "db2" + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db wal_retention_period 0 + sql = f"create database @db_name vgroups {self.vgroups1} replica {self.replicaVar} wal_retention_period 0 wal_retention_size 1" + self.exeDouble(sql) + + # create super talbe st + sql = f"create table @db_name.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + self.exeDouble(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table @db_name.t{i} using @db_name.st tags({i}) " + self.exeDouble(sql) + + # create normal table + sql = f"create table @db_name.ta(ts timestamp, c1 int, c2 bigint, c3 binary(32), c4 bool)" + self.exeDouble(sql) + + # insert data + self.insertData() + + # update + self.ts = 1680000000000 + 20000 + self.childRow = 1000 + + + # delete data + sql = "delete from @db_name.st where ts > 1680000019000 and ts < 1680000062000" + self.exeDouble(sql) + sql = "delete from @db_name.st where ts > 1680000099000 and ts < 1680000170000" + self.exeDouble(sql) + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.time() % 10000 + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # check query result same + def queryDouble(self, sql): + # sql + sql1 = sql.replace('@db_name', self.db1) + tdLog.info(sql1) + start1 = time.time() + rows1 = tdSql.query(sql1,queryTimes=2) + spend1 = time.time() - start1 + res1 = copy.deepcopy(tdSql.queryResult) + + sql2 = sql.replace('@db_name', self.db2) + tdLog.info(sql2) + start2 = time.time() + tdSql.query(sql2,queryTimes=2) + spend2 = time.time() - start2 + res2 = tdSql.queryResult + + rowlen1 = len(res1) + rowlen2 = len(res2) + errCnt = 0 + + if rowlen1 != rowlen2: + tdLog.exit(f"both row count not equal. rowlen1={rowlen1} rowlen2={rowlen2} ") + return False + + for i in range(rowlen1): + row1 = res1[i] + row2 = res2[i] + collen1 = len(row1) + collen2 = len(row2) + if collen1 != collen2: + tdLog.exit(f"both col count not equal. collen1={collen1} collen2={collen2}") + return False + for j in range(collen1): + if row1[j] != row2[j]: + tdLog.info(f"error both column value not equal. row={i} col={j} col1={row1[j]} col2={row2[j]} .") + errCnt += 1 + + if errCnt > 0: + tdLog.exit(f" db2 column value different with db2. different count ={errCnt} ") + + # warning performance + diff = (spend2 - spend1)*100/spend1 + tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff)) + if spend2 > spend1 and diff > 20: + tdLog.info("warning: the diff for performance after spliting is over 20%") + + return True + + + # check result + def checkResult(self): + # check vgroupid + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{self.db2}'" + tdSql.query(sql,queryTimes=2) + tdSql.checkRows(self.vgroups2) + + # check child table count same + sql = "select table_name from information_schema.ins_tables where db_name='@db_name' order by table_name" + self.queryDouble(sql) + + # check row value is ok + sql = "select * from @db_name.st order by ts, tbname" + self.queryDouble(sql) + + # where + sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts, tbname" + self.queryDouble(sql) + + # max + sql = "select max(c1) from @db_name.st" + self.queryDouble(sql) + + # min + sql = "select min(c2) from @db_name.st" + self.queryDouble(sql) + + # sum + sql = "select sum(c1) from @db_name.st" + self.queryDouble(sql) + + # normal table + + # count + sql = "select count(*) from @db_name.ta" + self.queryDouble(sql) + + # all rows + sql = "select * from @db_name.ta" + self.queryDouble(sql) + + # sum + sql = "select sum(c1) from @db_name.ta" + self.queryDouble(sql) + + + # get vgroup list + def getVGroup(self, db_name): + vgidList = [] + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + res = tdSql.getResult(sql) + rows = len(res) + for i in range(rows): + vgidList.append(res[i][0]) + + return vgidList; + + # split vgroup on db2 + def splitVGroup(self, db_name): + vgids = self.getVGroup(db_name) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdLog.info(sql) + tdSql.execute(sql) + + # wait end + seconds = 300 + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("split vgroup finished.") + return True + #tdLog.info(f"i={i} wait split vgroup ...") + time.sleep(1) + + tdLog.exit(f"split vgroup transaction is not finished after executing {seconds}s") + return False + + # split error + def expectSplitError(self, dbName): + vgids = self.getVGroup(dbName) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdLog.info(sql) + tdSql.error(sql) + + # expect split ok + def expectSplitOk(self, dbName): + # split vgroup + vgList1 = self.getVGroup(dbName) + self.splitVGroup(dbName) + vgList2 = self.getVGroup(dbName) + vgNum1 = len(vgList1) + 1 + vgNum2 = len(vgList2) + if vgNum1 != vgNum2: + tdLog.exit(f" vglist len={vgNum1} is not same for expect {vgNum2}") + return + + # split empty database + def splitEmptyDB(self): + dbName = "emptydb" + vgNum = 2 + # create database + sql = f"create database {dbName} vgroups {vgNum} replica {self.replicaVar }" + tdLog.info(sql) + tdSql.execute(sql) + + # split vgroup + self.expectSplitOk(dbName) + + + # forbid + def checkForbid(self): + # stream + tdLog.info("check forbid split having stream...") + tdSql.execute("create database streamdb;") + tdSql.execute("use streamdb;") + tdSql.execute("create table ta(ts timestamp, age int);") + tdSql.execute("create stream ma into sta as select count(*) from ta interval(1s);") + self.expectSplitError("streamdb") + tdSql.execute("drop stream ma;") + self.expectSplitOk("streamdb") + + # topic + tdLog.info("check forbid split having topic...") + tdSql.execute("create database topicdb wal_retention_period 10;") + tdSql.execute("use topicdb;") + tdSql.execute("create table ta(ts timestamp, age int);") + tdSql.execute("create topic toa as select * from ta;") + + #self.expectSplitError("topicdb") + tdSql.execute("drop topic toa;") + self.expectSplitOk("topicdb") + + # compact and check db2 + def compactAndCheck(self): + tdLog.info("compact db2 and check result ...") + # compact + tdSql.execute(f"compact database {self.db2};") + # check result + self.checkResult() + + # run + def run(self): + # prepare env + self.prepareEnv() + tdLog.info("generate at least two stt files of the same fileset (e.g. v4f1944) for db2 and db1 ") + for dbname in [self.db2, self.db1]: + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdLog.info("check db1 and db2 same after creating ...") + + self.checkResult() + + for i in range(3): + # split vgroup on db2 + start = time.time() + self.splitVGroup(self.db2) + end = time.time() + self.vgroups2 += 1 + + # insert the same data per tables into splited vgroups + tdLog.info("insert the same data per tables into splited vgroups(3,4)") + for dbname in [self.db2, self.db1]: + for tableid in range(self.childCnt): + tdSql.execute(f'insert into {dbname}.t{tableid} values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999") ;') + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'insert into {dbname}.ta values("2023-03-28 10:40:00.100",103,103,"2023-03-28 18:41:39.999",0);') + tdSql.execute(f'flush database {dbname}') + + # check two db query result same + self.checkResult() + spend = "%.3f"%(end-start) + tdLog.info(f"split vgroup i={i} passed. spend = {spend}s") + + # split empty db + self.splitEmptyDB() + + # check topic and stream forib + self.checkForbid() + + # compact database + self.compactAndCheck() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/splitVGroupRep1.py b/tests/system-test/0-others/splitVGroupWal.py similarity index 99% rename from tests/system-test/0-others/splitVGroupRep1.py rename to tests/system-test/0-others/splitVGroupWal.py index f7dab4fcf6..dfb2281e5e 100644 --- a/tests/system-test/0-others/splitVGroupRep1.py +++ b/tests/system-test/0-others/splitVGroupWal.py @@ -164,7 +164,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database @db_name vgroups {self.vgroups1} replica 1 wal_retention_period 0 wal_retention_size 1" + sql = f"create database @db_name vgroups {self.vgroups1} replica {self.replicaVar}" self.exeDouble(sql) # create super talbe st @@ -277,11 +277,11 @@ class TDTestCase: self.queryDouble(sql) # check row value is ok - sql = "select * from @db_name.st order by ts" + sql = "select * from @db_name.st order by ts, tbname" self.queryDouble(sql) # where - sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts" + sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts, tbname" self.queryDouble(sql) # max @@ -369,7 +369,7 @@ class TDTestCase: dbName = "emptydb" vgNum = 2 # create database - sql = f"create database {dbName} vgroups {vgNum} replica 1" + sql = f"create database {dbName} vgroups {vgNum} replica {self.replicaVar}" tdLog.info(sql) tdSql.execute(sql) diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index e261e18456..1ab0fee7bf 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -1,4 +1,5 @@ python3 ./test.py -f 2-query/tbname_vgroup.py +python3 ./test.py -f 2-query/compact-col.py python3 ./test.py -f 2-query/stbJoin.py python3 ./test.py -f 2-query/stbJoin.py -Q 2 python3 ./test.py -f 2-query/stbJoin.py -Q 3 @@ -205,8 +206,10 @@ python3 ./test.py -f 0-others/test_show_table_distributed.py python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 0-others/tag_index_basic.py python3 ./test.py -N 3 -f 0-others/walRetention.py -python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 -python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 +python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 1 +python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 1 +python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 3 +python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 3 python3 ./test.py -f 0-others/timeRangeWise.py -N 3 python3 ./test.py -f 0-others/delete_check.py python3 ./test.py -f 0-others/test_hot_refresh_configurations.py @@ -467,6 +470,7 @@ python3 ./test.py -f 2-query/concat2.py python3 ./test.py -f 2-query/json_tag.py python3 ./test.py -f 2-query/nestedQueryInterval.py python3 ./test.py -f 2-query/systable_func.py +python3 ./test.py -f 2-query/test_ts4382.py python3 ./test.py -f 2-query/stablity.py python3 ./test.py -f 2-query/stablity_1.py python3 ./test.py -f 2-query/elapsed.py From 35b8683262ecfbdcd3a7af15fd80bd74a39136b3 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 3 Jan 2024 14:52:24 +0800 Subject: [PATCH 072/198] test: add testcase for TD-28106 of copy file in snapshot --- tests/system-test/0-others/splitVGroupWal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/splitVGroupWal.py b/tests/system-test/0-others/splitVGroupWal.py index dfb2281e5e..d11d63b5a9 100644 --- a/tests/system-test/0-others/splitVGroupWal.py +++ b/tests/system-test/0-others/splitVGroupWal.py @@ -412,7 +412,7 @@ class TDTestCase: def run(self): # prepare env self.prepareEnv() - tdLog.info("add two same ver stt file like v4f1944ver3.stt and v4f1944ver4.stt to db2 and db1 ") + tdLog.info("generate at least two stt files of the same fileset (e.g. v4f1944) for db2 and db1 ") for dbname in [self.db2, self.db1]: tdSql.execute(f'insert into {dbname}.t1 values("2023-03-28 10:40:00.010",103,103,"2023-03-28 18:41:39.999") ;') tdSql.execute(f'flush database {dbname}') From e67c6764664b8f37504b7cc86f61555bade36601 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 3 Jan 2024 15:19:27 +0800 Subject: [PATCH 073/198] TS-4423: add test case --- tests/parallel_test/cases.task | 1 + .../system-test/1-insert/stt_blocks_check.py | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tests/system-test/1-insert/stt_blocks_check.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 093fec78ab..0e826304c1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -109,6 +109,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_stb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/stt_blocks_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py diff --git a/tests/system-test/1-insert/stt_blocks_check.py b/tests/system-test/1-insert/stt_blocks_check.py new file mode 100644 index 0000000000..5de6414453 --- /dev/null +++ b/tests/system-test/1-insert/stt_blocks_check.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from random import randint +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.boundary import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def stt_block_check(self): + tdSql.prepare() + tdSql.execute('use db') + + tdSql.execute('create table meters (ts timestamp, c1 int, c2 float) tags(t1 int)') + tdSql.execute("create table d0 using meters tags(1)") + + ts = 1704261670000 + + sql = "insert into d0 values " + for i in range(100): + sql = sql + f"({ts + i}, 1, 0.1)" + tdSql.execute(sql) + tdSql.execute("flush database db") + + ts = 1704261670099 + + sql = "insert into d0 values " + for i in range(100): + sql = sql + f"({ts + i}, 1, 0.1)" + tdSql.execute(sql) + tdSql.execute("flush database db") + + tdSql.execute(f"insert into d0 values({ts + 100}, 2, 1.0)") + tdSql.execute("flush database db") + + time.sleep(2) + + tdSql.query("show table distributed db.meters") + tdSql.query("select count(*) from db.meters") + tdSql.checkData(0, 0, 200) + + def run(self): + self.stt_block_check() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 589b046ecb366471018830268defd73b2735ce93 Mon Sep 17 00:00:00 2001 From: charles Date: Wed, 3 Jan 2024 17:05:55 +0800 Subject: [PATCH 074/198] update alter_database test case for arm64 ci --- tests/system-test/1-insert/alter_database.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 8de43b8ee8..6a831b88ff 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -4,7 +4,8 @@ import time import socket import os import threading - +import psutil +import platform from util.log import * from util.sql import * from util.cases import * @@ -17,6 +18,14 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.buffer_boundary = [3, 4097, 8193, 12289, 16384] + # remove the value > free_memory, 70% is the weight to calculate the max value + if platform.system() == "Linux" and platform.machine() == "aarch64": + mem = psutil.virtual_memory() + free_memory = mem.free * 0.7 / 1024 / 1024 + for item in self.buffer_boundary: + if item > free_memory: + self.buffer_boundary.remove(item) + self.buffer_error = [self.buffer_boundary[0] - 1, self.buffer_boundary[-1]+1] # pages_boundary >= 64 From b818faaa5eba3bf6fc28a4e059ea81d86a3b54c3 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 3 Jan 2024 17:51:27 +0800 Subject: [PATCH 075/198] remainGroups --- source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/scanoperator.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fe3528cce6..2e93dbd803 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2164,7 +2164,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool initRemainGroups = false; if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable) { + if (pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { initRemainGroups = true; } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8999340ddc..c2a194d150 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -690,7 +690,7 @@ void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { if (pInfo->countState == TABLE_COUNT_STATE_END) { return; } - if (pInfo->base.pTableListInfo->oneTableForEachGroup) { + if (pInfo->base.pTableListInfo->oneTableForEachGroup || pInfo->base.pTableListInfo->groupOffset) { pInfo->countState = TABLE_COUNT_STATE_PROCESSED; } else { taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId)); @@ -854,14 +854,14 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { STableListInfo* pTableListInfo = pTableScanInfo->base.pTableListInfo; - if (pTableListInfo->oneTableForEachGroup) { // group by tbname + if (pTableListInfo->oneTableForEachGroup || pTableListInfo->groupOffset) { // group by tbname, group by tag + sort if (pTableScanInfo->countState < TABLE_COUNT_STATE_PROCESSED) { pTableScanInfo->countState = TABLE_COUNT_STATE_PROCESSED; STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex); return getBlockForEmptyTable(pOperator, pStart); } - } else { // group by tag + } else { // group by tag + no sort int32_t numOfTables = tableListGetSize(pTableListInfo); if (pTableScanInfo->tableEndIndex + 1 >= numOfTables) { // get empty group, mark processed & rm from hash From 185fa75376c6fc9de82c02d9b4da50fbd49be1c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 17:59:42 +0800 Subject: [PATCH 076/198] enh(stream): add task into timer to be idle during exec when the corresponding output queue is full. --- include/libs/stream/tstream.h | 23 ++-- source/dnode/mnode/impl/src/mndStream.c | 4 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqStreamTask.c | 6 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 24 +++-- source/libs/stream/src/stream.c | 12 +-- source/libs/stream/src/streamExec.c | 118 +++++++++++++++++---- source/libs/stream/src/streamMeta.c | 4 +- source/libs/stream/src/streamQueue.c | 11 +- source/libs/stream/src/streamTask.c | 6 +- 10 files changed, 158 insertions(+), 52 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index a6dac7f5ba..79fc2a87a7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -50,10 +50,11 @@ extern "C" { (_t)->hTaskInfo.id.streamId = 0; \ } while (0) -#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) -#define STREAM_EXEC_START_ALL_TASKS_ID (-2) -#define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) -#define STREAM_EXEC_STOP_ALL_TASKS_ID (-4) +#define STREAM_EXEC_T_EXTRACT_WAL_DATA (-1) +#define STREAM_EXEC_T_START_ALL_TASKS (-2) +#define STREAM_EXEC_T_RESTART_ALL_TASKS (-3) +#define STREAM_EXEC_T_STOP_ALL_TASKS (-4) +#define STREAM_EXEC_T_RESUME_TASK (-5) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; @@ -81,14 +82,12 @@ typedef enum ETaskStatus { TASK_STATUS__HALT, // pause, but not be manipulated by user command TASK_STATUS__PAUSE, // pause TASK_STATUS__CK, // stream task is in checkpoint status, no data are allowed to put into inputQ anymore -// TASK_STATUS__STREAM_SCAN_HISTORY, } ETaskStatus; enum { TASK_SCHED_STATUS__INACTIVE = 1, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE, - TASK_SCHED_STATUS__FAILED, TASK_SCHED_STATUS__DROPPING, }; @@ -322,10 +321,11 @@ typedef struct SStreamStatus { int8_t taskStatus; int8_t downstreamReady; // downstream tasks are all ready now, if this flag is set int8_t schedStatus; - int8_t keepTaskStatus; - bool appendTranstateBlock; // has append the transfer state data block already, todo: remove it + int32_t schedIdleTime; // idle time before invoke again + int64_t lastExecTs; // last exec time stamp + int8_t statusBackup; + bool appendTranstateBlock; // has append the transfer state data block already int32_t timerActive; // timer is active - int8_t allowedAddInTimer; // allowed to add into timer int32_t inScanHistorySentinel; } SStreamStatus; @@ -366,7 +366,8 @@ typedef struct STaskQueue { typedef struct STaskSchedInfo { int8_t status; - void* pTimer; + tmr_h pDelayTimer; + tmr_h pIdleTimer; } STaskSchedInfo; typedef struct SSinkRecorder { @@ -541,6 +542,7 @@ typedef struct { SMsgHead head; int64_t streamId; int32_t taskId; + int32_t reqType; } SStreamTaskRunReq; struct SStreamDispatchReq { @@ -764,6 +766,7 @@ SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t void streamTaskInputFail(SStreamTask* pTask); int32_t streamExecTask(SStreamTask* pTask); +int32_t streamResumeTask(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); bool streamTaskShouldStop(const SStreamTask* pStatus); bool streamTaskShouldPause(const SStreamTask* pStatus); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3c700665d3..777a2a70b2 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1847,7 +1847,7 @@ int32_t mndPauseAllStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStre } if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) { - atomic_store_8(&pTask->status.keepTaskStatus, pTask->status.taskStatus); + atomic_store_8(&pTask->status.statusBackup, pTask->status.taskStatus); atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE); } } @@ -2005,7 +2005,7 @@ int32_t mndResumeAllStreamTasks(STrans *pTrans, SMnode *pMnode, SStreamObj *pStr } if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) { - atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); + atomic_store_8(&pTask->status.taskStatus, pTask->status.statusBackup); } } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 65cc56c0b4..2533c721ac 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1068,7 +1068,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { int32_t taskId = pReq->taskId; - if (taskId == STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID) { // all tasks are extracted submit data from the wal + if (taskId == STREAM_EXEC_T_EXTRACT_WAL_DATA) { // all tasks are extracted submit data from the wal tqScanWal(pTq); return 0; } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index aba3b0945f..b11e231c62 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -114,7 +114,8 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { pRunReq->head.vgId = vgId; pRunReq->streamId = 0; - pRunReq->taskId = STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID; + pRunReq->taskId = 0; + pRunReq->reqType = STREAM_EXEC_T_EXTRACT_WAL_DATA; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); @@ -138,7 +139,8 @@ int32_t tqStopStreamTasksAsync(STQ* pTq) { pRunReq->head.vgId = vgId; pRunReq->streamId = 0; - pRunReq->taskId = STREAM_EXEC_STOP_ALL_TASKS_ID; + pRunReq->taskId = 0; + pRunReq->reqType = STREAM_EXEC_T_STOP_ALL_TASKS; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a3d860fd78..b869d67bed 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -42,7 +42,8 @@ int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { tqDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); pRunReq->head.vgId = vgId; pRunReq->streamId = 0; - pRunReq->taskId = restart ? STREAM_EXEC_RESTART_ALL_TASKS_ID : STREAM_EXEC_START_ALL_TASKS_ID; + pRunReq->taskId = 0; + pRunReq->reqType = restart ? STREAM_EXEC_T_RESTART_ALL_TASKS : STREAM_EXEC_T_START_ALL_TASKS; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; tmsgPutToQueue(cb, STREAM_QUEUE, &msg); @@ -675,21 +676,30 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { SStreamTaskRunReq* pReq = pMsg->pCont; - int32_t taskId = pReq->taskId; + int32_t type = pReq->reqType; int32_t vgId = pMeta->vgId; - if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { + if (type == STREAM_EXEC_T_START_ALL_TASKS) { streamMetaStartAllTasks(pMeta); return 0; - } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { + } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { restartStreamTasks(pMeta, isLeader); return 0; - } else if (taskId == STREAM_EXEC_STOP_ALL_TASKS_ID) { + } else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) { streamMetaStopAllTasks(pMeta); return 0; + } else if (type == STREAM_EXEC_T_RESUME_TASK) { + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + if (pTask != NULL) { + ASSERT(streamTaskReadyToRun(pTask, NULL)); + streamResumeTask(pTask); + } + + streamMetaReleaseTask(pMeta, pTask); + return 0; } - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); if (pTask != NULL) { // even in halt status, the data in inputQ must be processed char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { @@ -706,7 +716,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead return 0; } else { // NOTE: pTask->status.schedStatus is not updated since it is not be handled by the run exec. // todo add one function to handle this - tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, taskId); + tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, pReq->taskId); return -1; } } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 1bef42bf14..f383f0f31d 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -57,7 +57,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { if (pTrigger == NULL) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); return; } @@ -68,7 +68,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); return; } @@ -77,7 +77,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { int32_t code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); if (code != TSDB_CODE_SUCCESS) { - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); return; } @@ -85,17 +85,17 @@ static void streamSchedByTimer(void* param, void* tmrId) { } } - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); } int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { if (pTask->info.triggerParam != 0 && pTask->info.fillHistory == 0) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); - ASSERT(ref == 2 && pTask->schedInfo.pTimer == NULL); + ASSERT(ref == 2 && pTask->schedInfo.pDelayTimer == NULL); stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.triggerParam); - pTask->schedInfo.pTimer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->info.triggerParam, pTask, streamTimer); + pTask->schedInfo.pDelayTimer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->info.triggerParam, pTask, streamTimer); pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b9839dfc0c..70b7ec3309 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -549,6 +549,13 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock return code; } +static void setTaskSchedInfo(SStreamTask* pTask, int32_t idleTime) { + SStreamStatus* pStatus = &pTask->status; + + pStatus->schedIdleTime = idleTime; + pStatus->lastExecTs = taosGetTimestampMs(); +} + /** * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the * appropriate batch of blocks should be handled in 5 to 10 sec. @@ -568,6 +575,12 @@ int32_t doStreamExecTask(SStreamTask* pTask) { break; } + if (streamQueueIsFull(pTask->outputq.queue)) { + stWarn("s-task:%s outputQ is full, idle for 500ms and retry", id); + setTaskSchedInfo(pTask, 500); + break; + } + /*int32_t code = */ streamTaskGetDataFromInputQ(pTask, &pInput, &numOfBlocks, &blockSize); if (pInput == NULL) { ASSERT(numOfBlocks == 0); @@ -582,7 +595,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { continue; } - if (pInput->type == STREAM_INPUT__TRANS_STATE) { + if (type == STREAM_INPUT__TRANS_STATE) { streamProcessTranstateBlock(pTask, (SStreamDataBlock*)pInput); continue; } @@ -671,27 +684,85 @@ bool streamTaskIsIdle(const SStreamTask* pTask) { } bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { - ETaskStatus st = streamTaskGetStatus(pTask, NULL); - return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY/* || st == TASK_STATUS__STREAM_SCAN_HISTORY*/ || - st == TASK_STATUS__CK); + ETaskStatus st = streamTaskGetStatus(pTask, pStatus); + return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__CK); } -int32_t streamExecTask(SStreamTask* pTask) { - // this function may be executed by multi-threads, so status check is required. +static void doStreamExecTaskHelper(void* param, void* tmrId) { + SStreamTask* pTask = (SStreamTask*)param; + + char* p = NULL; + ETaskStatus status = streamTaskGetStatus(pTask, &p); + if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) { + streamTaskSetSchedStatusInactive(pTask); + + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s status:%s not resume task, ref:%d", pTask->id.idStr, p, ref); + + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + // task resume running + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + /*int8_t status = */streamTaskSetSchedStatusInactive(pTask); + + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stError("failed to create msg to resume s-task:%s, reason out of memory, ref:%d", pTask->id.idStr, ref); + + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + pRunReq->head.vgId = pTask->info.nodeId; + pRunReq->streamId = pTask->id.streamId; + pRunReq->taskId = pTask->id.taskId; + pRunReq->reqType = STREAM_EXEC_T_RESUME_TASK; + + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pTask->id.idStr, pTask->status.schedIdleTime, ref); + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); + + // release the task ref count + streamMetaReleaseTask(pTask->pMeta, pTask); +} + +static int32_t schedTaskInFuture(SStreamTask* pTask) { + int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s task should idle, add into timer to retry in %dms, ref:%d", + pTask->id.idStr, DISPATCH_RETRY_INTERVAL_MS, ref); + + // add one ref count for task + SStreamTask* pAddRefTask = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId); + + if (pTask->schedInfo.pIdleTimer == NULL) { + pTask->schedInfo.pIdleTimer = taosTmrStart(doStreamExecTaskHelper, pTask->status.schedIdleTime, pTask, streamTimer); + } else { + taosTmrReset(doStreamExecTaskHelper, pTask->status.schedIdleTime, pTask, streamTimer, &pTask->schedInfo.pIdleTimer); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t streamResumeTask(SStreamTask* pTask) { + ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__ACTIVE); const char* id = pTask->id.idStr; - int8_t schedStatus = streamTaskSetSchedStatusActive(pTask); - if (schedStatus == TASK_SCHED_STATUS__WAITING) { - while (1) { - int32_t code = doStreamExecTask(pTask); - if (code < 0) { // todo this status should be removed - atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); - return -1; - } + while (1) { + /*int32_t code = */doStreamExecTask(pTask); + taosThreadMutexLock(&pTask->lock); - taosThreadMutexLock(&pTask->lock); - if ((streamQueueGetNumOfItems(pTask->inputq.queue) == 0) || streamTaskShouldStop(pTask) || - streamTaskShouldPause(pTask)) { + // check if this task needs to be idle for a while + if (pTask->status.schedIdleTime > 0) { + stDebug("s-task:%s idled, and will be invoked in %dms", id, pTask->status.schedIdleTime); + schedTaskInFuture(pTask); + } else { + int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); + if ((numOfItems == 0) || streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); taosThreadMutexUnlock(&pTask->lock); @@ -700,8 +771,19 @@ int32_t streamExecTask(SStreamTask* pTask) { stDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, p, pTask->status.schedStatus); return 0; } - taosThreadMutexUnlock(&pTask->lock); } + + taosThreadMutexUnlock(&pTask->lock); + } +} + +int32_t streamExecTask(SStreamTask* pTask) { + // this function may be executed by multi-threads, so status check is required. + const char* id = pTask->id.idStr; + + int8_t schedStatus = streamTaskSetSchedStatusActive(pTask); + if (schedStatus == TASK_SCHED_STATUS__WAITING) { + streamResumeTask(pTask); } else { char* p = NULL; streamTaskGetStatus(pTask, &p); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9c94799237..dc2e3bd651 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -452,7 +452,7 @@ void streamMetaClear(SStreamMeta* pMeta) { // release the ref by timer if (p->info.triggerParam != 0 && p->info.fillHistory == 0) { // one more ref in timer stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", p->id.idStr, p->refCnt); - taosTmrStop(p->schedInfo.pTimer); + taosTmrStop(p->schedInfo.pDelayTimer); p->info.triggerParam = 0; streamMetaReleaseTask(pMeta, p); } @@ -730,7 +730,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t if (pTask->info.triggerParam != 0 && pTask->info.fillHistory == 0) { stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt); - taosTmrStop(pTask->schedInfo.pTimer); + taosTmrStop(pTask->schedInfo.pDelayTimer); pTask->info.triggerParam = 0; streamMetaReleaseTask(pMeta, pTask); } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index d1610362f9..459efb838c 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -343,6 +343,7 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBlock) { STaosQueue* pQueue = pTask->outputq.queue->pQueue; +#if 0 // wait for the output queue is available for new data to dispatch while (streamQueueIsFull(pTask->outputq.queue)) { if (streamTaskShouldStop(pTask)) { @@ -358,6 +359,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc taosMsleep(OUTPUT_QUEUE_FULL_WAIT_DURATION); } +#endif int32_t code = taosWriteQitem(pQueue, pBlock); @@ -367,7 +369,14 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc stError("s-task:%s failed to put res into outputQ, outputQ items:%d, size:%.2fMiB code:%s, result lost", pTask->id.idStr, total + 1, size, tstrerror(code)); } else { - stDebug("s-task:%s data put into outputQ, outputQ items:%d, size:%.2fMiB", pTask->id.idStr, total, size); + if (streamQueueIsFull(pTask->outputq.queue)) { + stWarn( + "s-task:%s outputQ is full(outputQ items:%d, size:%.2fMiB), set the output status BLOCKING, wait for 500ms " + "after handle this batch of blocks", + pTask->id.idStr, total, size); + } else { + stDebug("s-task:%s data put into outputQ, outputQ items:%d, size:%.2fMiB", pTask->id.idStr, total, size); + } } return TSDB_CODE_SUCCESS; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 4d343a484d..32fd5cbb39 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -328,9 +328,9 @@ void tFreeStreamTask(SStreamTask* pTask) { taosMsleep(100); } - if (pTask->schedInfo.pTimer != NULL) { - taosTmrStop(pTask->schedInfo.pTimer); - pTask->schedInfo.pTimer = NULL; + if (pTask->schedInfo.pDelayTimer != NULL) { + taosTmrStop(pTask->schedInfo.pDelayTimer); + pTask->schedInfo.pDelayTimer = NULL; } if (pTask->hTaskInfo.pTimer != NULL) { From 4138692cea039e3e9abb7b4bb337987a365669ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 18:16:18 +0800 Subject: [PATCH 077/198] fix(stream): scan wal data files. --- source/dnode/vnode/src/tq/tq.c | 8 ++++---- source/libs/stream/src/streamExec.c | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2533c721ac..925671ae04 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1066,15 +1066,15 @@ int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRunReq* pReq = pMsg->pCont; - int32_t taskId = pReq->taskId; - - if (taskId == STREAM_EXEC_T_EXTRACT_WAL_DATA) { // all tasks are extracted submit data from the wal + if (pReq->reqType == STREAM_EXEC_T_EXTRACT_WAL_DATA) { // all tasks are extracted submit data from the wal tqScanWal(pTq); return 0; } int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); - if(code == 0 && taskId > 0){ + + // let's continue scan data in the wal files + if(code == 0 && pReq->reqType > 0){ tqScanWalAsync(pTq, false); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 70b7ec3309..e73c5ffddb 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -728,6 +728,7 @@ static void doStreamExecTaskHelper(void* param, void* tmrId) { tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); // release the task ref count + pTask->status.schedIdleTime = 0; // clear the idle time streamMetaReleaseTask(pTask->pMeta, pTask); } From 4399867e14eea095966dd3a3d31a0d342ca90ed1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 18:28:23 +0800 Subject: [PATCH 078/198] fix(stream): fix the not running bug. --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 925671ae04..851d3f967b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1074,7 +1074,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); // let's continue scan data in the wal files - if(code == 0 && pReq->reqType > 0){ + if(code == 0 && pReq->reqType >= 0){ tqScanWalAsync(pTq, false); } From b06ccc0ca75daa1b35fdec47cc6926c6d61c3101 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 19:18:06 +0800 Subject: [PATCH 079/198] fix(stream): cleanup the checkpoint trans & task info in execInfo --- source/dnode/mnode/impl/src/mndStream.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 777a2a70b2..b57ca009d3 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1500,6 +1500,16 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return -1; } #endif + + // kill the related checkpoint trans + int32_t transId = mndStreamGetRelCheckpointTrans(pMnode, pStream->uid); + if (transId != 0) { + killCheckpointTransImpl(pMnode, transId, pStream->sourceDb); + } + + // drop the stream obj in execInfo + removeStreamTasksInBuf(pStream, &execInfo); + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { sdbRelease(pSdb, pStream); sdbCancelFetch(pSdb, pIter); From 3fea717373a0cf0ec228cf1abab9651922fad68f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 19:28:49 +0800 Subject: [PATCH 080/198] fix(stream): discard the orphan tasks. --- source/client/test/clientTests.cpp | 24 +++++++++++----------- source/dnode/mnode/impl/src/mndStream.c | 27 +++++++++++++++---------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e78783cf3c..e6519a436e 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -939,8 +939,8 @@ TEST(clientCase, agg_query_tables) { } taos_free_result(pRes); - int64_t st = 1685959293000; - for (int32_t i = 0; i < 10000000; ++i) { + int64_t st = 1685959293299; + for (int32_t i = 0; i < 5; ++i) { char s[256] = {0}; while (1) { @@ -954,16 +954,16 @@ TEST(clientCase, agg_query_tables) { } } - while (1) { - sprintf(s, "insert into t2 values(%ld, %d)", st + i, i); - pRes = taos_query(pConn, s); - int32_t ret = taos_errno(pRes); - - taos_free_result(pRes); - if (ret == 0) { - break; - } - } +// while (1) { +// sprintf(s, "insert into t2 values(%ld, %d)", st + i, i); +// pRes = taos_query(pConn, s); +// int32_t ret = taos_errno(pRes); +// +// taos_free_result(pRes); +// if (ret == 0) { +// break; +// } +// } } // pRes = taos_query(pConn, "show table distributed tup"); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b57ca009d3..146f9f6fc4 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1642,10 +1642,18 @@ static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -static void setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlock *pBlock, int32_t numOfRows) { +static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDataBlock *pBlock, int32_t numOfRows) { SColumnInfoData *pColInfo; int32_t cols = 0; + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + + STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); + if (pe == NULL) { + mError("task:0x%" PRIx64 " not exists in vnode, no valid status/stage info", id.taskId); + return -1; + } + // stream name char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); @@ -1696,14 +1704,7 @@ static void setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDat colDataSetVal(pColInfo, numOfRows, (const char *)level, false); // status - char status[20 + VARSTR_HEADER_SIZE] = {0}; - STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - - STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); - if (pe == NULL) { - mError("task:0x%" PRIx64 " not exists in vnode, no valid status/stage info", id.taskId); - return; - } + char status[20 + VARSTR_HEADER_SIZE] = {0}; const char *pStatus = streamTaskGetStatusStr(pe->status); STR_TO_VARSTR(status, pStatus); @@ -1746,6 +1747,8 @@ static void setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDat pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + + return TSDB_CODE_SUCCESS; } static int32_t getNumOfTasks(SArray *pTaskList) { @@ -1787,8 +1790,10 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t numOfLevels = taosArrayGetSize(pLevel); for (int32_t j = 0; j < numOfLevels; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows); - numOfRows++; + int32_t code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows); + if (code == TSDB_CODE_SUCCESS) { + numOfRows++; + } } } From de631e20a5469ff80e638e2e99e742bd6b924aff Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 3 Jan 2024 19:57:22 +0800 Subject: [PATCH 081/198] fix: refactor logdebugs --- include/common/tglobal.h | 4 +- include/util/tconfig.h | 2 +- include/util/tdef.h | 7 ++- include/util/tlog.h | 1 + source/common/src/tglobal.c | 98 +++++++++++++++++++++------------- source/dnode/mgmt/exe/dmMain.c | 2 +- source/util/src/tconfig.c | 37 ++++++++++++- source/util/src/tlog.c | 1 + utils/tsim/inc/simInt.h | 1 - utils/tsim/src/simSystem.c | 2 - 10 files changed, 110 insertions(+), 45 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 887f4cb6dc..f006779a48 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -228,8 +228,8 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer); struct SConfig *taosGetCfg(); -void taosSetAllDebugFlag(int32_t flag, bool rewrite); -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite); +void taosSetAllDebugFlag(int32_t flag); +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(); diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 07abdd223d..f2a9446700 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -94,7 +94,7 @@ typedef struct SConfigItem { int64_t imax; double fmax; }; - SArray *array; // SDiskCfg + SArray *array; // SDiskCfg/SLogVar } SConfigItem; typedef struct { diff --git a/include/util/tdef.h b/include/util/tdef.h index 51b0b63da2..875a6f5738 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -260,7 +260,7 @@ typedef enum ELogicConditionType { #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 32 -#define TSDB_LABEL_LEN 12 +#define TSDB_LABEL_LEN 12 #define TSDB_JOB_STATUS_LEN 32 #define TSDB_CLUSTER_ID_LEN 40 @@ -272,6 +272,7 @@ typedef enum ELogicConditionType { #define TSDB_SHOW_SCHEMA_JSON_LEN TSDB_MAX_COLUMNS * 256 #define TSDB_SLOW_QUERY_SQL_LEN 512 #define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_LOG_VAR_LEN 32 #define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_TYPE_LEN 16 @@ -503,6 +504,10 @@ typedef struct { int32_t primary; } SDiskCfg; +typedef struct { + char name[TSDB_LOG_VAR_LEN]; +} SLogVar; + #define TMQ_SEPARATOR ':' enum { diff --git a/include/util/tlog.h b/include/util/tlog.h index 6d393bfefb..11ac0e1fae 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -67,6 +67,7 @@ extern int32_t smaDebugFlag; extern int32_t idxDebugFlag; extern int32_t tdbDebugFlag; extern int32_t sndDebugFlag; +extern int32_t simDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d6d1e1810d..79d21955d4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -423,7 +423,7 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", simDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; @@ -497,7 +497,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; @@ -962,6 +962,7 @@ static void taosSetClientLogCfg(SConfig *pCfg) { rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; } static void taosSetServerLogCfg(SConfig *pCfg) { @@ -1278,7 +1279,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetServerLogCfg(pCfg); } - taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false); + taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1356,6 +1357,8 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); if (taosSetFileHandlesLimit() != 0) return -1; + taosSetAllDebugFlag(cfgGetItem(tsCfg, "debugFlag")->i32); + cfgDumpCfg(tsCfg, tsc, false); if (taosCheckGlobalCfg() != 0) { @@ -1399,7 +1402,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, *pVar = flag; if (isDebugflag) { - taosSetDebugFlag(pOptions[d].optionVar, optName, flag, true); + taosSetDebugFlag(pOptions[d].optionVar, optName, flag); } terrno = TSDB_CODE_SUCCESS; } break; @@ -1447,8 +1450,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { } if (strncasecmp(name, "debugFlag", 9) == 0) { - int32_t flag = pItem->i32; - taosSetAllDebugFlag(flag, true); + taosSetAllDebugFlag(pItem->i32); return 0; } @@ -1460,7 +1462,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag}, - {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, + {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, }; static OptionNameAndVar options[] = { @@ -1524,8 +1526,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { switch (lowcaseName[0]) { case 'd': { if (strcasecmp("debugFlag", name) == 0) { - int32_t flag = pItem->i32; - taosSetAllDebugFlag(flag, true); + taosSetAllDebugFlag(pItem->i32); matched = true; } break; @@ -1691,7 +1692,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { {"cDebugFlag", &cDebugFlag}, {"dDebugFlag", &dDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"jniDebugFlag", &jniDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, - {"uDebugFlag", &uDebugFlag}, + {"uDebugFlag", &uDebugFlag}, {"simDebugFlag", &simDebugFlag}, }; static OptionNameAndVar options[] = { @@ -1736,9 +1737,9 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer) { return taosCfgDynamicOptionsForClient(pCfg, name); } -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite) { +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal) { SConfigItem *pItem = cfgGetItem(tsCfg, flagName); - if (pItem != NULL && (rewrite || pItem->i32 == 0)) { + if (pItem != NULL) { pItem->i32 = flagVal; } if (pFlagPtr != NULL) { @@ -1746,33 +1747,58 @@ void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, } } -void taosSetAllDebugFlag(int32_t flag, bool rewrite) { +static int taosLogVarComp(void const *lp, void const *rp) { + SLogVar *lpVar = (SLogVar *)lp; + SLogVar *rpVar = (SLogVar *)rp; + return strcasecmp(lpVar->name, rpVar->name); +} + +static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag, SArray *noNeedToSetVars) { + if (noNeedToSetVars != NULL && taosArraySearch(noNeedToSetVars, name, taosLogVarComp, TD_EQ) != NULL) { + return; + } + taosSetDebugFlag(pFlagPtr, name, flag); +} + +void taosSetAllDebugFlag(int32_t flag) { if (flag <= 0) return; - taosSetDebugFlag(NULL, "debugFlag", flag, rewrite); - taosSetDebugFlag(NULL, "simDebugFlag", flag, rewrite); - taosSetDebugFlag(NULL, "tmrDebugFlag", flag, rewrite); - taosSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, rewrite); - taosSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, rewrite); - taosSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, rewrite); - taosSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, rewrite); - taosSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, rewrite); - taosSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, rewrite); - taosSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, rewrite); - taosSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, rewrite); - taosSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, rewrite); - taosSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, rewrite); - taosSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, rewrite); - taosSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, rewrite); - taosSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, rewrite); - taosSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, rewrite); - taosSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, rewrite); - taosSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, rewrite); - taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite); - taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite); - taosSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, rewrite); - taosSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, rewrite); + SArray *noNeedToSetVars = NULL; + SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); + if (pItem != NULL) { + pItem->i32 = flag; + noNeedToSetVars = pItem->array; + } + + taosCheckAndSetDebugFlag(&simDebugFlag, "simDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tmrDebugFlag, "tmrDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, noNeedToSetVars); + + taosCheckAndSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, noNeedToSetVars); + + taosCheckAndSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, noNeedToSetVars); + taosCheckAndSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, noNeedToSetVars); + + taosArrayClear(noNeedToSetVars); // reset array + uInfo("all debug flag are set to %d", flag); + if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist } int8_t taosGranted() { return atomic_load_8(&tsGrant); } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 3c08714218..756ac8167e 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -68,7 +68,7 @@ static struct { int64_t startTime; } global = {0}; -static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143, true); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143); } static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } static void dmStopDnode(int signum, void *sigInfo, void *context) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index a78d930326..d656f0c14e 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -307,6 +307,34 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, return 0; } +static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) { + SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag"); + if (resetArray) { + // reset + if (pDebugFlagItem == NULL) return -1; + + // logflag names that should 'not' be set by 'debugFlag' + if (pDebugFlagItem->array == NULL) { + pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar)); + if (pDebugFlagItem->array == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + taosArrayClear(pDebugFlagItem->array); + return 0; + } + + // update + if (pDebugFlagItem == NULL) return -1; + if (pDebugFlagItem->array != NULL) { + SLogVar logVar = {0}; + strncpy(logVar.name, name, TSDB_LOG_VAR_LEN - 1); + taosArrayPush(pDebugFlagItem->array, &logVar); + } + return 0; +} + int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) { GRANT_CFG_SET; SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -661,7 +689,6 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; - if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); for (int32_t j = 0; j < CFG_SRC_PRINT_LEN; ++j) { if (src[j] == 0) src[j] = ' '; @@ -931,6 +958,14 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + + size_t len = strlen(name); + const char *debugFlagStr = "debugFlag"; + const size_t debugFlagLen = strlen(debugFlagStr); + if (len >= debugFlagLen && strcasecmp(name + len - debugFlagLen, debugFlagStr) == 0) { + code = cfgUpdateDebugFlagItem(pConfig, name, len == debugFlagLen); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } } taosCloseFile(&pFile); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e113a95fcc..bd6c37a7b5 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -111,6 +111,7 @@ int32_t udfDebugFlag = 131; int32_t smaDebugFlag = 131; int32_t idxDebugFlag = 131; int32_t sndDebugFlag = 131; +int32_t simDebugFlag = 131; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h index 02c7540cde..f2360277e0 100644 --- a/utils/tsim/inc/simInt.h +++ b/utils/tsim/inc/simInt.h @@ -184,7 +184,6 @@ extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; extern SCommand simCmdList[]; extern int32_t simScriptPos; extern int32_t simScriptSucceed; -extern int32_t simDebugFlag; extern char simScriptDir[]; extern bool abortExecution; extern bool useValgrind; diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index 98f9217fd6..dcf5d6ab12 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -21,7 +21,6 @@ SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; int32_t simScriptPos = -1; int32_t simScriptSucceed = 0; -int32_t simDebugFlag = 143; void simCloseTaosdConnect(SScript *script); char simScriptDir[PATH_MAX] = {0}; @@ -32,7 +31,6 @@ int32_t simInitCfg() { taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); SConfig *pCfg = taosGetCfg(); - simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX); return 0; } From 74df121cf5e36a1b364030eb38dc5e2e60f3c678 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 22:29:46 +0800 Subject: [PATCH 082/198] fix(stream): fix the deadlock. --- source/libs/stream/src/streamExec.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index e73c5ffddb..d68288222c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -734,8 +734,8 @@ static void doStreamExecTaskHelper(void* param, void* tmrId) { static int32_t schedTaskInFuture(SStreamTask* pTask) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s task should idle, add into timer to retry in %dms, ref:%d", - pTask->id.idStr, DISPATCH_RETRY_INTERVAL_MS, ref); + stDebug("s-task:%s task should idle, add into timer to retry in %dms, ref:%d", pTask->id.idStr, + pTask->status.schedIdleTime, ref); // add one ref count for task SStreamTask* pAddRefTask = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId); @@ -761,6 +761,9 @@ int32_t streamResumeTask(SStreamTask* pTask) { if (pTask->status.schedIdleTime > 0) { stDebug("s-task:%s idled, and will be invoked in %dms", id, pTask->status.schedIdleTime); schedTaskInFuture(pTask); + + taosThreadMutexUnlock(&pTask->lock); + return 0; } else { int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); if ((numOfItems == 0) || streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) { From 1e3aad88831a7d2e34a725df9d3ba5739a36e9d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 3 Jan 2024 23:57:33 +0800 Subject: [PATCH 083/198] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tqStreamTask.c | 26 +++++++++++------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index b11e231c62..38b240b47e 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -32,26 +32,24 @@ int32_t tqScanWal(STQ* pTq) { int64_t st = taosGetTimestampMs(); while (1) { - int32_t scan = pMeta->walScanCounter; - tqDebug("vgId:%d continue check if data in wal are available, walScanCounter:%d", vgId, scan); + tqDebug("vgId:%d continue check if data in wal are available, walScanCounter:%d", vgId, pMeta->walScanCounter); // check all tasks bool shouldIdle = true; - doScanWalForAllTasks(pTq->pStreamMeta, &shouldIdle); + doScanWalForAllTasks(pMeta, &shouldIdle); -// if (shouldIdle) { - streamMetaWLock(pMeta); - int32_t times = (--pMeta->walScanCounter); - ASSERT(pMeta->walScanCounter >= 0); - streamMetaWUnLock(pMeta); + streamMetaWLock(pMeta); + int32_t times = (--pMeta->walScanCounter); + ASSERT(pMeta->walScanCounter >= 0); + streamMetaWUnLock(pMeta); - if (times <= 0) { - break; - } else { - tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION); - } -// } + if (times > 0) { + tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION); + } else { // times <= 0 + break; + } + // todo: remove the sleep taosMsleep(SCAN_WAL_IDLE_DURATION); } From 860146d425dcea53420439ee2f2f591d7fb56400 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 3 Jan 2024 23:59:48 +0800 Subject: [PATCH 084/198] initRemainGroups --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2e93dbd803..e3f634f406 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2164,7 +2164,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* bool initRemainGroups = false; if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode)) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pScanNode; - if (pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { + if (tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable && !(groupSort || pScanNode->groupOrderScan)) { initRemainGroups = true; } } From f4ad4ea998d9e3b150172c39a7f1b809899c7823 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 10:01:05 +0800 Subject: [PATCH 085/198] enh: use learner in split vgroup --- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 20fb7a4d31..6eb5d7e592 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2707,8 +2707,16 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro if (newVg1.replica == 1) { if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER; + + newVg1.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER; + + newVg1.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER; + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; + + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; } else if (newVg1.replica == 3) { SVnodeGid del1 = {0}; if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER; From b85f36bcea67d833b65194333fec446413912ca3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 12:00:49 +0800 Subject: [PATCH 086/198] enh: allow to drop an empty dnode, no matter online or not --- source/dnode/mnode/impl/src/mndDnode.c | 29 +++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e224aceec2..6585e70533 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1094,6 +1094,32 @@ _OVER: return code; } +static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) { + bool isEmpty = false; + SMnodeObj *pMObj = NULL; + SQnodeObj *pQObj = NULL; + SSnodeObj *pSObj = NULL; + + pQObj = mndAcquireQnode(pMnode, dnodeId); + if (pQObj) goto _OVER; + + pSObj = mndAcquireSnode(pMnode, dnodeId); + if (pSObj) goto _OVER; + + pMObj = mndAcquireMnode(pMnode, dnodeId); + if (pMObj) goto _OVER; + + int32_t numOfVnodes = mndGetVnodesNum(pMnode, dnodeId); + if (numOfVnodes > 0) goto _OVER; + + isEmpty = true; +_OVER: + mndReleaseMnode(pMnode, pMObj); + mndReleaseQnode(pMnode, pQObj); + mndReleaseSnode(pMnode, pSObj); + return isEmpty; +} + static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -1155,7 +1181,8 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (!isonline && !force) { + bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id); + if (!isonline && !force && !isEmpty) { terrno = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); From c49425200271b920024277d9d532ef5f9defa005 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 17:44:05 +0800 Subject: [PATCH 087/198] fix(stream): record the related fill-history task failure. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 21 +++++++++++++++++++-- source/libs/stream/src/streamMeta.c | 11 ++++++++--- source/libs/stream/src/streamStart.c | 16 ++++------------ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index b869d67bed..95bc4f4131 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -438,19 +438,36 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); + int64_t initTs = 0; + int64_t now = taosGetTimestampMs(); if (!isLeader) { - streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, taosGetTimestampMs(), false); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); + if (pTask != NULL) { + initTs = pTask->execInfo.init; + + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pId = &pTask->hTaskInfo.id; + streamMetaUpdateTaskDownstreamStatus(pMeta, pId->streamId, pId->taskId, initTs, now, false); + } + + streamMetaReleaseTask(pMeta, pTask); + } + + streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, initTs, now, false); tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId); + return code; } SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); if (pTask == NULL) { - streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, taosGetTimestampMs(), false); + streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, now, false); tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", rsp.streamId, rsp.upstreamTaskId, vgId); terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; + + // failed to find the related fill-history task return -1; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index dc2e3bd651..8017a88dea 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1467,6 +1467,8 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { continue; } + // todo: may be we should find the related fill-history task and set it failed. + // fill-history task can only be launched by related stream tasks. STaskExecStatisInfo* pInfo = &pTask->execInfo; if (pTask->info.fillHistory == 1) { @@ -1486,13 +1488,16 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { continue; } - EStreamTaskEvent event = /*(HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; - int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); + int32_t ret = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT); if (ret != TSDB_CODE_SUCCESS) { - stError("vgId:%d failed to handle event:%d", pMeta->vgId, event); + stError("vgId:%d failed to handle event:%d", pMeta->vgId, TASK_EVENT_INIT); code = ret; streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->init, pInfo->start, false); + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pId = &pTask->hTaskInfo.id; + streamMetaUpdateTaskDownstreamStatus(pMeta, pId->streamId, pId->taskId, pInfo->init, pInfo->start, false); + } } streamMetaReleaseTask(pMeta, pTask); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 70f83e27a5..945d7844af 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -500,22 +500,14 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); } - streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, - taosGetTimestampMs(), false); + int32_t startTs = pTask->execInfo.init; + int64_t now = taosGetTimestampMs(); + streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, now, false); // automatically set the related fill-history task to be failed. if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { STaskId* pId = &pTask->hTaskInfo.id; - int64_t current = taosGetTimestampMs(); - - SStreamTask* pHTask = streamMetaAcquireTask(pTask->pMeta, pId->streamId, pId->taskId); - if (pHTask != NULL) { - streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pId->streamId, pId->taskId, pHTask->execInfo.init, current, - false); - streamMetaReleaseTask(pTask->pMeta, pHTask); - } else { - streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pId->streamId, pId->taskId, 0, current, false); - } + streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pId->streamId, pId->taskId, startTs, now, false); } } else { // TASK_DOWNSTREAM_NOT_READY, let's retry in 100ms STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp); From 6d37e596ec466f26c84b7ccd481adf24b6c35897 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 18:47:49 +0800 Subject: [PATCH 088/198] refactor: do some internal refactor. --- source/dnode/snode/src/snode.c | 4 +- source/dnode/vnode/src/tq/tq.c | 29 +++------ source/dnode/vnode/src/tq/tqStreamTask.c | 16 +++-- source/dnode/vnode/src/tqCommon/tqCommon.c | 10 +-- source/libs/stream/inc/streamsm.h | 5 -- source/libs/stream/src/stream.c | 4 +- source/libs/stream/src/streamCheckpoint.c | 7 +-- source/libs/stream/src/streamDispatch.c | 11 ++-- source/libs/stream/src/streamExec.c | 51 +++++++-------- source/libs/stream/src/streamMeta.c | 10 +-- source/libs/stream/src/streamStart.c | 72 ++++++++++------------ source/libs/stream/src/streamTask.c | 4 +- source/libs/stream/src/streamTaskSm.c | 13 ++-- 13 files changed, 103 insertions(+), 133 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 33eb9cd3ed..837c55c219 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -91,9 +91,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } - char *p = NULL; - streamTaskGetStatus(pTask, &p); - + char* p = streamTaskGetStatus(pTask)->name; if (pTask->info.fillHistory) { sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 851d3f967b..203c34f1ce 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -842,8 +842,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } - char* p = NULL; - streamTaskGetStatus(pTask, &p); + char* p = streamTaskGetStatus(pTask)->name; if (pTask->info.fillHistory) { tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 @@ -932,8 +931,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // do recovery step1 const char* id = pTask->id.idStr; - char* pStatus = NULL; - streamTaskGetStatus(pTask, &pStatus); + char* pStatus = streamTaskGetStatus(pTask)->name; // avoid multi-thread exec while (1) { @@ -990,15 +988,15 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { if (retInfo.ret == TASK_SCANHISTORY_REXEC) { streamReExecScanHistoryFuture(pTask, retInfo.idleTime); } else { - char* p = NULL; - ETaskStatus s = streamTaskGetStatus(pTask, &p); + SStreamTaskState* p = streamTaskGetStatus(pTask); + ETaskStatus s = p->state; if (s == TASK_STATUS__PAUSE) { tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr, el, pTask->execInfo.step1El, status); } else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) { - tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr, p, - pTask->execInfo.step1El); + tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr, + p->name, pTask->execInfo.step1El); } } @@ -1038,15 +1036,6 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pMeta, pStreamTask); } else { ASSERT(0); -// STimeWindow* pWindow = &pTask->dataRange.window; -// ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) || streamTaskShouldStop(pTask)); -// -// // Not update the fill-history time window until the state transfer is completed. -// tqDebug("s-task:%s scan-history in stream time window completed, start to handle data from WAL, startVer:%" PRId64 -// ", window:%" PRId64 " - %" PRId64, -// id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); -// -// code = streamTaskScanHistoryDataComplete(pTask); } atomic_store_32(&pTask->status.inScanHistorySentinel, 0); @@ -1138,7 +1127,7 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, } streamTaskResume(pTask); - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__SINK) { @@ -1275,7 +1264,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // todo save the checkpoint failed info taosThreadMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) { tqError("s-task:%s not ready for checkpoint, since it is halt, ignore this checkpoint:%" PRId64 ", set it failure", @@ -1362,7 +1351,7 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { } taosThreadMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; // if (status == TASK_STATUS__STREAM_SCAN_HISTORY) { // streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); // } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 38b240b47e..12f540b9cf 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -220,16 +220,15 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) { } // not in ready state, do not handle the data from wal - char* p = NULL; - int32_t status = streamTaskGetStatus(pTask, &p); - if (streamTaskGetStatus(pTask, &p) != TASK_STATUS__READY) { - tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, p); + SStreamTaskState* pState = streamTaskGetStatus(pTask); + if (pState->state != TASK_STATUS__READY) { + tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState->name); return false; } // fill-history task has entered into the last phase, no need to anything if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) { - ASSERT(status == TASK_STATUS__READY); + ASSERT(pState->state == TASK_STATUS__READY); // the maximum version of data in the WAL has reached already, the step2 is done tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr, pTask->dataRange.range.maxVer); @@ -342,10 +341,9 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { taosThreadMutexLock(&pTask->lock); - char* p = NULL; - ETaskStatus status = streamTaskGetStatus(pTask, &p); - if (status != TASK_STATUS__READY) { - tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, p); + SStreamTaskState* pState = streamTaskGetStatus(pTask); + if (pState->state != TASK_STATUS__READY) { + tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState->name); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pStreamMeta, pTask); continue; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 95bc4f4131..6a47cfc3eb 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -400,11 +400,11 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); streamMetaReleaseTask(pMeta, pTask); - char* p = NULL; - streamTaskGetStatus(pTask, &p); + SStreamTaskState* pState = streamTaskGetStatus(pTask); tqDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", - pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + pTask->id.idStr, pState->name, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, + rsp.status); } else { rsp.status = TASK_DOWNSTREAM_NOT_READY; tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 @@ -706,9 +706,11 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead streamMetaStopAllTasks(pMeta); return 0; } else if (type == STREAM_EXEC_T_RESUME_TASK) { + // task resume to run after idle for a while SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); if (pTask != NULL) { ASSERT(streamTaskReadyToRun(pTask, NULL)); + tqDebug("s-task:%s task resume to run after idle for a while", pTask->id.idStr); streamResumeTask(pTask); } @@ -772,7 +774,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); // clear flag set during do checkpoint, and open inputQ for all upstream tasks - if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) { + if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) { streamTaskClearCheckInfo(pTask, true); streamTaskSetStatusReady(pTask); } diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h index ea0522bd5a..abdafc0240 100644 --- a/source/libs/stream/inc/streamsm.h +++ b/source/libs/stream/inc/streamsm.h @@ -23,11 +23,6 @@ extern "C" { #endif // moore finite state machine for stream task -typedef struct SStreamTaskState { - ETaskStatus state; - char* name; -} SStreamTaskState; - typedef int32_t (*__state_trans_fn)(SStreamTask*); typedef int32_t (*__state_trans_succ_fn)(SStreamTask*); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index f383f0f31d..10b1b3fbbe 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -49,7 +49,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { return; } - if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) { + if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) { stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger); } else { if (status == TASK_TRIGGER_STATUS__ACTIVE) { @@ -247,7 +247,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } // disable the data from upstream tasks - if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__HALT) { + if (streamTaskGetStatus(pTask)->state == TASK_STATUS__HALT) { status = TASK_INPUT_STATUS__BLOCKED; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index b54adb0f96..c75185b0ea 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -187,7 +187,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc int32_t code = TSDB_CODE_SUCCESS; // set task status - if (streamTaskGetStatus(pTask, NULL) != TASK_STATUS__CK) { + if (streamTaskGetStatus(pTask)->state != TASK_STATUS__CK) { pTask->chkInfo.checkpointingId = checkpointId; code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); if (code != TSDB_CODE_SUCCESS) { @@ -309,8 +309,7 @@ int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { p->chkInfo.checkpointVer = p->chkInfo.processedVer; streamTaskClearCheckInfo(p, false); - char* str = NULL; - streamTaskGetStatus(p, &str); + SStreamTaskState* pState = streamTaskGetStatus(p); code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); taosThreadMutexUnlock(&p->lock); @@ -322,7 +321,7 @@ int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { stDebug("vgId:%d s-task:%s level:%d open upstream inputQ, save status after checkpoint, checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status: normal, prev:%s", - vgId, id, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, str); + vgId, id, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, pState->name); // save the task if not sink task if (p->info.taskLevel != TASK_LEVEL__SINK) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 00a8940b6a..a6d21e2625 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -696,10 +696,9 @@ int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask) { int32_t numOfVgs = taosArrayGetSize(vgInfo); pTask->notReadyTasks = numOfVgs; - char* p = NULL; - streamTaskGetStatus(pTask, &p); + SStreamTaskState* pState = streamTaskGetStatus(pTask); stDebug("s-task:%s send scan-history data complete msg to downstream (shuffle-dispatch) %d tasks, status:%s", - pTask->id.idStr, numOfVgs, p); + pTask->id.idStr, numOfVgs, pState->name); for (int32_t i = 0; i < numOfVgs; i++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); req.downstreamTaskId = pVgInfo->taskId; @@ -819,9 +818,9 @@ int32_t doDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHist initRpcMsg(&msg, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, buf, tlen + sizeof(SMsgHead)); tmsgSendReq(pEpSet, &msg); - char* p = NULL; - streamTaskGetStatus(pTask, &p); - stDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, p, + + SStreamTaskState* pState = streamTaskGetStatus(pTask); + stDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, pState->name, pReq->downstreamTaskId, vgId); return 0; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index d68288222c..d83d99f10f 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -24,12 +24,12 @@ static int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask); bool streamTaskShouldStop(const SStreamTask* pTask) { - ETaskStatus s = streamTaskGetStatus(pTask, NULL); - return (s == TASK_STATUS__STOP) || (s == TASK_STATUS__DROPPING); + SStreamTaskState* pState = streamTaskGetStatus(pTask); + return (pState->state == TASK_STATUS__STOP) || (pState->state == TASK_STATUS__DROPPING); } bool streamTaskShouldPause(const SStreamTask* pTask) { - return (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE); + return (streamTaskGetStatus(pTask)->state == TASK_STATUS__PAUSE); } static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBlock) { @@ -344,11 +344,14 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { streamMetaWUnLock(pMeta); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } else { - stDebug("s-task:%s fill-history task end, update related stream task:%s info, transfer exec state", id, - pStreamTask->id.idStr); + double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.; + stDebug( + "s-task:%s fill-history task end, scal wal elapsed time:%.2fSec,update related stream task:%s info, transfer " + "exec state", + id, el, pStreamTask->id.idStr); } - ETaskStatus status = streamTaskGetStatus(pStreamTask, NULL); + ETaskStatus status = streamTaskGetStatus(pStreamTask)->state; STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; // It must be halted for a source stream task, since when the related scan-history-data task start scan the history @@ -374,8 +377,10 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { // In case of sink tasks, no need to halt them. // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to // start the task state transfer procedure. - char* p = NULL; - status = streamTaskGetStatus(pStreamTask, &p); +// char* p = NULL; + SStreamTaskState* pState = streamTaskGetStatus(pStreamTask); + status = pState->state; + char* p = pState->name; if (status == TASK_STATUS__STOP || status == TASK_STATUS__DROPPING) { stError("s-task:%s failed to transfer state from fill-history task:%s, status:%s", id, pStreamTask->id.idStr, p); streamMetaReleaseTask(pMeta, pStreamTask); @@ -409,7 +414,7 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id); // 5. save to disk - pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask, NULL); + pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask)->state; // 6. add empty delete block if ((pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) && taosQueueEmpty(pStreamTask->inputq.queue->pQueue)) { @@ -570,7 +575,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { int32_t blockSize = 0; int32_t numOfBlocks = 0; SStreamQueueItem* pInput = NULL; - if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__UNINIT)) { + if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask)->state == TASK_STATUS__UNINIT)) { stDebug("s-task:%s stream task is stopped", id); break; } @@ -647,10 +652,9 @@ int32_t doStreamExecTask(SStreamTask* pTask) { if (type == STREAM_INPUT__CHECKPOINT) { // todo add lock - char* p = NULL; - ETaskStatus s = streamTaskGetStatus(pTask, &p); - if (s == TASK_STATUS__CK) { - stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); + SStreamTaskState* pState = streamTaskGetStatus(pTask); + if (pState->state == TASK_STATUS__CK) { + stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, pState->name); streamTaskBuildCheckpoint(pTask); } else { // todo refactor @@ -678,26 +682,27 @@ int32_t doStreamExecTask(SStreamTask* pTask) { // the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not // be updated by tryExec function, therefore, the schedStatus will always be the TASK_SCHED_STATUS__WAITING. bool streamTaskIsIdle(const SStreamTask* pTask) { - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || status == TASK_STATUS__STOP || status == TASK_STATUS__DROPPING); } bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { - ETaskStatus st = streamTaskGetStatus(pTask, pStatus); + SStreamTaskState* pState = streamTaskGetStatus(pTask); + ETaskStatus st = pState->state; + *pStatus = pState->name; return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__CK); } static void doStreamExecTaskHelper(void* param, void* tmrId) { SStreamTask* pTask = (SStreamTask*)param; - char* p = NULL; - ETaskStatus status = streamTaskGetStatus(pTask, &p); - if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) { + SStreamTaskState* p = streamTaskGetStatus(pTask); + if (p->state == TASK_STATUS__DROPPING || p->state == TASK_STATUS__STOP) { streamTaskSetSchedStatusInactive(pTask); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s not resume task, ref:%d", pTask->id.idStr, p, ref); + stDebug("s-task:%s status:%s not resume task, ref:%d", pTask->id.idStr, p->name, ref); streamMetaReleaseTask(pTask->pMeta, pTask); return; @@ -770,8 +775,7 @@ int32_t streamResumeTask(SStreamTask* pTask) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); taosThreadMutexUnlock(&pTask->lock); - char* p = NULL; - streamTaskGetStatus(pTask, &p); + char* p = streamTaskGetStatus(pTask)->name; stDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, p, pTask->status.schedStatus); return 0; } @@ -789,8 +793,7 @@ int32_t streamExecTask(SStreamTask* pTask) { if (schedStatus == TASK_SCHED_STATUS__WAITING) { streamResumeTask(pTask); } else { - char* p = NULL; - streamTaskGetStatus(pTask, &p); + char* p = streamTaskGetStatus(pTask)->name; stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, p, pTask->status.schedStatus); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8017a88dea..7ea42e62c1 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1093,7 +1093,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { STaskStatusEntry entry = { .id = *pId, - .status = streamTaskGetStatus(*pTask, NULL), + .status = streamTaskGetStatus(*pTask)->state, .nodeId = hbMsg.vgId, .stage = pMeta->stage, .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)), @@ -1362,13 +1362,13 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { } taosThreadMutexLock(&pTask->lock); - char* p = NULL; - ETaskStatus s = streamTaskGetStatus(pTask, &p); - if (s == TASK_STATUS__CK) { + + SStreamTaskState* pState = streamTaskGetStatus(pTask); + if (pState->state == TASK_STATUS__CK) { streamTaskSetCheckpointFailedId(pTask); stDebug("s-task:%s mark the checkpoint:%"PRId64" failed", pTask->id.idStr, pTask->chkInfo.checkpointingId); } else { - stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, p); + stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState->state); } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 945d7844af..45a571e93c 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -48,15 +48,13 @@ static void tryLaunchHistoryTask(void* param, void* tmrId); static void doProcessDownstreamReadyRsp(SStreamTask* pTask); int32_t streamTaskSetReady(SStreamTask* pTask) { - char* p = NULL; - int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); - ETaskStatus status = streamTaskGetStatus(pTask, &p); + int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); + SStreamTaskState* p = streamTaskGetStatus(pTask); - if ((status == TASK_STATUS__SCAN_HISTORY/* || status == TASK_STATUS__STREAM_SCAN_HISTORY*/) && - pTask->info.taskLevel != TASK_LEVEL__SOURCE) { + if ((p->state == TASK_STATUS__SCAN_HISTORY) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->upstreamInfo.pList); stDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", - pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream, p); + pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream, p->name); } ASSERT(pTask->status.downstreamReady == 0); @@ -93,12 +91,11 @@ static void doReExecScanhistory(void* param, void* tmrId) { SStreamTask* pTask = param; pTask->schedHistoryInfo.numOfTicks -= 1; - char* p = NULL; - ETaskStatus status = streamTaskGetStatus(pTask, &p); - if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) { + SStreamTaskState* p = streamTaskGetStatus(pTask); + if (p->state == TASK_STATUS__DROPPING || p->state == TASK_STATUS__STOP) { streamMetaReleaseTask(pTask->pMeta, pTask); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p, ref); + stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p->name, ref); return; } @@ -155,7 +152,7 @@ static int32_t doStartScanHistoryTask(SStreamTask* pTask) { int32_t streamTaskStartScanHistory(SStreamTask* pTask) { int32_t level = pTask->info.taskLevel; - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; ASSERT(pTask->status.downstreamReady == 1 && ((status == TASK_STATUS__SCAN_HISTORY)/* || (status == TASK_STATUS__STREAM_SCAN_HISTORY)*/)); @@ -315,7 +312,7 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ id, upstreamTaskId, vgId, stage, pInfo->stage); // record the checkpoint failure id and sent to mnode taosThreadMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; if (status == TASK_STATUS__CK) { streamTaskSetCheckpointFailedId(pTask); } @@ -325,7 +322,7 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ if (pInfo->stage != stage) { taosThreadMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; if (status == TASK_STATUS__CK) { streamTaskSetCheckpointFailedId(pTask); } @@ -346,9 +343,8 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { streamTaskSetReady(pTask); streamTaskSetRangeStreamCalc(pTask); - char* p = NULL; - ETaskStatus status = streamTaskGetStatus(pTask, &p); - ASSERT(status == TASK_STATUS__READY); + SStreamTaskState* p = streamTaskGetStatus(pTask); + ASSERT(p->state == TASK_STATUS__READY); if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { int64_t startVer = walReaderGetCurrentVer(pTask->exec.pWalReader); @@ -372,15 +368,14 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { streamTaskSetReady(pTask); streamTaskSetRangeStreamCalc(pTask); - char* p = NULL; - ETaskStatus status = streamTaskGetStatus(pTask, &p); - ASSERT(status == TASK_STATUS__SCAN_HISTORY/* || status == TASK_STATUS__STREAM_SCAN_HISTORY*/); + SStreamTaskState* p = streamTaskGetStatus(pTask); + ASSERT(p->state == TASK_STATUS__SCAN_HISTORY); if (pTask->info.fillHistory == 1) { - stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", id, p); + stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", id, p->name); streamTaskStartScanHistory(pTask); } else { - stDebug("s-task:%s scan wal data, status:%s", id, p); + stDebug("s-task:%s scan wal data, status:%s", id, p->name); } // NOTE: there will be an deadlock if launch fill history here. @@ -624,12 +619,11 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory ASSERT(taskLevel == TASK_LEVEL__AGG || taskLevel == TASK_LEVEL__SINK); const char* id = pTask->id.idStr; - char* p = NULL; - ETaskStatus status = streamTaskGetStatus(pTask, &p); + SStreamTaskState* p = streamTaskGetStatus(pTask); - if (status != TASK_STATUS__SCAN_HISTORY /*&& status != TASK_STATUS__STREAM_SCAN_HISTORY*/) { - stError("s-task:%s not in scan-history status, status:%s return upstream:0x%x scan-history finish directly", id, p, - pReq->upstreamTaskId); + if (p->state != TASK_STATUS__SCAN_HISTORY) { + stError("s-task:%s not in scan-history status, status:%s return upstream:0x%x scan-history finish directly", id, + p->name, pReq->upstreamTaskId); void* pBuf = NULL; int32_t len = 0; @@ -682,7 +676,7 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistory } int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) { - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; // task restart now, not handle the scan-history finish rsp if (status == TASK_STATUS__UNINIT) { @@ -741,8 +735,7 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { ASSERT((*ppTask)->status.timerActive >= 1); if (streamTaskShouldStop(*ppTask)) { - char* p = NULL; - streamTaskGetStatus((*ppTask), &p); + char* p = streamTaskGetStatus(*ppTask)->name; int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1); stDebug("s-task:%s status:%s should stop, quit launch fill-history task timer, retry:%d, ref:%d", (*ppTask)->id.idStr, p, (*ppTask)->hTaskInfo.retryTimes, ref); @@ -781,10 +774,9 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { // abort the timer if intend to stop task SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId); if (pHTask == NULL && (!streamTaskShouldStop(pTask))) { - char* p = NULL; + char* p = streamTaskGetStatus(pTask)->name; int32_t hTaskId = pHTaskInfo->id.taskId; - streamTaskGetStatus(pTask, &p); stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); @@ -878,7 +870,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { } int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask) { - if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__DROPPING) { + if (streamTaskGetStatus(pTask)->state == TASK_STATUS__DROPPING) { return 0; } @@ -1058,23 +1050,21 @@ void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta) { } void streamTaskResume(SStreamTask* pTask) { - char* p = NULL; - ETaskStatus status = streamTaskGetStatus(pTask, &p); + SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamMeta* pMeta = pTask->pMeta; - if (status == TASK_STATUS__PAUSE || status == TASK_STATUS__HALT) { + if (p->state == TASK_STATUS__PAUSE || p->state == TASK_STATUS__HALT) { streamTaskRestoreStatus(pTask); - char* pNew = NULL; - streamTaskGetStatus(pTask, &pNew); - if (status == TASK_STATUS__PAUSE) { + char* pNew = streamTaskGetStatus(pTask)->name; + if (p->state == TASK_STATUS__PAUSE) { int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); - stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, p, num); + stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, p->name, num); } else { - stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, p); + stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, p->name); } } else { - stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, p); + stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, p->name); } } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 32fd5cbb39..c14d355dc8 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -309,7 +309,9 @@ void tFreeStreamTask(SStreamTask* pTask) { ETaskStatus status1 = TASK_STATUS__UNINIT; taosThreadMutexLock(&pTask->lock); if (pTask->status.pSM != NULL) { - status1 = streamTaskGetStatus(pTask, &p); + SStreamTaskState* pStatus = streamTaskGetStatus(pTask); + p = pStatus->name; + status1 = pStatus->state; } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index d785932109..819cdbf374 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -65,8 +65,7 @@ static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus nex static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; } static int32_t attachEvent(SStreamTask* pTask, SAttachedEventInfo* pEvtInfo) { - char* p = NULL; - streamTaskGetStatus(pTask, &p); + char* p = streamTaskGetStatus(pTask)->name; stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p, GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); @@ -275,7 +274,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt while (1) { // wait for the task to be here taosThreadMutexLock(&pTask->lock); - ETaskStatus s = streamTaskGetStatus(pTask, NULL); + ETaskStatus s = streamTaskGetStatus(pTask)->state; taosThreadMutexUnlock(&pTask->lock); if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already @@ -400,12 +399,8 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even return TSDB_CODE_SUCCESS; } -ETaskStatus streamTaskGetStatus(const SStreamTask* pTask, char** pStr) { - SStreamTaskState s = pTask->status.pSM->current; // copy one obj in case of multi-thread environment - if (pStr != NULL) { - *pStr = s.name; - } - return s.state; +SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask) { + return &pTask->status.pSM->current; // copy one obj in case of multi-thread environment } const char* streamTaskGetStatusStr(ETaskStatus status) { From fa9df2a49591987e1c1dda5308945b0ecf91d6fa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 18:48:19 +0800 Subject: [PATCH 089/198] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 79fc2a87a7..47c5149795 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -730,6 +730,11 @@ typedef struct SStreamTaskNodeUpdateMsg { int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpdateMsg* pMsg); int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg); +typedef struct SStreamTaskState { + ETaskStatus state; + char* name; +} SStreamTaskState; + typedef struct { int64_t streamId; int32_t downstreamTaskId; @@ -773,11 +778,11 @@ bool streamTaskShouldPause(const SStreamTask* pStatus); bool streamTaskIsIdle(const SStreamTask* pTask); bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus); -char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); -ETaskStatus streamTaskGetStatus(const SStreamTask* pTask, char** pStr); -const char* streamTaskGetStatusStr(ETaskStatus status); -void streamTaskResetStatus(SStreamTask* pTask); -void streamTaskSetStatusReady(SStreamTask* pTask); +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); +SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask); +const char* streamTaskGetStatusStr(ETaskStatus status); +void streamTaskResetStatus(SStreamTask* pTask); +void streamTaskSetStatusReady(SStreamTask* pTask); void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); From bd64a78094f08aa5ad9bd3c0f1d03b7ab5e6acef Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 4 Jan 2024 18:53:37 +0800 Subject: [PATCH 090/198] fix:[TD-28155] set consumer lost if all current topic is 0 --- source/dnode/mnode/impl/src/mndConsumer.c | 24 ++++++++++------------- source/dnode/mnode/impl/src/mndTrans.c | 5 ++++- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 9d9473d883..a365a44d56 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -749,18 +749,6 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { return 0; } -static void updateConsumerStatus(SMqConsumerObj *pConsumer) { - int32_t status = pConsumer->status; - - if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS_REBALANCE) { - pConsumer->status = MQ_CONSUMER_STATUS_READY; - } else if (status == MQ_CONSUMER_STATUS_READY) { - pConsumer->status = MQ_CONSUMER_STATUS_LOST; - } - } -} - // remove from new topic static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); @@ -881,7 +869,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - updateConsumerStatus(pOldConsumer); + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && + taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && + taosArrayGetSize(pOldConsumer->currentTopics) != 0){ + pOldConsumer->status = MQ_CONSUMER_STATUS_READY; + } // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = taosGetTimestampMs(); @@ -905,7 +897,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - updateConsumerStatus(pOldConsumer); + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && + taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && + taosArrayGetSize(pOldConsumer->currentTopics) == 0){ + pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; + } pOldConsumer->rebalanceTime = taosGetTimestampMs(); atomic_add_fetch_32(&pOldConsumer->epoch, 1); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 99553fc57a..9075f0145c 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1119,7 +1119,10 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode, topHalf)) return TSDB_CODE_MND_TRANS_CTX_SWITCH; + if (mndCannotExecuteTransAction(pMnode, topHalf)) { + terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH; + return TSDB_CODE_MND_TRANS_CTX_SWITCH; + } int64_t signature = pTrans->id; signature = (signature << 32); From 4ea2cad0bb6824e315c68be890e52422b818fceb Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 4 Jan 2024 18:59:55 +0800 Subject: [PATCH 091/198] comment snode_restart_with_checkpoint.py --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1761189b19..ac21f0fbb2 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -33,7 +33,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py #,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 -,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py From 33f698e9265b2cbf041b575e0312cc152539bf9b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 19:09:45 +0800 Subject: [PATCH 092/198] fix(stream): remove invalid assert and add check for task status before launching related fill-history task. --- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/stream/src/streamStart.c | 27 +++++++++++++++++---------- source/libs/stream/src/streamTaskSm.c | 1 - 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 203c34f1ce..8cd16bbe60 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1160,7 +1160,7 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, } else if (status == TASK_STATUS__UNINIT) { // todo: fill-history task init ? if (pTask->info.fillHistory == 0) { - EStreamTaskEvent event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; + EStreamTaskEvent event = TASK_EVENT_INIT; streamTaskHandleEvent(pTask->status.pSM, event); } } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 45a571e93c..412f1a452c 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -388,12 +388,7 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { } void doProcessDownstreamReadyRsp(SStreamTask* pTask) { - EStreamTaskEvent event; - if (pTask->info.fillHistory == 0) { - event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; - } else { - event = TASK_EVENT_INIT_SCANHIST; - } + EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; streamTaskOnHandleEventSuccess(pTask->status.pSM, event); @@ -820,11 +815,23 @@ SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, int64_t streamId, in // an fill history task needs to be started. int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { SStreamMeta* pMeta = pTask->pMeta; - int32_t hTaskId = pTask->hTaskInfo.id.taskId; - ASSERT((hTaskId != 0) && (pTask->status.downstreamReady == 1)); - stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", pTask->id.idStr, - pTask->hTaskInfo.id.streamId, hTaskId); + int64_t streamId = pTask->hTaskInfo.id.streamId; + int32_t hTaskId = pTask->hTaskInfo.id.taskId; + ASSERT(hTaskId != 0); + + SStreamTaskState* pStatus = streamTaskGetStatus(pTask); + if (pStatus->state != TASK_STATUS__READY) { + STaskExecStatisInfo* pInfo = &pTask->execInfo; + stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", pTask->id.idStr, + pTask->hTaskInfo.id.streamId, hTaskId); + + streamMetaUpdateTaskDownstreamStatus(pMeta, streamId, hTaskId, pInfo->init, pInfo->start, false); + return -1;// todo set the correct error code + } else { + stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", pTask->id.idStr, + pTask->hTaskInfo.id.streamId, hTaskId); + } // Set the execute conditions, including the query time window and the version range SStreamTask** pHTask = taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 819cdbf374..dcfe4ba848 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -38,7 +38,6 @@ SStreamEventInfo StreamTaskEventList[12] = { {.event = 0, .name = ""}, // dummy event, place holder {.event = TASK_EVENT_INIT, .name = "initialize"}, {.event = TASK_EVENT_INIT_SCANHIST, .name = "scan-history-init"}, -// {.event = TASK_EVENT_INIT_STREAM_SCANHIST, .name = "stream-scan-history-init"}, {.event = TASK_EVENT_SCANHIST_DONE, .name = "scan-history-completed"}, {.event = TASK_EVENT_STOP, .name = "stopping"}, {.event = TASK_EVENT_GEN_CHECKPOINT, .name = "checkpoint"}, From 6286e42f5c561aa6f8b42def445feee5ca505bec Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 19:11:50 +0800 Subject: [PATCH 093/198] refactor: do some internal refactor. --- source/libs/stream/src/streamTaskSm.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index dcfe4ba848..f0dcc75c4c 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -108,12 +108,12 @@ int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) { // todo check rsp code for handle Event:TASK_EVENT_SCANHIST_DONE static bool isInvalidStateTransfer(ETaskStatus state, const EStreamTaskEvent event) { - if (/*event == TASK_EVENT_INIT_STREAM_SCANHIST || */event == TASK_EVENT_INIT || event == TASK_EVENT_INIT_SCANHIST) { + if (event == TASK_EVENT_INIT || event == TASK_EVENT_INIT_SCANHIST) { return (state != TASK_STATUS__UNINIT); } if (event == TASK_EVENT_SCANHIST_DONE) { - return (state != TASK_STATUS__SCAN_HISTORY/* && state != TASK_STATUS__STREAM_SCAN_HISTORY*/); + return (state != TASK_STATUS__SCAN_HISTORY); } if (event == TASK_EVENT_GEN_CHECKPOINT) { @@ -476,14 +476,10 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); taosArrayPush(streamTaskSMTrans, &trans); -// trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STREAM_SCAN_HISTORY, TASK_EVENT_INIT_STREAM_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false); -// taosArrayPush(streamTaskSMTrans, &trans); // scan-history related event trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); -// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true); -// taosArrayPush(streamTaskSMTrans, &trans); // halt stream task, from other task status trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); @@ -493,8 +489,6 @@ void doInitStateTransferTable(void) { SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; -// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); -// taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true); taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true); @@ -513,8 +507,6 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; -// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); -// taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true); @@ -548,8 +540,6 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); -// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true); -// taosArrayPush(streamTaskSMTrans, &trans); // dropping related event trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); @@ -568,7 +558,5 @@ void doInitStateTransferTable(void) { taosArrayPush(streamTaskSMTrans, &trans); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL, NULL, true); taosArrayPush(streamTaskSMTrans, &trans); -// trans = createStateTransform(TASK_STATUS__STREAM_SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true); -// taosArrayPush(streamTaskSMTrans, &trans); } //clang-format on \ No newline at end of file From 4df83e92cd2d48abdcae503fcf56033203f5b7c0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 19:14:24 +0800 Subject: [PATCH 094/198] fix(stream): fix syntax error. --- source/libs/stream/src/streamMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7ea42e62c1..16a9fb3274 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1368,7 +1368,7 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { streamTaskSetCheckpointFailedId(pTask); stDebug("s-task:%s mark the checkpoint:%"PRId64" failed", pTask->id.idStr, pTask->chkInfo.checkpointingId); } else { - stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState->state); + stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState->name); } taosThreadMutexUnlock(&pTask->lock); From 14e90eb16eb6c1b6ee1e7ab67562c861c3d07263 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 19:15:51 +0800 Subject: [PATCH 095/198] fix(stream): fix syntax error. --- source/libs/stream/src/streamStart.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 412f1a452c..5cae438eb1 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -355,7 +355,7 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { stDebug("s-task:%s no need to scan-history data, status:%s, sched-status:%d, ready for data from wal ver:%" PRId64, id, p, pTask->status.schedStatus, startVer); } else { - stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p, pTask->status.schedStatus); + stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p->name, pTask->status.schedStatus); } return TSDB_CODE_SUCCESS; @@ -824,7 +824,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { if (pStatus->state != TASK_STATUS__READY) { STaskExecStatisInfo* pInfo = &pTask->execInfo; stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", pTask->id.idStr, - pTask->hTaskInfo.id.streamId, hTaskId); + pTask->hTaskInfo.id.streamId, hTaskId, pStatus->name); streamMetaUpdateTaskDownstreamStatus(pMeta, streamId, hTaskId, pInfo->init, pInfo->start, false); return -1;// todo set the correct error code From 356b7093f50d80c6a94227764bfaed4bcb1092d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 19:18:10 +0800 Subject: [PATCH 096/198] fix(stream): fix syntax error. --- source/libs/stream/src/streamStart.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 5cae438eb1..a40fe0a2c2 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -353,7 +353,7 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { } stDebug("s-task:%s no need to scan-history data, status:%s, sched-status:%d, ready for data from wal ver:%" PRId64, - id, p, pTask->status.schedStatus, startVer); + id, p->name, pTask->status.schedStatus, startVer); } else { stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p->name, pTask->status.schedStatus); } From 0c8d5e3586ecebb7618733b2f85eb133e96a98b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Jan 2024 19:20:50 +0800 Subject: [PATCH 097/198] fix(stream): fix syntax error. --- source/libs/stream/src/streamStart.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index a40fe0a2c2..0dcc94a530 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -63,7 +63,7 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { pTask->execInfo.start = taosGetTimestampMs(); int64_t el = (pTask->execInfo.start - pTask->execInfo.init); stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", - pTask->id.idStr, numOfDowns, el, p); + pTask->id.idStr, numOfDowns, el, p->name); return TSDB_CODE_SUCCESS; } From a811a60ad497220fad7410afb779ba0e93cabbd5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 4 Jan 2024 16:07:58 +0800 Subject: [PATCH 098/198] enh: restore sync log store from snap on fsm state incomplete --- source/libs/sync/src/syncMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 52557f7b9c..9352d5662e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -790,7 +790,7 @@ int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) { SyncIndex commitIndex = snapshot.lastApplyIndex; SyncIndex firstVer = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore); SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); - if (lastVer < commitIndex || firstVer > commitIndex + 1) { + if ((lastVer < commitIndex || firstVer > commitIndex + 1) || pNode->fsmState == SYNC_FSM_STATE_INCOMPLETE) { if (pNode->pLogStore->syncLogRestoreFromSnapshot(pNode->pLogStore, commitIndex)) { sError("vgId:%d, failed to restore log store from snapshot since %s. lastVer:%" PRId64 ", snapshotVer:%" PRId64, pNode->vgId, terrstr(), lastVer, commitIndex); From 81d8b9b98b3775848c6398e136c260125f43fa3b Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 5 Jan 2024 00:35:29 +0800 Subject: [PATCH 099/198] test: add test case for create index --- source/libs/parser/src/parTranslater.c | 11 ++++------- tests/system-test/0-others/tag_index_basic.py | 8 ++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 40b869d2b7..c51e96d1b6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6930,10 +6930,11 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde } static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + int32_t code = 0; SName name; STableMeta* pMeta = NULL; - int32_t code = - getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); + + code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -6955,10 +6956,6 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS taosMemoryFree(pMeta); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, ((SColumnNode*)pNode)->colName); } - if (IS_IDX_ON(pSchema)) { - code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; - goto _exit; - } } SCreateTagIndexReq createTagIdxReq = {0}; @@ -6966,7 +6963,7 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSCreateTagIdxReq, &createTagIdxReq); } -_exit: +_exit: taosMemoryFree(pMeta); return code; } diff --git a/tests/system-test/0-others/tag_index_basic.py b/tests/system-test/0-others/tag_index_basic.py index c1e1d521d2..a55508e229 100644 --- a/tests/system-test/0-others/tag_index_basic.py +++ b/tests/system-test/0-others/tag_index_basic.py @@ -88,6 +88,13 @@ class TDTestCase: tdLog.info(f" create {count} child tables ok.") + def create_tagidx_check(self, stbname): + err_dict = {"NULL","",",","\"","\"\"","undef","t1,t2","t12,t12"} + for errs in err_dict: + sql = (f'create index idx_err_check on {stbname} (%s)'% (errs)) + tdLog.info(f' sql={sql}') + tdSql.error(f'{sql}') + tdLog.info(f' create tagidx check ok.') # create stable and child tables def create_tagidx(self, stbname): @@ -231,6 +238,7 @@ class TDTestCase: count = 1000 # do self.create_table(stable, tbname, count) + self.create_tagidx_check(stable) self.create_tagidx(stable) self.insert_data(tbname) self.show_tagidx(stable) From d3aee612e0cd4418c18c90b2c99f4acf3a4a8169 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 5 Jan 2024 13:08:16 +0800 Subject: [PATCH 100/198] chore: code optimization --- source/libs/parser/src/parTranslater.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c51e96d1b6..ea1a5f7d1d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6939,10 +6939,6 @@ static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexS taosMemoryFree(pMeta); return code; } - if (TSDB_SUPER_TABLE != pMeta->tableType) { - taosMemoryFree(pMeta); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only super table is allowed"); - } if (LIST_LENGTH(pStmt->pCols) != 1) { taosMemoryFree(pMeta); From 9eced80e49303929c8e21c1b80a2aca8a39c07b8 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Fri, 5 Jan 2024 14:33:42 +0800 Subject: [PATCH 101/198] updated cases.task --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ac21f0fbb2..7eedf505a5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -542,6 +542,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py From 1912de9b498ab510d84ee4e6ba7ac18243efda73 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Jan 2024 14:49:33 +0800 Subject: [PATCH 102/198] enh(stream): scan wal by using timer to trigger it. --- include/libs/stream/tstream.h | 7 +- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/tq/tq.c | 26 +++++-- source/dnode/vnode/src/tq/tqStreamTask.c | 98 ++++++++++++++++++------ source/libs/stream/src/streamMeta.c | 2 +- 5 files changed, 99 insertions(+), 35 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 47c5149795..2998c6d7b1 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -482,6 +482,11 @@ typedef struct STaskUpdateInfo { int32_t transId; } STaskUpdateInfo; +typedef struct SScanWalInfo { + int32_t scanCounter; + tmr_h scanTimer; +} SScanWalInfo; + // meta typedef struct SStreamMeta { char* path; @@ -499,7 +504,7 @@ typedef struct SStreamMeta { bool sendMsgBeforeClosing; // send hb to mnode before close all tasks when switch to follower. STaskStartInfo startInfo; TdThreadRwlock lock; - int32_t walScanCounter; + SScanWalInfo scanInfo; void* streamBackend; int64_t streamBackendRid; SHashObj* pTaskDbUnique; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 0ef29fcb3a..e4c673f533 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -42,6 +42,7 @@ extern "C" { // clang-format on typedef struct STqOffsetStore STqOffsetStore; +extern void* tqTimer; #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8cd16bbe60..33e9d97f8d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -19,13 +19,25 @@ // 0: not init // 1: already inited -// 2: wait to be inited or cleaup +// 2: wait to be inited or cleanup static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_EXEC; } static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_IDLE; } +int32_t tqTimerInit() { + tqTimer = taosTmrInit(100, 100, 1000, "TQ"); + if (tqTimer == NULL) { + return -1; + } + return 0; +} + +void tqTimerCleanUp() { + taosTmrCleanUp(tqTimer); +} + void tqDestroyTqHandle(void* data) { STqHandle* pData = (STqHandle*)data; qDestroyTask(pData->execHandle.task); @@ -106,6 +118,7 @@ int32_t tqInitialize(STQ* pTq) { return -1; } + tqTimerInit(); return 0; } @@ -136,6 +149,8 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq->path); tqMetaClose(pTq); streamMetaClose(pTq->pStreamMeta); + tqTimerCleanUp(); + qDebug("end to close tq"); taosMemoryFree(pTq); } @@ -1055,7 +1070,8 @@ int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRunReq* pReq = pMsg->pCont; - if (pReq->reqType == STREAM_EXEC_T_EXTRACT_WAL_DATA) { // all tasks are extracted submit data from the wal + // extracted submit data from wal files for all tasks + if (pReq->reqType == STREAM_EXEC_T_EXTRACT_WAL_DATA) { tqScanWal(pTq); return 0; } @@ -1351,14 +1367,8 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { } taosThreadMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask)->state; -// if (status == TASK_STATUS__STREAM_SCAN_HISTORY) { -// streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); -// } - SStreamTaskId id = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId}; streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &id); - taosThreadMutexUnlock(&pTask->lock); // clear the scheduler status diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 12f540b9cf..95a4474460 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -19,11 +19,14 @@ #define MAX_REPEAT_SCAN_THRESHOLD 3 #define SCAN_WAL_IDLE_DURATION 100 +void* tqTimer = NULL; + static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle); static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId); static bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver); static bool taskReadyForDataFromWal(SStreamTask* pTask); static bool doPutDataIntoInputQFromWal(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems); +static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration); // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. int32_t tqScanWal(STQ* pTq) { @@ -31,33 +34,78 @@ int32_t tqScanWal(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; int64_t st = taosGetTimestampMs(); - while (1) { - tqDebug("vgId:%d continue check if data in wal are available, walScanCounter:%d", vgId, pMeta->walScanCounter); + tqDebug("vgId:%d continue to check if data in wal are available, scanCounter:%d", vgId, pMeta->scanInfo.scanCounter); - // check all tasks - bool shouldIdle = true; - doScanWalForAllTasks(pMeta, &shouldIdle); + // check all tasks + int32_t numOfTasks = 0; + bool shouldIdle = true; + doScanWalForAllTasks(pMeta, &shouldIdle); - streamMetaWLock(pMeta); - int32_t times = (--pMeta->walScanCounter); - ASSERT(pMeta->walScanCounter >= 0); - streamMetaWUnLock(pMeta); + streamMetaWLock(pMeta); + int32_t times = (--pMeta->scanInfo.scanCounter); + ASSERT(pMeta->scanInfo.scanCounter >= 0); - if (times > 0) { - tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION); - } else { // times <= 0 - break; - } - - // todo: remove the sleep - taosMsleep(SCAN_WAL_IDLE_DURATION); - } + numOfTasks = taosArrayGetSize(pMeta->pTaskList); + streamMetaWUnLock(pMeta); int64_t el = (taosGetTimestampMs() - st); tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%" PRId64 " ms", vgId, el); + + if (times > 0) { + tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION); + tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION); + } return 0; } +typedef struct SBuildScanWalMsgParam { + STQ* pTq; + int32_t numOfTasks; +} SBuildScanWalMsgParam; + +static void doStartScanWal(void* param, void* tmrId) { + SBuildScanWalMsgParam* pParam = (SBuildScanWalMsgParam*) param; + + int32_t vgId = pParam->pTq->pStreamMeta->vgId; + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + taosMemoryFree(pParam); + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + return; + } + + tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks, + pParam->pTq->pVnode->restored); + + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = 0; + pRunReq->reqType = STREAM_EXEC_T_EXTRACT_WAL_DATA; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pParam->pTq->pVnode->msgCb, STREAM_QUEUE, &msg); + + taosMemoryFree(pParam); +} + +int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { + SStreamMeta* pMeta = pTq->pStreamMeta; + + SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam)); + + pParam->pTq = pTq; + pParam->numOfTasks = numOfTasks; + if (pMeta->scanInfo.scanTimer == NULL) { + pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, tqTimer); + } else { + taosTmrReset(doStartScanWal, idleDuration, pParam, tqTimer, &pMeta->scanInfo.scanTimer); + } + + return TSDB_CODE_SUCCESS; +} + int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; @@ -78,23 +126,23 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { return 0; } - pMeta->walScanCounter += 1; - if (pMeta->walScanCounter > MAX_REPEAT_SCAN_THRESHOLD) { - pMeta->walScanCounter = MAX_REPEAT_SCAN_THRESHOLD; + pMeta->scanInfo.scanCounter += 1; + if (pMeta->scanInfo.scanCounter > MAX_REPEAT_SCAN_THRESHOLD) { + pMeta->scanInfo.scanCounter = MAX_REPEAT_SCAN_THRESHOLD; } - if (pMeta->walScanCounter > 1) { - tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter); + if (pMeta->scanInfo.scanCounter > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->scanInfo.scanCounter); streamMetaWUnLock(pMeta); return 0; } - int32_t numOfPauseTasks = pTq->pStreamMeta->numOfPausedTasks; + int32_t numOfPauseTasks = pMeta->numOfPausedTasks; if (ckPause && numOfTasks == numOfPauseTasks) { tqDebug("vgId:%d ignore all submit, all streams had been paused, reset the walScanCounter", vgId); // reset the counter value, since we do not launch the scan wal operation. - pMeta->walScanCounter = 0; + pMeta->scanInfo.scanCounter = 0; streamMetaWUnLock(pMeta); return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 16a9fb3274..bf97bee5f2 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -359,7 +359,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - pMeta->walScanCounter = 0; + pMeta->scanInfo.scanCounter = 0; pMeta->vgId = vgId; pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; From 8ddff2eecbe476c84c1a7aba2364e7fb1dd0bc96 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 5 Jan 2024 07:02:16 +0000 Subject: [PATCH 103/198] fix/TD-28073 --- source/dnode/mnode/impl/src/mndCompact.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 6471fa817d..101022a44f 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -26,6 +26,7 @@ #include "mndTrans.h" #define MND_COMPACT_VER_NUMBER 1 +#define MND_COMPACT_ID_LEN 11 static int32_t mndProcessCompactTimer(SRpcMsg *pReq); @@ -451,7 +452,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){ code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[10] = {0}; + char obj[MND_COMPACT_ID_LEN] = {0}; sprintf(obj, "%d", pCompact->compactId); auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); From 0b9e3c9871d530fb9b19720284ed879f457a02c3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 5 Jan 2024 16:20:55 +0800 Subject: [PATCH 104/198] fix stream state buff issue --- source/libs/executor/src/executil.c | 4 +- source/libs/stream/src/streamSessionState.c | 2 +- tests/script/tsim/stream/state1.sim | 43 +++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c1ddd6e4f4..b713b9b112 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2274,10 +2274,10 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr, const char* taskIdStr) { if (!pBlock) { - qDebug("%s===stream===%s: Block is Null", taskIdStr, flag); + qDebug("%s===stream===%s %s: Block is Null", taskIdStr, flag, opStr); return; } else if (pBlock->info.rows == 0) { - qDebug("%s===stream===%s: Block is Empty. type:%d", taskIdStr, flag, pBlock->info.type); + qDebug("%s===stream===%s %s: Block is Empty. block type %d", taskIdStr, flag, opStr, pBlock->info.type); return; } if (qDebugFlag & DEBUG_DEBUG) { diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 765403b1aa..5f19b47e1f 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -557,7 +557,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch void* pFileStore = getStateFileStore(pFileState); void* p = NULL; int32_t code_file = streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); - if (code_file == TSDB_CODE_SUCCESS) { + if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim index 775528bd4e..e6f7a81851 100644 --- a/tests/script/tsim/stream/state1.sim +++ b/tests/script/tsim/stream/state1.sim @@ -152,6 +152,49 @@ endi print step 2 over +print step 3 + +sql create database test3 vgroups 1; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791213001,1,2,3,1.0); + +print create stream streams3 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into streamt3 as select _wstart, max(a), count(*) c1 from t1 state_window(a); +sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into streamt3 as select _wstart, max(a), count(*) c1 from t1 state_window(a); + +sleep 1000 + +sql insert into t1 values(1648791203000,2,2,3,1.0); + +sleep 500 + +sql insert into t1 values(1648791214000,1,2,3,1.0); + +$loop_count = 0 +loop6: + +sleep 1000 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print select * from streamt3 +sql select * from streamt3; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 + +if $rows != 2 then + print =====rows=$rows + goto loop6 +endi + +print step 3 over + print state1 end system sh/stop_dnodes.sh From db9655c54999f7830ab224ab397b014768a8e74f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 5 Jan 2024 17:07:24 +0800 Subject: [PATCH 105/198] test: save case.sql when case is failed and set default saved sqlfile True --- tests/parallel_test/run.sh | 4 ++++ tests/pytest/util/sql.py | 4 ++-- tests/system-test/1-insert/influxdb_line_taosc_insert.py | 2 +- tests/system-test/1-insert/insert_double.py | 2 +- tests/system-test/1-insert/insert_perf.py | 2 +- tests/system-test/1-insert/mutil_stage.py | 2 +- .../1-insert/opentsdb_telnet_line_taosc_insert.py | 2 +- tests/system-test/1-insert/time_range_wise.py | 2 +- tests/system-test/2-query/abs.py | 2 +- tests/system-test/2-query/and_or_for_byte.py | 2 +- tests/system-test/2-query/apercentile.py | 2 +- tests/system-test/2-query/case_when.py | 2 +- tests/system-test/2-query/check_tsdb.py | 2 +- tests/system-test/2-query/count.py | 2 +- tests/system-test/2-query/countAlwaysReturnValue.py | 2 +- tests/system-test/2-query/db.py | 2 +- tests/system-test/2-query/fill_with_group.py | 2 +- tests/system-test/2-query/func_to_char_timestamp.py | 2 +- tests/system-test/2-query/function_null.py | 2 +- tests/system-test/2-query/histogram.py | 2 +- tests/system-test/2-query/interval_limit_opt.py | 2 +- tests/system-test/2-query/interval_limit_opt_2.py | 2 +- tests/system-test/2-query/interval_unit.py | 2 +- tests/system-test/2-query/irate.py | 2 +- tests/system-test/2-query/join.py | 2 +- tests/system-test/2-query/join2.py | 2 +- tests/system-test/2-query/json_tag.py | 2 +- tests/system-test/2-query/json_tag_large_tables.py | 2 +- tests/system-test/2-query/last_cache_scan.py | 5 +++-- tests/system-test/2-query/like.py | 2 +- tests/system-test/2-query/limit.py | 2 +- tests/system-test/2-query/odbc.py | 2 +- tests/system-test/2-query/partition_by_col.py | 2 +- tests/system-test/2-query/partition_by_col_agg.py | 2 +- tests/system-test/2-query/qnodeCluster.py | 2 +- tests/system-test/2-query/query_cols_tags_and_or.py | 2 +- tests/system-test/2-query/state_window.py | 2 +- tests/system-test/2-query/substr.py | 2 +- tests/system-test/2-query/ts_3405_3398_3423.py | 2 +- tests/system-test/2-query/tsbsQuery.py | 2 +- tests/system-test/2-query/ttl_comment.py | 2 +- tests/system-test/2-query/varbinary.py | 2 +- tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py | 2 +- tests/system-test/7-tmq/dataFromTsdbNWal.py | 2 +- tests/system-test/7-tmq/stbTagFilter-1ctb.py | 2 +- tests/system-test/7-tmq/stbTagFilter-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqAutoCreateTbl.py | 2 +- tests/system-test/7-tmq/tmqClientConsLog.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py | 2 +- .../7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py | 2 +- .../7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1.py | 2 +- tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py | 2 +- tests/system-test/7-tmq/tmqDataPrecisionUnit.py | 2 +- tests/system-test/7-tmq/tmqDelete-1ctb.py | 2 +- tests/system-test/7-tmq/tmqDelete-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart1.py | 2 +- tests/system-test/7-tmq/tmqDropConsumer.py | 2 +- tests/system-test/7-tmq/tmqDropNtb-snapshot0.py | 2 +- tests/system-test/7-tmq/tmqDropNtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqDropStbCtb.py | 2 +- tests/system-test/7-tmq/tmqMaxGroupIds.py | 2 +- tests/system-test/7-tmq/tmqMaxTopic.py | 2 +- tests/system-test/7-tmq/tmqMultiConsumer.py | 2 +- tests/system-test/7-tmq/tmqOffset.py | 2 +- tests/system-test/7-tmq/tmqSeekAndCommit.py | 2 +- tests/system-test/7-tmq/tmqUpdate-1ctb.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqUpdateWithConsume.py | 2 +- tests/system-test/7-tmq/tmqVnodeReplicate.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-column.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-db.py | 2 +- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py | 2 +- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-stb.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform-db.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 2 +- tests/system-test/7-tmq/tmqVnodeTransform.py | 2 +- tests/system-test/7-tmq/tmq_per.py | 2 +- tests/system-test/8-stream/snode_restart.py | 2 +- tests/system-test/8-stream/snode_restart_with_checkpoint.py | 2 +- tests/system-test/8-stream/vnode_restart.py | 2 +- tests/system-test/99-TDcase/TD-17255.py | 2 +- 95 files changed, 101 insertions(+), 96 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index f05e0dfc83..cf21349513 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -200,6 +200,7 @@ function run_thread() { if [ -z "$case_file" ]; then continue fi + case_sql_file="$exec_dir/${case_file}.sql" case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}" count=$(( count + 1 )) local case_path=`dirname "$case_file"` @@ -324,12 +325,15 @@ function run_thread() { cmd="$tarcmd sh -c \"cd $remote_sim_dir; tar -czf sim.tar.gz sim\"" $cmd local remote_sim_tar="${workdirs[index]}/tmp/thread_volume/$thread_no/sim.tar.gz" + local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file)" scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" if [ -z ${passwords[index]} ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" $cmd + cmd="$scpcmd:${remote_case_sql_file} $log_dir/${case_file}.sql" + $cmd # backup source code (disabled) source_tar_dir=$log_dir/TDengine_${hosts[index]} source_tar_file=TDengine.tar.gz diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index f8a49983e1..000040d958 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -48,9 +48,9 @@ class TDSql: self.queryCols = 0 self.affectedRows = 0 - def init(self, cursor, log=False): + def init(self, cursor, log=True): self.cursor = cursor - + print(f"sqllog is :{log}") if (log): caller = inspect.getframeinfo(inspect.stack()[1][0]) self.cursor.log(caller.filename + ".sql") diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index 3e9bad4efd..85a0afed48 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self._conn = conn def createDb(self, name="test", db_update_tag=0): diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index b47b22ab44..b7af7237db 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -15,7 +15,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) self.database = "db1" tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_db(self): tdSql.execute(f"drop database if exists {self.database}") diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py index d96a031458..dac2afb3b2 100644 --- a/tests/system-test/1-insert/insert_perf.py +++ b/tests/system-test/1-insert/insert_perf.py @@ -30,7 +30,7 @@ class TDTestCase: self.tb3 = "t3" self.once = 1000 tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_db(self): tdSql.execute(f"drop database if exists {self.database}") diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 970554c6c1..ebaa1e7dc3 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -60,7 +60,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.taos_cfg_path = tdDnodes.dnodes[0].cfgPath self.taos_data_dir = tdDnodes.dnodes[0].dataDir diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index d419aee12c..2d9e34c287 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -33,7 +33,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self._conn = conn self.smlChildTableName_value = "id" diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 14b717d1da..a3df0eb6b0 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -137,7 +137,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.precision = "ms" self.sma_count = 0 self.sma_created_index = [] diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d64d550bc4..0eabd91535 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -17,7 +17,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 15e9110b3b..5b2fb51998 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -17,7 +17,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/apercentile.py b/tests/system-test/2-query/apercentile.py index e190bf9374..ee6835bd10 100644 --- a/tests/system-test/2-query/apercentile.py +++ b/tests/system-test/2-query/apercentile.py @@ -21,7 +21,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) self.rowNum = 10 self.ts = 1537146000000 self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/case_when.py b/tests/system-test/2-query/case_when.py index eebbcc3fa3..1ccd2b5076 100755 --- a/tests/system-test/2-query/case_when.py +++ b/tests/system-test/2-query/case_when.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) + tdSql.init(conn.cursor()) self.testcasePath = os.path.split(__file__)[0] self.testcaseFilename = os.path.split(__file__)[-1] diff --git a/tests/system-test/2-query/check_tsdb.py b/tests/system-test/2-query/check_tsdb.py index b3ff259cc5..c49c4f1481 100644 --- a/tests/system-test/2-query/check_tsdb.py +++ b/tests/system-test/2-query/check_tsdb.py @@ -15,7 +15,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_datas(self, dbname="db"): tdSql.execute( diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 3ba651622a..c06ee28d02 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -6,7 +6,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) self.setsql = TDSetSql() self.rowNum = 10 self.ts = 1537146000000 diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index 244964e486..f22fa3e01e 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepare_data(self, dbname="db"): tdSql.execute( diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index a48b4cf3bb..07514a0950 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): tdSql.execute("create database if not exists dbms precision 'ms'") diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index fce504d0f7..05be2ec9d2 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/func_to_char_timestamp.py b/tests/system-test/2-query/func_to_char_timestamp.py index d955e00a82..034d025fcd 100644 --- a/tests/system-test/2-query/func_to_char_timestamp.py +++ b/tests/system-test/2-query/func_to_char_timestamp.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/function_null.py b/tests/system-test/2-query/function_null.py index 44b86ee543..e5056b7c56 100644 --- a/tests/system-test/2-query/function_null.py +++ b/tests/system-test/2-query/function_null.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/histogram.py b/tests/system-test/2-query/histogram.py index cc6e8fe7e9..a69fb505d1 100644 --- a/tests/system-test/2-query/histogram.py +++ b/tests/system-test/2-query/histogram.py @@ -147,7 +147,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __create_hsg(self, sma:Hsgschema): return f"""{sma.histogram_flag}({sma.col}, '{sma.bin_type}', '{sma.bin_desc}', {sma.normalized})""" diff --git a/tests/system-test/2-query/interval_limit_opt.py b/tests/system-test/2-query/interval_limit_opt.py index 492f453de5..aa1702fe3c 100644 --- a/tests/system-test/2-query/interval_limit_opt.py +++ b/tests/system-test/2-query/interval_limit_opt.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/interval_limit_opt_2.py b/tests/system-test/2-query/interval_limit_opt_2.py index cadb32b388..cfabe46dca 100644 --- a/tests/system-test/2-query/interval_limit_opt_2.py +++ b/tests/system-test/2-query/interval_limit_opt_2.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/interval_unit.py b/tests/system-test/2-query/interval_unit.py index 9430c74cff..03917bfaf7 100644 --- a/tests/system-test/2-query/interval_unit.py +++ b/tests/system-test/2-query/interval_unit.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d', precision: str='ms'): if dropFlag == 1: diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index d882c9b8fb..82841541f0 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 120cc2eb30..c5c8f6c730 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -66,7 +66,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 7ddc5e57ba..1006033eae 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def __query_condition(self,tbname): query_condition = [] diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 8db92f38a2..33209620e1 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): # tdSql.prepare() diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py index ea3963640a..86bd6c2d82 100644 --- a/tests/system-test/2-query/json_tag_large_tables.py +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -36,7 +36,7 @@ class TDTestCase: self.testcaseFilename = os.path.split(__file__)[-1] # os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): # tdSql.prepare() diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 23433277c2..9b6dca56fd 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -376,7 +376,7 @@ class TDTestCase: tdSql.checkCols(1) p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"]) p.check_returncode() - tdSql.query_success_failed("select ts, last_row(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2") + tdSql.query_success_failed("select ts, last(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2") tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) tdSql.checkRows(1) tdSql.checkCols(3) @@ -413,7 +413,8 @@ class TDTestCase: tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) - tdSql.query_success_failed('select last(c2), c2, c3 from meters', queryTimes=10, expectErrInfo="Invalid column name: c2") + + def run(self): self.prepareTestEnv() diff --git a/tests/system-test/2-query/like.py b/tests/system-test/2-query/like.py index 95e2daf6d3..65d0a36b24 100644 --- a/tests/system-test/2-query/like.py +++ b/tests/system-test/2-query/like.py @@ -13,7 +13,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def initDB(self): tdSql.execute("drop database if exists db") diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index 961cff5087..e6736704f7 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1): if dropFlag == 1: diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index 8fbad93995..7daf5fd730 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -13,7 +13,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def check_ins_cols(self): tdSql.execute("create database if not exists db") diff --git a/tests/system-test/2-query/partition_by_col.py b/tests/system-test/2-query/partition_by_col.py index 230b7582d9..feb8693e3e 100644 --- a/tests/system-test/2-query/partition_by_col.py +++ b/tests/system-test/2-query/partition_by_col.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/partition_by_col_agg.py b/tests/system-test/2-query/partition_by_col_agg.py index 1bc7a2414a..405cff9663 100644 --- a/tests/system-test/2-query/partition_by_col_agg.py +++ b/tests/system-test/2-query/partition_by_col_agg.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index d15ef89eb8..851b5bb1d1 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): tsql.execute("use %s" %dbName) diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index b8d4e3c314..d86b5d07e7 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -20,7 +20,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) ## add for TD-6672 tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def insertData(self, tb_name): insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1, 2, 3, 4)', diff --git a/tests/system-test/2-query/state_window.py b/tests/system-test/2-query/state_window.py index 2bf363f0dc..7dedeb88f1 100644 --- a/tests/system-test/2-query/state_window.py +++ b/tests/system-test/2-query/state_window.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index 302711b13d..ad5a2bba55 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(),False) + tdSql.init(conn.cursor(),True) def __substr_condition(self): # sourcery skip: extract-method substr_condition = [] diff --git a/tests/system-test/2-query/ts_3405_3398_3423.py b/tests/system-test/2-query/ts_3405_3398_3423.py index dcd8fb5a85..814d9de394 100644 --- a/tests/system-test/2-query/ts_3405_3398_3423.py +++ b/tests/system-test/2-query/ts_3405_3398_3423.py @@ -11,7 +11,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): """This test case is used to verify the query performance for the merge scans process of diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 0e2975cdba..5a49f653ec 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_ctable(self,tsql=None, dbName='db',stbName='stb',ctbPrefix='ctb',ctbNum=1): tsql.execute("use %s" %dbName) diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py index 3ae602fa23..ba24579611 100644 --- a/tests/system-test/2-query/ttl_comment.py +++ b/tests/system-test/2-query/ttl_comment.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def run(self): dbname="db" diff --git a/tests/system-test/2-query/varbinary.py b/tests/system-test/2-query/varbinary.py index 8f8021c8ad..c2e2693762 100644 --- a/tests/system-test/2-query/varbinary.py +++ b/tests/system-test/2-query/varbinary.py @@ -10,7 +10,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def test(self): tdLog.info(" test") diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 808a4935e3..cf6aafc374 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 8386c22cd0..e0fbe89c36 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 1e6011d5db..15fd4ed875 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 5043c46f00..c7af55daae 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index 5d0af636b2..e3fe9ae0b6 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqClientConsLog.py b/tests/system-test/7-tmq/tmqClientConsLog.py index d708e6642c..ab0c5f6a72 100644 --- a/tests/system-test/7-tmq/tmqClientConsLog.py +++ b/tests/system-test/7-tmq/tmqClientConsLog.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index f372a2b742..2af61f4bbb 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index c7f95f6f41..d3087c0455 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 26dacf514d..275d80ca18 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index d6f100041b..0572499079 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 11fc7dbcc0..7b31019572 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -26,7 +26,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 8ed4a6df97..a05b6a23ec 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 117c3ce637..85adb50e88 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 2864240441..facbb6c228 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index a717c4966d..ede17130df 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index d8606efe58..fa7aad9dae 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index 05aa82c929..efc186c3bf 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index dcaa6ceb7c..f66605f5f4 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py b/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py index dd2420dcfb..d94f8df35c 100644 --- a/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py +++ b/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py @@ -29,7 +29,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqDataPrecisionUnit.py b/tests/system-test/7-tmq/tmqDataPrecisionUnit.py index f050116a1b..13b0fcbe5d 100644 --- a/tests/system-test/7-tmq/tmqDataPrecisionUnit.py +++ b/tests/system-test/7-tmq/tmqDataPrecisionUnit.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.db_name = "tmq_db" self.topic_name = "tmq_topic" diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index aa9c8d25d0..117bd66340 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index 7a47cd6025..0dc2dc35f0 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 0ac8482163..f4a63de53e 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index 2bde32800b..b398178c61 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqDropConsumer.py b/tests/system-test/7-tmq/tmqDropConsumer.py index e3e9906ecf..5208d14069 100644 --- a/tests/system-test/7-tmq/tmqDropConsumer.py +++ b/tests/system-test/7-tmq/tmqDropConsumer.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index c8bcdd6235..2c87e4083c 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) # drop some ntbs def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 3fc5a2fdc7..7a35867ce7 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) # drop some ntbs def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index c9e34136cc..eacacd913b 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqMaxGroupIds.py b/tests/system-test/7-tmq/tmqMaxGroupIds.py index 5049ee5bd7..eba15a4bef 100644 --- a/tests/system-test/7-tmq/tmqMaxGroupIds.py +++ b/tests/system-test/7-tmq/tmqMaxGroupIds.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index 05b699ca00..4f25c39627 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def modifyMaxTopics(self, tmqMaxTopicNum): # single dnode diff --git a/tests/system-test/7-tmq/tmqMultiConsumer.py b/tests/system-test/7-tmq/tmqMultiConsumer.py index cc217e0c4c..e9727042e2 100644 --- a/tests/system-test/7-tmq/tmqMultiConsumer.py +++ b/tests/system-test/7-tmq/tmqMultiConsumer.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqOffset.py b/tests/system-test/7-tmq/tmqOffset.py index 500c6f53e4..991d1185a9 100644 --- a/tests/system-test/7-tmq/tmqOffset.py +++ b/tests/system-test/7-tmq/tmqOffset.py @@ -31,7 +31,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getPath(self, tool="taosBenchmark"): if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqSeekAndCommit.py b/tests/system-test/7-tmq/tmqSeekAndCommit.py index 253edcd10d..65ee438a41 100644 --- a/tests/system-test/7-tmq/tmqSeekAndCommit.py +++ b/tests/system-test/7-tmq/tmqSeekAndCommit.py @@ -16,7 +16,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self.db_name = "tmq_db" self.topic_name = "tmq_topic" diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index 920e8e77e4..216f67ffe4 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index 2f1d3e2631..e51ac9f029 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index 6b8c10de27..3988565154 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index 3975013e74..86e49d378a 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index 6a9bb0ae92..e88e16eec0 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqVnodeReplicate.py b/tests/system-test/7-tmq/tmqVnodeReplicate.py index 0ee11781ed..3e40cb13ad 100644 --- a/tests/system-test/7-tmq/tmqVnodeReplicate.py +++ b/tests/system-test/7-tmq/tmqVnodeReplicate.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 1fe2b5809a..74213c4536 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py index f66acf4fcd..b9eac4b692 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py index 68fb07b813..f6b523e5f4 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 6140e8a544..d4c76c4f61 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py index 18b80b7f8d..e79a0ca77d 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -32,7 +32,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py index c203350322..a96d52ee2b 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db.py b/tests/system-test/7-tmq/tmqVnodeTransform-db.py index 5c61908d96..cef90ff65f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-db.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 64cdf6d153..95703c52ca 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index 3698297618..811b72c35f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -25,7 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/7-tmq/tmq_per.py b/tests/system-test/7-tmq/tmq_per.py index f3701dacab..bd319aa3a4 100644 --- a/tests/system-test/7-tmq/tmq_per.py +++ b/tests/system-test/7-tmq/tmq_per.py @@ -30,7 +30,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def getDataPath(self): selfPath = tdCom.getBuildPath() diff --git a/tests/system-test/8-stream/snode_restart.py b/tests/system-test/8-stream/snode_restart.py index 3657163ab0..e80ff54578 100644 --- a/tests/system-test/8-stream/snode_restart.py +++ b/tests/system-test/8-stream/snode_restart.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/8-stream/snode_restart_with_checkpoint.py b/tests/system-test/8-stream/snode_restart_with_checkpoint.py index d7bfd7b407..9eb8c09ca3 100644 --- a/tests/system-test/8-stream/snode_restart_with_checkpoint.py +++ b/tests/system-test/8-stream/snode_restart_with_checkpoint.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/8-stream/vnode_restart.py b/tests/system-test/8-stream/vnode_restart.py index a53432b77a..be0ee81787 100644 --- a/tests/system-test/8-stream/vnode_restart.py +++ b/tests/system-test/8-stream/vnode_restart.py @@ -20,7 +20,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def case1(self): diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 0f83468754..fa28372ae3 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -23,7 +23,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") From 0c4b91dc7233adeb662282c086f85bd70ec9001f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Jan 2024 17:55:39 +0800 Subject: [PATCH 106/198] enh(stream): start task async. --- include/dnode/vnode/tqCommon.h | 4 +- include/libs/stream/tstream.h | 8 +-- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 57 +++++++++++++++++----- source/libs/stream/src/streamMeta.c | 36 ++++++++++++++ 6 files changed, 91 insertions(+), 19 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index da2325f006..6c84c3ea09 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -18,6 +18,7 @@ // message process int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart); +int32_t tqStreamOneTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t streamId, int32_t taskId); int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored); int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); @@ -27,7 +28,8 @@ int32_t tqStreamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMs int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); -int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); +int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen, + bool isLeader, bool restored); int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2998c6d7b1..8c9eeb6f3a 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -52,9 +52,10 @@ extern "C" { #define STREAM_EXEC_T_EXTRACT_WAL_DATA (-1) #define STREAM_EXEC_T_START_ALL_TASKS (-2) -#define STREAM_EXEC_T_RESTART_ALL_TASKS (-3) -#define STREAM_EXEC_T_STOP_ALL_TASKS (-4) -#define STREAM_EXEC_T_RESUME_TASK (-5) +#define STREAM_EXEC_T_START_ONE_TASK (-3) +#define STREAM_EXEC_T_RESTART_ALL_TASKS (-4) +#define STREAM_EXEC_T_STOP_ALL_TASKS (-5) +#define STREAM_EXEC_T_RESUME_TASK (-6) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; @@ -880,6 +881,7 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool i int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int32_t streamMetaStartAllTasks(SStreamMeta* pMeta); int32_t streamMetaStopAllTasks(SStreamMeta* pMeta); +int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 837c55c219..4ad054be27 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -193,7 +193,7 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { case TDMT_STREAM_TASK_DEPLOY: { void * pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); - return tqStreamTaskProcessDeployReq(pSnode->pMeta, -1, pReq, len, true, true); + return tqStreamTaskProcessDeployReq(pSnode->pMeta, &pSnode->msgCb, -1, pReq, len, true, true); } case TDMT_STREAM_TASK_DROP: diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 33e9d97f8d..862d7e1d66 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -887,7 +887,8 @@ int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - return tqStreamTaskProcessDeployReq(pTq->pStreamMeta, sversion, msg, msgLen, vnodeIsRoleLeader(pTq->pVnode), pTq->pVnode->restored); + return tqStreamTaskProcessDeployReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, sversion, msg, msgLen, + vnodeIsRoleLeader(pTq->pVnode), pTq->pVnode->restored); } static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 6a47cfc3eb..a73c436044 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -50,6 +50,33 @@ int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { return 0; } +int32_t tqStreamOneTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t streamId, int32_t taskId) { + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + tqDebug("vgId:%d no stream tasks existed to run", vgId); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to start task:0x%x, code:%s", vgId, taskId, terrstr()); + return -1; + } + + tqDebug("vgId:%d start task:0x%x async", vgId, taskId); + pRunReq->head.vgId = vgId; + pRunReq->streamId = streamId; + pRunReq->taskId = taskId; + pRunReq->reqType = STREAM_EXEC_T_START_ONE_TASK; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(cb, STREAM_QUEUE, &msg); + return 0; +} + int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { int32_t vgId = pMeta->vgId; char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); @@ -507,8 +534,8 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) return code; } -int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, - bool restored) { +int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen, + bool isLeader, bool restored) { int32_t code = 0; int32_t vgId = pMeta->vgId; @@ -560,18 +587,19 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* // only handled in the leader node if (isLeader) { tqDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); - SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); - if (p != NULL && restored && p->info.fillHistory == 0) { - EStreamTaskEvent event = TASK_EVENT_INIT; - streamTaskHandleEvent(p->status.pSM, event); - } else if (!restored) { - tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); + if (restored) { + SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); + if (p != NULL && (p->info.fillHistory == 0)) { + tqStreamOneTaskStartAsync(pMeta, cb, streamId, taskId); + } + if (p != NULL) { + streamMetaReleaseTask(pMeta, p); + } + } else { + tqWarn("s-task:0x%x not launched since vnode(vgId:%d) not ready", taskId, vgId); } - if (p != NULL) { - streamMetaReleaseTask(pMeta, p); - } } else { tqDebug("vgId:%d not leader, not launch stream task s-task:0x%x", vgId, taskId); } @@ -696,7 +724,10 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t type = pReq->reqType; int32_t vgId = pMeta->vgId; - if (type == STREAM_EXEC_T_START_ALL_TASKS) { + if (type == STREAM_EXEC_T_START_ONE_TASK) { + streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); + return 0; + } else if (type == STREAM_EXEC_T_START_ALL_TASKS) { streamMetaStartAllTasks(pMeta); return 0; } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { @@ -755,7 +786,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); } else { streamMetaWUnLock(pMeta); - tqDebug("vgId:%d start all tasks completed", pMeta->vgId); + tqDebug("vgId:%d start all tasks completed in callbackFn", pMeta->vgId); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index bf97bee5f2..b8c5cf8e3f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1506,4 +1506,40 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { stInfo("vgId:%d start tasks completed", pMeta->vgId); taosArrayDestroy(pTaskList); return code; +} + +int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { + int32_t vgId = pMeta->vgId; + stInfo("vgId:%d start to task:0x%x by checking downstream status", vgId, taskId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, streamId, taskId); + if (pTask == NULL) { + stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, taskId); + streamMetaUpdateTaskDownstreamStatus(pMeta, streamId, taskId, 0, taosGetTimestampMs(), false); + return TSDB_CODE_STREAM_TASK_IVLD_STATUS; + } + + // todo: may be we should find the related fill-history task and set it failed. + + // fill-history task can only be launched by related stream tasks. + STaskExecStatisInfo* pInfo = &pTask->execInfo; + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; + } + + ASSERT(pTask->status.downstreamReady == 0); + + int32_t ret = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT); + if (ret != TSDB_CODE_SUCCESS) { + stError("vgId:%d failed to handle event:%d", pMeta->vgId, TASK_EVENT_INIT); + + streamMetaUpdateTaskDownstreamStatus(pMeta, streamId, taskId, pInfo->init, pInfo->start, false); + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + STaskId* pId = &pTask->hTaskInfo.id; + streamMetaUpdateTaskDownstreamStatus(pMeta, pId->streamId, pId->taskId, pInfo->init, pInfo->start, false); + } + } + + return ret; } \ No newline at end of file From 33253cbb54475e08f1a4ff12abbc9d0a8e5734a6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Jan 2024 18:13:35 +0800 Subject: [PATCH 107/198] fix(stream): add null ptr check. --- source/libs/stream/src/streamExec.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index d83d99f10f..5c38b24a77 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -689,8 +689,12 @@ bool streamTaskIsIdle(const SStreamTask* pTask) { bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { SStreamTaskState* pState = streamTaskGetStatus(pTask); + ETaskStatus st = pState->state; - *pStatus = pState->name; + if (pStatus != NULL) { + *pStatus = pState->name; + } + return (st == TASK_STATUS__READY || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__CK); } From 2386f842fb0e2094c7d5c3bcc6f3ab676f8dcf38 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Jan 2024 18:46:15 +0800 Subject: [PATCH 108/198] enh(stream): remove sleep to opt perf. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 13 +++++++------ source/libs/stream/src/streamExec.c | 20 +++++++++++++------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a73c436044..ad6d11aaf2 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -736,16 +736,17 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead } else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) { streamMetaStopAllTasks(pMeta); return 0; - } else if (type == STREAM_EXEC_T_RESUME_TASK) { - // task resume to run after idle for a while + } else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); if (pTask != NULL) { ASSERT(streamTaskReadyToRun(pTask, NULL)); - tqDebug("s-task:%s task resume to run after idle for a while", pTask->id.idStr); - streamResumeTask(pTask); - } + int64_t execTs = pTask->status.lastExecTs; + int32_t idle = taosGetTimestampMs() - execTs; + tqDebug("s-task:%s task resume to run after idle for:%dms from:%" PRId64, pTask->id.idStr, idle, execTs); - streamMetaReleaseTask(pMeta, pTask); + streamResumeTask(pTask); + streamMetaReleaseTask(pMeta, pTask); + } return 0; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 5c38b24a77..7b740a8e7a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -107,12 +107,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i return 0; } - if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { - stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry exec task", pTask->id.idStr); - taosMsleep(1000); - continue; - } - SSDataBlock* output = NULL; uint64_t ts = 0; if ((code = qExecTask(pExecutor, &output, &ts)) < 0) { @@ -561,6 +555,11 @@ static void setTaskSchedInfo(SStreamTask* pTask, int32_t idleTime) { pStatus->lastExecTs = taosGetTimestampMs(); } +static void clearTaskSchedInfo(SStreamTask* pTask) { + SStreamStatus* pStatus = &pTask->status; + pStatus->schedIdleTime = 0; +} + /** * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the * appropriate batch of blocks should be handled in 5 to 10 sec. @@ -586,6 +585,12 @@ int32_t doStreamExecTask(SStreamTask* pTask) { break; } + if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { + stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", pTask->id.idStr); + setTaskSchedInfo(pTask, 1000); + continue; + } + /*int32_t code = */ streamTaskGetDataFromInputQ(pTask, &pInput, &numOfBlocks, &blockSize); if (pInput == NULL) { ASSERT(numOfBlocks == 0); @@ -737,7 +742,7 @@ static void doStreamExecTaskHelper(void* param, void* tmrId) { tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); // release the task ref count - pTask->status.schedIdleTime = 0; // clear the idle time + clearTaskSchedInfo(pTask); streamMetaReleaseTask(pTask->pMeta, pTask); } @@ -747,6 +752,7 @@ static int32_t schedTaskInFuture(SStreamTask* pTask) { pTask->status.schedIdleTime, ref); // add one ref count for task + // todo this may be failed, and add ref may be failed. SStreamTask* pAddRefTask = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId); if (pTask->schedInfo.pIdleTimer == NULL) { From f2a1d24ab1a5a6c7b19a1ffdbafef4ebc44846b0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 5 Jan 2024 19:28:40 +0800 Subject: [PATCH 109/198] fix:[TD-28155] set consumer lost if all current topic is 0 --- source/dnode/mnode/impl/src/mndConsumer.c | 24 +++++++++++++--------- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++-- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index a365a44d56..4db000287c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -749,6 +749,18 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { return 0; } +static void updateConsumerStatus(SMqConsumerObj *pConsumer) { + int32_t status = pConsumer->status; + + if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { + if (status == MQ_CONSUMER_STATUS_REBALANCE) { + pConsumer->status = MQ_CONSUMER_STATUS_READY; + } else if (status == MQ_CONSUMER_STATUS_READY && taosArrayGetSize(pConsumer->currentTopics) == 0) { + pConsumer->status = MQ_CONSUMER_STATUS_LOST; + } + } +} + // remove from new topic static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); @@ -869,11 +881,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && - taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && - taosArrayGetSize(pOldConsumer->currentTopics) != 0){ - pOldConsumer->status = MQ_CONSUMER_STATUS_READY; - } + updateConsumerStatus(pOldConsumer); // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = taosGetTimestampMs(); @@ -897,11 +905,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && - taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0 && - taosArrayGetSize(pOldConsumer->currentTopics) == 0){ - pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; - } + updateConsumerStatus(pOldConsumer); pOldConsumer->rebalanceTime = taosGetTimestampMs(); atomic_add_fetch_32(&pOldConsumer->epoch, 1); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 16ae7f2548..62b671a12f 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -737,8 +737,6 @@ static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { SMqConsumerObj *pConsumer; void *pIter = NULL; - mInfo("start to process mq timer"); - // iterate all consumers, find all modification while (1) { pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); @@ -852,6 +850,8 @@ void mndRebCntDec() { static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { int code = 0; + mInfo("start to process mq timer"); + if (!mndRebTryStart()) { mInfo("mq rebalance already in progress, do nothing"); return code; From 522e688387d37184636f2c74a4109ac5514b895b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Jan 2024 22:16:58 +0800 Subject: [PATCH 110/198] fix(stream): check if in restart procedure in call back. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 29 +++++++++++++--------- source/libs/stream/src/streamQueue.c | 1 + 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index ad6d11aaf2..16d073797b 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -774,22 +774,27 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { STaskStartInfo* pStartInfo = &pMeta->startInfo; + int32_t vgId = pMeta->vgId; + streamMetaWLock(pMeta); + if (pStartInfo->taskStarting == 1) { + tqDebug("vgId:%d already in start tasks procedure in other thread, restartCounter:%d, do nothing", vgId, + pMeta->startInfo.restartCount); + } else { // not in starting procedure + if (pStartInfo->restartCount > 0) { + pStartInfo->restartCount -= 1; + tqDebug("vgId:%d role:%d need to restart all tasks again, restartCounter:%d", vgId, pMeta->role, + pStartInfo->restartCount); - if (pStartInfo->restartCount > 0) { - pStartInfo->restartCount -= 1; - - ASSERT(pStartInfo->taskStarting == 0); - tqDebug("vgId:%d role:%d need to restart all tasks again, restartCounter:%d", pMeta->vgId, pMeta->role, - pStartInfo->restartCount); - - streamMetaWUnLock(pMeta); - restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); - } else { - streamMetaWUnLock(pMeta); - tqDebug("vgId:%d start all tasks completed in callbackFn", pMeta->vgId); + streamMetaWUnLock(pMeta); + restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); + return TSDB_CODE_SUCCESS; + } else { + tqDebug("vgId:%d start all tasks completed in callbackFn", pMeta->vgId); + } } + streamMetaWUnLock(pMeta); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 459efb838c..479612d9d1 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -157,6 +157,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu *numOfBlocks = 0; *blockSize = 0; + // todo remove it // no available token in bucket for sink task, let's wait for a little bit if (taskLevel == TASK_LEVEL__SINK && (!streamTaskExtractAvailableToken(pTask->outputInfo.pTokenBucket, pTask->id.idStr))) { stDebug("s-task:%s no available token in bucket for sink data, wait for 10ms", id); From 445605ed2205c37afafbfb99cb520d7856ff8cef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Jan 2024 22:35:56 +0800 Subject: [PATCH 111/198] refactor: do some internal refactor. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 16d073797b..6ceb0a95ba 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -134,16 +134,11 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM return rsp.code; } -// streamMetaWUnLock(pMeta); - // todo for test purpose // the following two functions should not be executed within the scope of meta lock to avoid deadlock streamTaskUpdateEpsetInfo(pTask, req.pNodeList); streamTaskResetStatus(pTask); - // continue after lock the meta again -// streamMetaWLock(pMeta); - SStreamTask** ppHTask = NULL; if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); @@ -157,6 +152,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } } + tqDebug("s-task:%s start to save task", pTask->id.idStr); + { streamMetaSaveTask(pMeta, pTask); if (ppHTask != NULL) { @@ -168,6 +165,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } } + tqDebug("s-task:%s start to stop task after save task", pTask->id.idStr); + streamTaskStop(pTask); // keep the already handled info From 23341e1bdff0f89b64374b6514fcbc286cda4da8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Jan 2024 22:58:43 +0800 Subject: [PATCH 112/198] refactor: opt perf by avoid save tasks during restoring --- source/dnode/vnode/src/tqCommon/tqCommon.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 6ceb0a95ba..6e83c2d0a9 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -152,9 +152,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } } - tqDebug("s-task:%s start to save task", pTask->id.idStr); - - { + if (restored) { + tqDebug("s-task:%s vgId:%d start to save task", pTask->id.idStr, vgId); streamMetaSaveTask(pMeta, pTask); if (ppHTask != NULL) { streamMetaSaveTask(pMeta, *ppHTask); @@ -163,10 +162,11 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM if (streamMetaCommit(pMeta) < 0) { // persist to disk } + } else { + tqDebug("s-task:%s vgId:%d not save since restore not finish", pTask->id.idStr, vgId); } tqDebug("s-task:%s start to stop task after save task", pTask->id.idStr); - streamTaskStop(pTask); // keep the already handled info From cfda97beef8b4d99253c6af757b77c0ca159a702 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 6 Jan 2024 15:27:48 +0800 Subject: [PATCH 113/198] refactor: disable commit temp --- source/dnode/vnode/src/tqCommon/tqCommon.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 6e83c2d0a9..3989acad2e 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -134,7 +134,6 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM return rsp.code; } - // todo for test purpose // the following two functions should not be executed within the scope of meta lock to avoid deadlock streamTaskUpdateEpsetInfo(pTask, req.pNodeList); streamTaskResetStatus(pTask); @@ -159,9 +158,11 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM streamMetaSaveTask(pMeta, *ppHTask); } +#if 0 if (streamMetaCommit(pMeta) < 0) { // persist to disk } +#endif } else { tqDebug("s-task:%s vgId:%d not save since restore not finish", pTask->id.idStr, vgId); } From 02f6c971fd390fc405a21f1e03f2d5e131489f9e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 6 Jan 2024 19:43:39 +0800 Subject: [PATCH 114/198] fix(stream): check for not ready task when rsp returned. --- include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tqCommon/tqCommon.c | 94 ++++++++++++++++++---- source/libs/stream/src/streamMeta.c | 1 + 3 files changed, 80 insertions(+), 16 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8c9eeb6f3a..dbcdef57b0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -56,6 +56,7 @@ extern "C" { #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4) #define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_RESUME_TASK (-6) +#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 3989acad2e..aed1091c84 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -77,6 +77,33 @@ int32_t tqStreamOneTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t stream return 0; } +int32_t tqUpdateNodeEpsetAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t streamId, int32_t taskId) { + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + tqDebug("vgId:%d no stream tasks existed to run", vgId); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to start task:0x%x, code:%s", vgId, taskId, terrstr()); + return -1; + } + + tqDebug("vgId:%d update s-task:0x%x nodeEpset async", vgId, taskId); + pRunReq->head.vgId = vgId; + pRunReq->streamId = streamId; + pRunReq->taskId = taskId; + pRunReq->reqType = STREAM_EXEC_T_UPDATE_TASK_EPSET; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(cb, STREAM_QUEUE, &msg); + return 0; +} + int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { int32_t vgId = pMeta->vgId; char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); @@ -443,12 +470,22 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { return streamSendCheckRsp(pMeta, &req, &rsp, &pMsg->info, taskId); } +static void setParam(SStreamTask* pTask, int64_t* initTs, bool* hasHTask, STaskId* pId) { + *initTs = pTask->execInfo.init; + + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + *hasHTask = true; + pId->streamId = pTask->hTaskInfo.id.streamId; + pId->taskId = pTask->hTaskInfo.id.taskId; + } +} + int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader) { char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); int32_t vgId = pMeta->vgId; + int32_t code = TSDB_CODE_SUCCESS; - int32_t code; SStreamTaskCheckRsp rsp; SDecoder decoder; @@ -467,35 +504,60 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe int64_t initTs = 0; int64_t now = taosGetTimestampMs(); - if (!isLeader) { - SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); - if (pTask != NULL) { - initTs = pTask->execInfo.init; + STaskId id = {.streamId = rsp.streamId, .taskId = rsp.upstreamTaskId}; + STaskId fId = {0}; + bool hasHistoryTask = false; - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pId = &pTask->hTaskInfo.id; - streamMetaUpdateTaskDownstreamStatus(pMeta, pId->streamId, pId->taskId, initTs, now, false); + // todo extract method + if (!isLeader) { + // this task may have been stopped, so acquire task may failed. Retrieve it directly from the task hash map. + streamMetaRLock(pMeta); + + SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (ppTask != NULL) { + setParam(*ppTask, &initTs, &hasHistoryTask, &fId); + streamMetaRUnLock(pMeta); + + if (hasHistoryTask) { + streamMetaUpdateTaskDownstreamStatus(pMeta, fId.streamId, fId.taskId, initTs, now, false); } - streamMetaReleaseTask(pMeta, pTask); + tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, + rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId); + } else { + streamMetaRUnLock(pMeta); + + tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", + rsp.streamId, rsp.upstreamTaskId, vgId); + code = terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; } streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, initTs, now, false); - tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, - rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId); - return code; } SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); if (pTask == NULL) { - streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, 0, now, false); + streamMetaRLock(pMeta); + + SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (ppTask != NULL) { + setParam(*ppTask, &initTs, &hasHistoryTask, &fId); + streamMetaRUnLock(pMeta); + + if (hasHistoryTask) { + streamMetaUpdateTaskDownstreamStatus(pMeta, fId.streamId, fId.taskId, initTs, now, false); + } + } else { + streamMetaRUnLock(pMeta); + } + + streamMetaUpdateTaskDownstreamStatus(pMeta, rsp.streamId, rsp.upstreamTaskId, initTs, now, false); tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", rsp.streamId, rsp.upstreamTaskId, vgId); - terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; - // failed to find the related fill-history task - return -1; + code = terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; + return code; } code = streamProcessCheckRsp(pTask, &rsp); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b8c5cf8e3f..7e69610f80 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1472,6 +1472,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { // fill-history task can only be launched by related stream tasks. STaskExecStatisInfo* pInfo = &pTask->execInfo; if (pTask->info.fillHistory == 1) { + stDebug("s-task:%s fill-history task wait related stream task start", pTask->id.idStr); streamMetaReleaseTask(pMeta, pTask); continue; } From 0aef021d885e313c1df2b51ac7acf47c0cbce44b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 6 Jan 2024 21:01:34 +0800 Subject: [PATCH 115/198] fix: backslash add G only valid in string end --- tools/shell/src/shellEngine.c | 21 ++++++++++++++++++++- tools/shell/src/shellWebsocket.c | 3 ++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 60b1dfab41..cec4a9683c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -195,6 +195,25 @@ int32_t shellRunCommand(char *command, bool recordHistory) { return shellRunSingleCommand(cmd); } + +char * strendG(const char* pstr) { + if(pstr == NULL) { + return NULL; + } + + size_t len = strlen(pstr); + if(len < 3) { + return NULL; + } + + char * p = pstr + len - 2 + if (strcmp(p, "\\G") == 0) { + return p; + } + + return NULL; +} + void shellRunSingleCommandImp(char *command) { int64_t st, et; char *sptr = NULL; @@ -213,7 +232,7 @@ void shellRunSingleCommandImp(char *command) { } } - if ((sptr = strstr(command, "\\G")) != NULL) { + if ((sptr = strendG(command)) != NULL) { *sptr = '\0'; printMode = true; // When output to a file, the switch does not work. } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index bff2ef7592..8c3a46aa05 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -238,6 +238,7 @@ static int shellDumpWebsocket(WS_RES *wres, char *fname, return numOfRows; } +char * strendG(const char* pstr); void shellRunSingleCommandWebsocketImp(char *command) { int64_t st, et; char *sptr = NULL; @@ -256,7 +257,7 @@ void shellRunSingleCommandWebsocketImp(char *command) { } } - if ((sptr = strstr(command, "\\G")) != NULL) { + if ((sptr = strendG(command)) != NULL) { *sptr = '\0'; printMode = true; // When output to a file, the switch does not work. } From 96947e24d702d9ae28c473a723d5a249f6286127 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 6 Jan 2024 23:24:36 +0800 Subject: [PATCH 116/198] fix: modify build error --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index cec4a9683c..563481ede7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,7 +206,7 @@ char * strendG(const char* pstr) { return NULL; } - char * p = pstr + len - 2 + char * p = (char *)pstr + len - 2; if (strcmp(p, "\\G") == 0) { return p; } From 772a644de0a4262b6a0b5a9da90b496e2cd8c137 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 6 Jan 2024 23:41:52 +0800 Subject: [PATCH 117/198] fix: end must G; --- tools/shell/src/shellEngine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 563481ede7..d3a2dfdf79 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -202,7 +202,7 @@ char * strendG(const char* pstr) { } size_t len = strlen(pstr); - if(len < 3) { + if(len < 4) { return NULL; } @@ -210,6 +210,11 @@ char * strendG(const char* pstr) { if (strcmp(p, "\\G") == 0) { return p; } + + p = (char *)pstr + len - 3; + if (strcmp(p, "\\G;") == 0 ){ + return p; + } return NULL; } From b38a8102655d8211977b2c753e8e1e0df2133338 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 6 Jan 2024 23:43:55 +0800 Subject: [PATCH 118/198] fix: end must G; fix --- tools/shell/src/shellEngine.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d3a2dfdf79..060e28282e 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,12 +206,7 @@ char * strendG(const char* pstr) { return NULL; } - char * p = (char *)pstr + len - 2; - if (strcmp(p, "\\G") == 0) { - return p; - } - - p = (char *)pstr + len - 3; + char * p = (char *)pstr + len - 3; if (strcmp(p, "\\G;") == 0 ){ return p; } From 5e32aea1f96aaed454d3fd75844c6917a36d4405 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 7 Jan 2024 00:27:18 +0800 Subject: [PATCH 119/198] fix(stream): jump out of loop when downstream is blocking. --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7b740a8e7a..eb5a432235 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -588,7 +588,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", pTask->id.idStr); setTaskSchedInfo(pTask, 1000); - continue; + break; } /*int32_t code = */ streamTaskGetDataFromInputQ(pTask, &pInput, &numOfBlocks, &blockSize); From 29219ba9f47d343d3840cf04628963f71d2300a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 7 Jan 2024 01:32:50 +0800 Subject: [PATCH 120/198] fix(stream): add task number check. --- include/dnode/vnode/tqCommon.h | 2 +- source/dnode/snode/src/snode.c | 3 ++- source/dnode/vnode/src/tqCommon/tqCommon.c | 14 ++++++++------ source/dnode/vnode/src/vnd/vnodeSync.c | 8 +++++--- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 6c84c3ea09..7265d6f58f 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -32,7 +32,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve bool isLeader, bool restored); int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); -int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); +int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta, int32_t* numOfTasks); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 4ad054be27..613480f4bc 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -145,7 +145,8 @@ FAIL: } int32_t sndInit(SSnode *pSnode) { - tqStreamTaskResetStatus(pSnode->pMeta); + int32_t numOfTasks = 0; + tqStreamTaskResetStatus(pSnode->pMeta, &numOfTasks); streamMetaStartAllTasks(pSnode->pMeta); return 0; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index aed1091c84..6b1c7ccedb 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -706,16 +706,16 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen return 0; } -int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta) { +int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta, int32_t* numOfTasks) { int32_t vgId = pMeta->vgId; - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + *numOfTasks = taosArrayGetSize(pMeta->pTaskList); - tqDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); - if (numOfTasks == 0) { + tqDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, *numOfTasks); + if (*numOfTasks == 0) { return TSDB_CODE_SUCCESS; } - for (int32_t i = 0; i < numOfTasks; ++i) { + for (int32_t i = 0; i < (*numOfTasks); ++i) { SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; @@ -767,8 +767,10 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { } if (isLeader && !tsDisableStream) { - tqStreamTaskResetStatus(pMeta); + int32_t numOfTasks = 0; + tqStreamTaskResetStatus(pMeta, &numOfTasks); streamMetaWUnLock(pMeta); + streamMetaStartAllTasks(pMeta); } else { streamMetaResetStartInfo(&pMeta->startInfo); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 34ae6623a7..f0ef351184 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -570,10 +570,12 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) streamMetaWUnLock(pMeta); vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); } else { - vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); - tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta); + vInfo("vgId:%d sync restore finished, start to launch stream task(s)", pVnode->config.vgId); - { + int32_t numOfTasks = 0; + tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta, &numOfTasks); + + if (numOfTasks > 0) { if (pMeta->startInfo.taskStarting == 1) { pMeta->startInfo.restartCount += 1; tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, From a024fe960c2500f1f2519a1a183a78bb47dc934a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 7 Jan 2024 14:53:17 +0800 Subject: [PATCH 121/198] fix(stream): fix deadlock. --- source/dnode/vnode/src/vnd/vnodeSync.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index f0ef351184..26d232b443 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -567,31 +567,30 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) if (vnodeIsRoleLeader(pVnode)) { // start to restore all stream tasks if (tsDisableStream) { - streamMetaWUnLock(pMeta); vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); } else { vInfo("vgId:%d sync restore finished, start to launch stream task(s)", pVnode->config.vgId); - int32_t numOfTasks = 0; - tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta, &numOfTasks); + tqStreamTaskResetStatus(pMeta, &numOfTasks); if (numOfTasks > 0) { if (pMeta->startInfo.taskStarting == 1) { pMeta->startInfo.restartCount += 1; tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, pMeta->startInfo.restartCount); - streamMetaWUnLock(pMeta); } else { pMeta->startInfo.taskStarting = 1; streamMetaWUnLock(pMeta); tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); + return; } } } } else { - streamMetaWUnLock(pMeta); vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); } + + streamMetaWUnLock(pMeta); } static void vnodeBecomeFollower(const SSyncFSM *pFsm) { From fbfa1c9abe901f6fefc3d9e180461ff53d11110f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 7 Jan 2024 15:10:57 +0800 Subject: [PATCH 122/198] fix: use \G have no ; end --- tools/shell/src/shellEngine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 563481ede7..02a3084d4f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -202,7 +202,7 @@ char * strendG(const char* pstr) { } size_t len = strlen(pstr); - if(len < 3) { + if(len < 4) { return NULL; } @@ -211,6 +211,11 @@ char * strendG(const char* pstr) { return p; } + p = (char *)pstr + len - 3; + if (strcmp(p, "\\G;") == 0) { + return p; + } + return NULL; } From d3b13bbb84d41c1c72bea877509da431c5c9a1f3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 7 Jan 2024 15:15:56 +0800 Subject: [PATCH 123/198] fix: git add G no ; --- tools/shell/src/shellEngine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9f077c7817..f6f834f3d7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -206,8 +206,8 @@ char * strendG(const char* pstr) { return NULL; } - char * p = (char *)pstr + len - 3; - if (strcmp(p, "\\G;") == 0 ){ + char * p = (char *)pstr + len - 2; + if (strcmp(p, "\\G") == 0 ){ return p; } From 96460914923c2263accf2c2c774cfd00b52929db Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 7 Jan 2024 16:18:24 +0800 Subject: [PATCH 124/198] fix: sort error due to out of space --- source/libs/executor/src/tsort.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 205cd7d3ef..ee1d831a24 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -878,6 +878,9 @@ static int32_t blockCompareTsFn(const void* pLeft, const void* pRight, void* par static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk, SArray* aPgId) { int32_t pageId = -1; void* pPage = getNewBufPage(pHandle->pBuf, &pageId); + if (pPage == NULL) { + return terrno; + } taosArrayPush(aPgId, &pageId); int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t); @@ -944,16 +947,15 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO totalRows += blk->info.rows; } - SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); - SMultiwayMergeTreeInfo* pTree = NULL; code = tMergeTreeCreate(&pTree, taosArrayGetSize(aBlk), &sup, blockCompareTsFn); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(sup.aRowIdx); taosMemoryFree(sup.aTs); - return code; } + + SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); int32_t nRows = 0; int32_t nMergedRows = 0; bool mergeLimitReached = false; @@ -969,7 +971,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) { SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; - appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pTree); + taosArrayDestroy(aPgId); + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); + return code; + } nMergedRows += pHandle->pDataBlock->info.rows; blockDataCleanup(pHandle->pDataBlock); blkPgSz = pgHeaderSz; @@ -1001,7 +1010,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO if (!mergeLimitReached) { SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; - appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + code = appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(aPgId); + taosMemoryFree(pTree); + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); + return code; + } nMergedRows += pHandle->pDataBlock->info.rows; if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { mergeLimitReached = true; From 1a02bb4232a36f12bf13f3b88f1066c1673d4a7f Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 7 Jan 2024 18:25:30 +0800 Subject: [PATCH 125/198] doc: fix insert stb syntax --- docs/en/12-taos-sql/05-insert.md | 2 +- docs/zh/12-taos-sql/05-insert.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/05-insert.md b/docs/en/12-taos-sql/05-insert.md index 57e15746cc..462e7fc0ae 100644 --- a/docs/en/12-taos-sql/05-insert.md +++ b/docs/en/12-taos-sql/05-insert.md @@ -157,7 +157,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c Automatically creating table and the table name is specified through the `tbname` column ```sql -INSERT INTO meters(tbname, location, groupId, ts, current, phase) +INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index 1f31698d88..fa330cabae 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -157,7 +157,7 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c 自动建表, 表名通过tbname列指定 ```sql -INSERT INTO meters(tbname, location, groupId, ts, current, phase) +INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase) values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) From c5932c9d1c6226863b0e960ec7174a284044bf87 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 8 Jan 2024 01:13:53 +0800 Subject: [PATCH 126/198] fix: wildcard character in both target and pattern --- source/util/src/tcompare.c | 22 ++++++++++------ tests/system-test/2-query/like.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 2baa21f5d9..53fdc96c37 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1102,10 +1102,13 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t c1 = str[j++]; ++nMatchChar; - if (c == '\\' && pattern[i] == c1 && - (c1 == '_' || c1 == '%')) { - i++; - continue; + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + if (c1 != pattern[i]) { + return TSDB_PATTERN_NOMATCH; + } else { + i++; + continue; + } } if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { @@ -1176,10 +1179,13 @@ int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, c1 = str[j++]; nMatchChar++; - if (c == '\\' && pattern[i] == c1 && - (c1 == '_' || c1 == '%')) { - i++; - continue; + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + if (c1 != pattern[i]) { + return TSDB_PATTERN_NOMATCH; + } else { + i++; + continue; + } } if (c == c1 || towlower(c) == towlower(c1) || (c == pInfo->umatchOne && c1 != 0)) { diff --git a/tests/system-test/2-query/like.py b/tests/system-test/2-query/like.py index 95e2daf6d3..88c1e3b8dd 100644 --- a/tests/system-test/2-query/like.py +++ b/tests/system-test/2-query/like.py @@ -126,6 +126,47 @@ class TDTestCase: tdSql.query("select * from db.t4x where c1 like '\%\_%\%%'") tdSql.checkRows(1) + def like_wildcard_test2(self): + tdSql.execute("create table db.t5x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t5x values(now(), 'a\%c')") + tdSql.execute("insert into db.t5x values(now+1s, 'a\%bbbc')") + tdSql.execute("insert into db.t5x values(now()+2s, 'a%c')") + + tdSql.query("select * from db.t5x where c1 like 'a\%c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a%c") + + tdSql.execute("create table db.t6x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t6x values(now(), '\%c')") + tdSql.execute("insert into db.t6x values(now+1s, '\%bbbc')") + tdSql.execute("insert into db.t6x values(now()+2s, '%c')") + + tdSql.query("select * from db.t6x where c1 like '\%c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "%c") + + tdSql.execute("create table db.t7x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t7x values(now(), 'a\_c')") + tdSql.execute("insert into db.t7x values(now+1s, 'a\_bbbc')") + tdSql.execute("insert into db.t7x values(now()+2s, 'a_c')") + + tdSql.query("select * from db.t7x where c1 like 'a\_c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a_c") + + tdSql.execute("create table db.t8x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t8x values(now(), '\_c')") + tdSql.execute("insert into db.t8x values(now+1s, '\_bbbc')") + tdSql.execute("insert into db.t8x values(now()+2s, '_c')") + + tdSql.query("select * from db.t8x where c1 like '\_c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "_c") + def run(self): tdLog.printNoPrefix("==========start like_wildcard_test run ...............") tdSql.prepare(replica = self.replicaVar) @@ -135,6 +176,7 @@ class TDTestCase: self.like_wildcard_test() self.like_cnc_wildcard_test() self.like_multi_wildcard_test() + self.like_wildcard_test2() tdLog.printNoPrefix("==========end like_wildcard_test run ...............") self.stopTest() From 6b38e5b8fb083b8e784ae9cee778e00e40e732cb Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 8 Jan 2024 10:44:19 +0800 Subject: [PATCH 127/198] feat: websocket support bi with appending &conn_mode=1 string to dsn --- tools/shell/src/shellEngine.c | 4 +--- tools/shell/src/shellWebsocket.c | 11 ++++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index f6f834f3d7..23424cea98 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1318,9 +1318,7 @@ int32_t shellExecute() { if(shell.args.is_bi_mode) { // need set bi mode printf("Set BI mode is true.\n"); -#ifdef WEBSOCKET - //ws_taos_set_conn_mode(shell.ws_conn, TAOS_CONN_MODE_BI, 1); -#else +#ifndef WEBSOCKET taos_set_conn_mode(shell.conn, TAOS_CONN_MODE_BI, 1); #endif } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 8c3a46aa05..2d3aa27a3b 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -30,7 +30,16 @@ int shell_conn_ws_server(bool first) { fprintf(stdout, "trying to connect %s****** ", cuttedDsn); fflush(stdout); for (int i = 0; i < shell.args.timeout; i++) { - shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + if(shell.args.is_bi_mode) { + size_t len = strlen(shell.args.dsn); + char * dsn = taosMemoryMalloc(len + 32); + sprintf(dsn, "%s&conn_mode=1", shell.args.dsn); + shell.ws_conn = ws_connect_with_dsn(dsn); + taosMemoryFree(dsn); + } else { + shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + } + if (NULL == shell.ws_conn) { int errNo = ws_errno(NULL); if (0xE001 == errNo) { From b4f869d051d62d6b68755e80ce8193691ca9b8c1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 8 Jan 2024 11:19:54 +0800 Subject: [PATCH 128/198] case: s3_basic.py except arm64 --- tests/army/enterprise/s3/s3_basic.py | 3 +++ tests/army/frame/eos.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 389b8a52c7..112cb65b15 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -84,6 +84,9 @@ class TDTestCase(TBase): # run def run(self): tdLog.debug(f"start to excute {__file__}") + if eos.isArm64Cpu(): + tdLog.success(f"{__file__} arm64 ignore executed") + return # insert data self.insertData() diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index b24be74393..1c00f94380 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -22,10 +22,31 @@ import datetime import platform import subprocess +# +# platform +# + # if windows platform return True def isWin(): return platform.system().lower() == 'windows' +def isArm64Cpu(): + system = platform.system() + + if system == 'Linux': + machine = platform.machine().lower() + + # Check for ARM64 architecture on Linux systems + return machine in ['aarch64', 'armv8l'] + elif system == 'Darwin' or system == 'Windows': + processor = platform.processor().lower() + + # Check for ARM64 architecture on macOS and Windows systems + return processor in ['arm64', 'aarch64'] + else: + print("Unsupported operating system") + return False + # # execute programe # From bb9edde79ed8b7ab5768a73677aab1f2625fcac0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Jan 2024 11:33:54 +0800 Subject: [PATCH 129/198] fix(stream): add some logs. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 15 +++++++++++---- source/libs/stream/src/streamQueue.c | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 6b1c7ccedb..656ee86831 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -109,6 +109,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; + int64_t st = taosGetTimestampMs(); SStreamTaskNodeUpdateMsg req = {0}; @@ -194,7 +195,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM tqDebug("s-task:%s vgId:%d not save since restore not finish", pTask->id.idStr, vgId); } - tqDebug("s-task:%s start to stop task after save task", pTask->id.idStr); + tqDebug("s-task:%s vgId:%d start to stop task after save task", pTask->id.idStr, vgId); streamTaskStop(pTask); // keep the already handled info @@ -202,10 +203,16 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM if (ppHTask != NULL) { streamTaskStop(*ppHTask); - tqDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); + + int64_t now = taosGetTimestampMs(); + tqDebug("s-task:%s vgId:%d task nodeEp update completed, streamTask and related fill-history task closed, elapsed:%" PRId64 + " ms", + pTask->id.idStr, vgId, now-st); taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); } else { - tqDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); + int64_t now = taosGetTimestampMs(); + tqDebug("s-task:%s vgId:%d, task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms", pTask->id.idStr, + vgId, now - st); } rsp.code = 0; @@ -226,7 +233,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM pMeta->startInfo.tasksWillRestart = 0; streamMetaWUnLock(pMeta); } else { - tqDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); + tqDebug("vgId:%d all %d task(s) nodeEp updated and closed, transId:%d", vgId, numOfTasks, req.transId); #if 0 // for test purpose, to trigger the leader election taosMSleep(5000); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 479612d9d1..5f55285ab1 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -175,6 +175,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu if (qItem == NULL) { if ((taskLevel == TASK_LEVEL__SOURCE || taskLevel == TASK_LEVEL__SINK) && (++retryTimes) < MAX_RETRY_TIMES) { taosMsleep(WAIT_FOR_DURATION); + // todo remove it continue; } From 275e2d6f9457aae002829125612f294a6e6d54f1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Jan 2024 11:34:47 +0800 Subject: [PATCH 130/198] refactor: do some internal refactor. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 656ee86831..9a3fdbe512 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -205,9 +205,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM streamTaskStop(*ppHTask); int64_t now = taosGetTimestampMs(); - tqDebug("s-task:%s vgId:%d task nodeEp update completed, streamTask and related fill-history task closed, elapsed:%" PRId64 - " ms", - pTask->id.idStr, vgId, now-st); + tqDebug("s-task:%s vgId:%d task nodeEp update completed, streamTask/fill-history closed, elapsed:%" PRId64 " ms", + pTask->id.idStr, vgId, now - st); taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); } else { int64_t now = taosGetTimestampMs(); From 8cdd28131f642a5d087b04e6d3817db8c4377a0e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 12:29:09 +0800 Subject: [PATCH 131/198] test: save case.sql when case is failed and set default saved sqlfile True --- tests/parallel_test/run.sh | 330 +++++++++++++++++++----------------- tests/system-test/pytest.sh | 12 +- 2 files changed, 176 insertions(+), 166 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index cf21349513..ef9413a9d5 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -15,61 +15,61 @@ function usage() { ent=0 while getopts "m:t:b:l:o:w:eh" opt; do case $opt in - m) - config_file=$OPTARG - ;; - t) - t_file=$OPTARG - ;; - b) - branch=$OPTARG - ;; - l) - log_dir=$OPTARG - ;; - e) - ent=1 - ;; - o) - timeout_param="-o $OPTARG" - ;; - w) - web_server=$OPTARG - ;; - h) - usage - exit 0 - ;; - \?) - echo "Invalid option: -$OPTARG" - usage - exit 0 - ;; + m) + config_file=$OPTARG + ;; + t) + t_file=$OPTARG + ;; + b) + branch=$OPTARG + ;; + l) + log_dir=$OPTARG + ;; + e) + ent=1 + ;; + o) + timeout_param="-o $OPTARG" + ;; + w) + web_server=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; esac done #config_file=$1 -if [ -z $config_file ]; then +if [ -z "$config_file" ]; then usage exit 1 fi -if [ ! -f $config_file ]; then +if [ ! -f "$config_file" ]; then echo "$config_file not found" usage exit 1 fi #t_file=$2 -if [ -z $t_file ]; then +if [ -z "$t_file" ]; then usage exit 1 fi -if [ ! -f $t_file ]; then +if [ ! -f "$t_file" ]; then echo "$t_file not found" usage exit 1 fi -date_tag=`date +%Y%m%d-%H%M%S` +date_tag=$(date +%Y%m%d-%H%M%S) test_log_dir=${branch}_${date_tag} -if [ -z $log_dir ]; then +if [ -z "$log_dir" ]; then log_dir="log/${test_log_dir}" else log_dir="$log_dir/${test_log_dir}" @@ -82,42 +82,41 @@ workdirs=() threads=() i=0 -while [ 1 ]; do - host=`jq .[$i].host $config_file` +while true; do + host=$(jq .[$i].host "$config_file") if [ "$host" = "null" ]; then break fi - username=`jq .[$i].username $config_file` + username=$(jq .[$i].username "$config_file") if [ "$username" = "null" ]; then break fi - password=`jq .[$i].password $config_file` + password=$(jq .[$i].password "$config_file") if [ "$password" = "null" ]; then password="" fi - workdir=`jq .[$i].workdir $config_file` + workdir=$(jq .[$i].workdir "$config_file") if [ "$workdir" = "null" ]; then break fi - thread=`jq .[$i].thread $config_file` + thread=$(jq .[$i].thread "$config_file") if [ "$thread" = "null" ]; then break fi - hosts[i]=`echo $host|sed 's/\"$//'|sed 's/^\"//'` - usernames[i]=`echo $username|sed 's/\"$//'|sed 's/^\"//'` - passwords[i]=`echo $password|sed 's/\"$//'|sed 's/^\"//'` - workdirs[i]=`echo $workdir|sed 's/\"$//'|sed 's/^\"//'` + hosts[i]=$(echo "$host" | sed 's/\"$//' | sed 's/^\"//') + usernames[i]=$(echo "$username" | sed 's/\"$//' | sed 's/^\"//') + passwords[i]=$(echo "$password" | sed 's/\"$//' | sed 's/^\"//') + workdirs[i]=$(echo "$workdir" | sed 's/\"$//' | sed 's/^\"//') threads[i]=$thread - i=$(( i + 1 )) + i=$((i + 1)) done - function prepare_cases() { - cat $t_file >>$task_file + cat "$t_file" >>"$task_file" local i=0 - while [ $i -lt $1 ]; do - echo "%%FINISHED%%" >>$task_file - i=$(( i + 1 )) + while [ $i -lt "$1" ]; do + echo "%%FINISHED%%" >>"$task_file" + i=$((i + 1)) done } @@ -125,7 +124,7 @@ function clean_tmp() { # clean tmp dir local index=$1 local ssh_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then ssh_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi local cmd="${ssh_script} rm -rf ${workdirs[index]}/tmp" @@ -136,7 +135,7 @@ function run_thread() { local index=$1 local thread_no=$2 local runcase_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then runcase_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi local count=0 @@ -147,8 +146,9 @@ function run_thread() { local cmd="${runcase_script} ${script}" # script="echo" - while [ 1 ]; do - local line=`flock -x $lock_file -c "head -n1 $task_file;sed -i \"1d\" $task_file"` + while true; do + local line + line=$(flock -x "$lock_file" -c "head -n1 $task_file;sed -i \"1d\" $task_file") if [ "x$line" = "x%%FINISHED%%" ]; then # echo "$index . $thread_no EXIT" break @@ -156,15 +156,17 @@ function run_thread() { if [ -z "$line" ]; then continue fi - echo "$line"|grep -q "^#" - if [ $? -eq 0 ]; then + + if echo "$line" | grep -q "^#"; then continue fi - local case_redo_time=`echo "$line"|cut -d, -f2` + local case_redo_time + case_redo_time=$(echo "$line" | cut -d, -f2) if [ -z "$case_redo_time" ]; then case_redo_time=${DEFAULT_RETRY_TIME:-2} fi - local case_build_san=`echo "$line"|cut -d, -f3` + local case_build_san + case_build_san=$(echo "$line" | cut -d, -f3) if [ "${case_build_san}" == "y" ]; then case_build_san="y" DEBUGPATH="debugSan" @@ -175,140 +177,150 @@ function run_thread() { usage exit 1 fi - local exec_dir=`echo "$line"|cut -d, -f4` - local case_cmd=`echo "$line"|cut -d, -f5` + local exec_dir + exec_dir=$(echo "$line" | cut -d, -f4) + local case_cmd + case_cmd=$(echo "$line" | cut -d, -f5) local case_file="" - echo "$case_cmd"|grep -q "\.sh" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.sh"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "\.sh"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.sh" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "^python3" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "^python3"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.py" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "^./pytest.sh" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "^./pytest.sh"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.py" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "\.sim" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.sim"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "\.sim"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.sim" | awk '{print $NF}') fi if [ -z "$case_file" ]; then - case_file=`echo "$case_cmd"|awk '{print $NF}'` + case_file=$(echo "$case_cmd" | awk '{print $NF}') fi if [ -z "$case_file" ]; then continue fi case_sql_file="$exec_dir/${case_file}.sql" case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}" - count=$(( count + 1 )) - local case_path=`dirname "$case_file"` - if [ ! -z "$case_path" ]; then - mkdir -p $log_dir/$case_path + count=$((count + 1)) + local case_path + case_path=$(dirname "$case_file") + if [ -n "$case_path" ]; then + mkdir -p "$log_dir"/"$case_path" fi cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} -s ${case_build_san} ${timeout_param}" # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 local case_log_file=$log_dir/${case_file}.txt - start_time=`date +%s` - local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` - case_index=`printf "%5d" $case_index` - local case_info=`echo "$line"|cut -d, -f 3,4,5` + start_time=$(date +%s) + local case_index + case_index=$(flock -x "$lock_file" -c "sh -c \"echo \$(( \$( cat $index_file ) + 1 )) | tee $index_file\"") + case_index=$(printf "%5d" "$case_index") + local case_info + case_info=$(echo "$line" | cut -d, -f 3,4,5) while [ ${redo_count} -lt 6 ]; do - if [ -f $case_log_file ]; then - cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt + if [ -f "$case_log_file" ]; then + cp "$case_log_file" "$log_dir"/"$case_file".${redo_count}.redotxt fi - echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$case_log_file - local current_time=`date "+%Y-%m-%d %H:%M:%S"` + echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >"$case_log_file" + local current_time + current_time=$(date "+%Y-%m-%d %H:%M:%S") echo -e "$case_index \e[33m START >>>>> \e[0m ${case_info} \e[33m[$current_time]\e[0m" - echo "$current_time" >>$case_log_file - local real_start_time=`date +%s` + echo "$current_time" >>"$case_log_file" + local real_start_time + real_start_time=$(date +%s) # $cmd 2>&1 | tee -a $case_log_file # ret=${PIPESTATUS[0]} - $cmd >>$case_log_file 2>&1 + $cmd >>"$case_log_file" 2>&1 ret=$? - local real_end_time=`date +%s` - local time_elapsed=$(( real_end_time - real_start_time )) - echo "execute time: ${time_elapsed}s" >>$case_log_file - current_time=`date "+%Y-%m-%d %H:%M:%S"` - echo "${hosts[index]} $current_time exit code:${ret}" >>$case_log_file + local real_end_time + real_end_time=$(date +%s) + local time_elapsed + time_elapsed=$((real_end_time - real_start_time)) + echo "execute time: ${time_elapsed}s" >>"$case_log_file" + current_time=$(date "+%Y-%m-%d %H:%M:%S") + echo "${hosts[index]} $current_time exit code:${ret}" >>"$case_log_file" if [ $ret -eq 0 ]; then break fi redo=0 - grep -q "wait too long for taosd start" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "wait too long for taosd start" "$case_log_file"; then redo=1 fi - grep -q "kex_exchange_identification: Connection closed by remote host" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "kex_exchange_identification: Connection closed by remote host" "$case_log_file"; then redo=1 fi - grep -q "ssh_exchange_identification: Connection closed by remote host" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "ssh_exchange_identification: Connection closed by remote host" "$case_log_file"; then redo=1 fi - grep -q "kex_exchange_identification: read: Connection reset by peer" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "kex_exchange_identification: read: Connection reset by peer" "$case_log_file"; then redo=1 fi - grep -q "Database not ready" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "Database not ready" "$case_log_file"; then redo=1 fi - grep -q "Unable to establish connection" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "Unable to establish connection" "$case_log_file"; then redo=1 fi - if [ $redo_count -lt $case_redo_time ]; then + if [ $redo_count -lt "$case_redo_time" ]; then redo=1 fi if [ $redo -eq 0 ]; then break fi - redo_count=$(( redo_count + 1 )) + redo_count=$((redo_count + 1)) done - end_time=`date +%s` - echo >>$case_log_file - total_time=$(( end_time - start_time )) - echo "${hosts[index]} total time: ${total_time}s" >>$case_log_file + end_time=$(date +%s) + echo >>"$case_log_file" + total_time=$((end_time - start_time)) + echo "${hosts[index]} total time: ${total_time}s" >>"$case_log_file" # echo "$thread_no ${line} DONE" if [ $ret -eq 0 ]; then echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[32m success\e[0m" - flock -x $lock_file -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" + flock -x "$lock_file" -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" else - if [ ! -z ${web_server} ]; then - flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" + if [ -n "${web_server}" ]; then + flock -x "$lock_file" -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" else - flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" + flock -x "$lock_file" -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" fi - mkdir -p $log_dir/${case_file}.coredump + mkdir -p "${log_dir}"/"${case_file}".coredump local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump" local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" $cmd # 2>/dev/null - local corefile=`ls $log_dir/${case_file}.coredump/` + local corefile + corefile=$(ls "$log_dir/${case_file}.coredump/") echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[31m failed\e[0m" echo "=========================log============================" - cat $case_log_file + cat "$case_log_file" echo "=====================================================" echo -e "\e[34m log file: $case_log_file \e[0m" - if [ ! -z "${web_server}" ]; then + if [ -n "${web_server}" ]; then echo "${web_server}/$test_log_dir/${case_file}.txt" fi - if [ ! -z "$corefile" ]; then + if [ -n "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" # if [ $ent -ne 0 ]; then # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" # fi - mkdir $build_dir 2>/dev/null + mkdir "$build_dir" 2>/dev/null if [ $? -eq 0 ]; then # scp build binary cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" @@ -319,15 +331,15 @@ function run_thread() { # get remote sim dir local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no" local tarcmd="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then tarcmd="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi cmd="$tarcmd sh -c \"cd $remote_sim_dir; tar -czf sim.tar.gz sim\"" $cmd local remote_sim_tar="${workdirs[index]}/tmp/thread_volume/$thread_no/sim.tar.gz" - local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file)" + local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file}" scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" @@ -341,11 +353,9 @@ function run_thread() { source_tar_dir=$log_dir/TDinternal_${hosts[index]} source_tar_file=TDinternal.tar.gz fi - mkdir $source_tar_dir 2>/dev/null + mkdir "$source_tar_dir" 2>/dev/null if [ $? -eq 0 ]; then cmd="$scpcmd:${workdirs[index]}/$source_tar_file $source_tar_dir" - # echo "$cmd" - # $cmd fi fi done @@ -361,12 +371,12 @@ function run_thread() { i=0 while [ $i -lt ${#hosts[*]} ]; do clean_tmp $i & - i=$(( i + 1 )) + i=$((i + 1)) done wait -mkdir -p $log_dir -rm -rf $log_dir/* +mkdir -p "$log_dir" +rm -rf "${log_dir:?}"/* task_file=$log_dir/$$.task lock_file=$log_dir/$$.lock index_file=$log_dir/case_index.txt @@ -374,71 +384,71 @@ stat_file=$log_dir/stat.txt failed_case_file=$log_dir/failed.txt success_case_file=$log_dir/success.txt -echo "0" >$index_file +echo "0" >"$index_file" i=0 j=0 while [ $i -lt ${#hosts[*]} ]; do - j=$(( j + threads[i] )) - i=$(( i + 1 )) + j=$((j + threads[i])) + i=$((i + 1)) done prepare_cases $j i=0 while [ $i -lt ${#hosts[*]} ]; do j=0 - while [ $j -lt ${threads[i]} ]; do + while [ $j -lt "${threads[i]}" ]; do run_thread $i $j & - j=$(( j + 1 )) + j=$((j + 1)) done - i=$(( i + 1 )) + i=$((i + 1)) done wait -rm -f $lock_file -rm -f $task_file +rm -f "$lock_file" +rm -f "$task_file" # docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f echo "=====================================================================" echo "log dir: $log_dir" -total_cases=`cat $index_file` +total_cases=$(cat "$index_file") failed_cases=0 -if [ -f $failed_case_file ]; then - if [ ! -z "$web_server" ]; then - failed_cases=`grep -v "$web_server" $failed_case_file|wc -l` +if [ -f "$failed_case_file" ]; then + if [ -n "$web_server" ]; then + failed_cases=$(grep -c -v "$web_server" "$failed_case_file") else - failed_cases=`grep -v "log file:" $failed_case_file|wc -l` + failed_cases=$(grep -c -v "log file:" "$failed_case_file") fi fi -success_cases=$(( total_cases - failed_cases )) -echo "Total Cases: $total_cases" >$stat_file -echo "Successful: $success_cases" >>$stat_file -echo "Failed: $failed_cases" >>$stat_file -cat $stat_file +success_cases=$((total_cases - failed_cases)) +echo "Total Cases: $total_cases" >"$stat_file" +echo "Successful: $success_cases" >>"$stat_file" +echo "Failed: $failed_cases" >>"$stat_file" +cat "$stat_file" RET=0 i=1 if [ -f "${failed_case_file}" ]; then echo "=====================================================" - while read line; do - if [ ! -z "${web_server}" ]; then - echo "$line"|grep -q "${web_server}" - if [ $? -eq 0 ]; then + while read -r line; do + if [ -n "${web_server}" ]; then + + if echo "$line" | grep -q "${web_server}"; then echo " $line" continue fi else - echo "$line"|grep -q "log file:" - if [ $? -eq 0 ]; then + + if echo "$line" | grep -q "log file:"; then echo " $line" continue fi fi - line=`echo "$line"|cut -d, -f 3,4,5` + line=$(echo "$line" | cut -d, -f 3,4,5) echo -e "$i. $line \e[31m failed\e[0m" >&2 - i=$(( i + 1 )) - done <${failed_case_file} + i=$((i + 1)) + done <"${failed_case_file}" RET=1 fi diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index bae0fdf278..2837c817be 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -60,16 +60,16 @@ echo "ASAN_DIR : $ASAN_DIR" # prevent delete / folder or /usr/bin if [ ${#SIM_DIR} -lt 10 ]; then - echo "len(SIM_DIR) < 10 , danger so exit. SIM_DIR=$SIM_DIR" - exit 1 + echo "len(SIM_DIR) < 10 , danger so exit. SIM_DIR=$SIM_DIR" + exit 1 fi -rm -rf $SIM_DIR/* +rm -rf "${SIM_DIR:?}"/* mkdir -p $PRG_DIR mkdir -p $ASAN_DIR -cd $CODE_DIR +cd "$CODE_DIR" || exit ulimit -n 600000 ulimit -c unlimited @@ -78,10 +78,10 @@ ulimit -c unlimited echo "ExcuteCmd:" $* if [[ "$TD_OS" == "Alpine" ]]; then - $* + "$@" else AsanFile=$ASAN_DIR/psim.info - echo "AsanFile:" $AsanFile + echo "AsanFile:" "$AsanFile" unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5 From 3bcd038b7d87a18edf1bac625f34a444ddee14dc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Jan 2024 14:04:36 +0800 Subject: [PATCH 132/198] refactor: not reload info from disk when trying to restart stream tasks. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 50 +++++++++++++--------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 9a3fdbe512..5467e3dadd 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -25,7 +25,6 @@ typedef struct STaskUpdateEntry { int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { int32_t vgId = pMeta->vgId; - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); if (numOfTasks == 0) { tqDebug("vgId:%d no stream tasks existed to run", vgId); @@ -131,7 +130,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL || *ppTask == NULL) { - tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, + tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", vgId, req.taskId); rsp.code = TSDB_CODE_SUCCESS; streamMetaWUnLock(pMeta); @@ -141,21 +140,22 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } SStreamTask* pTask = *ppTask; + const char* idstr = pTask->id.idStr; if (pMeta->updateInfo.transId != req.transId) { pMeta->updateInfo.transId = req.transId; - tqInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId); + tqInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", idstr, req.transId); // info needs to be kept till the new trans to update the nodeEp arrived. taosHashClear(pMeta->updateInfo.pTasks); } else { - tqDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId); + tqDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", idstr, req.transId); } STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; - void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); - if (exist != NULL) { - tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, - req.transId); + + void* pReqTask = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); + if (pReqTask != NULL) { + tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", idstr, vgId, req.transId); rsp.code = TSDB_CODE_SUCCESS; streamMetaWUnLock(pMeta); taosArrayDestroy(req.pNodeList); @@ -171,7 +171,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); if (ppHTask == NULL || *ppHTask == NULL) { tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", - pMeta->vgId, req.taskId); + vgId, req.taskId); CLEAR_RELATED_FILLHISTORY_TASK(pTask); } else { tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); @@ -180,7 +180,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } if (restored) { - tqDebug("s-task:%s vgId:%d start to save task", pTask->id.idStr, vgId); + tqDebug("s-task:%s vgId:%d start to save task", idstr, vgId); streamMetaSaveTask(pMeta, pTask); if (ppHTask != NULL) { streamMetaSaveTask(pMeta, *ppHTask); @@ -192,10 +192,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } #endif } else { - tqDebug("s-task:%s vgId:%d not save since restore not finish", pTask->id.idStr, vgId); + tqDebug("s-task:%s vgId:%d not save since restore not finish", idstr, vgId); } - tqDebug("s-task:%s vgId:%d start to stop task after save task", pTask->id.idStr, vgId); + tqDebug("s-task:%s vgId:%d start to stop task after save task", idstr, vgId); streamTaskStop(pTask); // keep the already handled info @@ -206,11 +206,11 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM int64_t now = taosGetTimestampMs(); tqDebug("s-task:%s vgId:%d task nodeEp update completed, streamTask/fill-history closed, elapsed:%" PRId64 " ms", - pTask->id.idStr, vgId, now - st); + idstr, vgId, now - st); taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); } else { int64_t now = taosGetTimestampMs(); - tqDebug("s-task:%s vgId:%d, task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms", pTask->id.idStr, + tqDebug("s-task:%s vgId:%d, task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms", idstr, vgId, now - st); } @@ -237,6 +237,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM // for test purpose, to trigger the leader election taosMSleep(5000); #endif + tqStreamTaskStartAsync(pMeta, cb, true); streamMetaWUnLock(pMeta); } @@ -759,17 +760,24 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { } streamMetaWLock(pMeta); - streamMetaClear(pMeta); +// streamMetaClear(pMeta); int64_t el = taosGetTimestampMs() - st; tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el / 1000.); - code = streamMetaLoadAllTasks(pMeta); - if (code != TSDB_CODE_SUCCESS) { - tqError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); - streamMetaWUnLock(pMeta); - code = terrno; - return code; +// code = streamMetaLoadAllTasks(pMeta); +// if (code != TSDB_CODE_SUCCESS) { +// tqError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); +// streamMetaWUnLock(pMeta); +// code = terrno; +// return code; +// } + + { + STaskStartInfo* pStartInfo = &pMeta->startInfo; + taosHashClear(pStartInfo->pReadyTaskSet); + taosHashClear(pStartInfo->pFailedTaskSet); + pStartInfo->readyTs = 0; } if (isLeader && !tsDisableStream) { From 826c098aee032c9388bd63716c4188b11536775f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 14:46:04 +0800 Subject: [PATCH 133/198] test: test --- tests/system-test/2-query/last_cache_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 9b6dca56fd..8e8ac837e7 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -394,7 +394,7 @@ class TDTestCase: tdSql.query('select last(c2), c2, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) tdSql.checkRows(1) - tdSql.checkCols(3) + tdSql.checkCols(5) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) From 031bcaec8ff80f84d5ae47b9ef6e10f8d38ffac9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 14:47:25 +0800 Subject: [PATCH 134/198] test: test failed case --- tests/system-test/2-query/last_cache_scan.py | 2 +- tests/system-test/8-stream/stream_basic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 8e8ac837e7..9b6dca56fd 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -394,7 +394,7 @@ class TDTestCase: tdSql.query('select last(c2), c2, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) tdSql.checkRows(1) - tdSql.checkCols(5) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index 3ebc255114..04e635ea24 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -101,7 +101,7 @@ class TDTestCase: # check ts interval is correct sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;" tdSql.query(sql) - tdSql.checkRows(0) + tdSql.checkRows(1) # stop def stop(self): From 1eee7b44e08ce34693dc1d8783047bf98d2536f7 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 07:31:35 +0000 Subject: [PATCH 135/198] fix crash found on 3.1 --- include/libs/transport/trpc.h | 4 +- include/util/taoserror.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 9 ++- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 62 +++++++++--------- source/dnode/mnode/impl/src/mndProfile.c | 43 +++++++------ source/libs/executor/src/exchangeoperator.c | 70 +++++++++++---------- source/libs/executor/src/sysscanoperator.c | 15 +++-- source/libs/transport/inc/transComm.h | 5 +- source/libs/transport/src/trans.c | 12 +++- source/libs/transport/src/transCli.c | 12 +++- source/util/src/terror.c | 1 + 11 files changed, 140 insertions(+), 94 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index accb7e6f24..e7b0b25653 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -163,7 +163,8 @@ int rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc in // These functions will not be called in the child process int rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); int rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -int rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int32_t timeoutMs); +int rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated, + int32_t timeoutMs); int rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); void *rpcAllocHandle(); void rpcSetIpWhite(void *thandl, void *arg); @@ -171,6 +172,7 @@ void rpcSetIpWhite(void *thandl, void *arg); int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); +int32_t rpcCvtErrCode(int32_t code); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c4d61157b6..b41078dfbf 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -68,6 +68,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED TAOS_DEF_ERROR_CODE(0, 0x0020) // "Vgroup could not be connected" #define TSDB_CODE_RPC_SOMENODE_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0021) // #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) // +#define TSDB_CODE_RPC_NETWORK_ERROR TAOS_DEF_ERROR_CODE(0, 0x0023) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index ac94390619..a95ec42bd0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -159,13 +159,18 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq); SEpSet epSet = {0}; + int8_t epUpdated = 0; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, 5000); + rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, 5000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); - char tbuf[256]; + char tbuf[512]; dmEpSetToStr(tbuf, sizeof(tbuf), &epSet); dError("failed to send status req since %s, epSet:%s, inUse:%d", tstrerror(rpcRsp.code), tbuf, epSet.inUse); + } else { + if (epUpdated == 1) { + dmSetMnodeEpSet(pMgmt->pData, &epSet); + } } dmProcessStatusRsp(pMgmt, &rpcRsp); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index af43804db4..b79b3f9e11 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -22,10 +22,8 @@ static void *dmStatusThreadFp(void *param) { int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-status"); - const static int16_t TRIM_FREQ = 30; - int32_t trimCount = 0; - int32_t upTimeCount = 0; - int64_t upTime = 0; + int32_t upTimeCount = 0; + int64_t upTime = 0; while (1) { taosMsleep(200); @@ -38,11 +36,6 @@ static void *dmStatusThreadFp(void *param) { dmSendStatusReq(pMgmt); lastTime = curTime; - trimCount = (trimCount + 1) % TRIM_FREQ; - if (trimCount == 0) { - taosMemoryTrim(0); - } - if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { upTime = taosGetOsUptime() - tsDndStartOsUptime; tsDndUpTime = TMAX(tsDndUpTime, upTime); @@ -55,27 +48,27 @@ static void *dmStatusThreadFp(void *param) { SDmNotifyHandle dmNotifyHdl = {.state = 0}; static void *dmNotifyThreadFp(void *param) { - SDnodeMgmt *pMgmt = param; - setThreadName("dnode-notify"); + SDnodeMgmt *pMgmt = param; + setThreadName("dnode-notify"); - if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { - return NULL; + if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { + return NULL; } - bool wait = true; - while (1) { - if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; + bool wait = true; + while (1) { + if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; if (wait) tsem_wait(&dmNotifyHdl.sem); atomic_store_8(&dmNotifyHdl.state, 1); - dmSendNotifyReq(pMgmt); - if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { - wait = true; - continue; + dmSendNotifyReq(pMgmt); + if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { + wait = true; + continue; } - wait = false; + wait = false; } - return NULL; + return NULL; } static void *dmMonitorThreadFp(void *param) { @@ -83,6 +76,9 @@ static void *dmMonitorThreadFp(void *param) { int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-monitor"); + static int32_t TRIM_FREQ = 20; + int32_t trimCount = 0; + while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; @@ -93,6 +89,11 @@ static void *dmMonitorThreadFp(void *param) { if (interval >= tsMonitorInterval) { (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; + + trimCount = (trimCount + 1) % TRIM_FREQ; + if (trimCount == 0) { + taosMemoryTrim(0); + } } } @@ -126,14 +127,15 @@ static void *dmCrashReportThreadFp(void *param) { setThreadName("dnode-crashReport"); char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s%s.taosdCrashLog", tsLogDir, TD_DIRSEP); - char *pMsg = NULL; - int64_t msgLen = 0; + char *pMsg = NULL; + int64_t msgLen = 0; TdFilePtr pFile = NULL; - bool truncateFile = false; - int32_t sleepTime = 200; - int32_t reportPeriodNum = 3600 * 1000 / sleepTime;; + bool truncateFile = false; + int32_t sleepTime = 200; + int32_t reportPeriodNum = 3600 * 1000 / sleepTime; + ; int32_t loopTimes = reportPeriodNum; - + while (1) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; if (loopTimes++ < reportPeriodNum) { @@ -167,13 +169,13 @@ static void *dmCrashReportThreadFp(void *param) { pMsg = NULL; continue; } - + if (pFile) { taosReleaseCrashLogFile(pFile, truncateFile); pFile = NULL; truncateFile = false; } - + taosMsleep(sleepTime); loopTimes = 0; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 34ee18a5cc..e0559b4c48 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" +#include "audit.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -26,7 +27,6 @@ #include "mndView.h" #include "tglobal.h" #include "tversion.h" -#include "audit.h" typedef struct { uint32_t id; @@ -530,23 +530,24 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } #ifdef TD_ENTERPRISE - bool needCheck = true; - int32_t key = HEARTBEAT_KEY_DYN_VIEW; - SDynViewVersion* pDynViewVer = NULL; - SKv* pKv = taosHashGet(pHbReq->info, &key, sizeof(key)); + bool needCheck = true; + int32_t key = HEARTBEAT_KEY_DYN_VIEW; + SDynViewVersion *pDynViewVer = NULL; + SKv *pKv = taosHashGet(pHbReq->info, &key, sizeof(key)); if (NULL != pKv) { pDynViewVer = pKv->value; mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer); - SDynViewVersion* pRspVer = NULL; + SDynViewVersion *pRspVer = NULL; if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) { return -1; } - + if (needCheck) { SKv kv1 = {.key = HEARTBEAT_KEY_DYN_VIEW, .valueLen = sizeof(*pDynViewVer), .value = pRspVer}; taosArrayPush(hbRsp.info, &kv1); - mTrace("need to check view ver, lastest bootTs:%" PRId64 ", ver:%" PRIu64, pRspVer->svrBootTs, pRspVer->dynViewVer); + mTrace("need to check view ver, lastest bootTs:%" PRId64 ", ver:%" PRIu64, pRspVer->svrBootTs, + pRspVer->dynViewVer); } } #endif @@ -594,7 +595,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb if (!needCheck) { break; } - + void *rspMsg = NULL; int32_t rspLen = 0; mndValidateViewInfo(pMnode, kv->value, kv->valueLen / sizeof(SViewVersion), &rspMsg, &rspLen); @@ -814,8 +815,9 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl * @param offset skip [offset] queries in pConn * @param rowsToPack at most rows to pack * @return rows packed -*/ -static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBlock* pBlock, uint32_t offset, uint32_t rowsToPack) { + */ +static int32_t packQueriesIntoBlock(SShowObj *pShow, SConnObj *pConn, SSDataBlock *pBlock, uint32_t offset, + uint32_t rowsToPack) { int32_t cols = 0; taosRLockLatch(&pConn->queryLock); int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); @@ -826,7 +828,7 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc int32_t i = offset; for (; i < numOfQueries && (i - offset) < rowsToPack; ++i) { - int32_t curRowIndex = pBlock->info.rows; + int32_t curRowIndex = pBlock->info.rows; SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i); cols = 0; @@ -877,14 +879,19 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->subPlanNum, false); char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; + int64_t reserve = 64; int32_t strSize = sizeof(subStatus); int32_t offset = VARSTR_HEADER_SIZE; - for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) { + for (int32_t i = 0; i < pQuery->subPlanNum && offset + reserve < strSize; ++i) { if (i) { - offset += snprintf(subStatus + offset, strSize - offset - 1, ","); + offset += sprintf(subStatus + offset, ","); + } + if (offset + reserve < strSize) { + SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); + offset += sprintf(subStatus + offset, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); + } else { + break; } - SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); - offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); } varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -903,8 +910,8 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc } static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode * pMnode = pReq->info.node; - SSdb * pSdb = pMnode->pSdb; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SConnObj *pConn = NULL; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index ef8bf9674a..2797cb2d82 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -16,16 +16,14 @@ #include "executorInt.h" #include "filter.h" #include "function.h" -#include "index.h" #include "operator.h" -#include "os.h" #include "query.h" #include "querytask.h" #include "tdatablock.h" #include "thash.h" #include "tmsg.h" -#include "tname.h" #include "tref.h" +#include "trpc.h" typedef struct SFetchRspHandleWrapper { uint32_t exchangeId; @@ -131,14 +129,14 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, try next %d/%" PRIzu, + " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, totalSources); } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb", + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%" PRId64 + ", totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } @@ -232,7 +230,7 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { if (blockDataGetNumOfRows(pBlock) == 0) { continue; } - + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); @@ -261,7 +259,7 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const if (pInfo->dynamicOp) { return TSDB_CODE_SUCCESS; } - + for (int32_t i = 0; i < numOfSources; ++i) { SSourceDataInfo dataInfo = {0}; dataInfo.status = EX_SOURCE_DATA_NOT_READY; @@ -343,8 +341,8 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode goto _error; } - pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); return pOperator; _error: @@ -383,7 +381,7 @@ void doDestroyExchangeOperatorInfo(void* param) { blockDataDestroy(pExInfo->pDummyBlock); tSimpleHashCleanup(pExInfo->pHashSources); - + tsem_destroy(&pExInfo->ready); taosMemoryFreeClear(param); } @@ -411,13 +409,18 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, + pRsp->numOfBlocks, pRsp->numOfRows, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); - pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), - pExchangeInfo); + pSourceDataInfo->code = rpcCvtErrCode(code); + if (pSourceDataInfo->code != code) { + qError("%s fetch rsp received, index:%d, error:%s, cvted error: %s, %p", pSourceDataInfo->taskId, index, + tstrerror(code), tstrerror(pSourceDataInfo->code), pExchangeInfo); + } else { + qError("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), + pExchangeInfo); + } } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -450,7 +453,7 @@ int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, in return TSDB_CODE_OUT_OF_MEMORY; } pScan->tableSeq = tableSeq; - + (*ppRes)->opType = srcOpType; (*ppRes)->downstreamIdx = 0; (*ppRes)->value = pScan; @@ -459,9 +462,8 @@ int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, in return TSDB_CODE_SUCCESS; } - int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); if (EX_SOURCE_DATA_NOT_READY != pDataInfo->status) { return TSDB_CODE_SUCCESS; } @@ -490,7 +492,8 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas req.queryId = pTaskInfo->id.queryId; req.execId = pSource->execId; if (pDataInfo->pSrcUidList) { - int32_t code = buildTableScanOperatorParam(&req.pOpParam, pDataInfo->pSrcUidList, pDataInfo->srcOpType, pDataInfo->tableSeq); + int32_t code = + buildTableScanOperatorParam(&req.pOpParam, pDataInfo->pSrcUidList, pDataInfo->srcOpType, pDataInfo->tableSeq); taosArrayDestroy(pDataInfo->pSrcUidList); pDataInfo->pSrcUidList = NULL; if (TSDB_CODE_SUCCESS != code) { @@ -499,7 +502,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas return pTaskInfo->code; } } - + int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; @@ -741,8 +744,8 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 + ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); @@ -769,7 +772,7 @@ _error: } int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasicParam* pBasicParam) { - SExchangeInfo* pExchangeInfo = pOperator->info; + SExchangeInfo* pExchangeInfo = pOperator->info; SExchangeSrcIndex* pIdx = tSimpleHashGet(pExchangeInfo->pHashSources, &pBasicParam->vgId, sizeof(pBasicParam->vgId)); if (NULL == pIdx) { qError("No exchange source for vgId: %d", pBasicParam->vgId); @@ -784,7 +787,7 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic dataInfo.pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL); dataInfo.srcOpType = pBasicParam->srcOpType; dataInfo.tableSeq = pBasicParam->tableSeq; - + taosArrayPush(pExchangeInfo->pSourceDataInfo, &dataInfo); pIdx->inUseIdx = taosArrayGetSize(pExchangeInfo->pSourceDataInfo) - 1; } else { @@ -800,15 +803,14 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic return TSDB_CODE_SUCCESS; } - int32_t addDynamicExchangeSource(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - int32_t code = TSDB_CODE_SUCCESS; + SExchangeInfo* pExchangeInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; SExchangeOperatorBasicParam* pBasicParam = NULL; - SExchangeOperatorParam* pParam = (SExchangeOperatorParam*)pOperator->pOperatorGetParam->value; + SExchangeOperatorParam* pParam = (SExchangeOperatorParam*)pOperator->pOperatorGetParam->value; if (pParam->multiParams) { SExchangeOperatorBatchParam* pBatch = (SExchangeOperatorBatchParam*)pOperator->pOperatorGetParam->value; - int32_t iter = 0; + int32_t iter = 0; while (NULL != (pBasicParam = tSimpleHashIterate(pBatch->pBatchs, pBasicParam, &iter))) { code = addSingleExchangeSource(pOperator, pBasicParam); if (code) { @@ -826,11 +828,11 @@ int32_t addDynamicExchangeSource(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } - int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; - int32_t code = TSDB_CODE_SUCCESS; - if ((OPTR_IS_OPENED(pOperator) && !pExchangeInfo->dynamicOp) || (pExchangeInfo->dynamicOp && NULL == pOperator->pOperatorGetParam)) { + int32_t code = TSDB_CODE_SUCCESS; + if ((OPTR_IS_OPENED(pOperator) && !pExchangeInfo->dynamicOp) || + (pExchangeInfo->dynamicOp && NULL == pOperator->pOperatorGetParam)) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 76de204cab..0c421bf354 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -15,9 +15,7 @@ #include "executorInt.h" #include "filter.h" -#include "function.h" #include "functionMgt.h" -#include "os.h" #include "querynodes.h" #include "systable.h" #include "tname.h" @@ -32,6 +30,7 @@ #include "storageapi.h" #include "tcompare.h" #include "thash.h" +#include "trpc.h" #include "ttypes.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); @@ -1789,8 +1788,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); return pOperator; _error: @@ -1867,7 +1866,13 @@ int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->handle = htobe64(pRsp->handle); pRsp->compLen = htonl(pRsp->compLen); } else { - operator->pTaskInfo->code = code; + operator->pTaskInfo->code = rpcCvtErrCode(code); + if (operator->pTaskInfo->code != code) { + qError("load systable rsp received, error:%s, cvted error:%s", tstrerror(code), + tstrerror(operator->pTaskInfo->code)); + } else { + qError("load systable rsp received, error:%s", tstrerror(code)); + } } tsem_post(&pScanResInfo->ready); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 19b2dfc300..7483588593 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -122,6 +122,8 @@ typedef struct SExHandle { typedef struct { STransMsg* pRsp; + SEpSet epSet; + int8_t hasEpSet; tsem_t* pSem; int8_t inited; SRWLatch latch; @@ -317,7 +319,8 @@ int transReleaseSrvHandle(void* handle); int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); -int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int32_t timeoutMs); +int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs); int transSendResponse(const STransMsg* msg); int transRegisterMsg(const STransMsg* msg); int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index b23d229931..64302a78fc 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -169,8 +169,9 @@ int rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, in int rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { return transSendRecv(shandle, pEpSet, pMsg, pRsp); } -int rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int32_t timeoutMs) { - return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, timeoutMs); +int rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); } int rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } @@ -197,6 +198,13 @@ int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { return transUtilSWhiteListToStr(pWhiteList, ppBuf); } +int32_t rpcCvtErrCode(int32_t code) { + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return TSDB_CODE_RPC_NETWORK_ERROR; + } + return code; +} + int32_t rpcInit() { transInit(); return 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 936d11c151..4b3000b47e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2425,6 +2425,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STransSyncMsg* pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); if (pSyncMsg != NULL) { memcpy(pSyncMsg->pRsp, (char*)pResp, sizeof(*pResp)); + if (cliIsEpsetUpdated(pResp->code, pCtx)) { + pSyncMsg->hasEpSet = 1; + epsetAssign(&pSyncMsg->epSet, &pCtx->epSet); + } tsem_post(pSyncMsg->pSem); taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); } else { @@ -2640,10 +2644,12 @@ int64_t transCreateSyncMsg(STransMsg* pTransMsg) { pSyncMsg->inited = 0; pSyncMsg->pRsp = pTransMsg; pSyncMsg->pSem = sem; + pSyncMsg->hasEpSet = 0; return taosAddRef(transGetSyncMsgMgt(), pSyncMsg); } -int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int32_t timeoutMs) { +int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); STransMsg* pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); if (pTransInst == NULL) { @@ -2695,6 +2701,10 @@ int transSendRecvWithTimeout(void* shandle, const SEpSet* pEpSet, STransMsg* pRe ret = TSDB_CODE_TIMEOUT_ERROR; } else { memcpy(pRsp, pSyncMsg->pRsp, sizeof(STransMsg)); + if (pSyncMsg->hasEpSet == 1) { + epsetAssign(pEpSet, &pSyncMsg->epSet); + *epUpdated = 1; + } ret = 0; } _RETURN: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c294270e4a..360689cef3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -53,6 +53,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_BROKEN_LINK, "Conn is broken") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED, "some vnode/qnode/mnode(s) out of service") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, "rpc open too many session") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_ERROR, "rpc network error") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") From 1c28f535255f91b0430e1560719cf56de0cce02d Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 08:28:00 +0000 Subject: [PATCH 136/198] refactor stream code --- source/libs/stream/inc/streamBackendRocksdb.h | 3 - source/libs/stream/src/streamBackendRocksdb.c | 128 +++++++++--------- 2 files changed, 63 insertions(+), 68 deletions(-) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 9bfec5577c..b89664a6c1 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -19,10 +19,7 @@ #include "rocksdb/c.h" //#include "streamInt.h" #include "streamState.h" -#include "tcoding.h" #include "tcommon.h" -#include "tcompare.h" -#include "ttimer.h" typedef struct SCfComparator { rocksdb_comparator_t** comp; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index a5a4bd4dc1..4b2a9e4a65 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -14,8 +14,6 @@ */ #include "streamBackendRocksdb.h" -#include "executor.h" -#include "query.h" #include "streamInt.h" #include "tcommon.h" #include "tref.h" @@ -1059,14 +1057,14 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) { rocksdb_column_family_handle_t** ppCf = NULL; int32_t nCf = chkpGetAllDbCfHandle2(pTaskDb, &ppCf); - qDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); + stDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); if ((code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf)) == 0) { if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) { stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir); } else { - qDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, - taosGetTimestampMs() - st); + stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, + taosGetTimestampMs() - st); } } else { stError("stream backend:%p failed to flush db at:%s", pTaskDb, pChkpIdDir); @@ -1797,7 +1795,7 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { cfNames = NULL; } - qDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); + stDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); return pTaskDb; _EXIT: @@ -1826,7 +1824,7 @@ void taskDbDestroy(void* pDb, bool flush) { streamMetaRemoveDB(wrapper->pMeta, wrapper->idstr); - qDebug("succ to destroy stream backend:%p", wrapper); + stDebug("succ to destroy stream backend:%p", wrapper); int8_t nCf = sizeof(ginitDict) / sizeof(ginitDict[0]); @@ -2323,7 +2321,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { stError("failed to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); taosMemoryFree(err); } else { - qDebug("succ to open cf, %p %s_%s", pState, wrapper->idstr, funcName); + stDebug("succ to open cf, %p %s_%s", pState, wrapper->idstr, funcName); wrapper->pCf[idx] = cf; } } @@ -2365,14 +2363,14 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe char* err = NULL; \ int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ + stWarn("streamState failed to get cf name: %s", funcname); \ code = -1; \ break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ wrapper->dataWritten += 1; \ char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_writeoptions_t* opts = wrapper->writeOpt; \ @@ -2385,54 +2383,54 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe taosMemoryFree(err); \ code = -1; \ } else { \ - qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \ - ttlVLen, wrapper); \ + stTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d, %p", toString, funcname, vLen, \ + ttlVLen, wrapper); \ } \ taosMemoryFree(ttlV); \ } while (0); -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ - rocksdb_t* db = wrapper->db; \ - rocksdb_readoptions_t* opts = wrapper->readOpt; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL || len == 0) { \ - if (err == NULL) { \ - qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ - } else { \ - stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ - taosMemoryFreeClear(err); \ - } \ - code = -1; \ - } else { \ - char* p = NULL; \ - int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (tlen <= 0) { \ - stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ - funcname); \ - code = -1; \ - } else { \ - qTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, tlen, \ - wrapper); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = tlen; \ - } \ +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + stWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ + char toString[128] = {0}; \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->db; \ + rocksdb_readoptions_t* opts = wrapper->readOpt; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL || len == 0) { \ + if (err == NULL) { \ + stTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ + } else { \ + stError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFreeClear(err); \ + } \ + code = -1; \ + } else { \ + char* p = NULL; \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ + stError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ + funcname); \ + code = -1; \ + } else { \ + stTrace("streamState str: %s succ to read from %s_%s, valLen:%d, %p", toString, wrapper->idstr, funcname, \ + tlen, wrapper); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = tlen; \ + } \ } while (0); #define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ @@ -2442,14 +2440,14 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe char* err = NULL; \ int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ - qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ + stWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ code = -1; \ break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ wrapper->dataWritten += 1; \ char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pCf)[ginitDict[i].idx]; \ rocksdb_t* db = wrapper->db; \ @@ -2460,7 +2458,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe taosMemoryFree(err); \ code = -1; \ } else { \ - qTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ + stTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ } \ } while (0); @@ -2681,7 +2679,7 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { } #ifdef BUILD_NO_CALL SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateGetCur_rocksdb"); + stDebug("streamStateGetCur_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); @@ -2775,7 +2773,7 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k } SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) { - qDebug("streamStateSessionSeekToLast_rocksdb"); + stDebug("streamStateSessionSeekToLast_rocksdb"); int32_t code = 0; @@ -2812,7 +2810,7 @@ SStreamStateCur* streamStateSessionSeekToLast_rocksdb(SStreamState* pState) { } int32_t streamStateSessionCurPrev_rocksdb(SStreamStateCur* pCur) { - qDebug("streamStateCurPrev_rocksdb"); + stDebug("streamStateCurPrev_rocksdb"); if (!pCur) return -1; rocksdb_iter_prev(pCur->iter); @@ -2862,7 +2860,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta return pCur; } SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { - qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); + stDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { @@ -2900,7 +2898,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta } SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { - qDebug("streamStateSessionSeekKeyNext_rocksdb"); + stDebug("streamStateSessionSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { @@ -3004,7 +3002,7 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { } SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillGetCur_rocksdb"); + stDebug("streamStateFillGetCur_rocksdb"); SStreamStateCur* pCur = createStreamStateCursor(); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -3064,7 +3062,7 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, } SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillSeekKeyNext_rocksdb"); + stDebug("streamStateFillSeekKeyNext_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (!pCur) { @@ -3102,7 +3100,7 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const return NULL; } SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { - qDebug("streamStateFillSeekKeyPrev_rocksdb"); + stDebug("streamStateFillSeekKeyPrev_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; SStreamStateCur* pCur = createStreamStateCursor(); if (pCur == NULL) { From 4c5441da7ee711ac919a302306cf857c22b27f56 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Mon, 8 Jan 2024 08:38:16 +0000 Subject: [PATCH 137/198] refactor stream code --- source/libs/stream/src/streamMeta.c | 6 +- source/libs/stream/src/streamSessionState.c | 126 ++++++++++---------- source/libs/stream/src/streamSnapshot.c | 14 +-- source/libs/stream/src/streamUpdate.c | 6 +- 4 files changed, 77 insertions(+), 75 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9c94799237..7ea80ad8d8 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -201,7 +201,7 @@ int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) { void* key = taosHashGetKey(pIter, NULL); code = streamStateCvtDataFormat(pMeta->path, key, *(void**)pIter); if (code != 0) { - qError("failed to cvt data"); + stError("failed to cvt data"); goto _EXIT; } @@ -225,11 +225,11 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { if (compatible == STREAM_STATA_COMPATIBLE) { return 0; } else if (compatible == STREAM_STATA_NEED_CONVERT) { - qInfo("stream state need covert backend format"); + stInfo("stream state need covert backend format"); return streamMetaCvtDbFormat(pMeta); } else if (compatible == STREAM_STATA_NO_COMPATIBLE) { - qError( + stError( "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " "manually", tsDataDir); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 5f19b47e1f..2cb77c60dc 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -17,17 +17,14 @@ #include "query.h" #include "streamBackendRocksdb.h" -#include "taos.h" #include "tcommon.h" -#include "thash.h" #include "tsimplehash.h" - -typedef int (*__session_compare_fn_t) (const SSessionKey* pWin, const void* pDatas, int pos); +typedef int (*__session_compare_fn_t)(const SSessionKey* pWin, const void* pDatas, int pos); int sessionStateKeyCompare(const SSessionKey* pWin1, const void* pDatas, int pos) { SRowBuffPos* pPos2 = taosArrayGetP(pDatas, pos); - SSessionKey* pWin2 = (SSessionKey*) pPos2->pKey; + SSessionKey* pWin2 = (SSessionKey*)pPos2->pKey; return sessionWinKeyCmpr(pWin1, pWin2); } @@ -61,12 +58,12 @@ int32_t binarySearch(void* keyList, int num, const void* key, __session_compare_ int64_t getSessionWindowEndkey(void* data, int32_t index) { SArray* pWinInfos = (SArray*)data; SRowBuffPos** ppos = taosArrayGet(pWinInfos, index); - SSessionKey* pWin = (SSessionKey*)((*ppos)->pKey); + SSessionKey* pWin = (SSessionKey*)((*ppos)->pKey); return pWin->win.ekey; } bool inSessionWindow(SSessionKey* pKey, TSKEY ts, int64_t gap) { - if (ts + gap >= pKey->win.skey && ts - gap <= pKey->win.ekey) { + if (ts + gap >= pKey->win.skey && ts - gap <= pKey->win.ekey) { return true; } return false; @@ -80,7 +77,8 @@ static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pW return pNewPos; } -static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, int32_t index) { +static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, + int32_t index) { SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); @@ -97,11 +95,12 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe return pNewPos; } -int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen) { - int32_t code = TSDB_CODE_SUCCESS; +int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, + int32_t* pVLen) { + int32_t code = TSDB_CODE_SUCCESS; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -146,7 +145,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, if (index + 1 < size) { pPos = taosArrayGetP(pWinStates, index + 1); - if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) ) { + if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap))) { (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; @@ -157,9 +156,9 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, if (index + 1 == 0) { if (!isDeteled(pFileState, endTs) && isFlushedState(pFileState, endTs, gap)) { - void* p = NULL; - void* pFileStore = getStateFileStore(pFileState); - int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); + void* p = NULL; + void* pFileStore = getStateFileStore(pFileState); + int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); code = code_file; @@ -184,10 +183,10 @@ _end: } int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSessionKey* pKey = pPos->pKey; - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -202,7 +201,7 @@ int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) } // find the first position which is smaller than the pKey - int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); + int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); if (index >= 0) { taosArrayInsert(pWinStates, index, &pPos); } else { @@ -218,7 +217,7 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; - void* pBuff = NULL; + void* pBuff = NULL; int32_t code = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); if (code != TSDB_CODE_SUCCESS) { return code; @@ -229,16 +228,16 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v return TSDB_CODE_SUCCESS; } -int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen) { - SSHashObj* pSessionBuff = (SSHashObj*) pBuff; - SSessionKey* pWinKey = (SSessionKey*) key; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); +int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen) { + SSHashObj* pSessionBuff = (SSHashObj*)pBuff; + SSessionKey* pWinKey = (SSessionKey*)key; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (!ppBuff) { return TSDB_CODE_SUCCESS; } SArray* pWinStates = (SArray*)(*ppBuff); int32_t size = taosArrayGetSize(pWinStates); - TSKEY gap = 0; + TSKEY gap = 0; int32_t index = binarySearch(pWinStates, size, pWinKey, sessionStateKeyCompare); if (index >= 0) { SRowBuffPos* pPos = taosArrayGetP(pWinStates, index); @@ -251,15 +250,15 @@ int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen) } int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SSessionKey* pWinKey = (SSessionKey*) pPos->pKey; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SSessionKey* pWinKey = (SSessionKey*)pPos->pKey; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (!ppBuff) { return TSDB_CODE_SUCCESS; } SArray* pWinStates = (SArray*)(*ppBuff); int32_t size = taosArrayGetSize(pWinStates); - TSKEY gap = 0; + TSKEY gap = 0; int32_t index = binarySearch(pWinStates, size, pWinKey, sessionStateKeyCompare); if (index >= 0) { SRowBuffPos* pItemPos = taosArrayGetP(pWinStates, index); @@ -270,11 +269,12 @@ int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffP return TSDB_CODE_SUCCESS; } -int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { +int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, + const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { SRowBuffPos* pNewPos = NULL; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); - SArray* pWinStates = NULL; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; if (!ppBuff) { pWinStates = taosArrayInit(16, POINTER_BYTES); tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); @@ -297,10 +297,10 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream } else { if (size > 0) { SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0); - if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >=0) { + if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >= 0) { // pCur is invalid SSessionKey pTmpKey = *pWinKey; - int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); + int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); ASSERT(code == TSDB_CODE_FAILED); goto _end; } @@ -320,7 +320,7 @@ void sessionWinStateClear(SStreamFileState* pFileState) { void* pIte = NULL; size_t keyLen = 0; int32_t iter = 0; - void* pBuff = getRowStateBuff(pFileState); + void* pBuff = getRowStateBuff(pFileState); while ((pIte = tSimpleHashIterate(pBuff, pIte, &iter)) != NULL) { SArray* pWinStates = *((void**)pIte); int32_t size = taosArrayGetSize(pWinStates); @@ -336,7 +336,7 @@ void sessionWinStateCleanup(void* pBuff) { size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pBuff, pIte, &iter)) != NULL) { - SArray* pWinStates = (SArray*) (*(void**)pIte); + SArray* pWinStates = (SArray*)(*(void**)pIte); taosArrayDestroy(pWinStates); } tSimpleHashCleanup(pBuff); @@ -395,11 +395,13 @@ static void transformCursor(SStreamFileState* pFileState, SStreamStateCur* pCur) pCur->pStreamFileState = pFileState; } -static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t groupId, SArray* pWinStates, SStreamStateCur** ppCur) { +static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t groupId, SArray* pWinStates, + SStreamStateCur** ppCur) { SSessionKey key = {.groupId = groupId}; - int32_t code = streamStateSessionGetKVByCur_rocksdb(*ppCur, &key, NULL, NULL); - if (taosArrayGetSize(pWinStates) > 0 && (code == TSDB_CODE_FAILED || sessionStateKeyCompare(&key, pWinStates, 0) >= 0)) { - if ( !(*ppCur) ) { + int32_t code = streamStateSessionGetKVByCur_rocksdb(*ppCur, &key, NULL, NULL); + if (taosArrayGetSize(pWinStates) > 0 && + (code == TSDB_CODE_FAILED || sessionStateKeyCompare(&key, pWinStates, 0) >= 0)) { + if (!(*ppCur)) { (*ppCur) = createStreamStateCursor(); } transformCursor(pFileState, *ppCur); @@ -410,8 +412,8 @@ static void checkAndTransformCursor(SStreamFileState* pFileState, const uint64_t } SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey) { - SArray* pWinStates = NULL; - int32_t index = -1; + SArray* pWinStates = NULL; + int32_t index = -1; SStreamStateCur* pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index); if (pCur) { if (sessionStateKeyCompare(pWinKey, pWinStates, index) > 0) { @@ -427,8 +429,8 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, } SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey) { - SArray* pWinStates = NULL; - int32_t index = -1; + SArray* pWinStates = NULL; + int32_t index = -1; SStreamStateCur* pCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index); if (pCur) { sessionWinStateMoveToNext(pCur); @@ -449,7 +451,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void SSHashObj* pSessionBuff = getRowStateBuff(pCur->pStreamFileState); void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); - SArray* pWinStates = NULL; + SArray* pWinStates = NULL; if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } @@ -467,7 +469,8 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void } else { void* pData = NULL; code = streamStateSessionGetKVByCur_rocksdb(pCur, pKey, &pData, pVLen); - if (taosArrayGetSize(pWinStates) > 0 && (code == TSDB_CODE_FAILED || sessionStateKeyCompare(pKey, pWinStates, 0) >= 0)) { + if (taosArrayGetSize(pWinStates) > 0 && + (code == TSDB_CODE_FAILED || sessionStateKeyCompare(pKey, pWinStates, 0) >= 0)) { transformCursor(pCur->pStreamFileState, pCur); SRowBuffPos* pPos = taosArrayGetP(pWinStates, pCur->buffIndex); if (pVal) { @@ -498,9 +501,9 @@ int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) { int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey) { SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key); - SSessionKey tmpKey = *key; - int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); - bool hasCurrentPrev = true; + SSessionKey tmpKey = *key; + int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); + bool hasCurrentPrev = true; if (code == TSDB_CODE_FAILED) { streamStateFreeCur(pCur); pCur = sessionWinStateSeekKeyNext(pFileState, key); @@ -535,13 +538,13 @@ _end: } int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { SSessionKey* pWinKey = key; - TSKEY gap = 0; - int32_t code = TSDB_CODE_SUCCESS; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + TSKEY gap = 0; + int32_t code = TSDB_CODE_SUCCESS; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { @@ -560,7 +563,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; - qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); } else { (*pVal) = addNewSessionWindow(pFileState, pWinStates, key); code = TSDB_CODE_FAILED; @@ -589,7 +593,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (index + 1 < size) { pPos = taosArrayGetP(pWinStates, index + 1); void* stateKey = (char*)(pPos->pRowBuff) + (valSize - keyDataLen); - if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) || fn(pKeyData, stateKey) == true) { + if (inSessionWindow(pPos->pKey, startTs, gap) || (endTs != INT64_MIN && inSessionWindow(pPos->pKey, endTs, gap)) || + fn(pKeyData, stateKey) == true) { (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; @@ -607,7 +612,8 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); code = code_file; - qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); goto _end; } else { taosMemoryFree(p); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index e29f2ba7de..a69cb75dad 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -15,10 +15,8 @@ #include "streamSnapshot.h" #include "query.h" -#include "rocksdb/c.h" #include "streamBackendRocksdb.h" #include "streamInt.h" -#include "tcommon.h" enum SBackendFileType { ROCKSDB_OPTIONS_TYPE = 1, @@ -158,7 +156,7 @@ void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { } sprintf(buf + strlen(buf) - 1, "]"); - qInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, + stInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, pSnapFile->snapInfo.taskId, buf); taosMemoryFree(buf); } @@ -203,7 +201,7 @@ int32_t snapFileGenMeta(SBackendSnapFile2* pSnapFile) { int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { TdDirPtr pDir = taosOpenDir(pSnapFile->path); if (NULL == pDir) { - qError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path); + stError("%s failed to open %s", STREAM_STATE_TRANSFER, pSnapFile->path); return -1; } @@ -397,7 +395,7 @@ _NEXT: } item = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); - qDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 + stDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d, total set:%d, current set idx: %d", STREAM_STATE_TRANSFER, item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx, (int)taosArrayGetSize(pHandle->pDbSnapSet), pHandle->currIdx); @@ -515,7 +513,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); return code; } else { - qInfo("succ to write data %s", pItem->name); + stInfo("succ to write data %s", pItem->name); } pSnapFile->offset += bytes; } else { @@ -538,7 +536,7 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t } taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); - qInfo("succ to write data %s", pItem->name); + stInfo("succ to write data %s", pItem->name); pSnapFile->offset += pHdr->size; } code = 0; @@ -563,7 +561,7 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa "checkpoint", snapInfo.chkpId); if (!taosIsDir(path)) { code = taosMulMkDir(path); - qInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); + stInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); ASSERT(code == 0); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index f78f6f4df1..d607f26de3 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#include "query.h" -#include "tdatablock.h" #include "tencode.h" #include "tstreamUpdate.h" #include "ttime.h" @@ -387,8 +385,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); - bool res = true; - if ( pMapMaxTs && ts < *pMapMaxTs ) { + bool res = true; + if (pMapMaxTs && ts < *pMapMaxTs) { res = false; } else { taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), &ts, sizeof(TSKEY)); From 5e45ecd3fda3e3ea5530d800f672760f871184ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 8 Jan 2024 16:46:07 +0800 Subject: [PATCH 138/198] fix: possible race condition --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbMemTable.c | 3 ++ source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 48 +++++++++++++--------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 75f90af378..9d8d5013fa 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -397,6 +397,7 @@ struct STbData { tb_uid_t uid; TSKEY minKey; TSKEY maxKey; + SRWLatch lock; SDelData *pHead; SDelData *pTail; SMemSkipList sl; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index cc77474e79..4657b60f62 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -181,6 +181,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pDelData->sKey = sKey; pDelData->eKey = eKey; pDelData->pNext = NULL; + taosWLockLatch(&pTbData->lock); if (pTbData->pHead == NULL) { ASSERT(pTbData->pTail == NULL); pTbData->pHead = pTbData->pTail = pDelData; @@ -188,6 +189,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pTbData->pTail->pNext = pDelData; pTbData->pTail = pDelData; } + taosWUnLockLatch(&pTbData->lock); pMemTable->nDel++; pMemTable->minVer = TMIN(pMemTable->minVer, version); @@ -401,6 +403,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SL_NODE_BACKWARD(pTbData->sl.pHead, iLevel) = NULL; SL_NODE_FORWARD(pTbData->sl.pTail, iLevel) = NULL; } + taosInitRWLatch(&pTbData->lock); taosWLockLatch(&pMemTable->latch); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index a223a2dc2d..33b7667168 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -22,9 +22,14 @@ #include "tsdbUtil2.h" #include "tsimplehash.h" -#define INIT_TIMEWINDOW(_w) do { (_w)->skey = INT64_MAX; (_w)->ekey = INT64_MIN;} while(0); +#define INIT_TIMEWINDOW(_w) \ + do { \ + (_w)->skey = INT64_MAX; \ + (_w)->ekey = INT64_MIN; \ + } while (0); -static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order); +static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, + int32_t order); static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { int32_t num = numOfTables / pBuf->numPerBucket; @@ -449,7 +454,7 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); taosArrayPush(pBlockIter->blockList, &blockInfo); - STableBlockScanInfo *pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; + STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; if (pTableScanInfo->pBlockIdxList == NULL) { size_t szTableDataBlocks = taosArrayGetSize(pTableScanInfo->pBlockList); pTableScanInfo->pBlockIdxList = taosArrayInit(szTableDataBlocks, sizeof(STableDataBlockIdx)); @@ -504,7 +509,7 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ int32_t code = 0; STombRecord record = {0}; - uint64_t uid = pReader->status.uidList.tableUidList[*j]; + uint64_t uid = pReader->status.uidList.tableUidList[*j]; STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); if (pScanInfo->pFileDelData == NULL) { pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); @@ -595,12 +600,12 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs return code; } -// uint64_t uid = pReader->status.uidList.tableUidList[j]; + // uint64_t uid = pReader->status.uidList.tableUidList[j]; -// STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); -// if (pScanInfo->pFileDelData == NULL) { -// pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); -// } + // STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + // if (pScanInfo->pFileDelData == NULL) { + // pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); + // } ETombBlkCheckEnum ret = 0; code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret); @@ -650,6 +655,7 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT SDelData* p = NULL; if (pMemTbData != NULL) { + taosRLockLatch(&pMemTbData->lock); p = pMemTbData->pHead; while (p) { if (p->version <= ver) { @@ -658,6 +664,7 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT p = p->pNext; } + taosRUnLockLatch(&pMemTbData->lock); } if (piMemTbData != NULL) { @@ -681,7 +688,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo } int32_t i = 0; - while((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) { + while ((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) { ++i; } @@ -689,7 +696,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo return 0; } - SStatisBlk *p = &pStatisBlkArray->data[i]; + SStatisBlk* p = &pStatisBlkArray->data[i]; STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock)); tStatisBlockInit(pStatisBlock); @@ -759,14 +766,14 @@ void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { if (size < numOfFileObj) { int32_t inc = numOfFileObj - size; for (int32_t k = 0; k < inc; ++k) { - SLDataIter *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + SLDataIter* pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); taosArrayPush(pLDIterList, &pIter); } } else if (size > numOfFileObj) { // remove unused LDataIter int32_t inc = size - numOfFileObj; for (int i = 0; i < inc; ++i) { - SLDataIter *pIter = taosArrayPop(pLDIterList); + SLDataIter* pIter = taosArrayPop(pLDIterList); destroyLDataIter(pIter); } } @@ -781,9 +788,9 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) taosArrayPush(pSttFileBlockIterArray, &pList); } - for(int32_t j = 0; j < numOfLevels; ++j) { + for (int32_t j = 0; j < numOfLevels; ++j) { SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; - SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); + SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr)); } @@ -829,8 +836,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra } // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file - TStatisBlkArray *pStatisBlkArray = NULL; - int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pStatisBlkArray); + TStatisBlkArray* pStatisBlkArray = NULL; + int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); continue; @@ -899,11 +906,12 @@ static int32_t sortUidComparFn(const void* p1, const void* p2) { if (px1->skey == px2->skey) { return 0; } else { - return px1->skey < px2->skey? -1:1; + return px1->skey < px2->skey ? -1 : 1; } } -bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo *pScanInfo, int32_t order) { +bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, + int32_t order) { // check if it overlap with del skyline taosArraySort(pTimewindowList, sortUidComparFn); @@ -935,7 +943,7 @@ bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableB } static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, - int32_t startIndex) { + int32_t startIndex) { size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); for (int32_t i = startIndex; i < num; i += 1) { From 767feba65a24ec22b78b9aa48294667966cac19d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 16:49:02 +0800 Subject: [PATCH 139/198] test: revert failed case --- tests/system-test/8-stream/stream_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py index 04e635ea24..3ebc255114 100644 --- a/tests/system-test/8-stream/stream_basic.py +++ b/tests/system-test/8-stream/stream_basic.py @@ -101,7 +101,7 @@ class TDTestCase: # check ts interval is correct sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;" tdSql.query(sql) - tdSql.checkRows(1) + tdSql.checkRows(0) # stop def stop(self): From 75c25a194fd78626e3020afe080aa0dd90da3a5b Mon Sep 17 00:00:00 2001 From: zk66214 Date: Mon, 8 Jan 2024 17:19:48 +0800 Subject: [PATCH 140/198] add test cases for TD-28164 --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/backslash_g.py | 72 ++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 tests/system-test/2-query/backslash_g.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3db6eccffc..336f58afa3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -546,6 +546,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4348-td-27939.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/backslash_g.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py diff --git a/tests/system-test/2-query/backslash_g.py b/tests/system-test/2-query/backslash_g.py new file mode 100644 index 0000000000..69b06289c3 --- /dev/null +++ b/tests/system-test/2-query/backslash_g.py @@ -0,0 +1,72 @@ + +import taos + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def checksql(self, sql): + result = os.popen(f"taos -s \"{sql}\" ") + res = result.read() + print(res) + if ("Query OK" in res): + tdLog.info(f"checkEqual success") + else : + tdLog.exit(f"checkEqual error") + + def td_28164(self): + tdSql.execute("drop database if exists td_28164;") + tdSql.execute("create database td_28164;") + tdSql.execute("create table td_28164.test (ts timestamp, name varchar(10));") + tdSql.execute("insert into td_28164.test values (now(), 'ac\\\\G') (now() + 1s, 'ac\\\\G') (now()+2s, 'ac\\G') (now()+3s, 'acG') (now()+4s, 'acK') ;") + + tdSql.query(f"select * from td_28164.test;") + tdSql.checkRows(5) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\\\\\G';") + tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\\\G';") + tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac\\G';") + tdSql.checkRows(2) + + # tdSql.query(f"select * from td_28164.test where name like 'ac\\\g';") + # tdSql.checkRows(0) + # + # tdSql.query(f"select * from td_28164.test where name like 'ac\\g';") + # tdSql.checkRows(0) + + self.checksql(f'select * from td_28164.test where name like \'ac\\G\'\G;') + # tdSql.checkRows(2) + + self.checksql(f"select * from td_28164.test where name like \'ac\\G\' \G;") + # tdSql.checkRows(2) + + tdSql.query(f"select * from td_28164.test where name like 'ac/\\G';") + tdSql.checkRows(0) + + tdSql.query(f"select * from td_28164.test where name like 'ac/G';") + tdSql.checkRows(0) + + def run(self): + # tdSql.prepare() + self.td_28164() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 70c1e66959872c0fd0c0b75bc032a2a3b1bb8862 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Jan 2024 18:19:31 +0800 Subject: [PATCH 141/198] fix(stream): release task after launch stream tasks. --- source/libs/stream/src/streamMeta.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7e69610f80..6616fddaed 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1517,6 +1517,8 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, taskId); streamMetaUpdateTaskDownstreamStatus(pMeta, streamId, taskId, 0, taosGetTimestampMs(), false); + + streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } @@ -1542,5 +1544,6 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas } } + streamMetaReleaseTask(pMeta, pTask); return ret; } \ No newline at end of file From af9dc441b20e646882a8cbb4a23bfb34f9518f8f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Jan 2024 18:40:06 +0800 Subject: [PATCH 142/198] fix(stream): reset the saved value for fill-history task. --- include/libs/executor/executor.h | 1 + include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tqCommon/tqCommon.c | 13 ++---- source/libs/executor/src/executor.c | 51 ++++++++++++++++++++++ source/libs/stream/src/streamStart.c | 5 +++ 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index f78b7a3126..be11d04ff8 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -210,6 +210,7 @@ void* qExtractReaderFromStreamScanner(void* scanner); int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo); +int32_t qResetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo); int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow); int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow); int32_t qStreamRecoverFinish(qTaskInfo_t tinfo); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index dbcdef57b0..d11a4ad23b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -825,6 +825,7 @@ int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); // common int32_t streamRestoreParam(SStreamTask* pTask); +int32_t streamResetParamForScanHistory(SStreamTask* pTask); void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta); void streamTaskResume(SStreamTask* pTask); int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 5467e3dadd..2e83cd5bc5 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -728,6 +728,10 @@ int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta, int32_t* numOfTasks) { STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); streamTaskResetStatus(*pTask); + + if ((*pTask)->info.fillHistory == 1) { + streamResetParamForScanHistory(*pTask); + } } return 0; @@ -760,19 +764,10 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { } streamMetaWLock(pMeta); -// streamMetaClear(pMeta); int64_t el = taosGetTimestampMs() - st; tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el / 1000.); -// code = streamMetaLoadAllTasks(pMeta); -// if (code != TSDB_CODE_SUCCESS) { -// tqError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); -// streamMetaWUnLock(pMeta); -// code = terrno; -// return code; -// } - { STaskStartInfo* pStartInfo = &pMeta->startInfo; taosHashClear(pStartInfo->pReadyTaskSet); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6cee79bff2..c29fae0cef 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1023,6 +1023,57 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { return 0; } +int32_t qResetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + + while (1) { + int32_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + pSup->calTriggerSaved = 0; + pSup->deleteMarkSaved = 0; + qInfo("reset stream param for interval: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + pSup->calTriggerSaved = 0; + pSup->deleteMarkSaved = 0; + qInfo("reset stream param for session: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + pSup->calTriggerSaved = 0; + pSup->deleteMarkSaved = 0; + qInfo("reset stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) { + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + pSup->calTriggerSaved = 0; + pSup->deleteMarkSaved = 0; + qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + } + + // iterate operator tree + if (pOperator->numOfDownstream != 1 || pOperator->pDownstream[0] == NULL) { + return 0; + } else { + pOperator = pOperator->pDownstream[0]; + } + } +} + int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; const char* id = GET_TASKID(pTaskInfo); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 0dcc94a530..0c2993e296 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -544,6 +544,11 @@ int32_t streamSetParamForScanHistory(SStreamTask* pTask) { return qSetStreamOperatorOptionForScanHistory(pTask->exec.pExecutor); } +int32_t streamResetParamForScanHistory(SStreamTask* pTask) { + stDebug("s-task:%s reset operator option for scan-history data", pTask->id.idStr); + return qResetStreamOperatorOptionForScanHistory(pTask->exec.pExecutor); +} + int32_t streamRestoreParam(SStreamTask* pTask) { stDebug("s-task:%s restore operator param after scan-history", pTask->id.idStr); return qRestoreStreamOperatorOption(pTask->exec.pExecutor); From bbaaad6f1f65666beb6a0a70d99b4b48b719b43a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Jan 2024 18:45:36 +0800 Subject: [PATCH 143/198] fix(stream): add null ptr check. --- source/libs/stream/src/streamStart.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 0c2993e296..52c3c87431 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -154,8 +154,7 @@ int32_t streamTaskStartScanHistory(SStreamTask* pTask) { int32_t level = pTask->info.taskLevel; ETaskStatus status = streamTaskGetStatus(pTask)->state; - ASSERT(pTask->status.downstreamReady == 1 && - ((status == TASK_STATUS__SCAN_HISTORY)/* || (status == TASK_STATUS__STREAM_SCAN_HISTORY)*/)); + ASSERT((pTask->status.downstreamReady == 1) && (status == TASK_STATUS__SCAN_HISTORY)); if (level == TASK_LEVEL__SOURCE) { return doStartScanHistoryTask(pTask); @@ -546,7 +545,11 @@ int32_t streamSetParamForScanHistory(SStreamTask* pTask) { int32_t streamResetParamForScanHistory(SStreamTask* pTask) { stDebug("s-task:%s reset operator option for scan-history data", pTask->id.idStr); - return qResetStreamOperatorOptionForScanHistory(pTask->exec.pExecutor); + if (pTask->exec.pExecutor != NULL) { + return qResetStreamOperatorOptionForScanHistory(pTask->exec.pExecutor); + } else { + return TSDB_CODE_SUCCESS; + } } int32_t streamRestoreParam(SStreamTask* pTask) { From 5bd29a413b30df61024d4c6dfa5d1efbc669a02b Mon Sep 17 00:00:00 2001 From: fullhonest Date: Mon, 8 Jan 2024 15:04:04 +0800 Subject: [PATCH 144/198] Fix: not use last row scan when select tbname and last --- source/libs/planner/src/planOptimizer.c | 2 +- tests/system-test/2-query/last.py | 81 ++++++++++++++++++++ tests/system-test/2-query/last_cache_scan.py | 8 +- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index aa3181e166..035377d22e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2529,7 +2529,7 @@ static bool lastRowScanOptCheckFuncList(SLogicNode* pNode, bool* hasOtherFunc) { SNode* pParam = nodesListGetNode(pAggFunc->pParameterList, 0); if (QUERY_NODE_COLUMN == nodeType(pParam)) { SColumnNode* pCol = (SColumnNode*)pParam; - if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { + if (COLUMN_TYPE_COLUMN == pCol->colType && PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { if (selectNonPKColId != pCol->colId) { selectNonPKColId = pCol->colId; selectNonPKColNum++; diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index f4a1ab1790..a160aeab11 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -289,6 +289,82 @@ class TDTestCase: tdSql.query(f"select * from {stbname}") tdSql.checkRows(tables * rows) + def check_explain_res_has_row(self, plan_str_expect: str, rows, sql): + plan_found = False + for row in rows: + if str(row).find(plan_str_expect) >= 0: + tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row))) + plan_found = True + break + if not plan_found: + tdLog.exit("plan: %s not found in res: [%s] in sql: %s" % (plan_str_expect, str(rows), sql)) + + def check_explain_res_no_row(self, plan_str_not_expect: str, res, sql): + for row in res: + if str(row).find(plan_str_not_expect) >= 0: + tdLog.exit('plan: [%s] found in: [%s] for sql: %s' % (plan_str_not_expect, str(row), sql)) + + def explain_sql(self, sql: str): + sql = "explain " + sql + tdSql.query(sql, queryTimes=1) + return tdSql.queryResult + + def last_check_scan_type(self, cacheModel): + tdSql.execute("create database test_last_tbname cachemodel '%s';" % cacheModel) + tdSql.execute("use test_last_tbname;") + tdSql.execute("create stable test_last_tbname.st(ts timestamp, id int) tags(tid int);") + tdSql.execute("create table test_last_tbname.test_t1 using test_last_tbname.st tags(1);") + + maxRange = 100 + # 2023-11-13 00:00:00.000 + startTs = 1699804800000 + for i in range(maxRange): + insertSqlString = "insert into test_last_tbname.test_t1 values(%d, %d);" % (startTs + i, i) + tdSql.execute(insertSqlString) + + last_ts = startTs + maxRange + tdSql.execute("insert into test_last_tbname.test_t1 (ts) values(%d)" % (last_ts)) + sql = f'select tbname, last(ts) from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, "test_t1") + tdSql.checkData(0, 1, last_ts) + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + + sql = f'select last(ts), tbname from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, last_ts) + tdSql.checkData(0, 1, "test_t1") + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + sql = f'select tbname, last(ts), tbname from test_last_tbname.test_t1;' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, "test_t1") + tdSql.checkData(0, 1, last_ts) + tdSql.checkData(0, 2, "test_t1") + + explain_res = self.explain_sql(sql) + if cacheModel == "both" or cacheModel == "last_value": + self.check_explain_res_has_row("Last Row Scan", explain_res, sql) + else: + self.check_explain_res_has_row("Table Scan", explain_res, sql) + + tdSql.execute("drop table if exists test_last_tbname.test_t1 ;") + tdSql.execute("drop stable if exists test_last_tbname.st;") + tdSql.execute("drop database if exists test_last_tbname;") def run(self): self.last_check_stb_tb_base() @@ -296,6 +372,11 @@ class TDTestCase: self.last_check_stb_distribute() self.last_file_check() + self.last_check_scan_type("none") + self.last_check_scan_type("last_row") + self.last_check_scan_type("last_value") + self.last_check_scan_type("both") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 23433277c2..1269bd4fde 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -181,7 +181,7 @@ class TDTestCase: select_items = [ "last(ts), ts", "last(ts), c1", "last(ts), c2", "last(ts), c3",\ "last(ts), c4", "last(ts), tbname", "last(ts), t1", "last(ts), ts, ts"] - has_last_row_scan_res = [1, 0, 0, 0, 0, 0, 0, 1] + has_last_row_scan_res = [1, 0, 0, 0, 0, 1, 1, 1] res_expect = [ ["2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"], None, None, None, None, None, None, @@ -193,7 +193,7 @@ class TDTestCase: select_items = ["last(c1),ts", "last(c1), c1", "last(c1), c2", "last(c1), c3",\ "last(c1), c4", "last(c1), tbname", "last(c1), t1", "last(c1), ts, ts", "last(c1), c1, c1"] - has_last_row_scan_res = [1, 1, 0, 0, 0, 0, 0, 1, 1] + has_last_row_scan_res = [1, 1, 0, 0, 0, 1, 1, 1, 1] res_expect = [ [999, "2018-11-25 19:30:00.000"], [999, 999], None, None, None, None, None, @@ -207,7 +207,7 @@ class TDTestCase: sql_template = 'select %s from t1' select_items = ["last(c4),ts", "last(c4), c1", "last(c4), c2", "last(c4), c3",\ "last(c4), c4", "last(c4), tbname", "last(c4), t1"] - has_last_row_scan_res = [1, 0, 0, 0, 1, 0, 0] + has_last_row_scan_res = [1, 0, 0, 0, 1, 1, 1] res_expect = [ [4999.000000000000000, "2018-11-25 19:30:00.000"], None,None,None, @@ -220,7 +220,7 @@ class TDTestCase: sql_template = 'select %s from meters' select_items = ["last(c8), ts", "last(c8), c1", "last(c8), c8", "last(c8), tbname", \ "last(c8), t1", "last(c8), c8, c8", "last(c8), ts, ts"] - has_last_row_scan_res = [1, 0, 1, 0, 0, 1, 1] + has_last_row_scan_res = [1, 0, 1, 1, 1, 1, 1] res_expect = [ ["binary9999", "2018-11-25 19:30:00.000"], None, From f9d5c0d4039e8e00e685b8dd24f629c68f8268a7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Jan 2024 09:13:53 +0800 Subject: [PATCH 145/198] fix(stream): fix error in tq timer. --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 6 +----- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 5 ++++- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 14 +++++++------- source/libs/stream/inc/streamInt.h | 3 +++ source/libs/stream/src/stream.c | 5 +++++ source/libs/stream/src/streamMeta.c | 24 +++++++++++++----------- 7 files changed, 34 insertions(+), 25 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index f9bba19fbb..d173c83e97 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -28,9 +28,6 @@ } \ } while (0) -extern int32_t streamTimerInit(); -extern void streamTimerCleanUp(); - static SDnode globalDnode = {0}; SDnode *dmInstance() { return &globalDnode; } @@ -169,7 +166,6 @@ int32_t dmInit() { #if defined(USE_S3) if (s3Begin() != 0) return -1; #endif - if (streamTimerInit() != 0) return -1; dInfo("dnode env is initialized"); return 0; @@ -196,10 +192,10 @@ void dmCleanup() { udfStopUdfd(); taosStopCacheRefreshWorker(); dmDiskClose(); + #if defined(USE_S3) s3End(); #endif - streamTimerCleanUp(); dInfo("dnode env is cleaned up"); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 84465640c0..070d78fd69 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -90,6 +90,7 @@ int32_t dmInitDnode(SDnode *pDnode) { goto _OVER; } #endif + indexInit(tsNumOfCommitThreads); streamMetaInit(); @@ -108,7 +109,9 @@ _OVER: } void dmCleanupDnode(SDnode *pDnode) { - if (pDnode == NULL) return; + if (pDnode == NULL) { + return; + } dmCleanupClient(pDnode); dmCleanupStatusClient(pDnode); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e4c673f533..435b12bc06 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -42,7 +42,6 @@ extern "C" { // clang-format on typedef struct STqOffsetStore STqOffsetStore; -extern void* tqTimer; #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) @@ -108,6 +107,7 @@ struct STQ { TTB* pExecStore; TTB* pCheckStore; SStreamMeta* pStreamMeta; + void* tqTimer; }; int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 862d7e1d66..b3cb3c80f7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -26,16 +26,16 @@ static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_ static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_EXEC; } static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_IDLE; } -int32_t tqTimerInit() { - tqTimer = taosTmrInit(100, 100, 1000, "TQ"); - if (tqTimer == NULL) { +static int32_t tqTimerInit(STQ* pTq) { + pTq->tqTimer = taosTmrInit(100, 100, 1000, "TQ"); + if (pTq->tqTimer == NULL) { return -1; } return 0; } -void tqTimerCleanUp() { - taosTmrCleanUp(tqTimer); +static void tqTimerCleanUp(STQ* pTq) { + taosTmrCleanUp(pTq->tqTimer); } void tqDestroyTqHandle(void* data) { @@ -118,7 +118,7 @@ int32_t tqInitialize(STQ* pTq) { return -1; } - tqTimerInit(); + tqTimerInit(pTq); return 0; } @@ -149,7 +149,7 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq->path); tqMetaClose(pTq); streamMetaClose(pTq->pStreamMeta); - tqTimerCleanUp(); + tqTimerCleanUp(pTq); qDebug("end to close tq"); taosMemoryFree(pTq); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 9b5134d449..1d1eda74f6 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -92,6 +92,9 @@ extern int32_t streamBackendId; extern int32_t streamBackendCfWrapperId; extern int32_t taskDbWrapperId; +int32_t streamTimerInit(); +void streamTimerCleanUp(); + void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 10b1b3fbbe..7db4bec6a0 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -21,13 +21,18 @@ void* streamTimer = NULL; int32_t streamTimerInit() { streamTimer = taosTmrInit(1000, 100, 10000, "STREAM"); if (streamTimer == NULL) { + stError("init stream timer failed, code:%s", tstrerror(terrno)); return -1; } + + stInfo("init stream timer, %p", streamTimer); return 0; } void streamTimerCleanUp() { + stInfo("cleanup stream timer, %p", streamTimer); taosTmrCleanUp(streamTimer); + streamTimer = NULL; } char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 6616fddaed..f687841016 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -23,7 +23,7 @@ #include "ttimer.h" #include "wal.h" -static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; +TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; int32_t streamBackendId = 0; int32_t streamBackendCfWrapperId = 0; @@ -61,9 +61,10 @@ static void streamMetaEnvInit() { streamMetaId = taosOpenRef(64, streamMetaCloseImpl); metaRefMgtInit(); + streamTimerInit(); } -void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } +void streamMetaInit() { /*taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit);*/ streamMetaEnvInit();} void streamMetaCleanup() { taosCloseRef(streamBackendId); @@ -71,6 +72,7 @@ void streamMetaCleanup() { taosCloseRef(streamMetaId); metaRefMgtCleanup(); + streamTimerCleanUp(); } void metaRefMgtInit() { @@ -391,7 +393,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); metaRefMgtAdd(pMeta->vgId, pRid); - pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); +// pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); pMeta->pHbInfo->tickCounter = 0; pMeta->pHbInfo->stopFlag = 0; pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); @@ -1202,7 +1204,7 @@ void metaHbToMnode(void* param, void* tmrId) { } if (!waitForEnoughDuration(pMeta->pHbInfo)) { - taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); +// taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); return; } @@ -1212,7 +1214,7 @@ void metaHbToMnode(void* param, void* tmrId) { metaHeartbeatToMnodeImpl(pMeta); streamMetaRUnLock(pMeta); - taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); +// taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); } @@ -1262,11 +1264,11 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { // wait for the stream meta hb function stopping if (pMeta->role == NODE_ROLE_LEADER) { - pMeta->pHbInfo->stopFlag = STREAM_META_WILL_STOP; - while (pMeta->pHbInfo->stopFlag != STREAM_META_OK_TO_STOP) { - taosMsleep(100); - stDebug("vgId:%d wait for meta to stop timer", pMeta->vgId); - } +// pMeta->pHbInfo->stopFlag = STREAM_META_WILL_STOP; +// while (pMeta->pHbInfo->stopFlag != STREAM_META_OK_TO_STOP) { +// taosMsleep(100); +// stDebug("vgId:%d wait for meta to stop timer", pMeta->vgId); +// } } stDebug("vgId:%d start to check all tasks", vgId); @@ -1285,7 +1287,7 @@ void streamMetaStartHb(SStreamMeta* pMeta) { int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); metaRefMgtAdd(pMeta->vgId, pRid); *pRid = pMeta->rid; - metaHbToMnode(pRid, NULL); +// metaHbToMnode(pRid, NULL); } void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { From d93b4f88b38c7aa5616d9c58f0f097741bcc713a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Jan 2024 15:23:15 +0800 Subject: [PATCH 146/198] fix(tsdb/cache): destory brin block with fs loading --- source/dnode/vnode/src/tsdb/tsdbCache.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index a39c74a679..ab504acea7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1955,6 +1955,7 @@ typedef struct SFSNextRowIter { SArray *pIndexList; int32_t iBrinIndex; SBrinBlock brinBlock; + SBrinBlock *pBrinBlock; int32_t iBrinRecord; SBrinRecord brinRecord; SBlockData blockData; @@ -2143,6 +2144,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie pBrinBlk = taosArrayGet(state->pIndexList, state->iBrinIndex); } + if (!state->pBrinBlock) { + state->pBrinBlock = &state->brinBlock; + } else { + tBrinBlockClear(&state->brinBlock); + } code = tsdbDataFileReadBrinBlock(state->pr->pFileReader, pBrinBlk, &state->brinBlock); if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -2420,6 +2426,16 @@ int32_t clearNextRowFromFS(void *iter) { state->pBlockData = NULL; } + if (state->pBrinBlock) { + tBrinBlockDestroy(state->pBrinBlock); + state->pBrinBlock = NULL; + } + + if (state->pIndexList) { + taosArrayDestroy(state->pIndexList); + state->pIndexList = NULL; + } + if (state->pTSRow) { taosMemoryFree(state->pTSRow); state->pTSRow = NULL; From 404e17e67797f99f1c491171348ac463e0f99261 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 9 Jan 2024 16:18:55 +0800 Subject: [PATCH 147/198] fix: get expired user for each cluster --- source/client/src/clientHb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1de5941ac3..b3d9a9ced5 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -902,6 +902,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } + taosHashPut(clientHbMgr.appHbHash, &hbParam->clusterId, sizeof(uint64_t), NULL, 0); } // invoke after hbGetExpiredUserInfo @@ -1121,7 +1122,6 @@ static void *hbThreadFunc(void *param) { asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); - taosHashPut(clientHbMgr.appHbHash, &pAppHbMgr->pAppInstInfo->clusterId, sizeof(uint64_t), NULL, 0); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } From 6444af84e47f5ed90f1af99e7e549a2be968f97b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 9 Jan 2024 16:28:58 +0800 Subject: [PATCH 148/198] opti:[TD-28142]memory size if block num is too large --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 109 ++++++++++++--------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 25 ++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 19 +++- source/util/src/tarray.c | 11 ++- 4 files changed, 110 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index b57d2dfb45..255d5d4105 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -882,6 +882,21 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo } } +static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo){ + record->uid = pBlockInfo->uid; + record->firstKey = pBlockInfo->firstKey; + record->lastKey = pBlockInfo->lastKey; + record->minVer = pBlockInfo->minVer; + record->maxVer = pBlockInfo->maxVer; + record->blockOffset = pBlockInfo->blockOffset; + record->smaOffset = pBlockInfo->smaOffset; + record->blockSize = pBlockInfo->blockSize; + record->blockKeySize = pBlockInfo->blockKeySize; + record->smaSize = pBlockInfo->smaSize; + record->numRow = pBlockInfo->numRow; + record->count = pBlockInfo->count; +} + static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -899,7 +914,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pReader->info.order); int32_t step = asc ? 1 : -1; - SBrinRecord* pRecord = &pBlockInfo->record; + SBrinRecord tmp; + blockInfoToRecord(&tmp, pBlockInfo); + SBrinRecord* pRecord = &tmp; // no data exists, return directly. if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) { @@ -1092,14 +1109,16 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBrinRecord* pRecord = &pBlockInfo->record; + SBrinRecord tmp; + blockInfoToRecord(&tmp, pBlockInfo); + SBrinRecord* pRecord = &tmp; code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1], pSup->numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, code:%s %s", - pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.firstKey, - pBlockInfo->record.lastKey, pBlockInfo->record.numRow, tstrerror(code), pReader->idStr); + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->firstKey, + pBlockInfo->lastKey, pBlockInfo->numRow, tstrerror(code), pReader->idStr); return code; } @@ -1120,10 +1139,10 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI * This is an two rectangles overlap cases. */ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SFileDataBlockInfo* pBlock) { - return (pWindow->ekey < pBlock->record.lastKey && pWindow->ekey >= pBlock->record.firstKey) || - (pWindow->skey > pBlock->record.firstKey && pWindow->skey <= pBlock->record.lastKey) || - (pVerRange->minVer > pBlock->record.minVer && pVerRange->minVer <= pBlock->record.maxVer) || - (pVerRange->maxVer < pBlock->record.maxVer && pVerRange->maxVer >= pBlock->record.minVer); + return (pWindow->ekey < pBlock->lastKey && pWindow->ekey >= pBlock->firstKey) || + (pWindow->skey > pBlock->firstKey && pWindow->skey <= pBlock->lastKey) || + (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || + (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, @@ -1142,7 +1161,7 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); - memcpy(pRecord, &p->record, sizeof(SBrinRecord)); + blockInfoToRecord(pRecord, p); *nextIndex = pBlockInfo->tbBlockIdx + step; return true; @@ -1211,9 +1230,9 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order) { // it is the last block in current file, no chance to overlap with neighbor blocks. if (ASCENDING_TRAVERSE(order)) { - return pBlock->record.lastKey == pRec->firstKey; + return pBlock->lastKey == pRec->firstKey; } else { - return pBlock->record.firstKey == pRec->lastKey; + return pBlock->firstKey == pRec->lastKey; } } @@ -1223,9 +1242,9 @@ static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanI int64_t key = 0; if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; - key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt); + key = ascScan ? TMIN(pBlock->firstKey, keyInStt) : TMAX(pBlock->lastKey, keyInStt); } else { - key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey; + key = ascScan ? pBlock->firstKey : pBlock->lastKey; } return key; @@ -1241,8 +1260,8 @@ static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBloc } static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) { - return (key.ts >= pBlock->record.firstKey && key.ts <= pBlock->record.lastKey) && - (pBlock->record.maxVer >= pVerRange->minVer) && (pBlock->record.minVer <= pVerRange->maxVer); + return (key.ts >= pBlock->firstKey && key.ts <= pBlock->lastKey) && + (pBlock->maxVer >= pVerRange->minVer) && (pBlock->minVer <= pVerRange->maxVer); } static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, @@ -1258,18 +1277,20 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order); } + SBrinRecord pRecord; + blockInfoToRecord(&pRecord, pBlockInfo); // has duplicated ts of different version in this block - pInfo->hasDupTs = (pBlockInfo->record.numRow > pBlockInfo->record.count) || (pBlockInfo->record.count <= 0); - pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pBlockInfo->record, pReader->info.order); + pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0); + pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, pReader->info.order); ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey; pInfo->overlapWithSttBlock = - !(pBlockInfo->record.lastKey < nextProcKeyInStt || pBlockInfo->record.firstKey > nextProcKeyInStt); + !(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt); } - pInfo->moreThanCapcity = pBlockInfo->record.numRow > pReader->resBlockInfo.capacity; + pInfo->moreThanCapcity = pBlockInfo->numRow > pReader->resBlockInfo.capacity; pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo); pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange); } @@ -2361,17 +2382,14 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBrinRecord* pRecord = NULL; STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo == NULL) { return 0; } - pRecord = &pBlockInfo->record; - if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; } @@ -2380,13 +2398,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { goto _end; } - pRecord = &pBlockInfo->record; TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly int64_t cap = pReader->resBlockInfo.capacity; - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pRecord->numRow <= cap)) { - if (((asc && (pRecord->firstKey < keyInBuf.ts)) || (!asc && (pRecord->lastKey > keyInBuf.ts))) && + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pBlockInfo->numRow <= cap)) { + if (((asc && (pBlockInfo->firstKey < keyInBuf.ts)) || (!asc && (pBlockInfo->lastKey > keyInBuf.ts))) && (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA)) { code = copyBlockDataToSDataBlock(pReader); if (code) { @@ -2394,7 +2411,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } // record the last key value - pBlockScanInfo->lastProcKey = asc ? pRecord->lastKey : pRecord->firstKey; + pBlockScanInfo->lastProcKey = asc ? pBlockInfo->lastKey : pBlockInfo->firstKey; goto _end; } } @@ -2422,7 +2439,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { bool loadNeighbor = false; code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor); if ((!loadNeighbor) || (code != 0)) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); break; } } @@ -2441,7 +2458,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { - setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order); break; } @@ -2801,7 +2818,7 @@ static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockSc return true; } else { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; - return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt); + return (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt); } } @@ -2815,7 +2832,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); return code; } @@ -2850,20 +2867,20 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (notOverlapWithSttFiles(pBlockInfo, pScanInfo, asc)) { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; - pInfo->rows = pBlockInfo->record.numRow; + pInfo->rows = pBlockInfo->numRow; pInfo->id.uid = pScanInfo->uid; pInfo->dataLoad = 0; - pInfo->window = (STimeWindow){.skey = pBlockInfo->record.firstKey, .ekey = pBlockInfo->record.lastKey}; + pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey}; setComposedBlockFlag(pReader, false); - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); // update the last key for the corresponding table pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey; tsdbDebug("%p uid:%" PRIu64 " clean file block retrieved from file, global index:%d, " "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", - pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.numRow, - pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, + pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); } else { SBlockData* pBData = &pReader->status.fileBlockData; tBlockDataReset(pBData); @@ -2892,10 +2909,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // data in stt now overlaps with current active file data block, need to composed with file data block. int64_t lastKeyInStt = getCurrentKeyInSttBlock(pSttBlockReader); - if ((lastKeyInStt >= pBlockInfo->record.firstKey && asc) || - (lastKeyInStt <= pBlockInfo->record.lastKey && (!asc))) { + if ((lastKeyInStt >= pBlockInfo->firstKey && asc) || + (lastKeyInStt <= pBlockInfo->lastKey && (!asc))) { tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader, - lastKeyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + lastKeyInStt, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); break; } } @@ -3012,8 +3029,8 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) lastKey = pScanInfo->lastProcKey; } - pDumpInfo->totalRows = pBlockInfo->record.numRow; - pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->record.numRow - 1; + pDumpInfo->totalRows = pBlockInfo->numRow; + pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { pDumpInfo->totalRows = 0; pDumpInfo->rowIndex = 0; @@ -4632,7 +4649,9 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, // int64_t st = taosGetTimestampUs(); TARRAY2_CLEAR(&pSup->colAggArray, 0); - code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pFBlock->record, &pSup->colAggArray); + SBrinRecord pRecord; + blockInfoToRecord(&pRecord, pFBlock); + code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pRecord, &pSup->colAggArray); if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), pReader->idStr); @@ -4663,7 +4682,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pFBlock->record.numRow, numOfCols, pTsAgg); + doFillNullColSMA(pSup, pFBlock->numRow, numOfCols, pTsAgg); size_t size = pSup->colAggArray.size; @@ -4884,7 +4903,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT while (true) { if (hasNext) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); - int32_t numOfRows = pBlockInfo->record.numRow; + int32_t numOfRows = pBlockInfo->numRow; pTableBlockInfo->totalRows += numOfRows; @@ -4896,7 +4915,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT pTableBlockInfo->minRows = numOfRows; } - pTableBlockInfo->totalSize += pBlockInfo->record.blockSize; + pTableBlockInfo->totalSize += pBlockInfo->blockSize; int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets); pTableBlockInfo->blockRowsHisto[bucketIndex]++; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 33b7667168..b4b92c0008 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -359,6 +359,21 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v return pLeftBlock->offset > pRightBlock->offset ? 1 : -1; } +static void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record){ + pBlockInfo->uid = record->uid; + pBlockInfo->firstKey = record->firstKey; + pBlockInfo->lastKey = record->lastKey; + pBlockInfo->minVer = record->minVer; + pBlockInfo->maxVer = record->maxVer; + pBlockInfo->blockOffset = record->blockOffset; + pBlockInfo->smaOffset = record->smaOffset; + pBlockInfo->blockSize = record->blockSize; + pBlockInfo->blockKeySize = record->blockKeySize; + pBlockInfo->smaSize = record->smaSize; + pBlockInfo->numRow = record->numRow; + pBlockInfo->count = record->count; +} + int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->info.order); @@ -416,8 +431,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 pTableScanInfo->pBlockIdxList = taosArrayInit(numOfBlocks, sizeof(STableDataBlockIdx)); } for (int32_t i = 0; i < numOfBlocks; ++i) { - SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; - blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); + SFileDataBlockInfo blockInfo = {.tbBlockIdx = i}; + SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); + recordToBlockInfo(&blockInfo, record); taosArrayPush(pBlockIter->blockList, &blockInfo); STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i}; @@ -450,8 +466,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 int32_t pos = tMergeTreeGetChosenIndex(pTree); int32_t index = sup.indexPerTable[pos]++; - SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[pos][index].uid, .tbBlockIdx = index}; - blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + SFileDataBlockInfo blockInfo = {.tbBlockIdx = index}; + SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + recordToBlockInfo(&blockInfo, record); taosArrayPush(pBlockIter->blockList, &blockInfo); STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 39e65f22b1..cd9942a0a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -175,9 +175,22 @@ typedef struct SFilesetIter { typedef struct SFileDataBlockInfo { // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it - uint64_t uid; - int32_t tbBlockIdx; - SBrinRecord record; + // int64_t suid; + int64_t uid; + int64_t firstKey; +// int64_t firstKeyVer; + int64_t lastKey; +// int64_t lastKeyVer; + int64_t minVer; + int64_t maxVer; + int64_t blockOffset; + int64_t smaOffset; + int32_t blockSize; + int32_t blockKeySize; + int32_t smaSize; + int32_t numRow; + int32_t count; + int32_t tbBlockIdx; } SFileDataBlockInfo; typedef struct SDataBlockIter { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index bc98461592..88eb51d500 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -18,6 +18,9 @@ #include "tcoding.h" // todo refactor API +#define BOUNDARY_SIZE 1024*1024*1024 // 1G +#define BOUNDARY_SMALL_FACTOR 1.2 +#define BOUNDARY_BIG_FACTOR 2 SArray* taosArrayInit(size_t size, size_t elemSize) { if (elemSize == 0) { @@ -85,9 +88,13 @@ static int32_t taosArrayResize(SArray* pArray) { int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { if (newCap > pArray->capacity) { - size_t tsize = (pArray->capacity << 1u); + float factor = BOUNDARY_BIG_FACTOR; + if(newCap * pArray->elemSize > BOUNDARY_SIZE){ + factor = BOUNDARY_SMALL_FACTOR; + } + size_t tsize = (pArray->capacity * factor); while (newCap > tsize) { - tsize = (tsize << 1u); + tsize = (tsize * factor); } pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); From 7979d4532bfd9ce3e0c6291e7bd80ad3eec20ed9 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 9 Jan 2024 16:49:27 +0800 Subject: [PATCH 149/198] enh: let max num of concurrent msgs of snap replication be configurable --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 5 +++++ source/libs/sync/src/syncSnapshot.c | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index f006779a48..8f8ddc632f 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -74,6 +74,7 @@ extern int64_t tsRpcQueueMemoryAllowed; extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; extern int32_t tsHeartbeatTimeout; +extern int32_t tsSnapReplicationMsgNLimit; // vnode extern int64_t tsVndCommitMaxIntervalMs; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 79d21955d4..5228226235 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -72,6 +72,7 @@ int32_t tsPQSortMemThreshold = 16; // M int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; int32_t tsHeartbeatTimeout = 20 * 1000; +int32_t tsSnapReplicationMsgNLimit = 128; // mnode int64_t tsMndSdbWriteDelta = 200; @@ -673,6 +674,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "syncSnapReplicationMsgNLimit", tsSnapReplicationMsgNLimit, 16, + (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; @@ -1183,6 +1187,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsElectInterval = cfgGetItem(pCfg, "syncElectInterval")->i32; tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; + tsSnapReplicationMsgNLimit = cfgGetItem(pCfg, "syncSnapReplicationMsgNLimit")->i32; tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index f0e457ef8d..564d28b0f4 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -22,6 +22,7 @@ #include "syncRaftStore.h" #include "syncReplication.h" #include "syncUtil.h" +#include "tglobal.h" static SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths); @@ -1186,7 +1187,7 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp pSndBuf->start = ack + 1; } - while (pSender->seq != SYNC_SNAPSHOT_SEQ_END && pSender->seq - pSndBuf->start < (pSndBuf->size >> 2)) { + while (pSender->seq != SYNC_SNAPSHOT_SEQ_END && pSender->seq - pSndBuf->start < tsSnapReplicationMsgNLimit) { if (snapshotSend(pSender) != 0) { code = terrno; goto _out; From 220cef4a61c1c2a8d857c812549b2ab378e6dd47 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Tue, 9 Jan 2024 11:02:10 +0000 Subject: [PATCH 150/198] opt msg on mnode --- source/dnode/mnode/impl/src/mndMain.c | 164 +++++++++++++++++--------- 1 file changed, 109 insertions(+), 55 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 38a92b43e7..c0fd93c65d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" #include "mndCluster.h" +#include "mndCompact.h" +#include "mndCompactDetail.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -42,8 +44,6 @@ #include "mndUser.h" #include "mndVgroup.h" #include "mndView.h" -#include "mndCompact.h" -#include "mndCompactDetail.h" static inline int32_t mndAcquireRpc(SMnode *pMnode) { int32_t code = 0; @@ -158,16 +158,16 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { } } -static void mndStreamCheckpointRemain(SMnode* pMnode) { +static void mndStreamCheckpointRemain(SMnode *pMnode) { int32_t contLen = 0; - void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); + void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } } -static void mndStreamCheckNode(SMnode* pMnode) { +static void mndStreamCheckNode(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -275,6 +275,100 @@ static void mndCheckDnodeOffline(SMnode *pMnode) { mndReleaseRpc(pMnode); } +static bool mnodeIsNotLeader(SMnode *pMnode) { + terrno = 0; + taosThreadRwlockRdlock(&pMnode->lock); + SSyncState state = syncGetState(pMnode->syncMgmt.sync); + if (terrno != 0) { + taosThreadRwlockUnlock(&pMnode->lock); + return true; + } + + if (state.state != TAOS_SYNC_STATE_LEADER) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_NOT_LEADER; + return true; + } + if (!state.restored || !pMnode->restored) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_RESTORING; + return true; + } + taosThreadRwlockUnlock(&pMnode->lock); + return false; +} + +static int32_t minCronTime() { + int64_t min = INT64_MAX; + min = TMIN(min, tsTtlPushIntervalSec); + min = TMIN(min, tsTrimVDbIntervalSec); + min = TMIN(min, tsTransPullupInterval); + min = TMIN(min, tsCompactPullupInterval); + min = TMIN(min, tsMqRebalanceInterval); + min = TMIN(min, tsStreamCheckpointInterval); + min = TMIN(min, 5); // checkpointRemain + min = TMIN(min, tsStreamNodeCheckInterval); + + int64_t telemInt = TMIN(60, (tsTelemInterval - 1)); + min = TMIN(min, telemInt); + min = TMIN(min, tsGrantHBInterval); + min = TMIN(min, tsUptimeInterval); + + return min <= 1 ? 2 : min; +} +void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { + if (sec % tsTtlPushIntervalSec == 0) { + mndPullupTtl(pMnode); + } + + if (sec % tsTrimVDbIntervalSec == 0) { + mndPullupTrimDb(pMnode); + } + + if (sec % tsTransPullupInterval == 0) { + mndPullupTrans(pMnode); + } + + if (sec % tsCompactPullupInterval == 0) { + mndPullupCompacts(pMnode); + } + + if (sec % tsMqRebalanceInterval == 0) { + mndCalMqRebalance(pMnode); + } + + if (sec % tsStreamCheckpointInterval == 0) { + mndStreamCheckpointTick(pMnode, sec); + } + + if (sec % 5 == 0) { + mndStreamCheckpointRemain(pMnode); + } + + if (sec % tsStreamNodeCheckInterval == 0) { + mndStreamCheckNode(pMnode); + } + + if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) { + mndPullupTelem(pMnode); + } + + if (sec % tsGrantHBInterval == 0) { + mndPullupGrant(pMnode); + } + + if (sec % tsUptimeInterval == 0) { + mndIncreaseUpTime(pMnode); + } +} +void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) { + if (sec % (tsStatusInterval * 5) == 0) { + mndCheckDnodeOffline(pMnode); + } + if (sec % (MNODE_TIMEOUT_SEC / 2) == 0) { + mndSyncCheckTimeout(pMnode); + } +} static void *mndThreadFp(void *param) { SMnode *pMnode = param; int64_t lastTime = 0; @@ -287,57 +381,15 @@ static void *mndThreadFp(void *param) { if (lastTime % 10 != 0) continue; int64_t sec = lastTime / 10; - if (sec % tsTtlPushIntervalSec == 0) { - mndPullupTtl(pMnode); - } + mndDoTimerCheckTask(pMnode, sec); - if (sec % tsTrimVDbIntervalSec == 0) { - mndPullupTrimDb(pMnode); - } - - if (sec % tsTransPullupInterval == 0) { - mndPullupTrans(pMnode); - } - - if (sec % tsCompactPullupInterval == 0) { - mndPullupCompacts(pMnode); - } - - if (sec % tsMqRebalanceInterval == 0) { - mndCalMqRebalance(pMnode); - } - - if (sec % tsStreamCheckpointInterval == 0) { - mndStreamCheckpointTick(pMnode, sec); - } - - if (sec % 5 == 0) { - mndStreamCheckpointRemain(pMnode); - } - - if (sec % tsStreamNodeCheckInterval == 0) { - mndStreamCheckNode(pMnode); - } - - if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) { - mndPullupTelem(pMnode); - } - - if (sec % tsGrantHBInterval == 0) { - mndPullupGrant(pMnode); - } - - if (sec % tsUptimeInterval == 0) { - mndIncreaseUpTime(pMnode); - } - - if (sec % (tsStatusInterval * 5) == 0) { - mndCheckDnodeOffline(pMnode); - } - - if (sec % (MNODE_TIMEOUT_SEC / 2) == 0) { - mndSyncCheckTimeout(pMnode); + int64_t minCron = minCronTime(); + if (sec % minCron == 0 && mnodeIsNotLeader(pMnode)) { + // not leader, do nothing + mTrace("timer not process since mnode is not leader, reason: %s", tstrerror(terrno)) terrno = 0; + continue; } + mndDoTimerPullupTask(pMnode, sec); } return NULL; @@ -712,7 +764,9 @@ _OVER: if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER || - pMsg->msgType == TDMT_MND_COMPACT_TIMER) { + pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER || + pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE || + pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, pMnode->stopped, state.restored, syncStr(state.state)); return -1; From dc0c5539a0f8fc2478bef1ba874f439f2d94790b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Jan 2024 08:48:49 +0800 Subject: [PATCH 151/198] fix(stream): remove invalid timer controllers. --- source/dnode/vnode/src/tq/tqStreamTask.c | 6 ++---- source/libs/stream/src/streamMeta.c | 10 +++++----- source/libs/stream/src/streamStart.c | 7 ++++--- source/libs/wal/src/walRead.c | 12 ++++++------ 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 95a4474460..2a600d08ef 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -19,8 +19,6 @@ #define MAX_REPEAT_SCAN_THRESHOLD 3 #define SCAN_WAL_IDLE_DURATION 100 -void* tqTimer = NULL; - static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle); static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId); static bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver); @@ -98,9 +96,9 @@ int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { pParam->pTq = pTq; pParam->numOfTasks = numOfTasks; if (pMeta->scanInfo.scanTimer == NULL) { - pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, tqTimer); + pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTq->tqTimer); } else { - taosTmrReset(doStartScanWal, idleDuration, pParam, tqTimer, &pMeta->scanInfo.scanTimer); + taosTmrReset(doStartScanWal, idleDuration, pParam, pTq->tqTimer, &pMeta->scanInfo.scanTimer); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index f687841016..15256ececf 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -23,7 +23,7 @@ #include "ttimer.h" #include "wal.h" -TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; +static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; int32_t streamBackendId = 0; int32_t streamBackendCfWrapperId = 0; @@ -64,7 +64,7 @@ static void streamMetaEnvInit() { streamTimerInit(); } -void streamMetaInit() { /*taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit);*/ streamMetaEnvInit();} +void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit);} void streamMetaCleanup() { taosCloseRef(streamBackendId); @@ -393,7 +393,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); metaRefMgtAdd(pMeta->vgId, pRid); -// pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); + pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); pMeta->pHbInfo->tickCounter = 0; pMeta->pHbInfo->stopFlag = 0; pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); @@ -1204,7 +1204,7 @@ void metaHbToMnode(void* param, void* tmrId) { } if (!waitForEnoughDuration(pMeta->pHbInfo)) { -// taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); return; } @@ -1214,7 +1214,7 @@ void metaHbToMnode(void* param, void* tmrId) { metaHeartbeatToMnodeImpl(pMeta); streamMetaRUnLock(pMeta); -// taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 52c3c87431..c852215d16 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -345,16 +345,17 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { SStreamTaskState* p = streamTaskGetStatus(pTask); ASSERT(p->state == TASK_STATUS__READY); + int8_t schedStatus = pTask->status.schedStatus; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { int64_t startVer = walReaderGetCurrentVer(pTask->exec.pWalReader); if (startVer == -1) { startVer = pTask->chkInfo.nextProcessVer; } - stDebug("s-task:%s no need to scan-history data, status:%s, sched-status:%d, ready for data from wal ver:%" PRId64, - id, p->name, pTask->status.schedStatus, startVer); + stDebug("s-task:%s status:%s, sched-status:%d, ready for data from wal ver:%" PRId64, id, p->name, schedStatus, + startVer); } else { - stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p->name, pTask->status.schedStatus); + stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p->name, schedStatus); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1660a0ecf0..6748d161ae 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -305,8 +305,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) { } int32_t walSkipFetchBody(SWalReader *pRead) { - wDebug("vgId:%d, skip fetch body %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64", 0x%"PRIx64, + wDebug("vgId:%d, skip fetch body:%" PRId64 ", first:%" PRId64 ", commit:%" PRId64 ", last:%" PRId64 + ", applied:%" PRId64 ", 0x%" PRIx64, pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId); @@ -325,11 +325,11 @@ int32_t walFetchBody(SWalReader *pRead) { int64_t ver = pReadHead->version; int32_t vgId = pRead->pWal->cfg.vgId; int64_t id = pRead->readerId; + SWalVer *pVer = &pRead->pWal->vers; - wDebug("vgId:%d, fetch body %" PRId64 ", first ver:%" PRId64 ", commit ver:%" PRId64 ", last ver:%" PRId64 - ", applied ver:%" PRId64 ", 0x%" PRIx64, - vgId, ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, - pRead->pWal->vers.appliedVer, id); + wDebug("vgId:%d, fetch body:%" PRId64 ", first:%" PRId64 ", commit:%" PRId64 ", last:%" PRId64 ", applied:%" PRId64 + ", 0x%" PRIx64, + vgId, ver, pVer->firstVer, pVer->commitVer, pVer->lastVer, pVer->appliedVer, id); if (pRead->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); From 9dbddeed16ad722f56735f4f993dfe02f121b588 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 10 Jan 2024 10:03:38 +0800 Subject: [PATCH 152/198] opt msg on mnd --- source/common/src/tglobal.c | 137 +++++++++++++------------- source/dnode/mnode/impl/src/mndMain.c | 39 ++++---- 2 files changed, 86 insertions(+), 90 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 79d21955d4..76e1ab46b7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -95,9 +95,9 @@ int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; // audit -bool tsEnableAudit = true; -bool tsEnableAuditCreateTable = true; -int32_t tsAuditInterval = 5000; +bool tsEnableAudit = true; +bool tsEnableAuditCreateTable = true; +int32_t tsAuditInterval = 5000; // telem #ifdef TD_ENTERPRISE @@ -252,7 +252,7 @@ int32_t tsCompactPullupInterval = 10; int32_t tsMqRebalanceInterval = 2; int32_t tsStreamCheckpointInterval = 60; float tsSinkDataRate = 2.0; -int32_t tsStreamNodeCheckInterval = 15; +int32_t tsStreamNodeCheckInterval = 16; int32_t tsTtlUnit = 86400; int32_t tsTtlPushIntervalSec = 10; int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups @@ -282,7 +282,7 @@ int32_t tsS3BlockCacheSize = 16; // number of blocks int32_t tsS3PageCacheSize = 4096; // number of pages int32_t tsS3UploadDelaySec = 60 * 60 * 24; -bool tsExperimental = true; +bool tsExperimental = true; #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -691,8 +691,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1; @@ -711,8 +710,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != - 0) + if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) @@ -1465,40 +1464,38 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, }; - static OptionNameAndVar options[] = { - {"audit", &tsEnableAudit}, - {"asynclog", &tsAsyncLog}, - {"disableStream", &tsDisableStream}, - {"enableWhiteList", &tsEnableWhiteList}, - {"telemetryReporting", &tsEnableTelem}, - {"monitor", &tsEnableMonitor}, + static OptionNameAndVar options[] = {{"audit", &tsEnableAudit}, + {"asynclog", &tsAsyncLog}, + {"disableStream", &tsDisableStream}, + {"enableWhiteList", &tsEnableWhiteList}, + {"telemetryReporting", &tsEnableTelem}, + {"monitor", &tsEnableMonitor}, - {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, - {"minDiskFreeSize", &tsMinDiskFreeSize}, + {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, + {"minDiskFreeSize", &tsMinDiskFreeSize}, - {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, - {"checkpointInterval", &tsStreamCheckpointInterval}, - {"keepAliveIdle", &tsKeepAliveIdle}, - {"logKeepDays", &tsLogKeepDays}, - {"maxStreamBackendCache", &tsMaxStreamBackendCache}, - {"mqRebalanceInterval", &tsMqRebalanceInterval}, - {"numOfLogLines", &tsNumOfLogLines}, - {"queryRspPolicy", &tsQueryRspPolicy}, - {"timeseriesThreshold", &tsTimeSeriesThreshold}, - {"tmqMaxTopicNum", &tmqMaxTopicNum}, - {"transPullupInterval", &tsTransPullupInterval}, - {"compactPullupInterval", &tsCompactPullupInterval}, - {"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, - {"ttlBatchDropNum", &tsTtlBatchDropNum}, - {"ttlFlushThreshold", &tsTtlFlushThreshold}, - {"ttlPushInterval", &tsTtlPushIntervalSec}, - //{"s3BlockSize", &tsS3BlockSize}, - {"s3BlockCacheSize", &tsS3BlockCacheSize}, - {"s3PageCacheSize", &tsS3PageCacheSize}, - {"s3UploadDelaySec", &tsS3UploadDelaySec}, - {"supportVnodes", &tsNumOfSupportVnodes}, - {"experimental", &tsExperimental} - }; + {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, + {"checkpointInterval", &tsStreamCheckpointInterval}, + {"keepAliveIdle", &tsKeepAliveIdle}, + {"logKeepDays", &tsLogKeepDays}, + {"maxStreamBackendCache", &tsMaxStreamBackendCache}, + {"mqRebalanceInterval", &tsMqRebalanceInterval}, + {"numOfLogLines", &tsNumOfLogLines}, + {"queryRspPolicy", &tsQueryRspPolicy}, + {"timeseriesThreshold", &tsTimeSeriesThreshold}, + {"tmqMaxTopicNum", &tmqMaxTopicNum}, + {"transPullupInterval", &tsTransPullupInterval}, + {"compactPullupInterval", &tsCompactPullupInterval}, + {"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, + {"ttlBatchDropNum", &tsTtlBatchDropNum}, + {"ttlFlushThreshold", &tsTtlFlushThreshold}, + {"ttlPushInterval", &tsTtlPushIntervalSec}, + //{"s3BlockSize", &tsS3BlockSize}, + {"s3BlockCacheSize", &tsS3BlockCacheSize}, + {"s3PageCacheSize", &tsS3PageCacheSize}, + {"s3UploadDelaySec", &tsS3UploadDelaySec}, + {"supportVnodes", &tsNumOfSupportVnodes}, + {"experimental", &tsExperimental}}; if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { taosCfgSetOption(options, tListLen(options), pItem, false); @@ -1692,36 +1689,34 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { {"cDebugFlag", &cDebugFlag}, {"dDebugFlag", &dDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"jniDebugFlag", &jniDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, - {"uDebugFlag", &uDebugFlag}, {"simDebugFlag", &simDebugFlag}, + {"uDebugFlag", &uDebugFlag}, {"simDebugFlag", &simDebugFlag}, }; - static OptionNameAndVar options[] = { - {"asyncLog", &tsAsyncLog}, - {"assert", &tsAssert}, - {"compressMsgSize", &tsCompressMsgSize}, - {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, - {"crashReporting", &tsEnableCrashReport}, - {"enableCoreFile", &tsAsyncLog}, - {"enableQueryHb", &tsEnableQueryHb}, - {"keepColumnName", &tsKeepColumnName}, - {"keepAliveIdle", &tsKeepAliveIdle}, - {"logKeepDays", &tsLogKeepDays}, - {"maxInsertBatchRows", &tsMaxInsertBatchRows}, - {"maxRetryWaitTime", &tsMaxRetryWaitTime}, - {"minSlidingTime", &tsMinSlidingTime}, - {"minIntervalTime", &tsMinIntervalTime}, - {"numOfLogLines", &tsNumOfLogLines}, - {"querySmaOptimize", &tsQuerySmaOptimize}, - {"queryPolicy", &tsQueryPolicy}, - {"queryPlannerTrace", &tsQueryPlannerTrace}, - {"queryNodeChunkSize", &tsQueryNodeChunkSize}, - {"queryUseNodeAllocator", &tsQueryUseNodeAllocator}, - {"smlDot2Underline", &tsSmlDot2Underline}, - {"shellActivityTimer", &tsShellActivityTimer}, - {"slowLogThreshold", &tsSlowLogThreshold}, - {"useAdapter", &tsUseAdapter}, - {"experimental", &tsExperimental} - }; + static OptionNameAndVar options[] = {{"asyncLog", &tsAsyncLog}, + {"assert", &tsAssert}, + {"compressMsgSize", &tsCompressMsgSize}, + {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, + {"crashReporting", &tsEnableCrashReport}, + {"enableCoreFile", &tsAsyncLog}, + {"enableQueryHb", &tsEnableQueryHb}, + {"keepColumnName", &tsKeepColumnName}, + {"keepAliveIdle", &tsKeepAliveIdle}, + {"logKeepDays", &tsLogKeepDays}, + {"maxInsertBatchRows", &tsMaxInsertBatchRows}, + {"maxRetryWaitTime", &tsMaxRetryWaitTime}, + {"minSlidingTime", &tsMinSlidingTime}, + {"minIntervalTime", &tsMinIntervalTime}, + {"numOfLogLines", &tsNumOfLogLines}, + {"querySmaOptimize", &tsQuerySmaOptimize}, + {"queryPolicy", &tsQueryPolicy}, + {"queryPlannerTrace", &tsQueryPlannerTrace}, + {"queryNodeChunkSize", &tsQueryNodeChunkSize}, + {"queryUseNodeAllocator", &tsQueryUseNodeAllocator}, + {"smlDot2Underline", &tsSmlDot2Underline}, + {"shellActivityTimer", &tsShellActivityTimer}, + {"slowLogThreshold", &tsSlowLogThreshold}, + {"useAdapter", &tsUseAdapter}, + {"experimental", &tsExperimental}}; if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { taosCfgSetOption(options, tListLen(options), pItem, false); @@ -1763,7 +1758,7 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag void taosSetAllDebugFlag(int32_t flag) { if (flag <= 0) return; - SArray *noNeedToSetVars = NULL; + SArray * noNeedToSetVars = NULL; SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); if (pItem != NULL) { pItem->i32 = flag; @@ -1798,7 +1793,7 @@ void taosSetAllDebugFlag(int32_t flag) { taosArrayClear(noNeedToSetVars); // reset array uInfo("all debug flag are set to %d", flag); - if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist + if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist } int8_t taosGranted() { return atomic_load_8(&tsGrant); } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index c0fd93c65d..4bd0776ecd 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -107,7 +107,7 @@ static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) { static void mndPullupTrans(SMnode *pMnode) { mTrace("pullup trans msg"); int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); @@ -117,7 +117,7 @@ static void mndPullupTrans(SMnode *pMnode) { static void mndPullupCompacts(SMnode *pMnode) { mTrace("pullup compact timer msg"); int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_COMPACT_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); @@ -127,7 +127,7 @@ static void mndPullupCompacts(SMnode *pMnode) { static void mndPullupTtl(SMnode *pMnode) { mTrace("pullup ttl"); int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } @@ -135,14 +135,14 @@ static void mndPullupTtl(SMnode *pMnode) { static void mndPullupTrimDb(SMnode *pMnode) { mTrace("pullup trim"); int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRIM_DB_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } static void mndCalMqRebalance(SMnode *pMnode) { int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -151,7 +151,7 @@ static void mndCalMqRebalance(SMnode *pMnode) { static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { int32_t contLen = 0; - void *pReq = mndBuildCheckpointTickMsg(&contLen, sec); + void * pReq = mndBuildCheckpointTickMsg(&contLen, sec); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -160,7 +160,7 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { static void mndStreamCheckpointRemain(SMnode *pMnode) { int32_t contLen = 0; - void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); + void * pReq = mndBuildCheckpointTickMsg(&contLen, 0); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -169,7 +169,7 @@ static void mndStreamCheckpointRemain(SMnode *pMnode) { static void mndStreamCheckNode(SMnode *pMnode) { int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_NODECHECK_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -179,7 +179,7 @@ static void mndStreamCheckNode(SMnode *pMnode) { static void mndPullupTelem(SMnode *pMnode) { mTrace("pullup telem msg"); int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -189,7 +189,7 @@ static void mndPullupTelem(SMnode *pMnode) { static void mndPullupGrant(SMnode *pMnode) { mTrace("pullup grant msg"); int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = { .msgType = TDMT_MND_GRANT_HB_TIMER, .pCont = pReq, .contLen = contLen, .info.ahandle = (void *)0x9527}; @@ -200,7 +200,7 @@ static void mndPullupGrant(SMnode *pMnode) { static void mndIncreaseUpTime(SMnode *pMnode) { mTrace("increate uptime"); int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); + void * pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = { .msgType = TDMT_MND_UPTIME_TIMER, .pCont = pReq, .contLen = contLen, .info.ahandle = (void *)0x9528}; @@ -254,7 +254,7 @@ static void mndCheckDnodeOffline(SMnode *pMnode) { mTrace("check dnode offline"); if (mndAcquireRpc(pMnode) != 0) return; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; int64_t curMs = taosGetTimestampMs(); void *pIter = NULL; @@ -299,14 +299,14 @@ static bool mnodeIsNotLeader(SMnode *pMnode) { } static int32_t minCronTime() { - int64_t min = INT64_MAX; + int32_t min = INT32_MAX; min = TMIN(min, tsTtlPushIntervalSec); min = TMIN(min, tsTrimVDbIntervalSec); min = TMIN(min, tsTransPullupInterval); min = TMIN(min, tsCompactPullupInterval); min = TMIN(min, tsMqRebalanceInterval); min = TMIN(min, tsStreamCheckpointInterval); - min = TMIN(min, 5); // checkpointRemain + min = TMIN(min, 6); // checkpointRemain min = TMIN(min, tsStreamNodeCheckInterval); int64_t telemInt = TMIN(60, (tsTelemInterval - 1)); @@ -386,7 +386,8 @@ static void *mndThreadFp(void *param) { int64_t minCron = minCronTime(); if (sec % minCron == 0 && mnodeIsNotLeader(pMnode)) { // not leader, do nothing - mTrace("timer not process since mnode is not leader, reason: %s", tstrerror(terrno)) terrno = 0; + mTrace("timer not process since mnode is not leader, reason: %s", tstrerror(terrno)); + terrno = 0; continue; } mndDoTimerPullupTask(pMnode, sec); @@ -700,7 +701,7 @@ void mndStop(SMnode *pMnode) { } int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; + SMnode * pMnode = pMsg->info.node; SSyncMgmt *pMgmt = &pMnode->syncMgmt; const STraceId *trace = &pMsg->info.traceId; @@ -802,7 +803,7 @@ _OVER: } int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; + SMnode * pMnode = pMsg->info.node; const STraceId *trace = &pMsg->info.traceId; MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)]; @@ -857,7 +858,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { if (mndAcquireRpc(pMnode) != 0) return -1; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc)); @@ -941,7 +942,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries; tstrncpy(desc.status, "unsynced", sizeof(desc.status)); for (int32_t i = 0; i < pVgroup->replica; ++i) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SVnodeGid * pVgid = &pVgroup->vnodeGid[i]; SMonVnodeDesc *pVnDesc = &desc.vnodes[i]; pVnDesc->dnode_id = pVgid->dnodeId; tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role)); From eac0d148685fd67faa91cdc1b92d2a4813d7a74c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Jan 2024 11:02:44 +0800 Subject: [PATCH 153/198] fix(stream): add check when merging rows from files and buffer. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 104 +++------------------ source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 9 +- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 6 ++ 3 files changed, 21 insertions(+), 98 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 255d5d4105..68ba00eefc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1490,7 +1490,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLast = INT64_MIN; - if (hasDataInSttBlock(pBlockScanInfo)) { + if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { tsLast = getCurrentKeyInSttBlock(pSttBlockReader); } @@ -1509,7 +1509,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* int64_t minKey = 0; if (pReader->info.order == TSDB_ORDER_ASC) { minKey = INT64_MAX; // chosen the minimum value - if (minKey > tsLast && hasDataInSttBlock(pBlockScanInfo)) { + if (minKey > tsLast && hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { minKey = tsLast; } @@ -1522,7 +1522,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } else { minKey = INT64_MIN; - if (minKey < tsLast && hasDataInSttBlock(pBlockScanInfo)) { + if (minKey < tsLast && hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { minKey = tsLast; } @@ -1626,91 +1626,6 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } -static int32_t doMergeFileBlockAndLastBlock(SSttBlockReader* pSttBlockReader, STsdbReader* pReader, - STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, - bool mergeBlockData) { - SRowMerger* pMerger = &pReader->status.merger; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - - int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); - bool copied = false; - int32_t code = TSDB_CODE_SUCCESS; - SRow* pTSRow = NULL; - TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - - // create local variable to hold the row value - TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; - - tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid, - fRow.pBlockData->aTSKEY[fRow.iRow], pReader->idStr); - - // only stt block exists - if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); - if (code) { - return code; - } - - if (copied) { - pBlockScanInfo->lastProcKey = tsLastBlock; - return TSDB_CODE_SUCCESS; - } else { - code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, pRow1, NULL); - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, - pReader->idStr); - - code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); - - taosMemoryFree(pTSRow); - tsdbRowMergerClear(pMerger); - - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } else { // not merge block data - code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, - pReader->idStr); - - // merge with block data if ts == key - if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); - } - - code = tsdbRowMergerGetRow(pMerger, &pTSRow); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); - - taosMemoryFree(pTSRow); - tsdbRowMergerClear(pMerger); - - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - return TSDB_CODE_SUCCESS; -} - static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* pSttBlockReader, int64_t key, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -1816,7 +1731,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); int64_t tsLast = INT64_MIN; - if (hasDataInSttBlock(pBlockScanInfo)) { + if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { tsLast = getCurrentKeyInSttBlock(pSttBlockReader); } @@ -1865,7 +1780,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = key; } - if (minKey > tsLast && hasDataInSttBlock(pBlockScanInfo)) { + if (minKey > tsLast && hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { minKey = tsLast; } } else { @@ -1882,7 +1797,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = key; } - if (minKey < tsLast && hasDataInSttBlock(pBlockScanInfo)) { + if (minKey < tsLast && hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { minKey = tsLast; } } @@ -2169,10 +2084,15 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pScanInfo->sttKeyInfo.nextProcKey = ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; hasData = true; - } else { + } else { // not clean stt blocks + INIT_TIMEWINDOW(pScanInfo->sttWindow); //reset the time window + pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); } } else { + pScanInfo->cleanSttBlocks = false; + INIT_TIMEWINDOW(pScanInfo->sttWindow); //reset the time window + pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index b4b92c0008..33604d21de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -22,12 +22,6 @@ #include "tsdbUtil2.h" #include "tsimplehash.h" -#define INIT_TIMEWINDOW(_w) \ - do { \ - (_w)->skey = INT64_MAX; \ - (_w)->ekey = INT64_MIN; \ - } while (0); - static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order); @@ -165,6 +159,9 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf INIT_TIMEWINDOW(&pScanInfo->sttWindow); INIT_TIMEWINDOW(&pScanInfo->filesetWindow); + pScanInfo->cleanSttBlocks = false; + pScanInfo->sttBlockReturned = false; + pUidList->tableUidList[j] = idList[j].uid; if (ASCENDING_TRAVERSE(pTsdbReader->info.order)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index cd9942a0a6..7a93e73a07 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -26,6 +26,12 @@ extern "C" { #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define INIT_TIMEWINDOW(_w) \ + do { \ + (_w)->skey = INT64_MAX; \ + (_w)->ekey = INT64_MIN; \ + } while (0); + typedef enum { READER_STATUS_SUSPEND = 0x1, READER_STATUS_NORMAL = 0x2, From 1b6d55616fd4c5e036b525d13464449db115768f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Jan 2024 11:28:06 +0800 Subject: [PATCH 154/198] fix(stream): fix syntax error. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 68ba00eefc..7360f59301 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2085,13 +2085,13 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; hasData = true; } else { // not clean stt blocks - INIT_TIMEWINDOW(pScanInfo->sttWindow); //reset the time window + INIT_TIMEWINDOW(&pScanInfo->sttWindow); //reset the time window pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); } } else { pScanInfo->cleanSttBlocks = false; - INIT_TIMEWINDOW(pScanInfo->sttWindow); //reset the time window + INIT_TIMEWINDOW(&pScanInfo->sttWindow); //reset the time window pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); } From 7744a31deb3dce64623334a67aaa0949b6f4cd8b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 10 Jan 2024 11:29:24 +0800 Subject: [PATCH 155/198] case: fix memory leak with return cause --- tests/army/enterprise/s3/s3_basic.py | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 112cb65b15..8e7c1f2d35 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -86,30 +86,29 @@ class TDTestCase(TBase): tdLog.debug(f"start to excute {__file__}") if eos.isArm64Cpu(): tdLog.success(f"{__file__} arm64 ignore executed") - return + else: + # insert data + self.insertData() - # insert data - self.insertData() + # check insert data correct + self.checkInsertCorrect() - # check insert data correct - self.checkInsertCorrect() + # save + self.snapshotAgg() - # save - self.snapshotAgg() + # do action + self.doAction() - # do action - self.doAction() + # check save agg result correct + self.checkAggCorrect() - # check save agg result correct - self.checkAggCorrect() + # check insert correct again + self.checkInsertCorrect() - # check insert correct again - self.checkInsertCorrect() + # drop database and free s3 file + self.dropDb() - # drop database and free s3 file - self.dropDb() - - tdLog.success(f"{__file__} successfully executed") + tdLog.success(f"{__file__} successfully executed") From 485bad1ebd1fc90651b48b31687fb2f5533f8de1 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 10 Jan 2024 11:30:42 +0800 Subject: [PATCH 156/198] refact: use config syncSnapReplMaxWaitN instead --- include/common/tglobal.h | 2 +- source/common/src/tglobal.c | 6 +++--- source/libs/sync/src/syncSnapshot.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8f8ddc632f..f23bb4d51b 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -74,7 +74,7 @@ extern int64_t tsRpcQueueMemoryAllowed; extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; extern int32_t tsHeartbeatTimeout; -extern int32_t tsSnapReplicationMsgNLimit; +extern int32_t tsSnapReplMaxWaitN; // vnode extern int64_t tsVndCommitMaxIntervalMs; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5228226235..dd51c5267a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -72,7 +72,7 @@ int32_t tsPQSortMemThreshold = 16; // M int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; int32_t tsHeartbeatTimeout = 20 * 1000; -int32_t tsSnapReplicationMsgNLimit = 128; +int32_t tsSnapReplMaxWaitN = 128; // mnode int64_t tsMndSdbWriteDelta = 200; @@ -674,7 +674,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "syncSnapReplicationMsgNLimit", tsSnapReplicationMsgNLimit, 16, + if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -1187,7 +1187,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsElectInterval = cfgGetItem(pCfg, "syncElectInterval")->i32; tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; - tsSnapReplicationMsgNLimit = cfgGetItem(pCfg, "syncSnapReplicationMsgNLimit")->i32; + tsSnapReplMaxWaitN = cfgGetItem(pCfg, "syncSnapReplMaxWaitN")->i32; tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 564d28b0f4..578e6798e0 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1187,7 +1187,7 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp pSndBuf->start = ack + 1; } - while (pSender->seq != SYNC_SNAPSHOT_SEQ_END && pSender->seq - pSndBuf->start < tsSnapReplicationMsgNLimit) { + while (pSender->seq != SYNC_SNAPSHOT_SEQ_END && pSender->seq - pSndBuf->start < tsSnapReplMaxWaitN) { if (snapshotSend(pSender) != 0) { code = terrno; goto _out; From e8aea4249816a7a2201439f7fa5828809fc2f8dc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Jan 2024 21:40:27 +0800 Subject: [PATCH 157/198] fix(tsdb): open reader during resume task for main reader after suspending reader. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 100 ++++++++++++++---------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 255d5d4105..3f11546cf9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -676,6 +676,10 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { + if (pBlockIter->blockList == NULL) { + return NULL; + } + size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { ASSERT(pBlockIter->numOfBlocks == num); @@ -2054,30 +2058,34 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea return TSDB_CODE_SUCCESS; } -static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, - STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { +static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STableBlockScanInfo* pBlockScanInfo, bool asc, + STsdbReaderInfo* pInfo) { // it is an multi-table data block if (pBlockData->aUid != NULL) { - uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex]; + uint64_t uid = pBlockData->aUid[rowIndex]; if (uid != pBlockScanInfo->uid) { // move to next row return false; } } // check for version and time range - int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex]; - if (ver > pReader->info.verRange.maxVer || ver < pReader->info.verRange.minVer) { + int64_t ver = pBlockData->aVersion[rowIndex]; + if (ver > pInfo->verRange.maxVer || ver < pInfo->verRange.minVer) { return false; } - int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - if (ts > pReader->info.window.ekey || ts < pReader->info.window.skey) { + int64_t ts = pBlockData->aTSKEY[rowIndex]; + if (ts > pInfo->window.ekey || ts < pInfo->window.skey) { + return false; + } + + if ((asc && (ts <= pBlockScanInfo->lastProcKey)) || ((!asc) && (ts >= pBlockScanInfo->lastProcKey))) { return false; } if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) { bool dropped = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, - pReader->info.order, &pReader->info.verRange); + pInfo->order, &pInfo->verRange); if (dropped) { return false; } @@ -2424,7 +2432,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { { while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block - if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + if (isValidFileBlockRow(pBlockData, pDumpInfo->rowIndex, pBlockScanInfo, asc, &pReader->info)) { hasBlockData = true; break; } @@ -4210,37 +4218,21 @@ void tsdbReaderClose2(STsdbReader* pReader) { taosMemoryFreeClear(pReader); } -int32_t tsdbReaderSuspend2(STsdbReader* pReader) { - // save reader's base state & reset top state to be reconstructed from base state - int32_t code = 0; - SReaderStatus* pStatus = &pReader->status; - STableBlockScanInfo* pBlockScanInfo = NULL; - - pReader->status.suspendInvoked = true; // record the suspend status +static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { + SReaderStatus* pStatus = &pCurrentReader->status; if (pStatus->loadFromFile) { - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); - if (pBlockInfo != NULL) { - pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { - goto _err; - } - } else { - pBlockScanInfo = *pStatus->pTableIter; - } + tsdbDataFileReaderClose(&pCurrentReader->pFileReader); - tsdbDataFileReaderClose(&pReader->pFileReader); - - SReadCostSummary* pCost = &pReader->cost; - pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); - pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + SReadCostSummary* pCost = &pCurrentReader->cost; + pStatus->pLDataIterArray = destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); + pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); } // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; - + int32_t step = ASCENDING_TRAVERSE(pCurrentReader->info.order) ? 1 : -1; int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; @@ -4251,6 +4243,26 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { pStatus->uidList.currentIndex = 0; initReaderStatus(pStatus); + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbReaderSuspend2(STsdbReader* pReader) { + // save reader's base state & reset top state to be reconstructed from base state + int32_t code = 0; + pReader->status.suspendInvoked = true; // record the suspend status + + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + if (pReader->step == EXTERNAL_ROWS_PREV) { + doSuspendCurrentReader(pReader->innerReader[0]); + } else if (pReader->step == EXTERNAL_ROWS_MAIN) { + doSuspendCurrentReader(pReader); + } else { + doSuspendCurrentReader(pReader->innerReader[1]); + } + } else { + doSuspendCurrentReader(pReader); + } + tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false); pReader->pReadSnap = NULL; if (pReader->bFilesetDelimited) { @@ -4259,16 +4271,16 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { } pReader->flag = READER_STATUS_SUSPEND; + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + clearSharedPtr(pReader->innerReader[0]); + clearSharedPtr(pReader->innerReader[1]); + } + #if SUSPEND_RESUME_TEST tsem_post(&pReader->resumeAfterSuspend); #endif - tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, - pReader->idStr); - return code; - -_err: - tsdbError("failed to suspend data reader, code:%s %s", tstrerror(code), pReader->idStr); + tsdbDebug("reader: %p suspended in this query %s, step:%d", pReader, pReader->idStr, pReader->step); return code; } @@ -4299,8 +4311,7 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { int32_t code = 0; STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; - // restore reader's state - // task snapshot + // restore reader's state, task snapshot int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { qTrace("tsdb/reader: %p, take snapshot", pReader); @@ -4325,7 +4336,14 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { pNextReader->resBlockInfo.capacity = 1; setSharedPtr(pNextReader, pReader); - code = doOpenReaderImpl(pPrevReader); + if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) { + code = doOpenReaderImpl(pPrevReader); + } else if (pReader->step == EXTERNAL_ROWS_MAIN) { + code = doOpenReaderImpl(pReader); + } else { + code = doOpenReaderImpl(pNextReader); + } + if (code != TSDB_CODE_SUCCESS) { return code; } From b0202bb8536718d117e89aa527b77603ddaf9c57 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Jan 2024 21:40:27 +0800 Subject: [PATCH 158/198] fix(tsdb): open reader during resume task for main reader after suspending reader. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 100 ++++++++++++++---------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 7360f59301..62be0968e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -676,6 +676,10 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { + if (pBlockIter->blockList == NULL) { + return NULL; + } + size_t num = TARRAY_SIZE(pBlockIter->blockList); if (num == 0) { ASSERT(pBlockIter->numOfBlocks == num); @@ -1969,30 +1973,34 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea return TSDB_CODE_SUCCESS; } -static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, - STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { +static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STableBlockScanInfo* pBlockScanInfo, bool asc, + STsdbReaderInfo* pInfo) { // it is an multi-table data block if (pBlockData->aUid != NULL) { - uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex]; + uint64_t uid = pBlockData->aUid[rowIndex]; if (uid != pBlockScanInfo->uid) { // move to next row return false; } } // check for version and time range - int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex]; - if (ver > pReader->info.verRange.maxVer || ver < pReader->info.verRange.minVer) { + int64_t ver = pBlockData->aVersion[rowIndex]; + if (ver > pInfo->verRange.maxVer || ver < pInfo->verRange.minVer) { return false; } - int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - if (ts > pReader->info.window.ekey || ts < pReader->info.window.skey) { + int64_t ts = pBlockData->aTSKEY[rowIndex]; + if (ts > pInfo->window.ekey || ts < pInfo->window.skey) { + return false; + } + + if ((asc && (ts <= pBlockScanInfo->lastProcKey)) || ((!asc) && (ts >= pBlockScanInfo->lastProcKey))) { return false; } if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) { bool dropped = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, - pReader->info.order, &pReader->info.verRange); + pInfo->order, &pInfo->verRange); if (dropped) { return false; } @@ -2344,7 +2352,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { { while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block - if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + if (isValidFileBlockRow(pBlockData, pDumpInfo->rowIndex, pBlockScanInfo, asc, &pReader->info)) { hasBlockData = true; break; } @@ -4130,37 +4138,21 @@ void tsdbReaderClose2(STsdbReader* pReader) { taosMemoryFreeClear(pReader); } -int32_t tsdbReaderSuspend2(STsdbReader* pReader) { - // save reader's base state & reset top state to be reconstructed from base state - int32_t code = 0; - SReaderStatus* pStatus = &pReader->status; - STableBlockScanInfo* pBlockScanInfo = NULL; - - pReader->status.suspendInvoked = true; // record the suspend status +static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { + SReaderStatus* pStatus = &pCurrentReader->status; if (pStatus->loadFromFile) { - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); - if (pBlockInfo != NULL) { - pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); - if (pBlockScanInfo == NULL) { - goto _err; - } - } else { - pBlockScanInfo = *pStatus->pTableIter; - } + tsdbDataFileReaderClose(&pCurrentReader->pFileReader); - tsdbDataFileReaderClose(&pReader->pFileReader); - - SReadCostSummary* pCost = &pReader->cost; - pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); - pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + SReadCostSummary* pCost = &pCurrentReader->cost; + pStatus->pLDataIterArray = destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost); + pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); } // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; - + int32_t step = ASCENDING_TRAVERSE(pCurrentReader->info.order) ? 1 : -1; int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; @@ -4171,6 +4163,26 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { pStatus->uidList.currentIndex = 0; initReaderStatus(pStatus); + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbReaderSuspend2(STsdbReader* pReader) { + // save reader's base state & reset top state to be reconstructed from base state + int32_t code = 0; + pReader->status.suspendInvoked = true; // record the suspend status + + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + if (pReader->step == EXTERNAL_ROWS_PREV) { + doSuspendCurrentReader(pReader->innerReader[0]); + } else if (pReader->step == EXTERNAL_ROWS_MAIN) { + doSuspendCurrentReader(pReader); + } else { + doSuspendCurrentReader(pReader->innerReader[1]); + } + } else { + doSuspendCurrentReader(pReader); + } + tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false); pReader->pReadSnap = NULL; if (pReader->bFilesetDelimited) { @@ -4179,16 +4191,16 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { } pReader->flag = READER_STATUS_SUSPEND; + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + clearSharedPtr(pReader->innerReader[0]); + clearSharedPtr(pReader->innerReader[1]); + } + #if SUSPEND_RESUME_TEST tsem_post(&pReader->resumeAfterSuspend); #endif - tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, - pReader->idStr); - return code; - -_err: - tsdbError("failed to suspend data reader, code:%s %s", tstrerror(code), pReader->idStr); + tsdbDebug("reader: %p suspended in this query %s, step:%d", pReader, pReader->idStr, pReader->step); return code; } @@ -4219,8 +4231,7 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { int32_t code = 0; STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; - // restore reader's state - // task snapshot + // restore reader's state, task snapshot int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { qTrace("tsdb/reader: %p, take snapshot", pReader); @@ -4245,7 +4256,14 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { pNextReader->resBlockInfo.capacity = 1; setSharedPtr(pNextReader, pReader); - code = doOpenReaderImpl(pPrevReader); + if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) { + code = doOpenReaderImpl(pPrevReader); + } else if (pReader->step == EXTERNAL_ROWS_MAIN) { + code = doOpenReaderImpl(pReader); + } else { + code = doOpenReaderImpl(pNextReader); + } + if (code != TSDB_CODE_SUCCESS) { return code; } From 2537adb72887dda2d0eb2bae6e015a2fcb75d6d3 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Wed, 10 Jan 2024 06:00:44 +0000 Subject: [PATCH 159/198] opt msg on mnode --- source/common/src/tglobal.c | 8 +++---- source/dnode/mnode/impl/src/mndMain.c | 32 +++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 76e1ab46b7..4b32fa676e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -108,7 +108,7 @@ bool tsEnableTelem = true; int32_t tsTelemInterval = 43200; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.tdengine.com"; uint16_t tsTelemPort = 80; -char * tsTelemUri = "/report"; +char *tsTelemUri = "/report"; #ifdef TD_ENTERPRISE bool tsEnableCrashReport = false; @@ -1376,7 +1376,7 @@ void taosCleanupCfg() { typedef struct { const char *optionName; - void * optionVar; + void *optionVar; } OptionNameAndVar; static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) { @@ -1389,7 +1389,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, switch (pItem->dtype) { case CFG_DTYPE_BOOL: { int32_t flag = pItem->i32; - bool * pVar = pOptions[d].optionVar; + bool *pVar = pOptions[d].optionVar; uInfo("%s set from %d to %d", optName, *pVar, flag); *pVar = flag; terrno = TSDB_CODE_SUCCESS; @@ -1758,7 +1758,7 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag void taosSetAllDebugFlag(int32_t flag) { if (flag <= 0) return; - SArray * noNeedToSetVars = NULL; + SArray *noNeedToSetVars = NULL; SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); if (pItem != NULL) { pItem->i32 = flag; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 4bd0776ecd..aa54de9ae7 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -107,7 +107,7 @@ static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) { static void mndPullupTrans(SMnode *pMnode) { mTrace("pullup trans msg"); int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); @@ -117,7 +117,7 @@ static void mndPullupTrans(SMnode *pMnode) { static void mndPullupCompacts(SMnode *pMnode) { mTrace("pullup compact timer msg"); int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_COMPACT_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); @@ -127,7 +127,7 @@ static void mndPullupCompacts(SMnode *pMnode) { static void mndPullupTtl(SMnode *pMnode) { mTrace("pullup ttl"); int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } @@ -135,14 +135,14 @@ static void mndPullupTtl(SMnode *pMnode) { static void mndPullupTrimDb(SMnode *pMnode) { mTrace("pullup trim"); int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRIM_DB_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } static void mndCalMqRebalance(SMnode *pMnode) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -151,7 +151,7 @@ static void mndCalMqRebalance(SMnode *pMnode) { static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { int32_t contLen = 0; - void * pReq = mndBuildCheckpointTickMsg(&contLen, sec); + void *pReq = mndBuildCheckpointTickMsg(&contLen, sec); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -160,7 +160,7 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { static void mndStreamCheckpointRemain(SMnode *pMnode) { int32_t contLen = 0; - void * pReq = mndBuildCheckpointTickMsg(&contLen, 0); + void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -169,7 +169,7 @@ static void mndStreamCheckpointRemain(SMnode *pMnode) { static void mndStreamCheckNode(SMnode *pMnode) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_NODECHECK_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -179,7 +179,7 @@ static void mndStreamCheckNode(SMnode *pMnode) { static void mndPullupTelem(SMnode *pMnode) { mTrace("pullup telem msg"); int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); @@ -189,7 +189,7 @@ static void mndPullupTelem(SMnode *pMnode) { static void mndPullupGrant(SMnode *pMnode) { mTrace("pullup grant msg"); int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = { .msgType = TDMT_MND_GRANT_HB_TIMER, .pCont = pReq, .contLen = contLen, .info.ahandle = (void *)0x9527}; @@ -200,7 +200,7 @@ static void mndPullupGrant(SMnode *pMnode) { static void mndIncreaseUpTime(SMnode *pMnode) { mTrace("increate uptime"); int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = { .msgType = TDMT_MND_UPTIME_TIMER, .pCont = pReq, .contLen = contLen, .info.ahandle = (void *)0x9528}; @@ -254,7 +254,7 @@ static void mndCheckDnodeOffline(SMnode *pMnode) { mTrace("check dnode offline"); if (mndAcquireRpc(pMnode) != 0) return; - SSdb * pSdb = pMnode->pSdb; + SSdb *pSdb = pMnode->pSdb; int64_t curMs = taosGetTimestampMs(); void *pIter = NULL; @@ -701,7 +701,7 @@ void mndStop(SMnode *pMnode) { } int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { - SMnode * pMnode = pMsg->info.node; + SMnode *pMnode = pMsg->info.node; SSyncMgmt *pMgmt = &pMnode->syncMgmt; const STraceId *trace = &pMsg->info.traceId; @@ -803,7 +803,7 @@ _OVER: } int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { - SMnode * pMnode = pMsg->info.node; + SMnode *pMnode = pMsg->info.node; const STraceId *trace = &pMsg->info.traceId; MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)]; @@ -858,7 +858,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { if (mndAcquireRpc(pMnode) != 0) return -1; - SSdb * pSdb = pMnode->pSdb; + SSdb *pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc)); @@ -942,7 +942,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries; tstrncpy(desc.status, "unsynced", sizeof(desc.status)); for (int32_t i = 0; i < pVgroup->replica; ++i) { - SVnodeGid * pVgid = &pVgroup->vnodeGid[i]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SMonVnodeDesc *pVnDesc = &desc.vnodes[i]; pVnDesc->dnode_id = pVgid->dnodeId; tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role)); From 3f0236ec171801f3ca09f55d0746611a6a50d139 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Wed, 10 Jan 2024 06:18:39 +0000 Subject: [PATCH 160/198] opt msg on mnode --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 29 +++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index b79b3f9e11..7ca19d7725 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -47,28 +47,29 @@ static void *dmStatusThreadFp(void *param) { } SDmNotifyHandle dmNotifyHdl = {.state = 0}; -static void *dmNotifyThreadFp(void *param) { - SDnodeMgmt *pMgmt = param; - setThreadName("dnode-notify"); - if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { - return NULL; +static void *dmNotifyThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + setThreadName("dnode-notify"); + + if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { + return NULL; } - bool wait = true; - while (1) { - if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; + bool wait = true; + while (1) { + if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; if (wait) tsem_wait(&dmNotifyHdl.sem); atomic_store_8(&dmNotifyHdl.state, 1); - dmSendNotifyReq(pMgmt); - if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { - wait = true; - continue; + dmSendNotifyReq(pMgmt); + if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { + wait = true; + continue; } - wait = false; + wait = false; } - return NULL; + return NULL; } static void *dmMonitorThreadFp(void *param) { From ba2ba0a8cbb292fd69b8011e45efe5d13c818f88 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 10 Jan 2024 16:19:20 +0800 Subject: [PATCH 161/198] fix: time window computation error --- source/common/src/ttime.c | 13 ++--- source/libs/executor/test/timewindowTest.cpp | 23 ++++++++ tests/parallel_test/cases.task | 1 + tests/system-test/2-query/count_interval.py | 60 ++++++++++++++++++++ 4 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 tests/system-test/2-query/count_interval.py diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 023a425df2..376cfce255 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -872,18 +872,17 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { ASSERT(pInterval->offset >= 0); if (pInterval->offset > 0) { - start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); - // try to move current window to the left-hande-side, due to the offset effect. int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newe = end; + int64_t slidingEnd = end; while (newe >= ts) { - end = newe; - newe = taosTimeAdd(newe, -pInterval->sliding, pInterval->slidingUnit, precision); + end = slidingEnd; + slidingEnd = taosTimeAdd(slidingEnd, -pInterval->sliding, pInterval->slidingUnit, precision); + newe = taosTimeAdd(slidingEnd, pInterval->offset, pInterval->offsetUnit, precision); } - - start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; + int64_t slidingStart = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; + start = taosTimeAdd(slidingStart, pInterval->offset, pInterval->offsetUnit, precision); } return start; diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index 2894c66587..7a83b64fac 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -158,4 +158,27 @@ TEST(testCase, timewindow_gen) { } +TEST(testCase, timewindow_natural) { + osSetTimezone("CST"); + + int32_t precision = TSDB_TIME_PRECISION_MILLI; + + SInterval interval2 = createInterval(17, 17, 13392000000, 'n', 'n', 0, precision); + int64_t key1 = 1633446027072; + STimeWindow w1 = {0}; + getInitialStartTimeWindow(&interval2, key1, &w1, true); + printTimeWindow(&w1, precision, key1); + STimeWindow w3 = getAlignQueryTimeWindow(&interval2, key1); + printf("%ld win %ld, %ld\n", key1, w3.skey, w3.ekey); + + int64_t key2 = 1648758398208; + STimeWindow w2 = {0}; + getInitialStartTimeWindow(&interval2, key2, &w2, true); + printTimeWindow(&w2, precision, key2); + STimeWindow w4 = getAlignQueryTimeWindow(&interval2, key2); + printf("%ld win %ld, %ld\n", key2, w3.skey, w3.ekey); + + ASSERT_EQ(w3.skey, w4.skey); + ASSERT_EQ(w3.ekey, w4.ekey); +} #pragma GCC diagnostic pop \ No newline at end of file diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3db6eccffc..752234bbf5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -36,6 +36,7 @@ #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2 diff --git a/tests/system-test/2-query/count_interval.py b/tests/system-test/2-query/count_interval.py new file mode 100644 index 0000000000..b37cc1db22 --- /dev/null +++ b/tests/system-test/2-query/count_interval.py @@ -0,0 +1,60 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +import random + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use d") + + def run(self): + tdSql.execute("drop database if exists d"); + tdSql.execute("create database d"); + tdSql.execute("use d"); + tdSql.execute("create table st(ts timestamp, f int) tags (t int)") + + for i in range(-2048, 2047): + ts = 1626624000000 + i; + tdSql.execute(f"insert into ct1 using st tags(1) values({ts}, {i})") + + tdSql.execute("flush database d") + for i in range(1638): + ts = 1648758398208 + i + tdSql.execute(f"insert into ct1 using st tags(1) values({ts}, {i})") + tdSql.execute("insert into ct1 using st tags(1) values(1648970742528, 1638)") + tdSql.execute("flush database d") + + tdSql.query("select count(ts) from ct1 interval(17n, 5n)") + self.restartTaosd() + tdSql.query("select count(ts) from ct1 interval(17n, 5n)") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 27b63b721fb40e96a74f920a50edbf9040d5d5bf Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Jan 2024 17:36:51 +0800 Subject: [PATCH 162/198] fix:decode mqDataMsg error for the message with old version --- source/common/src/tmsg.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3ff814dda9..c9e2908e8a 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -8334,7 +8334,7 @@ int32_t tDecodeMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { return 0; } -int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { +int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; @@ -8356,11 +8356,16 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { } } } + return 0; +} + +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { + if (tEncodeMqDataRspCommon(pEncoder, pRsp) < 0) return -1; if (tEncodeI64(pEncoder, pRsp->sleepTime) < 0) return -1; return 0; } -int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { +int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; @@ -8402,7 +8407,15 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } } } - if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1; + + return 0; +} + +int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { + if (tDecodeMqDataRspCommon(pDecoder, pRsp) < 0) return -1; + if (!tDecodeIsEnd(pDecoder)) { + if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1; + } return 0; } @@ -8418,7 +8431,7 @@ void tDeleteMqDataRsp(SMqDataRsp *pRsp) { } int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { - if (tEncodeMqDataRsp(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1; + if (tEncodeMqDataRspCommon(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { @@ -8432,7 +8445,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { } int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { - if (tDecodeMqDataRsp(pDecoder, (SMqDataRsp *)pRsp) < 0) return -1; + if (tDecodeMqDataRspCommon(pDecoder, (SMqDataRsp*)pRsp) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { @@ -8447,6 +8460,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { taosArrayPush(pRsp->createTableReq, &pCreate); } } + return 0; } From b5e5167b8fb845ffc0bd266e457c4abda533222e Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Wed, 10 Jan 2024 09:57:08 +0000 Subject: [PATCH 163/198] split sync/status channel --- include/common/tmsgcb.h | 4 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 4 + source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 5 + source/dnode/mgmt/node_mgmt/src/dmTransport.c | 93 +++++++++++++++++-- source/dnode/mnode/impl/src/mndDump.c | 9 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 1 + source/dnode/vnode/src/vnd/vnodeSync.c | 12 +-- source/libs/transport/src/tmsgcb.c | 8 ++ 9 files changed, 119 insertions(+), 19 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 311bffb7da..dd1e54318e 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -53,9 +53,12 @@ typedef struct { void* mgmt; void* clientRpc; void* serverRpc; + void* statusRpc; + void* syncRpc; PutToQueueFp putToQueueFp; GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; + SendReqFp sendSyncReqFp; SendRspFp sendRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; @@ -67,6 +70,7 @@ void tmsgSetDefault(const SMsgCb* msgcb); int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg); int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); +int32_t tmsgSendSyncReq(const SEpSet* epSet, SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a95ec42bd0..545a5fc870 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -161,7 +161,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SEpSet epSet = {0}; int8_t epUpdated = 0; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, 5000); + rpcSendRecvWithTimeout(pMgmt->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, tsStatusInterval * 10); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); char tbuf[512]; diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index c646bb4bdd..cb3395dcc2 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -48,6 +48,8 @@ typedef struct { typedef struct { void *serverRpc; void *clientRpc; + void *statusRpc; + void *syncRpc; SDnodeHandle msgHandles[TDMT_MAX]; } SDnodeTrans; @@ -136,8 +138,10 @@ int32_t dmInitServer(SDnode *pDnode); void dmCleanupServer(SDnode *pDnode); int32_t dmInitClient(SDnode *pDnode); int32_t dmInitStatusClient(SDnode *pDnode); +int32_t dmInitSyncClient(SDnode *pDnode); void dmCleanupClient(SDnode *pDnode); void dmCleanupStatusClient(SDnode *pDnode); +void dmCleanupSyncClient(SDnode *pDnode); SMsgCb dmGetMsgcb(SDnode *pDnode); #ifdef TD_MODULE_OPTIMIZE int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 84465640c0..63122171b1 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -93,6 +93,9 @@ int32_t dmInitDnode(SDnode *pDnode) { indexInit(tsNumOfCommitThreads); streamMetaInit(); + dmInitStatusClient(pDnode); + dmInitSyncClient(pDnode); + dmReportStartup("dnode-transport", "initialized"); dDebug("dnode is created, ptr:%p", pDnode); code = 0; @@ -112,7 +115,9 @@ void dmCleanupDnode(SDnode *pDnode) { dmCleanupClient(pDnode); dmCleanupStatusClient(pDnode); + dmCleanupSyncClient(pDnode); dmCleanupServer(pDnode); + dmClearVars(pDnode); rpcCleanup(); streamMetaCleanup(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 1ea61f0e93..512f090ab0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -322,6 +322,23 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { return 0; } } +static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { + SDnode *pDnode = dmInstance(); + if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + if (pDnode->status == DND_STAT_INIT) { + terrno = TSDB_CODE_APP_IS_STARTING; + } else { + terrno = TSDB_CODE_APP_IS_STOPPING; + } + dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); + return -1; + } else { + rpcSendRequest(pDnode->trans.syncRpc, pEpSet, pMsg, NULL); + return 0; + } +} static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLinkArg(pMsg); } @@ -421,16 +438,61 @@ int32_t dmInitStatusClient(SDnode *pDnode) { rpcInit.timeToGetConn = tsTimeToGetAvailableConn; taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); - // pTrans->statusClientRpc = rpcOpen(&rpcInit); - // if (pTrans->statusClientRpc == NULL) { - // dError("failed to init dnode rpc status client"); - // return -1; - // } + pTrans->statusRpc = rpcOpen(&rpcInit); + if (pTrans->statusRpc == NULL) { + dError("failed to init dnode rpc status client"); + return -1; + } dDebug("dnode rpc status client is initialized"); return 0; } +int32_t dmInitSyncClient(SDnode *pDnode) { + SDnodeTrans *pTrans = &pDnode->trans; + + SRpcInit rpcInit = {0}; + rpcInit.label = "DND-SYNC"; + rpcInit.numOfThreads = tsNumOfRpcThreads; + rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = TSDB_DEFAULT_USER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.parent = pDnode; + rpcInit.rfp = rpcRfp; + rpcInit.compressSize = tsCompressMsgSize; + + rpcInit.retryMinInterval = tsRedirectPeriod; + rpcInit.retryStepFactor = tsRedirectFactor; + rpcInit.retryMaxInterval = tsRedirectMaxPeriod; + rpcInit.retryMaxTimeout = tsMaxRetryWaitTime; + + rpcInit.failFastInterval = 5000; // interval threshold(ms) + rpcInit.failFastThreshold = 3; // failed threshold + rpcInit.ffp = dmFailFastFp; + + int32_t connLimitNum = 100; + connLimitNum = TMAX(connLimitNum, 10); + connLimitNum = TMIN(connLimitNum, 500); + + rpcInit.connLimitNum = connLimitNum; + rpcInit.connLimitLock = 1; + rpcInit.supportBatch = 1; + rpcInit.batchSize = 8 * 1024; + rpcInit.timeToGetConn = tsTimeToGetAvailableConn; + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); + + pTrans->syncRpc = rpcOpen(&rpcInit); + if (pTrans->syncRpc == NULL) { + dError("failed to init dnode rpc sync client"); + return -1; + } + + dDebug("dnode rpc sync client is initialized"); + return 0; +} + void dmCleanupClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; if (pTrans->clientRpc) { @@ -441,11 +503,19 @@ void dmCleanupClient(SDnode *pDnode) { } void dmCleanupStatusClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; - // if (pTrans->statusClientRpc) { - // rpcClose(pTrans->statusClientRpc); - // pTrans->statusClientRpc = NULL; - // dDebug("dnode rpc status client is closed"); - // } + if (pTrans->statusRpc) { + rpcClose(pTrans->statusRpc); + pTrans->statusRpc = NULL; + dDebug("dnode rpc status client is closed"); + } +} +void dmCleanupSyncClient(SDnode *pDnode) { + SDnodeTrans *pTrans = &pDnode->trans; + if (pTrans->syncRpc) { + rpcClose(pTrans->syncRpc); + pTrans->syncRpc = NULL; + dDebug("dnode rpc sync client is closed"); + } } int32_t dmInitServer(SDnode *pDnode) { @@ -486,7 +556,10 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) { SMsgCb msgCb = { .clientRpc = pDnode->trans.clientRpc, .serverRpc = pDnode->trans.serverRpc, + .statusRpc = pDnode->trans.statusRpc, + .syncRpc = pDnode->trans.syncRpc, .sendReqFp = dmSendReq, + .sendSyncReqFp = dmSendSyncReq, .sendRspFp = dmSendRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 5efebbc16e..c68b11d184 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -32,6 +32,10 @@ int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { terrno = TSDB_CODE_INVALID_PTR; return -1; } +int32_t sendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; +} char *i642str(int64_t val) { static char str[24] = {0}; @@ -568,6 +572,7 @@ void mndDumpSdb() { SMsgCb msgCb = {0}; msgCb.reportStartupFp = reportStartup; msgCb.sendReqFp = sendReq; + msgCb.sendSyncReqFp = sendSyncReq; msgCb.sendRspFp = sendRsp; msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack tmsgSetDefault(&msgCb); @@ -590,7 +595,7 @@ void mndDumpSdb() { dumpTopic(pSdb, json); dumpConsumer(pSdb, json); dumpSubscribe(pSdb, json); -// dumpOffset(pSdb, json); + // dumpOffset(pSdb, json); dumpStream(pSdb, json); dumpAcct(pSdb, json); dumpAuth(pSdb, json); @@ -605,7 +610,7 @@ void mndDumpSdb() { char *pCont = tjsonToString(json); int32_t contLen = strlen(pCont); char file[] = "sdb.json"; - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC| TD_FILE_WRITE_THROUGH); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to write %s since %s", file, terrstr()); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 2d03631a37..4d0d53ced0 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -61,6 +61,7 @@ class MndTestTrans2 : public ::testing::Test { static SMsgCb msgCb = {0}; msgCb.reportStartupFp = reportStartup; msgCb.sendReqFp = sendReq; + msgCb.sendSyncReqFp = sendSyncReq; msgCb.sendRspFp = sendRsp; msgCb.queueFps[SYNC_QUEUE] = putToQueue; msgCb.queueFps[WRITE_QUEUE] = putToQueue; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 82baaa652b..09becffdaa 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -14,11 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "tq.h" #include "sync.h" +#include "tq.h" +#include "tqCommon.h" #include "tsdb.h" #include "vnd.h" -#include "tqCommon.h" #define BATCH_ENABLE 0 @@ -411,7 +411,7 @@ static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { } static int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { - int32_t code = tmsgSendReq(pEpSet, pMsg); + int32_t code = tmsgSendSyncReq(pEpSet, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; @@ -477,8 +477,8 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta } static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) { - SVnode *pVnode = pFsm->data; - int32_t code = vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader); + SVnode *pVnode = pFsm->data; + int32_t code = vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader); return code; } @@ -555,7 +555,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) walApplyVer(pVnode->pWal, commitIdx); pVnode->restored = true; - SStreamMeta* pMeta = pVnode->pTq->pStreamMeta; + SStreamMeta *pMeta = pVnode->pTq->pStreamMeta; streamMetaWLock(pMeta); if (pMeta->startInfo.tasksWillRestart) { diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 1b1fa8cc1c..e44328c683 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -48,6 +48,14 @@ int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { } return code; } +int32_t tmsgSendSyncReq(const SEpSet* epSet, SRpcMsg* pMsg) { + int32_t code = (*defaultMsgCb.sendSyncReqFp)(epSet, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; +} void tmsgSendRsp(SRpcMsg* pMsg) { #if 1 From df69a89da202a60df6915c2653e9af00f74aa6a7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Jan 2024 18:46:10 +0800 Subject: [PATCH 164/198] fix:[TD-28185]add pause & resume logic for snode --- include/dnode/vnode/tqCommon.h | 2 + source/dnode/snode/src/snode.c | 6 +- source/dnode/vnode/src/tq/tq.c | 101 +------------------- source/dnode/vnode/src/tqCommon/tqCommon.c | 105 +++++++++++++++++++++ source/libs/stream/src/streamMeta.c | 8 +- 5 files changed, 118 insertions(+), 104 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index da2325f006..82c4102369 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -33,5 +33,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg); +int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode); #endif // TDENGINE_TQ_COMMON_H diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 33eb9cd3ed..9177559f61 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -195,7 +195,7 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { case TDMT_STREAM_TASK_DEPLOY: { void * pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); - return tqStreamTaskProcessDeployReq(pSnode->pMeta, -1, pReq, len, true, true); + return tqStreamTaskProcessDeployReq(pSnode->pMeta, pMsg->info.conn.applyIndex, pReq, len, true, true); } case TDMT_STREAM_TASK_DROP: @@ -204,6 +204,10 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return tqStreamTaskProcessUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); case TDMT_VND_STREAM_TASK_RESET: return tqStreamTaskProcessTaskResetReq(pSnode->pMeta, pMsg); + case TDMT_STREAM_TASK_PAUSE: + return tqStreamTaskProcessTaskPauseReq(pSnode->pMeta, pMsg->pCont); + case TDMT_STREAM_TASK_RESUME: + return tqStreamTaskProcessTaskResumeReq(pSnode->pMeta, pMsg->info.conn.applyIndex, pMsg->pCont, false); default: ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 65cc56c0b4..275bbffe25 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1094,108 +1094,11 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { } int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)msg; - - SStreamMeta* pMeta = pTq->pStreamMeta; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask == NULL) { - tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, - pReq->taskId); - // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active - return TSDB_CODE_SUCCESS; - } - - tqDebug("s-task:%s receive pause msg from mnode", pTask->id.idStr); - streamTaskPause(pTask, pMeta); - - SStreamTask* pHistoryTask = NULL; - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); - if (pHistoryTask == NULL) { - tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 - ", it may have been dropped already", - pMeta->vgId, pTask->hTaskInfo.id.taskId); - streamMetaReleaseTask(pMeta, pTask); - - // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active - return TSDB_CODE_SUCCESS; - } - - tqDebug("s-task:%s fill-history task handle paused along with related stream task", pHistoryTask->id.idStr); - - streamTaskPause(pHistoryTask, pMeta); - streamMetaReleaseTask(pMeta, pHistoryTask); - } - - streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; -} - -int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, int8_t igUntreated) { - int32_t vgId = pTq->pStreamMeta->vgId; - if (pTask == NULL) { - return -1; - } - - streamTaskResume(pTask); - ETaskStatus status = streamTaskGetStatus(pTask, NULL); - - int32_t level = pTask->info.taskLevel; - if (level == TASK_LEVEL__SINK) { - if (status == TASK_STATUS__UNINIT) { - } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; - } - - if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) { - // no lock needs to secure the access of the version - if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { - // discard all the data when the stream task is suspended. - walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); - tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 - ", schedStatus:%d", - vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); - } else { // from the previous paused version and go on - tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", - vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); - } - - if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { - streamStartScanHistoryAsync(pTask, igUntreated); - } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { - tqScanWalAsync(pTq, false); - } else { - streamSchedExec(pTask); - } - } else if (status == TASK_STATUS__UNINIT) { - // todo: fill-history task init ? - if (pTask->info.fillHistory == 0) { - EStreamTaskEvent event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; - streamTaskHandleEvent(pTask->status.pSM, event); - } - } - - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; + return tqStreamTaskProcessTaskPauseReq(pTq->pStreamMeta, msg); } int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, pReq->taskId); - int32_t code = tqProcessTaskResumeImpl(pTq, pTask, sversion, pReq->igUntreated); - if (code != 0) { - return code; - } - - STaskId* pHTaskId = &pTask->hTaskInfo.id; - SStreamTask* pHistoryTask = streamMetaAcquireTask(pTq->pStreamMeta, pHTaskId->streamId, pHTaskId->taskId); - if (pHistoryTask) { - code = tqProcessTaskResumeImpl(pTq, pHistoryTask, sversion, pReq->igUntreated); - } - - return code; + return tqStreamTaskProcessTaskResumeReq(pTq, sversion, msg, true); } int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a3d860fd78..70bfa00609 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -752,4 +752,109 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; +} + +int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg){ + SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + if (pTask == NULL) { + tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, + pReq->taskId); + // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active + return TSDB_CODE_SUCCESS; + } + + tqDebug("s-task:%s receive pause msg from mnode", pTask->id.idStr); + streamTaskPause(pTask, pMeta); + + SStreamTask* pHistoryTask = NULL; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); + if (pHistoryTask == NULL) { + tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 + ", it may have been dropped already", + pMeta->vgId, pTask->hTaskInfo.id.taskId); + streamMetaReleaseTask(pMeta, pTask); + + // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active + return TSDB_CODE_SUCCESS; + } + + tqDebug("s-task:%s fill-history task handle paused along with related stream task", pHistoryTask->id.idStr); + + streamTaskPause(pHistoryTask, pMeta); + streamMetaReleaseTask(pMeta, pHistoryTask); + } + + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; +} + +static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t sversion, int8_t igUntreated, bool fromVnode) { + SStreamMeta *pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; + int32_t vgId = pMeta->vgId; + if (pTask == NULL) { + return -1; + } + + streamTaskResume(pTask); + ETaskStatus status = streamTaskGetStatus(pTask, NULL); + + int32_t level = pTask->info.taskLevel; + if (level == TASK_LEVEL__SINK) { + if (status == TASK_STATUS__UNINIT) { + } + streamMetaReleaseTask(pMeta, pTask); + return 0; + } + + if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) { + // no lock needs to secure the access of the version + if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { + // discard all the data when the stream task is suspended. + walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); + tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 + ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); + } else { // from the previous paused version and go on + tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); + } + + if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { + streamStartScanHistoryAsync(pTask, igUntreated); + } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { + tqScanWalAsync((STQ*)handle, false); + } else { + streamSchedExec(pTask); + } + } else if (status == TASK_STATUS__UNINIT) { + // todo: fill-history task init ? + if (pTask->info.fillHistory == 0) { + EStreamTaskEvent event = /*HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : */TASK_EVENT_INIT; + streamTaskHandleEvent(pTask->status.pSM, event); + } + } + + streamMetaReleaseTask(pMeta, pTask); + return 0; +} + +int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode){ + SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; + SStreamMeta *pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + int32_t code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); + if (code != 0) { + return code; + } + + STaskId* pHTaskId = &pTask->hTaskInfo.id; + SStreamTask* pHistoryTask = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); + if (pHistoryTask) { + code = tqProcessTaskResumeImpl(pMeta, pHistoryTask, sversion, pReq->igUntreated, fromVnode); + } + + return code; } \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7ea80ad8d8..4e30dc2555 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -377,16 +377,16 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->rid = taosAddRef(streamMetaId, pMeta); // set the attribute when running on Linux OS -#if defined LINUX TdThreadRwlockAttr attr; taosThreadRwlockAttrInit(&attr); +#ifdef LINUX pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); - taosThreadRwlockInit(&pMeta->lock, &attr); - - taosThreadRwlockAttrDestroy(&attr); #endif + taosThreadRwlockInit(&pMeta->lock, &attr); + taosThreadRwlockAttrDestroy(&attr); + int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); metaRefMgtAdd(pMeta->vgId, pRid); From dcbe963d326972ed1c29f63790abd3db09e9c99f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Jan 2024 19:19:53 +0800 Subject: [PATCH 165/198] fix(stream): keep the previous status in stack object. --- source/libs/stream/src/streamStart.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index c852215d16..d39528312d 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1066,21 +1066,21 @@ void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta) { } void streamTaskResume(SStreamTask* pTask) { - SStreamTaskState* p = streamTaskGetStatus(pTask); + SStreamTaskState prevState = *streamTaskGetStatus(pTask); SStreamMeta* pMeta = pTask->pMeta; - if (p->state == TASK_STATUS__PAUSE || p->state == TASK_STATUS__HALT) { + if (prevState.state == TASK_STATUS__PAUSE || prevState.state == TASK_STATUS__HALT) { streamTaskRestoreStatus(pTask); char* pNew = streamTaskGetStatus(pTask)->name; - if (p->state == TASK_STATUS__PAUSE) { + if (prevState.state == TASK_STATUS__PAUSE) { int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); - stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, p->name, num); + stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); } else { - stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, p->name); + stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, prevState.name); } } else { - stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, p->name); + stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, prevState.name); } } From fb42c6afd5a06d7a748afa69df6e9436a31528f6 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 10 Jan 2024 19:59:44 +0800 Subject: [PATCH 166/198] coverage: update coverage script file and BlockSMA trigger is 1 --- tests/script/coverage_test.sh | 363 +++++++++++--------------- tests/system-test/2-query/blockSMA.py | 2 +- 2 files changed, 160 insertions(+), 205 deletions(-) mode change 100755 => 100644 tests/script/coverage_test.sh diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh old mode 100755 new mode 100644 index 535b6802f2..1b9e9753d3 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -12,7 +12,6 @@ fi today=`date +"%Y%m%d"` TDENGINE_DIR=/root/TDengine JDBC_DIR=/root/taos-connector-jdbc -TAOSKEEPER_DIR=/root/taoskeeper TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log # Color setting @@ -23,52 +22,52 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' function buildTDengine() { - echo "check if TDengine need build" - cd $TDENGINE_DIR - git remote prune origin > /dev/null - git remote update > /dev/null - REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` - LOCAL_COMMIT=`git rev-parse --short @` - echo " LOCAL: $LOCAL_COMMIT" - echo "REMOTE: $REMOTE_COMMIT" + echo "check if TDengine need build" + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" - # reset counter - lcov -d . --zerocounters + # reset counter + lcov -d . --zerocounters - if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then - echo "repo up-to-date" - else - echo "repo need to pull" - fi + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + fi - git reset --hard - git checkout -- . - git checkout $branch - git checkout -- . - git clean -dfx - git pull - - [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug - cd $TDENGINE_DIR/debug + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -dfx + git pull + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug echo "rebuild.." - LOCAL_COMMIT=`git rev-parse --short @` + LOCAL_COMMIT=`git rev-parse --short @` - rm -rf * - if [ "$branch" == "3.0" ]; then - echo "3.0 =============" - cmake -DCOVER=true -DBUILD_TEST=true -DBUILD_HTTP=false -DBUILD_TOOLS=true .. > /dev/null - else - cmake -DCOVER=true -DBUILD_TEST=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null - fi - make -j - make install + rm -rf * + if [ "$branch" == "3.0" ]; then + echo "3.0 =============" + cmake -DCOVER=true -DBUILD_TEST=true -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_CONTRIB=true .. + else + cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null + fi + make -j + make install } function runCasesOneByOne () { - while read -r line; do - if [[ "$line" != "#"* ]]; then - cmd=`echo $line | cut -d',' -f 5` + while read -r line; do + if [[ "$line" != "#"* ]]; then + cmd=`echo $line | cut -d',' -f 5` if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then case=`echo $cmd | cut -d' ' -f 3` start_time=`date +%s` @@ -77,18 +76,7 @@ function runCasesOneByOne () { || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_COVERAGE_REPORT - elif [[ "$2" == "system-test" ]] && [[ $line == *"system-test"* ]]; then - if [[ "$cmd" == *"pytest.sh"* ]]; then - cmd=`echo $cmd | cut -d' ' -f 2-20` - fi - case=`echo $cmd | cut -d' ' -f 4-20` - start_time=`date +%s` - date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && $cmd > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \ - echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT - end_time=`date +%s` - echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_COVERAGE_REPORT - elif [[ "$2" == "develop-test" ]] && [[ $line == *"develop-test"* ]]; then + elif [$line == *"$2"* ]; then if [[ "$cmd" == *"pytest.sh"* ]]; then cmd=`echo $cmd | cut -d' ' -f 2-20` fi @@ -101,209 +89,176 @@ function runCasesOneByOne () { echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_COVERAGE_REPORT fi fi - done < $1 + done < $1 } function runUnitTest() { - echo "=== Run unit test case ===" - echo " $TDENGINE_DIR/debug" - cd $TDENGINE_DIR/debug - ctest -j12 - - echo " $TDENGINE_DIR/tests/script/api" - cd $TDENGINE_DIR/tests/script/api - make clean && make - - stopTaosd - stopTaosadapter - - nohup taosd -c /etc/taos >> /dev/null 2>&1 & - ./batchprepare 127.0.0.1 - ./dbTableRoute 127.0.0.1 - ./stopquery 127.0.0.1 demo t1 - - echo "3.0 unit test done" + echo "=== Run unit test case ===" + echo " $TDENGINE_DIR/debug" + cd $TDENGINE_DIR/debug + ctest -j12 + echo "3.0 unit test done" } function runSimCases() { - echo "=== Run sim cases ===" - - cd $TDENGINE_DIR/tests/script - runCasesOneByOne ../parallel_test/cases.task sim - - totalSuccess=`grep 'sim success' $TDENGINE_COVERAGE_REPORT | wc -l` - if [ "$totalSuccess" -gt "0" ]; then - echo "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_COVERAGE_REPORT - fi + echo "=== Run sim cases ===" - totalFailed=`grep 'sim failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` - if [ "$totalFailed" -ne "0" ]; then - echo "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_COVERAGE_REPORT - fi + cd $TDENGINE_DIR/tests/script + runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/cases.task sim + + totalSuccess=`grep 'sim success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + echo "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'sim failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + echo "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_COVERAGE_REPORT + fi } function runPythonCases() { - echo "=== Run python cases ===" + echo "=== Run python cases ===" - cd $TDENGINE_DIR/tests/parallel_test - sed -i '/compatibility.py/d' cases.task - - cd $TDENGINE_DIR/tests/system-test - runCasesOneByOne ../parallel_test/cases.task system-test + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' cases.task - cd $TDENGINE_DIR/tests/develop-test - runCasesOneByOne ../parallel_test/cases.task develop-test + # army + cd $TDENGINE_DIR/tests/army + runCasesOneByOne ../parallel_test/cases.task army - totalSuccess=`grep 'py success' $TDENGINE_COVERAGE_REPORT | wc -l` - if [ "$totalSuccess" -gt "0" ]; then - echo "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_COVERAGE_REPORT - fi + # system-test + cd $TDENGINE_DIR/tests/system-test + runCasesOneByOne ../parallel_test/cases.task system-test - totalFailed=`grep 'py failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` - if [ "$totalFailed" -ne "0" ]; then - echo "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_COVERAGE_REPORT - fi + # develop-test + cd $TDENGINE_DIR/tests/develop-test + runCasesOneByOne ../parallel_test/cases.task develop-test + + totalSuccess=`grep 'py success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + echo "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'py failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + echo "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_COVERAGE_REPORT + fi } function runJDBCCases() { - echo "=== Run JDBC cases ===" + echo "=== Run JDBC cases ===" - cd $JDBC_DIR - git checkout -- . - git reset --hard HEAD - git checkout main - git pull + cd $JDBC_DIR + git checkout -- . + git reset --hard HEAD + git checkout main + git pull - stopTaosd - stopTaosadapter + stopTaosd + stopTaosadapter - nohup taosd -c /etc/taos >> /dev/null 2>&1 & - nohup taosadapter >> /dev/null 2>&1 & + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos >> /dev/null 2>&1 & + nohup taosadapter >> /dev/null 2>&1 & - mvn clean test > result.txt 2>&1 - summary=`grep "Tests run:" result.txt | tail -n 1` - echo -e "### JDBC test result: $summary ###" | tee -a $TDENGINE_COVERAGE_REPORT -} - -function runTaosKeeperCases() { - echo "=== Run taoskeeper cases ===" - - cd $TAOSKEEPER_DIR - git checkout -- . - git reset --hard HEAD - git checkout master - git pull - - stopTaosd - stopTaosadapter - - taosd -c /etc/taos >> /dev/null 2>&1 & - taosadapter >> /dev/null 2>&1 & - - go mod tidy && go test -v ./... + mvn clean test > result.txt 2>&1 + summary=`grep "Tests run:" result.txt | tail -n 1` + echo -e "### JDBC test result: $summary ###" | tee -a $TDENGINE_COVERAGE_REPORT } function runTest() { - echo "run Test" - - cd $TDENGINE_DIR - [ -d sim ] && rm -rf sim + echo "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT - runUnitTest - runSimCases - runPythonCases - runJDBCCases - runTaosKeeperCases - - stopTaosd - cd $TDENGINE_DIR/tests/script - find . -name '*.sql' | xargs rm -f + runUnitTest + runSimCases + runPythonCases + runJDBCCases - cd $TDENGINE_DIR/tests/pytest - find . -name '*.sql' | xargs rm -f + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f } function lcovFunc { - echo "collect data by lcov" - cd $TDENGINE_DIR + echo "collect data by lcov" + cd $TDENGINE_DIR - # collect data - lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + # collect data + lcov --ignore-errors negative --ignore-errors mismatch -d . --capture --rc lcov_branch_coverage=1 --rc branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info - # remove exclude paths - if [ "$branch" == "main" ] ; then - lcov --remove coverage.info \ - '*/contrib/*' '*/tests/*' '*/test/*' '*/tools/*' '*/libs/sync/*'\ - '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\ - '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ - '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\ - '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.c' '*/shellAuto.c' '*/shellTire.c'\ - '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' '*/catalog.c'\ - '*/tqSnapshot.c' '*/tsdbSnapshot.c''*/metaSnapshot.c' '*/smaSnapshot.c' '*/tqOffsetSnapshot.c'\ - '*/vnodeSnapshot.c' '*/metaSnapshot.c' '*/tsdbSnapshot.c' '*/mndGrant.c' '*/mndSnode.c' '*/streamRecover.c'\ - '*/osAtomic.c' '*/osDir.c' '*/osFile.c' '*/osMath.c' '*/osSignal.c' '*/osSleep.c' '*/osString.c' '*/osSystem.c'\ - '*/osThread.c' '*/osTime.c' '*/osTimezone.c' \ - --rc lcov_branch_coverage=1 -o coverage.info - else - lcov --remove coverage.info \ - '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c' '*/clientJniConnector.c'\ - --rc lcov_branch_coverage=1 -o coverage.info - fi + # remove exclude paths + lcov --remove coverage.info \ + '*/contrib/*' '*/tests/*' '*/test/*' '*/packaging/*' '*/taos-tools/*' '*/taosadapter/*' '*/TSZ/*' \ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/examples/*' '*/tidpool.c' '*/tmempool.c'\ + '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.c'\ + '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ + '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ + --rc branch_coverage=1 -o coverage.info + # generate result + echo "generate result" + lcov -l --rc branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT - # generate result - echo "generate result" - lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + sed -i 's/\/root\/TDengine\/sql.c/\/root\/TDengine\/source\/libs\/parser\/inc\/sql.c/g' coverage.info + sed -i 's/\/root\/TDengine\/sql.y/\/root\/TDengine\/source\/libs\/parser\/inc\/sql.y/g' coverage.info - # push result to coveralls.io - echo "push result to coveralls.io" - /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + # push result to coveralls.io + echo "push result to coveralls.io" + /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + + #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT + #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT + cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT - #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT - #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT - cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT - } function sendReport { - echo "send report" - receiver="develop@taosdata.com" - mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + echo "send report" + receiver="develop@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" - cd $TDENGINE_DIR + cd $TDENGINE_DIR - sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT - BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` - echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ - (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ - /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT + BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` + echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ + /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" } function stopTaosd { - echo "Stop taosd start" + echo "Stop taosd start" systemctl stop taosd - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - while [ -n "$PID" ] - do + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do pkill -TERM -x taosd sleep 1 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - done - echo "Stop tasod end" + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasod end" } function stopTaosadapter { - echo "Stop taosadapter" - systemctl stop taosadapter.service - PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` - while [ -n "$PID" ] - do - pkill -TERM -x taosadapter - sleep 1 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` - done - echo "Stop tasoadapter end" + echo "Stop taosadapter" + systemctl stop taosadapter.service + PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosadapter + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasoadapter end" } @@ -318,7 +273,7 @@ buildTDengine runTest lcovFunc -sendReport +#sendReport stopTaosd date >> $WORK_DIR/cron.log diff --git a/tests/system-test/2-query/blockSMA.py b/tests/system-test/2-query/blockSMA.py index 85c0189e27..605542cb3c 100644 --- a/tests/system-test/2-query/blockSMA.py +++ b/tests/system-test/2-query/blockSMA.py @@ -16,7 +16,7 @@ class TDTestCase: def run(self): dbname = "db" - tdSql.prepare() + tdSql.prepare(dbname=dbname, drop=True, stt_trigger=1) tdSql.execute(f'''create table {dbname}.ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') From 216830f8b4e35254e707fbf23189e77d4ee05f83 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Jan 2024 20:33:13 +0800 Subject: [PATCH 167/198] fix:[TD-28185]add pause & resume logic for snode --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 70bfa00609..399788e731 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -853,7 +853,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m STaskId* pHTaskId = &pTask->hTaskInfo.id; SStreamTask* pHistoryTask = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); if (pHistoryTask) { - code = tqProcessTaskResumeImpl(pMeta, pHistoryTask, sversion, pReq->igUntreated, fromVnode); + code = tqProcessTaskResumeImpl(handle, pHistoryTask, sversion, pReq->igUntreated, fromVnode); } return code; From b4b44009587f42a0eb61fc50c507e9d450fda76b Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 10 Jan 2024 21:36:09 +0800 Subject: [PATCH 168/198] fix: recalcuate window ekey --- source/libs/executor/src/executorInt.c | 8 +++++++- source/libs/executor/test/timewindowTest.cpp | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 95d26fdd0e..cb4988c31f 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -449,7 +449,13 @@ STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + if (pInterval->offset > 0) { + int64_t slideStart = taosTimeAdd(win.skey, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); + int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + win.ekey = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision); + } else { + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + } if (win.ekey < win.skey) { win.ekey = INT64_MAX; } diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index 7a83b64fac..3639bf15e1 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -164,6 +164,11 @@ TEST(testCase, timewindow_natural) { int32_t precision = TSDB_TIME_PRECISION_MILLI; SInterval interval2 = createInterval(17, 17, 13392000000, 'n', 'n', 0, precision); + int64_t key = 1648970865984; + STimeWindow w0 = getAlignQueryTimeWindow(&interval2, key); + printTimeWindow(&w0, precision, key); + ASSERT_GE(w0.ekey, key); + int64_t key1 = 1633446027072; STimeWindow w1 = {0}; getInitialStartTimeWindow(&interval2, key1, &w1, true); From ab137a56f0107734a766023bd91eebfabba20659 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 11 Jan 2024 08:31:16 +0800 Subject: [PATCH 169/198] fix: more interval end calculation with slide-start/slide-end as anchor --- include/common/ttime.h | 1 + source/common/src/ttime.c | 12 ++++++++++++ source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/executorInt.c | 8 +------- source/libs/executor/src/timewindowoperator.c | 4 ++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index ed4d1a9290..4db25d2f71 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -75,6 +75,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); +int64_t taosTimeGetIntervalEnd(int64_t ts, const SInterval* pInterval); int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision, int32_t order); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 376cfce255..f683baee7c 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -888,6 +888,18 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { return start; } +// used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point +int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) { + if (pInterval->offset > 0) { + int64_t slideStart = taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); + int64_t end = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + int64_t result = taosTimeAdd(end, pInterval->offset, pInterval->offsetUnit, pInterval->precision); + return result; + } else { + int64_t result = taosTimeAdd(intervalStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return result; + } +} // internal function, when program is paused in debugger, // one can call this function from debugger to print a // timestamp as human readable string, for example (gdb): diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b713b9b112..019483a015 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1838,7 +1838,7 @@ static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { STimeWindow w = {0}; w.skey = taosTimeTruncate(ts, pInterval); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + w.ekey = taosTimeGetIntervalEnd(w.skey, pInterval); return w; } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index cb4988c31f..ff4d3d0d27 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -449,13 +449,7 @@ STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - if (pInterval->offset > 0) { - int64_t slideStart = taosTimeAdd(win.skey, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); - int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - win.ekey = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision); - } else { - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - } + win.ekey = taosTimeGetIntervalEnd(win.skey, pInterval); if (win.ekey < win.skey) { win.ekey = INT64_MAX; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index afe1921d30..c459e4d8b9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -450,7 +450,7 @@ int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBl TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { pNext->skey = taosTimeTruncate(next, pInterval); - pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + pNext->ekey = taosTimeGetIntervalEnd(pNext->skey, pInterval); } else { pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; pNext->skey = pNext->ekey - pInterval->interval + 1; @@ -459,7 +459,7 @@ int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBl TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { pNext->skey = taosTimeTruncate(next, pInterval); - pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + pNext->ekey = taosTimeGetIntervalEnd(pNext->skey, pInterval); } else { pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; pNext->ekey = pNext->skey + pInterval->interval - 1; From 9294c8d4eea9a6344a92669191b29459b1ecb4e6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Jan 2024 09:20:34 +0800 Subject: [PATCH 170/198] fix(stream): enable the hb to mnode for stream meta when it becomes the leader. --- source/libs/stream/src/streamMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 3522feb244..30a40ffc9f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1287,7 +1287,7 @@ void streamMetaStartHb(SStreamMeta* pMeta) { int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); metaRefMgtAdd(pMeta->vgId, pRid); *pRid = pMeta->rid; -// metaHbToMnode(pRid, NULL); + metaHbToMnode(pRid, NULL); } void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { From 3b36dbbe189f625896c6b4e13a0badd9ace42afb Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 11 Jan 2024 09:40:54 +0800 Subject: [PATCH 171/198] opt recover buff --- source/libs/stream/src/tstreamFileState.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index d31e4cbfcd..aac78cf03b 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -26,6 +26,7 @@ #define FLUSH_NUM 4 #define DEFAULT_MAX_STREAM_BUFFER_SIZE (128 * 1024 * 1024) #define MIN_NUM_OF_ROW_BUFF 10240 +#define MIN_NUM_OF_RECOVER_ROW_BUFF 128 #define TASK_KEY "streamFileState" #define STREAM_STATE_INFO_NAME "StreamStateCheckPoint" @@ -660,7 +661,7 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { if (pCur == NULL) { return -1; } - int32_t recoverNum = TMIN(MIN_NUM_OF_ROW_BUFF, pFileState->maxRowCount); + int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; @@ -694,7 +695,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { if (pCur == NULL) { return -1; } - int32_t recoverNum = TMIN(MIN_NUM_OF_ROW_BUFF, pFileState->maxRowCount); + int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; From 1c562bd535215f8292f993c8253501302ec84cee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Jan 2024 09:41:40 +0800 Subject: [PATCH 172/198] refactor: enable the restart of stream tasks. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 15 ++++++++++++--- source/libs/stream/src/streamMeta.c | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 2e83cd5bc5..b7b3893dfe 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -729,9 +729,9 @@ int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta, int32_t* numOfTasks) { SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); streamTaskResetStatus(*pTask); - if ((*pTask)->info.fillHistory == 1) { - streamResetParamForScanHistory(*pTask); - } +// if ((*pTask)->info.fillHistory == 1) { +// streamResetParamForScanHistory(*pTask); +// } } return 0; @@ -764,10 +764,19 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { } streamMetaWLock(pMeta); + streamMetaClear(pMeta); int64_t el = taosGetTimestampMs() - st; tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el / 1000.); + code = streamMetaLoadAllTasks(pMeta); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + { STaskStartInfo* pStartInfo = &pMeta->startInfo; taosHashClear(pStartInfo->pReadyTaskSet); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 30a40ffc9f..6bb90a9c25 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -469,6 +469,7 @@ void streamMetaClear(SStreamMeta* pMeta) { taosArrayClear(pMeta->pTaskList); taosArrayClear(pMeta->chkpSaved); taosArrayClear(pMeta->chkpInUse); + pMeta->numOfStreamTasks = 0; pMeta->numOfPausedTasks = 0; From f6c06491084fa1ce5a24fe2d28f7abcfbb0ad2da Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Jan 2024 09:53:19 +0800 Subject: [PATCH 173/198] fix(stream): fix syntax error. --- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 9112d9bda7..4815dfe65c 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -194,7 +194,7 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { case TDMT_STREAM_TASK_DEPLOY: { void * pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); - return tqStreamTaskProcessDeployReq(pSnode->pMeta, pMsg->info.conn.applyIndex, pReq, len, true, true); + return tqStreamTaskProcessDeployReq(pSnode->pMeta, &pSnode->msgCb,pMsg->info.conn.applyIndex, pReq, len, true, true); } case TDMT_STREAM_TASK_DROP: diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a745f7996e..da1c2ecf16 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -947,7 +947,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t } streamTaskResume(pTask); - ETaskStatus status = streamTaskGetStatus(pTask, NULL); + ETaskStatus status = streamTaskGetStatus(pTask)->state; int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__SINK) { From 686108050ba6bcde7946423ff1d6f9246dd9715f Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 10 Jan 2024 18:44:04 +0800 Subject: [PATCH 174/198] fix: table merge scan return disordered rows --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 15 ++++++++++++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index b57d2dfb45..256ef10c5e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -73,6 +73,7 @@ static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order static void resetTableListIndex(SReaderStatus* pStatus); static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey); static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo); +static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -3095,6 +3096,17 @@ static ERetrieveType doReadDataFromSttFiles(STsdbReader* pReader) { return TSDB_READ_RETURN; } + if (pReader->status.bProcMemPreFileset) { + code = buildFromPreFilesetBuffer(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (pResBlock->info.rows > 0) { + pReader->status.processingMemPreFileSet = true; + return TSDB_READ_RETURN; + } + } + if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. return TSDB_READ_CONTINUE; } else { // all blocks in data file are checked, let's check the data in last files @@ -4342,6 +4354,7 @@ static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) { } else { tsdbDebug("finished pre-fileset %d buffer processing. %s", fid, pReader->idStr); pStatus->bProcMemPreFileset = false; + pStatus->processingMemPreFileSet = false; if (pReader->notifyFn) { STsdReaderNotifyInfo info = {0}; info.duration.filesetId = fid; @@ -4374,7 +4387,7 @@ static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { pStatus->bProcMemFirstFileset, pReader->idStr); if (pStatus->bProcMemPreFileset) { if (pBlock->info.rows > 0) { - if (pReader->notifyFn) { + if (pReader->notifyFn && !pReader->status.processingMemPreFileSet) { int32_t fid = pReader->status.pCurrentFileset->fid; STsdReaderNotifyInfo info = {0}; info.duration.filesetId = fid; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 39e65f22b1..2157c7a423 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -219,6 +219,7 @@ typedef struct SReaderStatus { int64_t prevFilesetStartKey; int64_t prevFilesetEndKey; bool bProcMemFirstFileset; + bool processingMemPreFileSet; STableUidList procMemUidList; STableBlockScanInfo** pProcMemTableIter; } SReaderStatus; From df235b0adf1918842740496c0d54e6546907d413 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 11 Jan 2024 03:22:34 +0000 Subject: [PATCH 175/198] split sync/status channel --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/libs/transport/src/transCli.c | 1 + source/libs/transport/src/transComm.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 545a5fc870..d83ee541df 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -161,7 +161,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SEpSet epSet = {0}; int8_t epUpdated = 0; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, tsStatusInterval * 10); + rpcSendRecvWithTimeout(pMgmt->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, tsStatusInterval * 10 * 1000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); char tbuf[512]; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 4b3000b47e..1ccf708703 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -2701,6 +2701,7 @@ int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STr ret = TSDB_CODE_TIMEOUT_ERROR; } else { memcpy(pRsp, pSyncMsg->pRsp, sizeof(STransMsg)); + pSyncMsg->pRsp->pCont = NULL; if (pSyncMsg->hasEpSet == 1) { epsetAssign(pEpSet, &pSyncMsg->epSet); *epUpdated = 1; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 759a4d79db..b1fb9a2450 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -667,7 +667,7 @@ void transDestroySyncMsg(void* msg) { STransSyncMsg* pSyncMsg = msg; tsem_destroy(pSyncMsg->pSem); taosMemoryFree(pSyncMsg->pSem); - + transFreeMsg(pSyncMsg->pRsp->pCont); taosMemoryFree(pSyncMsg->pRsp); taosMemoryFree(pSyncMsg); } From 58694c67dd73b187c6972d11c686997e2308f1f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Jan 2024 11:50:52 +0800 Subject: [PATCH 176/198] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 18 +++++++++--------- include/util/tworker.h | 5 ++--- source/common/src/systable.c | 2 +- source/common/src/tglobal.c | 4 ++-- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 8 ++------ source/dnode/mnode/impl/src/mndStream.c | 2 +- source/libs/stream/src/streamMeta.c | 2 +- 7 files changed, 18 insertions(+), 23 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index d11a4ad23b..9ea655c15c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -685,18 +685,18 @@ typedef struct STaskStatusEntry { int32_t statusLastDuration; // to record the last duration of current status int64_t stage; int32_t nodeId; - int64_t verStart; // start version in WAL, only valid for source task - int64_t verEnd; // end version in WAL, only valid for source task - int64_t processedVer; // only valid for source task + int64_t verStart; // start version in WAL, only valid for source task + int64_t verEnd; // end version in WAL, only valid for source task + int64_t processedVer; // only valid for source task int64_t activeCheckpointId; // current active checkpoint id - int32_t chkpointTransId; // checkpoint trans id - bool checkpointFailed; // denote if the checkpoint is failed or not - bool inputQChanging; // inputQ is changing or not + int32_t chkpointTransId; // checkpoint trans id + bool checkpointFailed; // denote if the checkpoint is failed or not + bool inputQChanging; // inputQ is changing or not int64_t inputQUnchangeCounter; - double inputQUsed; // in MiB + double inputQUsed; // in MiB double inputRate; - double sinkQuota; // existed quota size for sink task - double sinkDataSize; // sink to dst data size + double sinkQuota; // existed quota size for sink task + double sinkDataSize; // sink to dst data size } STaskStatusEntry; typedef struct SStreamHbMsg { diff --git a/include/util/tworker.h b/include/util/tworker.h index 8508adf052..f39540d24b 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -23,7 +23,6 @@ extern "C" { #endif -typedef struct SQWorkerPool SQWorkerPool; typedef struct SWWorkerPool SWWorkerPool; typedef struct SQueueWorker { @@ -60,14 +59,14 @@ typedef struct SWWorker { SWWorkerPool *pool; } SWWorker; -typedef struct SWWorkerPool { +struct SWWorkerPool { int32_t max; // max number of workers int32_t num; int32_t nextId; // from 0 to max-1, cyclic const char *name; SWWorker *workers; TdThreadMutex mutex; -} SWWorkerPool; +}; int32_t tQWorkerInit(SQWorkerPool *pool); void tQWorkerCleanup(SQWorkerPool *pool); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 2e52c77080..75a54a0cd5 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -166,7 +166,7 @@ static const SSysDbTableSchema streamTaskSchema[] = { {.name = "node_type", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "node_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "level", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "status", .bytes = 15 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 79d21955d4..b72c4fe077 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -float tsRatioOfVnodeStreamThreads = 4.0; +float tsRatioOfVnodeStreamThreads = 1.0; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; @@ -621,7 +621,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { 0) return -1; - if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER, + if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 10, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 9a792a2774..8b80527447 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -407,12 +407,8 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { if (tWWorkerInit(pFPool) != 0) return -1; SSingleWorkerCfg mgmtCfg = { - .min = 1, - .max = 1, - .name = "vnode-mgmt", - .fp = (FItem)vmProcessMgmtQueue, - .param = pMgmt, - }; + .min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1; dDebug("vnode workers are initialized"); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 146f9f6fc4..95150b9d6e 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1728,7 +1728,7 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); // output queue - // sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate); +// sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate); // STR_TO_VARSTR(vbuf, buf); // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index de435c63a3..04d34b0945 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1102,7 +1102,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)), }; - entry.inputRate = entry.inputQUsed * 100.0 / STREAM_TASK_QUEUE_CAPACITY_IN_SIZE; + entry.inputRate = entry.inputQUsed * 100.0 / (2*STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { entry.sinkQuota = (*pTask)->outputInfo.pTokenBucket->quotaRate; entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize); From 5f7ce21530cfc7535dc8fc80aedb03a27d60235b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Jan 2024 11:59:29 +0800 Subject: [PATCH 177/198] refactor: do some internal refactor. --- source/dnode/mnode/impl/inc/mndStream.h | 2 +- source/dnode/mnode/impl/src/mndStream.c | 21 ++++++++++---------- source/dnode/mnode/impl/src/mndStreamTrans.c | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 7b71ad873b..58a4c92d3e 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -72,7 +72,7 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamUid); int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId); bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamUid, const char *pTransName, bool lock); -int32_t mndStreamGetRelCheckpointTrans(SMnode *pMnode, int64_t streamUid); +int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamUid); // for sma // TODO refactor diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 95150b9d6e..a69543a6d6 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -81,7 +81,7 @@ static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExe static void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); -static void killCheckpointTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); +static void killTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); static void freeCheckpointCandEntry(void *); @@ -95,9 +95,6 @@ static int32_t mndStreamSeqActionInsert(SSdb *pSdb, SStreamSeq *pStream); static int32_t mndStreamSeqActionDelete(SSdb *pSdb, SStreamSeq *pStream); static int32_t mndStreamSeqActionUpdate(SSdb *pSdb, SStreamSeq *pOldStream, SStreamSeq *pNewStream); -static SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); -static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); - int32_t mndInitStream(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_STREAM, @@ -1455,9 +1452,10 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { } // kill the related checkpoint trans - int32_t transId = mndStreamGetRelCheckpointTrans(pMnode, pStream->uid); + int32_t transId = mndStreamGetRelTrans(pMnode, pStream->uid); if (transId != 0) { - killCheckpointTransImpl(pMnode, transId, pStream->sourceDb); + mDebug("drop active related transId:%d due to stream:%s dropped", transId, pStream->name); + killTransImpl(pMnode, transId, pStream->sourceDb); } removeStreamTasksInBuf(pStream, &execInfo); @@ -1502,9 +1500,10 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { #endif // kill the related checkpoint trans - int32_t transId = mndStreamGetRelCheckpointTrans(pMnode, pStream->uid); + int32_t transId = mndStreamGetRelTrans(pMnode, pStream->uid); if (transId != 0) { - killCheckpointTransImpl(pMnode, transId, pStream->sourceDb); + mDebug("drop active related transId:%d due to stream:%s dropped", transId, pStream->name); + killTransImpl(pMnode, transId, pStream->sourceDb); } // drop the stream obj in execInfo @@ -2836,10 +2835,10 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return TSDB_CODE_ACTION_IN_PROGRESS; } -void killCheckpointTransImpl(SMnode* pMnode, int32_t transId, const char* pDbName) { +void killTransImpl(SMnode* pMnode, int32_t transId, const char* pDbName) { STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { - mInfo("kill checkpoint transId:%d in Db:%s", transId, pDbName); + mInfo("kill active transId:%d in Db:%s", transId, pDbName); mndKillTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); } @@ -2859,7 +2858,7 @@ int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { } char* pDupDBName = strndup(pDBName, len); - killCheckpointTransImpl(pMnode, pTransInfo->transId, pDupDBName); + killTransImpl(pMnode, pTransInfo->transId, pDupDBName); taosMemoryFree(pDupDBName); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index f5047acc49..bfea4349b0 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -113,7 +113,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamUid, const char* return false; } -int32_t mndStreamGetRelCheckpointTrans(SMnode* pMnode, int64_t streamUid) { +int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) { taosThreadMutexLock(&execInfo.lock); int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { @@ -127,7 +127,7 @@ int32_t mndStreamGetRelCheckpointTrans(SMnode* pMnode, int64_t streamUid) { SStreamTransInfo tInfo = *pEntry; taosThreadMutexUnlock(&execInfo.lock); - if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) { + if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0) { return tInfo.transId; } } else { From 44146caa9a73c947c69a215d640dbcd8bd671d09 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 11 Jan 2024 13:46:54 +0800 Subject: [PATCH 178/198] opt msg on mnd --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index d83ee541df..14853009e0 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -45,7 +45,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) { SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer}; int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); + void * pHead = rpcMallocCont(contLen); tSerializeRetrieveIpWhite(pHead, contLen, &req); SRpcMsg rpcMsg = {.pCont = pHead, @@ -144,7 +144,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.ipWhiteVer = pMgmt->pData->ipWhiteVer; int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); + void * pHead = rpcMallocCont(contLen); tSerializeSStatusReq(pHead, contLen, &req); tFreeSStatusReq(&req); @@ -161,7 +161,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SEpSet epSet = {0}; int8_t epUpdated = 0; dmGetMnodeEpSet(pMgmt->pData, &epSet); - rpcSendRecvWithTimeout(pMgmt->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, tsStatusInterval * 10 * 1000); + rpcSendRecvWithTimeout(pMgmt->msgCb.statusRpc, &epSet, &rpcMsg, &rpcRsp, &epUpdated, tsStatusInterval * 5 * 1000); if (rpcRsp.code != 0) { dmRotateMnodeEpSet(pMgmt->pData); char tbuf[512]; @@ -189,7 +189,7 @@ void dmSendNotifyReq(SDnodeMgmt *pMgmt) { req.pVloads = vinfo.pVloads; int32_t contLen = tSerializeSNotifyReq(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); + void * pHead = rpcMallocCont(contLen); tSerializeSNotifyReq(pHead, contLen, &req); tFreeSNotifyReq(&req); @@ -284,7 +284,7 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } SSDataBlock *dmBuildVariablesBlock(void) { - SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + SSDataBlock * pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); size_t size = 0; const SSysTableMeta *pMeta = NULL; getInfosDbMeta(&pMeta, &size); From 423fa762074105df4bc6d03d4e8a6e91646c160f Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 11 Jan 2024 06:01:06 +0000 Subject: [PATCH 179/198] split sync/status channel --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 512f090ab0..4f72ac24b4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -364,7 +364,7 @@ int32_t dmInitClient(SDnode *pDnode) { SRpcInit rpcInit = {0}; rpcInit.label = "DND-C"; - rpcInit.numOfThreads = tsNumOfRpcThreads; + rpcInit.numOfThreads = tsNumOfRpcThreads / 2; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; @@ -383,7 +383,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.failFastThreshold = 3; // failed threshold rpcInit.ffp = dmFailFastFp; - int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3); + int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3) / 2; connLimitNum = TMAX(connLimitNum, 10); connLimitNum = TMIN(connLimitNum, 500); @@ -453,7 +453,7 @@ int32_t dmInitSyncClient(SDnode *pDnode) { SRpcInit rpcInit = {0}; rpcInit.label = "DND-SYNC"; - rpcInit.numOfThreads = tsNumOfRpcThreads; + rpcInit.numOfThreads = tsNumOfRpcThreads / 2; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; @@ -472,7 +472,7 @@ int32_t dmInitSyncClient(SDnode *pDnode) { rpcInit.failFastThreshold = 3; // failed threshold rpcInit.ffp = dmFailFastFp; - int32_t connLimitNum = 100; + int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3) / 2; connLimitNum = TMAX(connLimitNum, 10); connLimitNum = TMIN(connLimitNum, 500); From 7d46458bb70c38a5f5739e2ff6a5e510e8d4aa1f Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 11 Jan 2024 14:50:14 +0800 Subject: [PATCH 180/198] fix: use slide start to calculate previous slide start --- source/common/src/ttime.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index f683baee7c..bbda31efb1 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -873,16 +873,15 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->offset > 0) { // try to move current window to the left-hande-side, due to the offset effect. - int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newe = end; - int64_t slidingEnd = end; + int64_t newe = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + int64_t slidingStart = start; while (newe >= ts) { - end = slidingEnd; - slidingEnd = taosTimeAdd(slidingEnd, -pInterval->sliding, pInterval->slidingUnit, precision); + start = slidingStart; + slidingStart = taosTimeAdd(slidingStart, -pInterval->sliding, pInterval->slidingUnit, precision); + int64_t slidingEnd = taosTimeAdd(slidingStart, pInterval->interval, pInterval->intervalUnit, precision) - 1; newe = taosTimeAdd(slidingEnd, pInterval->offset, pInterval->offsetUnit, precision); } - int64_t slidingStart = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; - start = taosTimeAdd(slidingStart, pInterval->offset, pInterval->offsetUnit, precision); + start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); } return start; @@ -892,8 +891,8 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) { if (pInterval->offset > 0) { int64_t slideStart = taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); - int64_t end = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - int64_t result = taosTimeAdd(end, pInterval->offset, pInterval->offsetUnit, pInterval->precision); + int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision); return result; } else { int64_t result = taosTimeAdd(intervalStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; From d4fd544c745a3f46ef26ca7d9557945a8010d375 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Jan 2024 15:18:11 +0800 Subject: [PATCH 181/198] refactor: remove sleep. --- source/libs/stream/src/streamExec.c | 49 ++++++++++--------- source/libs/stream/src/streamQueue.c | 33 +++---------- .../8-stream/at_once_state_window.py | 3 ++ 3 files changed, 35 insertions(+), 50 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index eb5a432235..18f7ed061a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -548,17 +548,9 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock return code; } -static void setTaskSchedInfo(SStreamTask* pTask, int32_t idleTime) { - SStreamStatus* pStatus = &pTask->status; - - pStatus->schedIdleTime = idleTime; - pStatus->lastExecTs = taosGetTimestampMs(); -} - -static void clearTaskSchedInfo(SStreamTask* pTask) { - SStreamStatus* pStatus = &pTask->status; - pStatus->schedIdleTime = 0; -} +static void setTaskSchedInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } +static void clearTaskSchedInfo(SStreamTask* pTask) { pTask->status.schedIdleTime = 0; } +static void setLastExecTs(SStreamTask* pTask, int64_t ts) { pTask->status.lastExecTs = ts; } /** * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the @@ -574,21 +566,28 @@ int32_t doStreamExecTask(SStreamTask* pTask) { int32_t blockSize = 0; int32_t numOfBlocks = 0; SStreamQueueItem* pInput = NULL; + if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask)->state == TASK_STATUS__UNINIT)) { stDebug("s-task:%s stream task is stopped", id); - break; + return 0; } if (streamQueueIsFull(pTask->outputq.queue)) { stWarn("s-task:%s outputQ is full, idle for 500ms and retry", id); setTaskSchedInfo(pTask, 500); - break; + return 0; } if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { - stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", pTask->id.idStr); + stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", id); setTaskSchedInfo(pTask, 1000); - break; + return 0; + } + + if (taosGetTimestampMs() - pTask->status.lastExecTs < 50) { + stDebug("s-task:%s invoke with high frequency, idle and retry exec in 50ms", id); + setTaskSchedInfo(pTask, 50); + return 0; } /*int32_t code = */ streamTaskGetDataFromInputQ(pTask, &pInput, &numOfBlocks, &blockSize); @@ -597,9 +596,8 @@ int32_t doStreamExecTask(SStreamTask* pTask) { return 0; } - int32_t type = pInput->type; - // dispatch checkpoint msg to all downstream tasks + int32_t type = pInput->type; if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { streamProcessCheckpointBlock(pTask, (SStreamDataBlock*)pInput); continue; @@ -646,7 +644,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { if (ver != pInfo->processedVer) { stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64 " ckpt:%" PRId64, - pTask->id.idStr, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer); + id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer); pInfo->processedVer = ver; } @@ -659,7 +657,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { // todo add lock SStreamTaskState* pState = streamTaskGetStatus(pTask); if (pState->state == TASK_STATUS__CK) { - stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, pState->name); + stDebug("s-task:%s checkpoint block received, set status:%s", id, pState->name); streamTaskBuildCheckpoint(pTask); } else { // todo refactor @@ -672,8 +670,8 @@ int32_t doStreamExecTask(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", pTask->id.idStr, - 0, tstrerror(code)); + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", id, 0, + tstrerror(code)); } } @@ -774,19 +772,24 @@ int32_t streamResumeTask(SStreamTask* pTask) { // check if this task needs to be idle for a while if (pTask->status.schedIdleTime > 0) { - stDebug("s-task:%s idled, and will be invoked in %dms", id, pTask->status.schedIdleTime); schedTaskInFuture(pTask); taosThreadMutexUnlock(&pTask->lock); + setLastExecTs(pTask, taosGetTimestampMs()); return 0; } else { int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); + if ((numOfItems == 0) || streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); taosThreadMutexUnlock(&pTask->lock); + setLastExecTs(pTask, taosGetTimestampMs()); + char* p = streamTaskGetStatus(pTask)->name; - stDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, p, pTask->status.schedStatus); + stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p, + pTask->status.schedStatus, pTask->status.lastExecTs); + return 0; } } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 5f55285ab1..d3287a6b96 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -17,8 +17,7 @@ #define MAX_STREAM_EXEC_BATCH_NUM 32 #define MAX_SMOOTH_BURST_RATIO 5 // 5 sec -#define WAIT_FOR_DURATION 40 -#define OUTPUT_QUEUE_FULL_WAIT_DURATION 500 // 500 ms +#define WAIT_FOR_DURATION 10 // todo refactor: // read data from input queue @@ -161,7 +160,7 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu // no available token in bucket for sink task, let's wait for a little bit if (taskLevel == TASK_LEVEL__SINK && (!streamTaskExtractAvailableToken(pTask->outputInfo.pTokenBucket, pTask->id.idStr))) { stDebug("s-task:%s no available token in bucket for sink data, wait for 10ms", id); - taosMsleep(10); + taosMsleep(WAIT_FOR_DURATION); return TSDB_CODE_SUCCESS; } @@ -173,11 +172,10 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputq.queue); if (qItem == NULL) { - if ((taskLevel == TASK_LEVEL__SOURCE || taskLevel == TASK_LEVEL__SINK) && (++retryTimes) < MAX_RETRY_TIMES) { - taosMsleep(WAIT_FOR_DURATION); - // todo remove it - continue; - } +// if ((taskLevel == TASK_LEVEL__SOURCE || taskLevel == TASK_LEVEL__SINK) && (++retryTimes) < MAX_RETRY_TIMES) { +// taosMsleep(WAIT_FOR_DURATION); +// continue; +// } // restore the token to bucket if (*numOfBlocks > 0) { @@ -344,25 +342,6 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) // the result should be put into the outputQ in any cases, the result may be lost otherwise. int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBlock) { STaosQueue* pQueue = pTask->outputq.queue->pQueue; - -#if 0 - // wait for the output queue is available for new data to dispatch - while (streamQueueIsFull(pTask->outputq.queue)) { - if (streamTaskShouldStop(pTask)) { - stInfo("s-task:%s discard result block due to task stop", pTask->id.idStr); - return TSDB_CODE_STREAM_EXEC_CANCELLED; - } - - int32_t total = streamQueueGetNumOfItems(pTask->outputq.queue); - double size = SIZE_IN_MiB(taosQueueMemorySize(pQueue)); - // let's wait for there are enough space to hold this result pBlock - stDebug("s-task:%s outputQ is full, wait for %dms and retry, outputQ items:%d, size:%.2fMiB", pTask->id.idStr, - OUTPUT_QUEUE_FULL_WAIT_DURATION, total, size); - - taosMsleep(OUTPUT_QUEUE_FULL_WAIT_DURATION); - } -#endif - int32_t code = taosWriteQitem(pQueue, pBlock); int32_t total = streamQueueGetNumOfItems(pTask->outputq.queue); diff --git a/tests/system-test/8-stream/at_once_state_window.py b/tests/system-test/8-stream/at_once_state_window.py index fa9f4ddd78..60a4f4e890 100644 --- a/tests/system-test/8-stream/at_once_state_window.py +++ b/tests/system-test/8-stream/at_once_state_window.py @@ -59,6 +59,9 @@ class TDTestCase: self.tdCom.create_stream(stream_name=f'{self.tb_name}{self.tdCom.stream_suffix}', des_table=self.tb_stream_des_table, source_sql=f'select _wstart AS wstart, {self.tdCom.tb_source_select_str} from {self.tb_name} partition by {partition} {partition_elm_alias} state_window({stream_state_window})', trigger_mode="at_once", subtable_value=tb_subtable_value, fill_history_value=fill_history_value) range_times = self.tdCom.range_count state_window_max = self.tdCom.dataDict['state_window_max'] + + time.sleep(2) + for i in range(range_times): state_window_value = random.randint(int((i)*state_window_max/range_times), int((i+1)*state_window_max/range_times)) for i in range(2, range_times+3): From 5e1a7c694b790f2ece43112dd9127935e4d7079c Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 11 Jan 2024 07:23:46 +0000 Subject: [PATCH 182/198] split sync/status channel --- include/util/tdef.h | 34 +++++++++---------- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 6 ++-- source/libs/transport/src/transCli.c | 7 +++- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 875a6f5738..aee20514ad 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -109,12 +109,12 @@ extern const int32_t TYPE_BYTES[21]; #define TSDB_INS_USER_STABLES_DBNAME_COLID 2 static const int64_t TICK_PER_SECOND[] = { - 1000LL, // MILLISECOND - 1000000LL, // MICROSECOND - 1000000000LL, // NANOSECOND - 0LL, // HOUR - 0LL, // MINUTE - 1LL // SECOND + 1000LL, // MILLISECOND + 1000000LL, // MICROSECOND + 1000000000LL, // NANOSECOND + 0LL, // HOUR + 0LL, // MINUTE + 1LL // SECOND }; #define TSDB_TICK_PER_SECOND(precision) \ @@ -239,8 +239,8 @@ typedef enum ELogicConditionType { #define TSDB_MAX_SQL_SHOW_LEN 1024 #define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb -#define TSDB_VIEW_NAME_LEN 193 -#define TSDB_VIEW_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_VIEW_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_VIEW_NAME_LEN 193 +#define TSDB_VIEW_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_VIEW_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_APP_NAME_LEN TSDB_UNI_LEN #define TSDB_TB_COMMENT_LEN 1025 @@ -260,7 +260,7 @@ typedef enum ELogicConditionType { #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 32 -#define TSDB_LABEL_LEN 12 +#define TSDB_LABEL_LEN 16 #define TSDB_JOB_STATUS_LEN 32 #define TSDB_CLUSTER_ID_LEN 40 @@ -288,7 +288,7 @@ typedef enum ELogicConditionType { #define TSDB_ACTIVE_KEY_LEN 109 #define TSDB_CONN_ACTIVE_KEY_LEN 255 -#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE +#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE #define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024) #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE @@ -397,13 +397,13 @@ typedef enum ELogicConditionType { #define TSDB_MAX_STT_TRIGGER 1 #define TSDB_DEFAULT_SST_TRIGGER 1 #endif -#define TSDB_STT_TRIGGER_ARRAY_SIZE 16 // maximum of TSDB_MAX_STT_TRIGGER of TD_ENTERPRISE and TD_COMMUNITY -#define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN) -#define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2) -#define TSDB_DEFAULT_HASH_PREFIX 0 -#define TSDB_MIN_HASH_SUFFIX (2 - TSDB_TABLE_NAME_LEN) -#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) -#define TSDB_DEFAULT_HASH_SUFFIX 0 +#define TSDB_STT_TRIGGER_ARRAY_SIZE 16 // maximum of TSDB_MAX_STT_TRIGGER of TD_ENTERPRISE and TD_COMMUNITY +#define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN) +#define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2) +#define TSDB_DEFAULT_HASH_PREFIX 0 +#define TSDB_MIN_HASH_SUFFIX (2 - TSDB_TABLE_NAME_LEN) +#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) +#define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 #define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600 diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 4f72ac24b4..ceafdbd2e2 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -363,7 +363,7 @@ int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; - rpcInit.label = "DND-C"; + rpcInit.label = "DND-CLI"; rpcInit.numOfThreads = tsNumOfRpcThreads / 2; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; @@ -407,7 +407,7 @@ int32_t dmInitStatusClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; - rpcInit.label = "DND-STATUS"; + rpcInit.label = "DND-STATUS-CLI"; rpcInit.numOfThreads = 1; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; @@ -452,7 +452,7 @@ int32_t dmInitSyncClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; - rpcInit.label = "DND-SYNC"; + rpcInit.label = "DND-SYNC-CLI"; rpcInit.numOfThreads = tsNumOfRpcThreads / 2; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 1ccf708703..8c65a3ac1b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1907,7 +1907,12 @@ bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) { static void* cliWorkThread(void* arg) { SCliThrd* pThrd = (SCliThrd*)arg; pThrd->pid = taosGetSelfPthreadId(); - setThreadName("trans-cli-work"); + + char threadName[TSDB_LABEL_LEN] = {0}; + STrans* pInst = pThrd->pTransInst; + strntolower(threadName, pInst->label, sizeof(threadName)); + setThreadName(threadName); + uv_run(pThrd->loop, UV_RUN_DEFAULT); tDebug("thread quit-thread:%08" PRId64, pThrd->pid); From 5a1510aa2fd7769d6916cac544455d20268fc4d2 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 11 Jan 2024 07:30:03 +0000 Subject: [PATCH 183/198] split sync/status channel --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index ceafdbd2e2..d79e9ff8ef 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -407,7 +407,7 @@ int32_t dmInitStatusClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; - rpcInit.label = "DND-STATUS-CLI"; + rpcInit.label = "DNODE-STA-CLI"; rpcInit.numOfThreads = 1; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; @@ -452,7 +452,7 @@ int32_t dmInitSyncClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; - rpcInit.label = "DND-SYNC-CLI"; + rpcInit.label = "DNODE-SYNC-CLI"; rpcInit.numOfThreads = tsNumOfRpcThreads / 2; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; From b0f670947452c47d6560a540ebdaf7edcc1a3222 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 11 Jan 2024 07:30:39 +0000 Subject: [PATCH 184/198] split sync/status channel --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index d79e9ff8ef..3b7ecce77c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -363,7 +363,7 @@ int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; - rpcInit.label = "DND-CLI"; + rpcInit.label = "DNODE-CLI"; rpcInit.numOfThreads = tsNumOfRpcThreads / 2; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; From 3cea4fd2c79e8f1727acf837228219cf9a8985f3 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 11 Jan 2024 16:41:11 +0800 Subject: [PATCH 185/198] fix: getNextTimeWindow use new time algorithm --- source/common/src/ttime.c | 12 +++++++++- source/libs/executor/src/executil.c | 34 +++++++++-------------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index bbda31efb1..775cff6670 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -683,6 +683,10 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati return getDuration(*duration, *unit, duration, timePrecision); } +static bool taosIsLeapYear(int32_t year) { + return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); +} + int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; @@ -702,7 +706,13 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - + int daysOfMonth[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + if (taosIsLeapYear(1900 + tm.tm_year)) { + daysOfMonth[1] = 29; + } + if (tm.tm_mday > daysOfMonth[tm.tm_mon]) { + tm.tm_mday = daysOfMonth[tm.tm_mon]; + } return (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision) + fraction); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 019483a015..efab09f02b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1887,31 +1887,17 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI } void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { + int64_t slidingStart = 0; + if (pInterval->offset > 0) { + slidingStart = taosTimeAdd(tw->skey, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); + } else { + slidingStart = tw->skey; + } int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (!IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { - tw->skey += pInterval->sliding * factor; - tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - return; - } - - // convert key to second - int64_t key = convertTimePrecision(tw->skey, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - int64_t duration = pInterval->sliding; - if (pInterval->slidingUnit == 'y') { - duration *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm, NULL); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + duration * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + slidingStart = taosTimeAdd(slidingStart, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + tw->skey = taosTimeAdd(slidingStart, pInterval->offset, pInterval->offsetUnit, pInterval->precision); + int64_t slidingEnd = taosTimeAdd(slidingStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + tw->ekey = taosTimeAdd(slidingEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision); } bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { From 9703018f561fedc209e23ef7829187a37cd8556c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Jan 2024 16:47:51 +0800 Subject: [PATCH 186/198] fix(stream): add checkpoint status check. --- source/dnode/mnode/impl/src/mndStream.c | 13 -------- source/libs/stream/src/streamCheckpoint.c | 40 ++++++++++++++--------- source/libs/stream/src/streamQueue.c | 5 --- 3 files changed, 25 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index a69543a6d6..f4ece1a141 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -3116,19 +3116,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (p->status != TASK_STATUS__READY) { mDebug("received s-task:0x%" PRIx64 " not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status)); - -// if (p->status == TASK_STATUS__STREAM_SCAN_HISTORY) { -// bool drop = needDropRelatedFillhistoryTask(pTaskEntry, &execInfo); -// if (drop) { -// SStreamObj *pStreamObj = mndGetStreamObj(pMnode, pTaskEntry->id.streamId); -// if (pStreamObj == NULL) { -// mError("failed to acquire the streamObj:0x%" PRIx64 " it may have been dropped", pStreamObj->uid); -// } else { -// mndDropRelatedFillhistoryTask(pMnode, pTaskEntry, pStreamObj); -// mndReleaseStream(pMnode, pStreamObj); -// } -// } -// } } } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index c75185b0ea..1c9361fa61 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -288,10 +288,11 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { } int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { - SStreamMeta* pMeta = p->pMeta; - int32_t vgId = pMeta->vgId; - const char* id = p->id.idStr; - int32_t code = 0; + SStreamMeta* pMeta = p->pMeta; + int32_t vgId = pMeta->vgId; + const char* id = p->id.idStr; + int32_t code = 0; + SCheckpointInfo* pCKInfo = &p->chkInfo; if (p->info.fillHistory == 1) { return code; @@ -303,25 +304,33 @@ int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { taosThreadMutexLock(&p->lock); - ASSERT(p->chkInfo.checkpointId <= p->chkInfo.checkpointingId && p->chkInfo.checkpointingId == checkpointId && - p->chkInfo.checkpointVer <= p->chkInfo.processedVer); - p->chkInfo.checkpointId = p->chkInfo.checkpointingId; - p->chkInfo.checkpointVer = p->chkInfo.processedVer; + SStreamTaskState* pStatus = streamTaskGetStatus(p); + if (pStatus->state == TASK_STATUS__CK) { + ASSERT(pCKInfo->checkpointId <= pCKInfo->checkpointingId && pCKInfo->checkpointingId == checkpointId && + pCKInfo->checkpointVer <= pCKInfo->processedVer); - streamTaskClearCheckInfo(p, false); - SStreamTaskState* pState = streamTaskGetStatus(p); + pCKInfo->checkpointId = pCKInfo->checkpointingId; + pCKInfo->checkpointVer = pCKInfo->processedVer; - code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); - taosThreadMutexUnlock(&p->lock); + streamTaskClearCheckInfo(p, false); + code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + taosThreadMutexUnlock(&p->lock); + } else { + stDebug("s-task:%s vgId:%d status:%s not keep the checkpoint metaInfo, checkpoint:%" PRId64 " failed", id, vgId, + pStatus->name, pCKInfo->checkpointingId); + taosThreadMutexUnlock(&p->lock); + + return TSDB_CODE_STREAM_TASK_IVLD_STATUS; + } if (code != TSDB_CODE_SUCCESS) { stDebug("s-task:%s vgId:%d handle event:checkpoint-done failed", id, vgId); - return -1; + return code; } stDebug("vgId:%d s-task:%s level:%d open upstream inputQ, save status after checkpoint, checkpointId:%" PRId64 ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status: normal, prev:%s", - vgId, id, p->info.taskLevel, checkpointId, p->chkInfo.checkpointVer, p->chkInfo.nextProcessVer, pState->name); + vgId, id, p->info.taskLevel, checkpointId, pCKInfo->checkpointVer, pCKInfo->nextProcessVer, pStatus->name); // save the task if not sink task if (p->info.taskLevel != TASK_LEVEL__SINK) { @@ -343,6 +352,7 @@ int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { streamMetaWUnLock(pMeta); } + return code; } @@ -474,7 +484,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { code = streamSaveTaskCheckpointInfo(pTask, ckId); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s commit taskInfo failed, checkpoint:%" PRId64 " failed, code:%s", pTask->id.idStr, ckId, - tstrerror(terrno)); + tstrerror(code)); } else { code = streamTaskUploadChkp(pTask, ckId, (char*)pTask->id.idStr); if (code != 0) { diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index d3287a6b96..8b5b2da061 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -160,7 +160,6 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu // no available token in bucket for sink task, let's wait for a little bit if (taskLevel == TASK_LEVEL__SINK && (!streamTaskExtractAvailableToken(pTask->outputInfo.pTokenBucket, pTask->id.idStr))) { stDebug("s-task:%s no available token in bucket for sink data, wait for 10ms", id); - taosMsleep(WAIT_FOR_DURATION); return TSDB_CODE_SUCCESS; } @@ -172,10 +171,6 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputq.queue); if (qItem == NULL) { -// if ((taskLevel == TASK_LEVEL__SOURCE || taskLevel == TASK_LEVEL__SINK) && (++retryTimes) < MAX_RETRY_TIMES) { -// taosMsleep(WAIT_FOR_DURATION); -// continue; -// } // restore the token to bucket if (*numOfBlocks > 0) { From d3cfa9774040391e22a614571f3c0dee0858afe9 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 11 Jan 2024 09:41:02 +0000 Subject: [PATCH 187/198] fix invalid read --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8c65a3ac1b..246605694b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1910,7 +1910,7 @@ static void* cliWorkThread(void* arg) { char threadName[TSDB_LABEL_LEN] = {0}; STrans* pInst = pThrd->pTransInst; - strntolower(threadName, pInst->label, sizeof(threadName)); + strtolower(threadName, pInst->label); setThreadName(threadName); uv_run(pThrd->loop, UV_RUN_DEFAULT); From e13ee9a37fe6aad16a3ed8a9a133fe2bb5899704 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 11 Jan 2024 17:46:50 +0800 Subject: [PATCH 188/198] coverage : modify coverage_test.sh to 2.0 --- tests/script/coverage_test.sh | 49 ++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index 1b9e9753d3..136530e451 100644 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -9,8 +9,14 @@ else exit 1 fi +# work main path +TDENGINE_DIR=/root/TDinternal/community +if [ x$2 != x ];then + TDENGINE_DIR=$2 +fi + +echo "TDENGINE_DIR=$TDENGINE_DIR" today=`date +"%Y%m%d"` -TDENGINE_DIR=/root/TDengine JDBC_DIR=/root/taos-connector-jdbc TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log @@ -23,7 +29,16 @@ NC='\033[0m' function buildTDengine() { echo "check if TDengine need build" + + # pull parent code + cd "$TDENGINE_DIR/../" + echo "git pull parent code..." + git remote prune origin > /dev/null + git remote update > /dev/null + + # pull tdengine code cd $TDENGINE_DIR + echo "git pull tdengine code..." git remote prune origin > /dev/null git remote update > /dev/null REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` @@ -54,13 +69,11 @@ function buildTDengine() { LOCAL_COMMIT=`git rev-parse --short @` rm -rf * - if [ "$branch" == "3.0" ]; then - echo "3.0 =============" - cmake -DCOVER=true -DBUILD_TEST=true -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_CONTRIB=true .. - else - cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null - fi - make -j + makecmd="cmake -DCOVER=true -DBUILD_TEST=true -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_CONTRIB=true ../../" + echo "$makecmd" + $makecmd + + make -j 8 make install } @@ -71,18 +84,18 @@ function runCasesOneByOne () { if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then case=`echo $cmd | cut -d' ' -f 3` start_time=`date +%s` - date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && $cmd > /dev/null 2>&1 && \ + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && timeout 30m $cmd > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_COVERAGE_REPORT - elif [$line == *"$2"* ]; then + elif [[ "$line" == *"$2"* ]]; then if [[ "$cmd" == *"pytest.sh"* ]]; then cmd=`echo $cmd | cut -d' ' -f 2-20` fi case=`echo $cmd | cut -d' ' -f 4-20` start_time=`date +%s` - date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && $cmd > /dev/null 2>&1 && \ + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && timeout 30m $cmd > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \ echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT end_time=`date +%s` @@ -178,7 +191,7 @@ function runTest() { runPythonCases runJDBCCases - stopTaosd + stopTaosd cd $TDENGINE_DIR/tests/script find . -name '*.sql' | xargs rm -f @@ -191,10 +204,10 @@ function lcovFunc { cd $TDENGINE_DIR # collect data - lcov --ignore-errors negative --ignore-errors mismatch -d . --capture --rc lcov_branch_coverage=1 --rc branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + lcov --capture -d $TDENGINE_DIR -b $TDENGINE_DIR -o coverage.info --ignore-errors negative,mismatch,inconsistent,source --branch-coverage --function-coverage --no-external # remove exclude paths - lcov --remove coverage.info \ + lcov --remove coverage1.info \ '*/contrib/*' '*/tests/*' '*/test/*' '*/packaging/*' '*/taos-tools/*' '*/taosadapter/*' '*/TSZ/*' \ '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\ '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ @@ -202,23 +215,23 @@ function lcovFunc { '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.c'\ '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ - --rc branch_coverage=1 -o coverage.info + '*/sql.c' '*/sql.y'\ + --branch-coverage --function-coverage -o coverage.info # generate result echo "generate result" - lcov -l --rc branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + lcov -l --branch-coverage --function-coverage coverage.info | tee -a $TDENGINE_COVERAGE_REPORT sed -i 's/\/root\/TDengine\/sql.c/\/root\/TDengine\/source\/libs\/parser\/inc\/sql.c/g' coverage.info sed -i 's/\/root\/TDengine\/sql.y/\/root\/TDengine\/source\/libs\/parser\/inc\/sql.y/g' coverage.info # push result to coveralls.io echo "push result to coveralls.io" - /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + /usr/local/bin/coveralls-lcov coverage.info -b 3.0 -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT - } function sendReport { From d4900919798b5ea4f9f5bff59e58a479cd020dca Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 11 Jan 2024 18:10:29 +0800 Subject: [PATCH 189/198] fix: tweak timeout from 30m to 20m --- tests/script/coverage_test.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index 136530e451..2a93fe69ef 100644 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -73,8 +73,7 @@ function buildTDengine() { echo "$makecmd" $makecmd - make -j 8 - make install + make -j 8 install } function runCasesOneByOne () { @@ -84,7 +83,7 @@ function runCasesOneByOne () { if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then case=`echo $cmd | cut -d' ' -f 3` start_time=`date +%s` - date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && timeout 30m $cmd > /dev/null 2>&1 && \ + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && timeout 20m $cmd > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT end_time=`date +%s` @@ -95,7 +94,7 @@ function runCasesOneByOne () { fi case=`echo $cmd | cut -d' ' -f 4-20` start_time=`date +%s` - date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && timeout 30m $cmd > /dev/null 2>&1 && \ + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && timeout 20m $cmd > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \ echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT end_time=`date +%s` @@ -207,7 +206,7 @@ function lcovFunc { lcov --capture -d $TDENGINE_DIR -b $TDENGINE_DIR -o coverage.info --ignore-errors negative,mismatch,inconsistent,source --branch-coverage --function-coverage --no-external # remove exclude paths - lcov --remove coverage1.info \ + lcov --remove coverage.info \ '*/contrib/*' '*/tests/*' '*/test/*' '*/packaging/*' '*/taos-tools/*' '*/taosadapter/*' '*/TSZ/*' \ '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\ '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ From 5e4d8ed26358ab8112a6e00e14c0b054d073bc76 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 11 Jan 2024 18:38:16 +0800 Subject: [PATCH 190/198] fix: change 3.0 to dynamic variant name --- tests/script/coverage_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index 2a93fe69ef..c5e0c31f83 100644 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -226,7 +226,7 @@ function lcovFunc { # push result to coveralls.io echo "push result to coveralls.io" - /usr/local/bin/coveralls-lcov coverage.info -b 3.0 -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + /usr/local/bin/coveralls-lcov coverage.info -b $branch -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT From d3659db2fff9a97f0535c9b081dc1e760a294cb7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 11 Jan 2024 19:22:54 +0800 Subject: [PATCH 191/198] fix:[TD-28202]move tq timer to write thread in mnode --- source/client/src/clientTmq.c | 4 ++-- source/dnode/mnode/impl/src/mndMain.c | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 15c8903978..28e269ee10 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1968,7 +1968,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { void* rspObj = NULL; int64_t startTime = taosGetTimestampMs(); - tscInfo("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, + tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, timeout); // in no topic status, delayed task also need to be processed @@ -2015,7 +2015,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; if (elapsedTime > timeout) { - tscInfo("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, + tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index c0fd93c65d..acc65c0650 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -145,7 +145,7 @@ static void mndCalMqRebalance(SMnode *pMnode) { void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { SRpcMsg rpcMsg = {.msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen}; - tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 62b671a12f..0909003201 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -747,7 +747,7 @@ static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); int32_t status = atomic_load_32(&pConsumer->status); - mInfo("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", + mDebug("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, hbStatus); From aa410a6deb5b9d71e30f3dd9dd2c82a884e65595 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 12 Jan 2024 09:12:56 +0800 Subject: [PATCH 192/198] fix(tsdb/cache): clear fs state before next open --- source/dnode/vnode/src/tsdb/tsdbCache.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ab504acea7..9b29329cf3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1990,9 +1990,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (SFSNEXTROW_FILESET == state->state) { _next_fileset: - if (--state->iFileSet < 0) { - clearLastFileSet(state); + clearLastFileSet(state); + if (--state->iFileSet < 0) { *ppRow = NULL; return code; } else { @@ -2862,7 +2862,9 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC taosArraySet(pColArray, iCol, &lastCol); int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ); - taosArrayRemove(aColArray, aColIndex); + if (aColIndex >= 0) { + taosArrayRemove(aColArray, aColIndex); + } } else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { noneCol = iCol; setNoneCol = true; From a9fe3ecaa90606efde37d86e0aded1c24c0d9467 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Fri, 12 Jan 2024 10:21:13 +0800 Subject: [PATCH 193/198] fix characters problem --- .../system-test/1-insert/insert_timestamp.py | 42 +++++++------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/tests/system-test/1-insert/insert_timestamp.py b/tests/system-test/1-insert/insert_timestamp.py index bc3b1c0275..310a27dae0 100644 --- a/tests/system-test/1-insert/insert_timestamp.py +++ b/tests/system-test/1-insert/insert_timestamp.py @@ -13,25 +13,13 @@ class TDTestCase: tdSql.init(conn.cursor(), True) def run(self): - """ - timestamp输入插入规则： - 对于插入的字段类型为timestamp类型的字段，只允许这么几种情况： - timestamp - timestamp +/- interval - interval + timestamp - timestamp可以是字符串譬如："2023-12-05 00:00:00.000", 也可以是int型， 譬如：1701619200000 - interval支持：b, u, a, s, m, h, d, w 不支持n, y,譬如：1h, 2d - - 仅支持2元表达式，譬如：timestamp + 2h， 不支持2元以上表达，譬如timestamp + 2h + 1d - """ - tdSql.execute("create database test_insert_timestamp PRECISION 'ns';") tdSql.execute("use test_insert_timestamp;") tdSql.execute("create stable st(ts timestamp, c1 int) tags(id int);") tdSql.execute("create table test_t using st tags(1);") expectErrInfo = "syntax error" - # 异常场景：timestamp + timestamp + # abnormal scenario: timestamp + timestamp tdSql.error("insert into test_t values(now + today(), 1 );", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values(now - today(), 1 );", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values(today() + now(), 1 ); ", expectErrInfo=expectErrInfo, fullMatched=False) @@ -40,24 +28,24 @@ class TDTestCase: tdSql.error("insert into test_t values('2023-11-28 00:00:00.000' + 1701111600000, 1 ); ", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values(1701111500000 + 1701111600000, 1 ); ", expectErrInfo=expectErrInfo, fullMatched=False) - # 异常场景：timestamp + interval + interval + # abnormal scenario: timestamp + interval + interval tdSql.error("insert into test_t values(today() + 1d + 1s, 1);", expectErrInfo=expectErrInfo, fullMatched=False) - # 异常场景：interval - timestamp + # abnormal scenario: interval - timestamp tdSql.error("insert into test_t values(2h - now(), 1 ); ", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values(2h - today(), 1 ); ", expectErrInfo=expectErrInfo, fullMatched=False) - # 异常场景：interval + interval + # abnormal scenario: interval + interval tdSql.error("insert into test_t values(2h - 1h, 1 ); ", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values(2h + 1h, 1 ); ", expectErrInfo=expectErrInfo, fullMatched=False) - # 异常场景：非法interval类型n + # abnormal scenario: non-support datatype - n tdSql.error("insert into test_t values(today() + 2n, 7); ", expectErrInfo=expectErrInfo, fullMatched=False) - # 异常场景：非法interval类型y + # abnormal scenario: non-support datatype - y tdSql.error("insert into test_t values(today() - 2y, 8);", expectErrInfo=expectErrInfo, fullMatched=False) - # 异常场景：数据类型不对 + # abnormal scenario: non-support datatype tdSql.error("insert into test_t values('a1701619200000', 8);", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values('ss2023-12-05 00:00:00.000' + '1701619200000', 1);", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values(123456, 1);", expectErrInfo="Timestamp data out of range") @@ -66,31 +54,31 @@ class TDTestCase: tdSql.error("insert into test_t values(None, 1);", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values(null, 1);", expectErrInfo=expectErrInfo, fullMatched=False) - # 异常场景：格式不对 + # abnormal scenario: incorrect format tdSql.error("insert into test_t values('2023-122-05 00:00:00.000' + '1701619200000', 1);", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values('2023-12--05 00:00:00.000' + '1701619200000', 1);", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values('12/12/2023' + 10a, 1);", expectErrInfo=expectErrInfo, fullMatched=False) tdSql.error("insert into test_t values(1701619200000111, 1);", expectErrInfo="Timestamp data out of range", fullMatched=False) - # 正常场景：timestamp + interval + # normal scenario:timestamp + interval tdSql.execute("insert into test_t values(today() + 2b, 1);") tdSql.execute("insert into test_t values(1701619200000000000 + 2u, 2);") tdSql.execute("insert into test_t values(today + 2a, 3);") tdSql.execute("insert into test_t values('2023-12-05 23:59:59.999' + 2a, 4);") tdSql.execute("insert into test_t values(1701921599000000000 + 3a, 5);") - # 正常场景：timestamp - interval + # normal scenario:timestamp - interval tdSql.execute("insert into test_t values(today() - 2s, 6);") tdSql.execute("insert into test_t values(now() - 2m, 7);") tdSql.execute("insert into test_t values(today - 2h, 8);") tdSql.execute("insert into test_t values('2023-12-05 00:00:00.000000000' - 2a, 9);") tdSql.execute("insert into test_t values(1701669000000000000 - 2a, 10);") - # 正常场景：interval + timestamp + # normal scenario:interval + timestamp tdSql.execute("insert into test_t values(2d + now, 11);") tdSql.execute("insert into test_t values(2w + today, 12);") - # 正常场景：timestamp + # normal scenario:timestamp tdSql.execute("insert into test_t values('2023-12-05 00:00:00.000', 13);") tdSql.execute("insert into test_t values(1701629100000000000, 14);") tdSql.execute("insert into test_t values(now() + 2s, 15);") @@ -102,7 +90,7 @@ class TDTestCase: tdSql.execute("insert into test_t values(1701619200000000000, -5);") tdSql.execute("insert into test_t values('2023-12-05 12:12:12' + 10a, 19);") - # 验证数据 + # data verification tdSql.query(f'select ts,c1 from test_t order by c1;') tdSql.checkRows(22) tdSql.checkEqual(tdSql.queryResult[0][0], 1699977600000000000) # c1=-15 @@ -133,12 +121,10 @@ class TDTestCase: tdSql.execute("drop database if exists test_insert_timestamp;") def __convert_ts_to_date(self, ts: int) -> str: - # 创建datetime对象并进行转换 dt_object = datetime.datetime.fromtimestamp(ts / 1e9) - # 格式化日期字符串 formatted_date = dt_object.strftime('%Y-%m-%d') - # print("转换后的日期为：", formatted_date) + return formatted_date def __get_today_ts(self) -> int: From b36cc972367fa9a3520ddf3bc266ec76801bee8c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 12 Jan 2024 10:44:49 +0800 Subject: [PATCH 194/198] fix(stream): close the inputQ of the related stream task when the fill-history task recv the trans-state msg from upstream. --- include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tq/tq.c | 1 - source/libs/executor/src/executor.c | 6 ++-- source/libs/stream/src/stream.c | 30 +++++++++++++++---- source/libs/stream/src/streamDispatch.c | 3 ++ source/libs/stream/src/streamExec.c | 39 +++++++++++++++++-------- source/libs/stream/src/streamQueue.c | 2 -- source/libs/stream/src/streamStart.c | 14 +++++++-- 8 files changed, 69 insertions(+), 27 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9ea655c15c..a115783b70 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -562,6 +562,7 @@ struct SStreamDispatchReq { int32_t upstreamTaskId; int32_t upstreamChildId; int32_t upstreamNodeId; + int32_t upstreamRelTaskId; int32_t blockNum; int64_t totalLen; SArray* dataLen; // SArray diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 10420ec463..a53b322753 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -963,7 +963,6 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // let's decide which step should be executed now if (pTask->execInfo.step1Start == 0) { int64_t ts = taosGetTimestampMs(); - pTask->execInfo.step1Start = ts; tqDebug("s-task:%s start scan-history stage(step 1), status:%s, step1 startTs:%" PRId64, id, pStatus, ts); } else { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6b38457d70..d27daad550 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -922,8 +922,8 @@ int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRan pStreamInfo->fillHistoryWindow = *pWindow; pStreamInfo->recoverStep = STREAM_RECOVER_STEP__PREPARE2; - qDebug("%s step 2. set param for stream scanner for scan-history data, verRange:%" PRId64 " - %" PRId64 - ", window:%" PRId64 " - %" PRId64, + qDebug("%s step 2. set param for stream scanner scan wal, verRange:%" PRId64 " - %" PRId64 ", window:%" PRId64 + " - %" PRId64, GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey, pWindow->ekey); return 0; @@ -1129,7 +1129,7 @@ int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; STimeWindow* pWindow = &pTaskInfo->streamInfo.fillHistoryWindow; - qDebug("%s remove scan-history filter window:%" PRId64 "-%" PRId64 ", set new window:%" PRId64 "-%" PRId64, + qDebug("%s remove timeWindow filter:%" PRId64 "-%" PRId64 ", set new window:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWindow->skey, pWindow->ekey, INT64_MIN, INT64_MAX); pWindow->skey = INT64_MIN; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 7db4bec6a0..7d507a1394 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -214,8 +214,9 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, } int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { - int32_t status = 0; - const char* id = pTask->id.idStr; + int32_t status = 0; + SStreamMeta* pMeta = pTask->pMeta; + const char* id = pTask->id.idStr; stDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64 ", msgId:%d", id, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId); @@ -223,7 +224,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId); ASSERT(pInfo != NULL); - if (pTask->pMeta->role == NODE_ROLE_FOLLOWER) { + if (pMeta->role == NODE_ROLE_FOLLOWER) { stError("s-task:%s task on follower received dispatch msgs, dispatch msg rejected", id); status = TASK_INPUT_STATUS__REFUSED; } else { @@ -244,6 +245,22 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); streamTaskCloseUpstreamInput(pTask, pReq->upstreamTaskId); stDebug("s-task:%s close inputQ for upstream:0x%x, msgId:%d", id, pReq->upstreamTaskId, pReq->msgId); + } else if (pReq->type == STREAM_INPUT__TRANS_STATE) { + atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); + streamTaskCloseUpstreamInput(pTask, pReq->upstreamTaskId); + + // disable the related stream task here to avoid it to receive the newly arrived data after the transfer-state + STaskId* pRelTaskId = &pTask->streamTaskId; + SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pRelTaskId->streamId, pRelTaskId->taskId); + if (pStreamTask != NULL) { + atomic_add_fetch_32(&pStreamTask->upstreamInfo.numOfClosed, 1); + streamTaskCloseUpstreamInput(pStreamTask, pReq->upstreamRelTaskId); + streamMetaReleaseTask(pMeta, pStreamTask); + } + + stDebug("s-task:%s close inputQ for upstream:0x%x since trans-state msgId:%d recv, rel stream-task:0x%" PRIx64 + " close inputQ for upstream:0x%x", + id, pReq->upstreamTaskId, pReq->msgId, pTask->streamTaskId.taskId, pReq->upstreamRelTaskId); } status = streamTaskAppendInputBlocks(pTask, pReq); @@ -252,9 +269,9 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } // disable the data from upstream tasks - if (streamTaskGetStatus(pTask)->state == TASK_STATUS__HALT) { - status = TASK_INPUT_STATUS__BLOCKED; - } +// if (streamTaskGetStatus(pTask)->state == TASK_STATUS__HALT) { +// status = TASK_INPUT_STATUS__BLOCKED; +// } { // do send response with the input status @@ -295,6 +312,7 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) { } pTask->upstreamInfo.numOfClosed = 0; + stDebug("s-task:%s opening up inputQ from upstream tasks", pTask->id.idStr); } void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index a6d21e2625..e2c8704b1c 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -58,6 +58,7 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->upstreamChildId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamRelTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1; if (tEncodeI64(pEncoder, pReq->totalLen) < 0) return -1; ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum); @@ -84,6 +85,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->upstreamChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamRelTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->totalLen) < 0) return -1; @@ -114,6 +116,7 @@ static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTas pReq->upstreamTaskId = pTask->id.taskId; pReq->upstreamChildId = pTask->info.selfChildId; pReq->upstreamNodeId = pTask->info.nodeId; + pReq->upstreamRelTaskId = pTask->streamTaskId.taskId; pReq->blockNum = numOfBlocks; pReq->taskId = dstTaskId; pReq->type = type; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 18f7ed061a..c69fe2a79c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -371,7 +371,6 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { // In case of sink tasks, no need to halt them. // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to // start the task state transfer procedure. -// char* p = NULL; SStreamTaskState* pState = streamTaskGetStatus(pStreamTask); status = pState->state; char* p = pState->name; @@ -392,8 +391,12 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { } // 1. expand the query time window for stream task of WAL scanner - pTimeWindow->skey = INT64_MIN; - qStreamInfoResetTimewindowFilter(pStreamTask->exec.pExecutor); + if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { + pTimeWindow->skey = INT64_MIN; + qStreamInfoResetTimewindowFilter(pStreamTask->exec.pExecutor); + } else { + stDebug("s-task:%s non-source task no need to reset filter window", pStreamTask->id.idStr); + } // 2. transfer the ownership of executor state streamTaskReleaseState(pTask); @@ -407,10 +410,13 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { // 4. free it and remove fill-history task from disk meta-store streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id); - // 5. save to disk + // 5. assign the status to the value that will be kept in disk pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask)->state; - // 6. add empty delete block + // 6. open the inputQ for all upstream tasks + streamTaskOpenAllUpstreamInput(pStreamTask); + + // 7. add empty delete block if ((pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) && taosQueueEmpty(pStreamTask->inputq.queue->pQueue)) { SStreamRefDataBlock* pItem = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); @@ -430,6 +436,8 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { int32_t code = TSDB_CODE_SUCCESS; + SStreamMeta* pMeta = pTask->pMeta; + ASSERT(pTask->status.appendTranstateBlock == 1); int32_t level = pTask->info.taskLevel; @@ -439,8 +447,14 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { if (level == TASK_LEVEL__AGG || level == TASK_LEVEL__SOURCE) { // do transfer task operator states. code = streamDoTransferStateToStreamTask(pTask); - } else { // drop fill-history task - streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pTask->pMeta->vgId, &pTask->id); + } else { // drop fill-history task and open inputQ of sink task + SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + if (pStreamTask != NULL) { + streamTaskOpenAllUpstreamInput(pStreamTask); + streamMetaReleaseTask(pMeta, pStreamTask); + } + + streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id); } return code; @@ -496,16 +510,17 @@ static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_ } } -int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { +int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { const char* id = pTask->id.idStr; int32_t code = TSDB_CODE_SUCCESS; + int32_t level = pTask->info.taskLevel; - int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__AGG || level == TASK_LEVEL__SINK) { int32_t remain = streamAlignTransferState(pTask); + if (remain > 0) { streamFreeQitem((SStreamQueueItem*)pBlock); - stDebug("s-task:%s receive upstream transfer state msg, remain:%d", id, remain); + stDebug("s-task:%s receive upstream trans-state msg, not sent remain:%d", id, remain); return 0; } } @@ -536,7 +551,7 @@ int32_t streamProcessTranstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock } } else { // non-dispatch task, do task state transfer directly streamFreeQitem((SStreamQueueItem*)pBlock); - stDebug("s-task:%s non-dispatch task, level:%d start to transfer state directly", id, pTask->info.taskLevel); + stDebug("s-task:%s non-dispatch task, level:%d start to transfer state directly", id, level); ASSERT(pTask->info.fillHistory == 1); code = streamTransferStateToStreamTask(pTask); @@ -604,7 +619,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { } if (type == STREAM_INPUT__TRANS_STATE) { - streamProcessTranstateBlock(pTask, (SStreamDataBlock*)pInput); + streamProcessTransstateBlock(pTask, (SStreamDataBlock*)pInput); continue; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 8b5b2da061..90823cd937 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -147,8 +147,6 @@ const char* streamQueueItemGetTypeStr(int32_t type) { int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, int32_t* blockSize) { - int32_t retryTimes = 0; - int32_t MAX_RETRY_TIMES = 5; const char* id = pTask->id.idStr; int32_t taskLevel = pTask->info.taskLevel; diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index d39528312d..24909a9776 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1029,6 +1029,11 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer); } } else { + ASSERT(pTask->info.fillHistory == 0); + if (pTask->info.taskLevel >= TASK_LEVEL__AGG) { + return; + } + int64_t ekey = 0; if (pRange->window.ekey < INT64_MAX) { ekey = pRange->window.ekey + 1; @@ -1043,10 +1048,13 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { pRange->range.minVer = 0; pRange->range.maxVer = ver; - stDebug("s-task:%s level:%d related fill-history task exists, update stream calc time window:%" PRId64 " - %" PRId64 + stDebug("s-task:%s level:%d related fill-history task exists, set stream task timeWindow:%" PRId64 " - %" PRId64 ", verRang:%" PRId64 " - %" PRId64, - pTask->id.idStr, pTask->info.taskLevel, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, - pRange->range.maxVer); + pTask->id.idStr, pTask->info.taskLevel, pRange->window.skey, pRange->window.ekey, ver, INT64_MAX); + + SVersionRange verRange = {.minVer = ver, .maxVer = INT64_MAX}; + STimeWindow win = pRange->window; + streamSetParamForStreamScannerStep2(pTask, &verRange, &win); } } From cf941f74884288b783139e5dc14e8ef2f4a5d391 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 12 Jan 2024 11:12:23 +0800 Subject: [PATCH 195/198] fix(stream): check status when resume to run. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index da1c2ecf16..7883c858f0 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -820,15 +820,23 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead return 0; } else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask != NULL) { - ASSERT(streamTaskReadyToRun(pTask, NULL)); - int64_t execTs = pTask->status.lastExecTs; - int32_t idle = taosGetTimestampMs() - execTs; - tqDebug("s-task:%s task resume to run after idle for:%dms from:%" PRId64, pTask->id.idStr, idle, execTs); - streamResumeTask(pTask); + if (pTask != NULL) { + char* pStatus = NULL; + if (streamTaskReadyToRun(pTask, &pStatus)) { + int64_t execTs = pTask->status.lastExecTs; + int32_t idle = taosGetTimestampMs() - execTs; + tqDebug("s-task:%s task resume to run after idle for:%dms from:%" PRId64, pTask->id.idStr, idle, execTs); + + streamResumeTask(pTask); + } else { + int8_t status = streamTaskSetSchedStatusInactive(pTask); + tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, + pTask->id.idStr, pStatus, status); + } streamMetaReleaseTask(pMeta, pTask); } + return 0; } From 5fa8f85c425e03a2f86eb80f76b1b21fce50ad78 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 12 Jan 2024 11:28:41 +0800 Subject: [PATCH 196/198] fix(stream): remove invalid release. --- source/libs/stream/src/streamMeta.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 04d34b0945..68348f462f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1520,8 +1520,6 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, taskId); streamMetaUpdateTaskDownstreamStatus(pMeta, streamId, taskId, 0, taosGetTimestampMs(), false); - - streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } From 52d8811bdca4a7622c1ee74417f9ffd7df5a2aec Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 12 Jan 2024 13:49:32 +0800 Subject: [PATCH 197/198] code coverage --- include/libs/function/functionMgt.h | 4 ++ .../executor/src/streamtimewindowoperator.c | 6 +++ source/libs/executor/src/timewindowoperator.c | 10 ---- source/libs/function/inc/builtins.h | 2 + source/libs/function/inc/builtinsimpl.h | 16 +++++- source/libs/function/src/builtins.c | 50 +++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 11 ++-- .../libs/function/src/detail/tavgfunction.c | 2 + source/libs/function/src/functionMgt.c | 2 + 9 files changed, 87 insertions(+), 16 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 865f1b2295..a0b5d938e3 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -259,9 +259,13 @@ EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, STimeWindow* int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); + +#ifdef BUILD_NO_CALL int32_t fmSetInvertFunc(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet); bool fmIsInvertible(int32_t funcId); +#endif + char* fmGetFuncName(int32_t funcId); #ifdef __cplusplus diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index b2d0f25466..22339aa384 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -427,6 +427,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { taosMemoryFreeClear(param); } +#ifdef BUILD_NO_CALL static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; i++) { if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) { @@ -435,6 +436,7 @@ static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { } return true; } +#endif void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo* pInfo) { SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; @@ -3846,6 +3848,7 @@ _error: return NULL; } +#ifdef BUILD_NO_CALL static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) { for (int i = 0; i < num; i++) { if (type == STREAM_INVERT) { @@ -3855,6 +3858,7 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type } } } +#endif static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; @@ -3947,9 +3951,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { // caller. Note that all the time window are not close till now. // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); +#ifdef BUILD_NO_CALL if (pInfo->invertible) { setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); } +#endif doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c459e4d8b9..b86253a8d1 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1079,16 +1079,6 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { return (rows == 0) ? NULL : pBlock; } -static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) { - for (int i = 0; i < num; i++) { - if (type == STREAM_INVERT) { - fmSetInvertFunc(pCtx[i].functionId, &(pCtx[i].fpSet)); - } else if (type == STREAM_NORMAL) { - fmSetNormalFunc(pCtx[i].functionId, &(pCtx[i].fpSet)); - } - } -} - static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false); if (NULL == pResult) { diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index e7fcc38818..6181a9b929 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -40,7 +40,9 @@ typedef struct SBuiltinFuncDefinition { FExecProcess processFunc; FScalarExecProcess sprocessFunc; FExecFinalize finalizeFunc; +#ifdef BUILD_NO_CALL FExecProcess invertFunc; +#endif FExecCombine combineFunc; const char* pPartialFunc; const char* pMergeFunc; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index d2f19ed2eb..ba7bf72aea 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -59,12 +59,19 @@ int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t countFunction(SqlFunctionCtx* pCtx); + +#ifdef BUILD_NO_CALL int32_t countInvertFunction(SqlFunctionCtx* pCtx); +#endif EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); bool getSumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t sumFunction(SqlFunctionCtx* pCtx); + +#ifdef BUILD_NO_CALL int32_t sumInvertFunction(SqlFunctionCtx* pCtx); +#endif + int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); @@ -81,7 +88,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx); int32_t avgFunctionMerge(SqlFunctionCtx* pCtx); int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + +#ifdef BUILD_NO_CALL int32_t avgInvertFunction(SqlFunctionCtx* pCtx); +#endif + int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getAvgInfoSize(); @@ -91,7 +102,11 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx); int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + +#ifdef BUILD_NO_CALL int32_t stddevInvertFunction(SqlFunctionCtx* pCtx); +#endif + int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getStddevInfoSize(); @@ -99,7 +114,6 @@ bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t leastSQRFunction(SqlFunctionCtx* pCtx); int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); -int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx); int32_t leastSQRCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 98fda024fa..61149f593a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2368,7 +2368,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = countFunction, .sprocessFunc = countScalarFunction, .finalizeFunc = functionFinalize, +#ifdef BUILD_NO_CALL .invertFunc = countInvertFunction, +#endif .combineFunc = combineFunction, .pPartialFunc = "count", .pMergeFunc = "sum" @@ -2384,7 +2386,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = sumFunction, .sprocessFunc = sumScalarFunction, .finalizeFunc = functionFinalize, +#ifdef BUILD_NO_CALL .invertFunc = sumInvertFunction, +#endif .combineFunc = sumCombine, .pPartialFunc = "sum", .pMergeFunc = "sum" @@ -2429,7 +2433,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = stddevFunction, .sprocessFunc = stddevScalarFunction, .finalizeFunc = stddevFinalize, +#ifdef BUILD_NO_CALL .invertFunc = stddevInvertFunction, +#endif .combineFunc = stddevCombine, .pPartialFunc = "_stddev_partial", .pMergeFunc = "_stddev_merge" @@ -2443,7 +2449,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = stddevFunctionSetup, .processFunc = stddevFunction, .finalizeFunc = stddevPartialFinalize, +#ifdef BUILD_NO_CALL .invertFunc = stddevInvertFunction, +#endif .combineFunc = stddevCombine, }, { @@ -2455,7 +2463,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = stddevFunctionSetup, .processFunc = stddevFunctionMerge, .finalizeFunc = stddevFinalize, +#ifdef BUILD_NO_CALL .invertFunc = stddevInvertFunction, +#endif .combineFunc = stddevCombine, }, { @@ -2468,7 +2478,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = leastSQRFunction, .sprocessFunc = leastSQRScalarFunction, .finalizeFunc = leastSQRFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = leastSQRCombine, }, { @@ -2482,7 +2494,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = avgFunction, .sprocessFunc = avgScalarFunction, .finalizeFunc = avgFinalize, +#ifdef BUILD_NO_CALL .invertFunc = avgInvertFunction, +#endif .combineFunc = avgCombine, .pPartialFunc = "_avg_partial", .pMergeFunc = "_avg_merge" @@ -2497,7 +2511,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = avgFunctionSetup, .processFunc = avgFunction, .finalizeFunc = avgPartialFinalize, +#ifdef BUILD_NO_CALL .invertFunc = avgInvertFunction, +#endif .combineFunc = avgCombine, }, { @@ -2509,7 +2525,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = avgFunctionSetup, .processFunc = avgFunctionMerge, .finalizeFunc = avgFinalize, +#ifdef BUILD_NO_CALL .invertFunc = avgInvertFunction, +#endif .combineFunc = avgCombine, }, { @@ -2523,7 +2541,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = percentileFunction, .sprocessFunc = percentileScalarFunction, .finalizeFunc = percentileFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = NULL, }, { @@ -2536,7 +2556,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = apercentileFunction, .sprocessFunc = apercentileScalarFunction, .finalizeFunc = apercentileFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = apercentileCombine, .pPartialFunc = "_apercentile_partial", .pMergeFunc = "_apercentile_merge", @@ -2551,7 +2573,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = apercentileFunctionSetup, .processFunc = apercentileFunction, .finalizeFunc = apercentilePartialFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = apercentileCombine, }, { @@ -2563,7 +2587,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = apercentileFunctionSetup, .processFunc = apercentileFunctionMerge, .finalizeFunc = apercentileFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = apercentileCombine, }, { @@ -2609,7 +2635,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = spreadFunction, .sprocessFunc = spreadScalarFunction, .finalizeFunc = spreadFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = spreadCombine, .pPartialFunc = "_spread_partial", .pMergeFunc = "_spread_merge" @@ -2624,7 +2652,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = spreadFunctionSetup, .processFunc = spreadFunction, .finalizeFunc = spreadPartialFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = spreadCombine, }, { @@ -2637,7 +2667,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = spreadFunctionSetup, .processFunc = spreadFunctionMerge, .finalizeFunc = spreadFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = spreadCombine, }, { @@ -2651,7 +2683,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = elapsedFunctionSetup, .processFunc = elapsedFunction, .finalizeFunc = elapsedFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = elapsedCombine, }, { @@ -2664,7 +2698,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = elapsedFunctionSetup, .processFunc = elapsedFunction, .finalizeFunc = elapsedPartialFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = elapsedCombine, }, { @@ -2677,7 +2713,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = elapsedFunctionSetup, .processFunc = elapsedFunctionMerge, .finalizeFunc = elapsedFinalize, + #ifdef BUILD_NO_CALL .invertFunc = NULL, + #endif .combineFunc = elapsedCombine, }, { @@ -2907,7 +2945,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = histogramFunction, .sprocessFunc = histogramScalarFunction, .finalizeFunc = histogramFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = histogramCombine, .pPartialFunc = "_histogram_partial", .pMergeFunc = "_histogram_merge", @@ -2921,7 +2961,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = histogramFunctionSetup, .processFunc = histogramFunctionPartial, .finalizeFunc = histogramPartialFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = histogramCombine, }, { @@ -2933,7 +2975,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = histogramFunctionMerge, .finalizeFunc = histogramFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = histogramCombine, }, { @@ -2946,7 +2990,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = hllFunction, .sprocessFunc = hllScalarFunction, .finalizeFunc = hllFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = hllCombine, .pPartialFunc = "_hyperloglog_partial", .pMergeFunc = "_hyperloglog_merge" @@ -2960,7 +3006,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = hllFunction, .finalizeFunc = hllPartialFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = hllCombine, }, { @@ -2972,7 +3020,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = hllFunctionMerge, .finalizeFunc = hllFinalize, +#ifdef BUILD_NO_CALL .invertFunc = NULL, +#endif .combineFunc = hllCombine, }, { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 8a2e118fe7..390190e8db 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -549,6 +549,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +#ifdef BUILD_NO_CALL int32_t countInvertFunction(SqlFunctionCtx* pCtx) { int64_t numOfElem = getNumOfElems(pCtx); @@ -559,6 +560,7 @@ int32_t countInvertFunction(SqlFunctionCtx* pCtx) { SET_VAL(pResInfo, *((int64_t*)buf), 1); return TSDB_CODE_SUCCESS; } +#endif int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); @@ -642,6 +644,7 @@ _sum_over: return TSDB_CODE_SUCCESS; } +#ifdef BUILD_NO_CALL int32_t sumInvertFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; @@ -699,6 +702,7 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx) { SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; } +#endif int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); @@ -1230,6 +1234,7 @@ int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +#ifdef BUILD_NO_CALL int32_t stddevInvertFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; @@ -1294,6 +1299,7 @@ int32_t stddevInvertFunction(SqlFunctionCtx* pCtx) { SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; } +#endif int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; @@ -1578,11 +1584,6 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } -int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx) { - // TODO - return TSDB_CODE_SUCCESS; -} - int32_t leastSQRCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SLeastSQRInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index e626c937da..6bcbd1c3a7 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -724,6 +724,7 @@ int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +#ifdef BUILD_NO_CALL int32_t avgInvertFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; @@ -786,6 +787,7 @@ int32_t avgInvertFunction(SqlFunctionCtx* pCtx) { SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; } +#endif int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 036e4238d4..7bb863839a 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -281,6 +281,7 @@ void fmFuncMgtDestroy() { } } +#ifdef BUILD_NO_CALL int32_t fmSetInvertFunc(int32_t funcId, SFuncExecFuncs* pFpSet) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { return TSDB_CODE_FAILED; @@ -314,6 +315,7 @@ bool fmIsInvertible(int32_t funcId) { } return res; } +#endif // function has same input/output type bool fmIsSameInOutType(int32_t funcId) { From bc7434e2b5b4ddc539d194e67a2fd1da272061c4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 12 Jan 2024 14:49:16 +0800 Subject: [PATCH 198/198] fix(stream): set error code for return. --- source/dnode/mnode/impl/src/mndStreamTrans.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index bfea4349b0..847f55a8fd 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -92,6 +92,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamUid, const char* if (strcmp(pTransName, MND_STREAM_DROP_NAME) != 0) { mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); + terrno = TSDB_CODE_MND_TRANS_CONFLICT; return true; } else { mDebug("not conflict with checkpoint trans, name:%s, continue create trans", pTransName); @@ -100,6 +101,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamUid, const char* (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0)) { mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); + terrno = TSDB_CODE_MND_TRANS_CONFLICT; return true; } } else {