From 3e2c24d987234ab1f5bc6e62ad9daeab0c525224 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 27 Jul 2022 20:56:33 +0800 Subject: [PATCH 1/3] other: add debug logs --- source/libs/wal/src/walWrite.c | 7 +++++++ tests/script/tsim/sync/vnodesnapshot-test.sim | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 491e5b0e08..67b2b90d6a 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -250,11 +250,13 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver) { } int32_t walEndSnapshot(SWal *pWal) { + uInfo("%s:%d rsma: WAL walEndSnapshot entry for %s", __func__, __LINE__, pWal->path); int32_t code = 0; taosThreadMutexLock(&pWal->mutex); int64_t ver = pWal->vers.verInSnapshotting; if (ver == -1) { code = -1; + uInfo("%s:%d rsma: WAL walEndSnapshot code = -1 for %s", __func__, __LINE__, pWal->path); goto END; }; @@ -291,12 +293,15 @@ int32_t walEndSnapshot(SWal *pWal) { } char fnameStr[WAL_FILE_LEN]; // remove file + uInfo("%s:%d rsma: WAL walEndSnapshot deleteCnt=%d %s", __func__, __LINE__, (int32_t)deleteCnt, pWal->path); for (int i = 0; i < deleteCnt; i++) { pInfo = taosArrayGet(pWal->fileInfoSet, i); walBuildLogName(pWal, pInfo->firstVer, fnameStr); taosRemoveFile(fnameStr); + uInfo("rsma: removed WAL log file %s", fnameStr); walBuildIdxName(pWal, pInfo->firstVer, fnameStr); taosRemoveFile(fnameStr); + uInfo("rsma: removed WAL idx file %s", fnameStr); } // make new array, remove files @@ -307,6 +312,8 @@ int32_t walEndSnapshot(SWal *pWal) { } else { pWal->vers.firstVer = ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } + } else { + uInfo("%s:%d rsma: WAL walEndSnapshot %s", __func__, __LINE__, pWal->path); } pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; pWal->totSize = newTotSize; diff --git a/tests/script/tsim/sync/vnodesnapshot-test.sim b/tests/script/tsim/sync/vnodesnapshot-test.sim index c3d8a243d3..a0c804179c 100644 --- a/tests/script/tsim/sync/vnodesnapshot-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-test.sim @@ -49,7 +49,7 @@ $replica = 3 $vgroups = 1 print ============= create database -sql create database db replica $replica vgroups $vgroups +sql create database db replica $replica vgroups $vgroups retentions 3s:7d,5s:21d,15s:365d $loop_cnt = 0 check_db_ready: @@ -113,7 +113,7 @@ endi vg_ready: print ====> create stable/child table -sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) rollup(max) sql show stables if $rows != 1 then @@ -132,7 +132,7 @@ print ===> write 100 records $N = 100 $count = 0 while $count < $N - $ms = 1591200000000 + $count + $ms = 1658924000000 + $count sql insert into ct1 values( $ms , $count , 2.1, 3.1) $count = $count + 1 endw @@ -149,7 +149,7 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT - +sleep 10000 ######################################################## print ===> start dnode1 dnode2 dnode3 dnode4 From 0dcb3a5da154f24c0051d7b42a26b77a8941462c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 28 Jul 2022 17:07:47 +0800 Subject: [PATCH 2/3] fix: assign uid for rsma by physical plan --- source/common/src/tdatablock.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 4 ++-- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executor.c | 6 ++++++ source/libs/executor/src/scanoperator.c | 5 +++++ source/libs/nodes/src/nodesCodeFuncs.c | 8 +++++++- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e516bddac1..faee6cc2fa 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1713,7 +1713,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, + printf("%s |block type %d |child id %d|group id %" PRIu64 "\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index a6fde1e2d2..eecb0e6621 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -599,14 +599,14 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche SSubmitReq *pReq = NULL; // TODO: the schema update should be handled if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) < 0) { - smaError("vgId:%d, build submit req for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma), + smaError("vgId:%d, build submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr()); goto _err; } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { taosMemoryFreeClear(pReq); - smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", + smaError("vgId:%d, process submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr()); goto _err; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d5486d62b1..a80c2c2fea 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -318,6 +318,7 @@ typedef struct STableScanInfo { int32_t currentTable; int8_t scanMode; int8_t noTable; + int8_t assignBlockUid; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 8b1cbb5ae8..4d47eda52b 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -270,6 +270,12 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); + if (pTaskInfo->tableqinfoList.map == NULL) { + pTaskInfo->tableqinfoList.map = + taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + } + + taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); } if (keyBuf != NULL) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f07256e88e..4a2f57d628 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -408,6 +408,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pBlock->info.groupId = *groupId; } + if (pTableScanInfo->assignBlockUid) { + pBlock->info.groupId = pBlock->info.uid; + } + pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; @@ -616,6 +620,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; pInfo->currentGroupId = -1; + pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pOperator->name = "TableScanOperator"; // for debug purpose pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c499d6e7cc..9722d1fc10 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1522,6 +1522,7 @@ static const char* jkTableScanPhysiPlanWatermark = "Watermark"; static const char* jkTableScanPhysiPlanIgnoreExpired = "IgnoreExpired"; static const char* jkTableScanPhysiPlanGroupTags = "GroupTags"; static const char* jkTableScanPhysiPlanGroupSort = "GroupSort"; +static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1578,6 +1579,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanAssignBlockUid, pNode->assignBlockUid); + } return code; } @@ -1637,6 +1641,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanAssignBlockUid, &pNode->assignBlockUid); + } return code; } @@ -4518,7 +4525,6 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: return jsonToPhysiScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: - return jsonToPhysiLastRowScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: From 0bb234278798447ccdc627d0f50e251e81b26c33 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 28 Jul 2022 17:18:20 +0800 Subject: [PATCH 3/3] other: revert the test case --- tests/script/tsim/sync/vnodesnapshot-test.sim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/sync/vnodesnapshot-test.sim b/tests/script/tsim/sync/vnodesnapshot-test.sim index a0c804179c..9f4cd37b6d 100644 --- a/tests/script/tsim/sync/vnodesnapshot-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-test.sim @@ -49,7 +49,7 @@ $replica = 3 $vgroups = 1 print ============= create database -sql create database db replica $replica vgroups $vgroups retentions 3s:7d,5s:21d,15s:365d +sql create database db replica $replica vgroups $vgroups $loop_cnt = 0 check_db_ready: @@ -113,7 +113,7 @@ endi vg_ready: print ====> create stable/child table -sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) rollup(max) +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) sql show stables if $rows != 1 then @@ -149,7 +149,7 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 10000 +sleep 3000 ######################################################## print ===> start dnode1 dnode2 dnode3 dnode4