From e18f056c9f5b0a4639a2f2d6c16a9f52706a1280 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 21 Mar 2024 10:52:00 +0800 Subject: [PATCH 1/6] enh: meta pause/resume for sys table scan --- source/libs/executor/src/sysscanoperator.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 5735ee298b..390cde96f0 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -522,6 +522,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + } else { + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); } if (pInfo->pSchema == NULL) { @@ -599,6 +601,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { pInfo->restore = true; if (pInfo->pRes->info.rows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); break; } } else { @@ -607,6 +610,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } if (numOfRows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; } @@ -702,6 +706,8 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + } else { + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); } bool blockFull = false; @@ -741,6 +747,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { numOfRows = 0; if (pInfo->pRes->info.rows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); break; } @@ -749,6 +756,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } if (numOfRows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; } From 6b2821cd288b6f611517ecd20f9c62b529bbf0eb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Mar 2024 15:45:28 +0800 Subject: [PATCH 2/6] fix(query): not lock when reading meta, and do some internal refactor. --- source/libs/executor/src/sysscanoperator.c | 183 +++++++++++---------- 1 file changed, 100 insertions(+), 83 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 390cde96f0..5e8c98c5c4 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -433,93 +433,109 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { return false; } -static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; - +static SSDataBlock* doOptimizeTableNameFilter(SOperatorInfo* pOperator, SSDataBlock* dataBlock, char* dbname) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; - if (pOperator->status == OP_EXEC_DONE) { + int32_t numOfRows = 0; + + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->req.filterTb); + + SMetaReader smrTable = {0}; + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); + pInfo->loadInfo.totalRows = 0; return NULL; } - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; + if (smrTable.me.type == TSDB_SUPER_TABLE) { + pAPI->metaReaderFn.clearReader(&smrTable); + pInfo->loadInfo.totalRows = 0; + return NULL; + } - SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); - blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); + if (smrTable.me.type == TSDB_CHILD_TABLE) { + int64_t suid = smrTable.me.ctbEntry.suid; + pAPI->metaReaderFn.clearReader(&smrTable); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + } + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + SSchemaWrapper* schemaRow = NULL; + if (smrTable.me.type == TSDB_SUPER_TABLE) { + schemaRow = &smrTable.me.stbEntry.schemaRow; + STR_TO_VARSTR(typeName, "CHILD_TABLE"); + } else if (smrTable.me.type == TSDB_NORMAL_TABLE) { + schemaRow = &smrTable.me.ntbEntry.schemaRow; + STR_TO_VARSTR(typeName, "NORMAL_TABLE"); + } + + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); + pAPI->metaReaderFn.clearReader(&smrTable); + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + setOperatorCompleted(pOperator); + + qDebug("get cols success, total rows:%" PRIu64 ", current:%d %s", pInfo->loadInfo.totalRows, pInfo->pRes->info.rows, + GET_TASKID(pTaskInfo)); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +int32_t doExtractDbName(char* dbname, SSysTableScanInfo* pInfo, SStorageAPI* pAPI) { + SName sn = {0}; const char* db = NULL; int32_t vgId = 0; pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, varDataVal(dbname)); varDataSetLen(dbname, strlen(varDataVal(dbname))); - // optimize when sql like where table_name='tablename' and xxx. - if (pInfo->req.filterTb[0]) { - char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tableName, pInfo->req.filterTb); + return TSDB_CODE_SUCCESS; +} - SMetaReader smrTable = {0}; - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); - int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly - pAPI->metaReaderFn.clearReader(&smrTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } +static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; + int32_t numOfRows = 0; + int32_t ret = 0; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + SSDataBlock* pDataBlock = NULL; - if (smrTable.me.type == TSDB_SUPER_TABLE) { - pAPI->metaReaderFn.clearReader(&smrTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } - - if (smrTable.me.type == TSDB_CHILD_TABLE) { - int64_t suid = smrTable.me.ctbEntry.suid; - pAPI->metaReaderFn.clearReader(&smrTable); - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); - code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly - pAPI->metaReaderFn.clearReader(&smrTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } - } - - char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - SSchemaWrapper* schemaRow = NULL; - if (smrTable.me.type == TSDB_SUPER_TABLE) { - schemaRow = &smrTable.me.stbEntry.schemaRow; - STR_TO_VARSTR(typeName, "CHILD_TABLE"); - } else if (smrTable.me.type == TSDB_NORMAL_TABLE) { - schemaRow = &smrTable.me.ntbEntry.schemaRow; - STR_TO_VARSTR(typeName, "NORMAL_TABLE"); - } - - sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); - pAPI->metaReaderFn.clearReader(&smrTable); - - if (numOfRows > 0) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - } - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - setOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + blockDataCleanup(pInfo->pRes); + + pDataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); + blockDataEnsureCapacity(pDataBlock, pOperator->resultInfo.capacity); + doExtractDbName(dbname, pInfo, pAPI); + + // optimize when sql like where table_name='tablename' and xxx. + if (pInfo->req.filterTb[0]) { + SSDataBlock* p = doOptimizeTableNameFilter(pOperator, pDataBlock, dbname); + blockDataDestroy(pDataBlock); + return p; } - int32_t ret = 0; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } else { @@ -534,7 +550,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (!pInfo->pCur || !pInfo->pSchema) { terrno = TSDB_CODE_OUT_OF_MEMORY; qError("sysTableScanUserCols failed since %s", terrstr(terrno)); - blockDataDestroy(dataBlock); + blockDataDestroy(pDataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } @@ -553,7 +569,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { SSchemaWrapper* schemaRow = NULL; if (pInfo->pCur->mr.me.type == TSDB_SUPER_TABLE) { - qDebug("sysTableScanUserCols cursor get super table"); + qDebug("sysTableScanUserCols cursor get super table, %s", GET_TASKID(pTaskInfo)); void* schema = taosHashGet(pInfo->pSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t)); if (schema == NULL) { SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&pInfo->pCur->mr.me.stbEntry.schemaRow); @@ -561,7 +577,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } continue; } else if (pInfo->pCur->mr.me.type == TSDB_CHILD_TABLE) { - qDebug("sysTableScanUserCols cursor get child table"); + qDebug("sysTableScanUserCols cursor get child table, %s", GET_TASKID(pTaskInfo)); + STR_TO_VARSTR(typeName, "CHILD_TABLE"); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); int64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; @@ -570,13 +587,14 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly - qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); + qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d, %s", suid, code, GET_TASKID(pTaskInfo)); + pAPI->metaReaderFn.clearReader(&smrSuperTable); - blockDataDestroy(dataBlock); + blockDataDestroy(pDataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } @@ -586,17 +604,17 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { pAPI->metaReaderFn.clearReader(&smrSuperTable); } } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { - qDebug("sysTableScanUserCols cursor get normal table"); + qDebug("sysTableScanUserCols cursor get normal table, %s", GET_TASKID(pTaskInfo)); schemaRow = &pInfo->pCur->mr.me.ntbEntry.schemaRow; STR_TO_VARSTR(typeName, "NORMAL_TABLE"); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); } else { - qDebug("sysTableScanUserCols cursor get invalid table"); + qDebug("sysTableScanUserCols cursor get invalid table, %s", GET_TASKID(pTaskInfo)); continue; } if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, pDataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; pInfo->restore = true; @@ -605,17 +623,17 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { break; } } else { - sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, pDataBlock, tableName, schemaRow, typeName); } } if (numOfRows > 0) { pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, pDataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; } - blockDataDestroy(dataBlock); + blockDataDestroy(pDataBlock); if (ret != 0) { pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; @@ -623,8 +641,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - qDebug("sysTableScanUserCols get cols success, rows:%" PRIu64, pInfo->loadInfo.totalRows); - + qDebug("get cols success, rows:%" PRIu64 " %s", pInfo->loadInfo.totalRows, GET_TASKID(pTaskInfo)); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } From f58902180a78087a63395a53c1574de3622836d7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Mar 2024 15:51:39 +0800 Subject: [PATCH 3/6] fix(query): fix syntax error. --- source/libs/executor/src/sysscanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 5e8c98c5c4..76103a8fb7 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -492,8 +492,8 @@ static SSDataBlock* doOptimizeTableNameFilter(SOperatorInfo* pOperator, SSDataBl pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; setOperatorCompleted(pOperator); - qDebug("get cols success, total rows:%" PRIu64 ", current:%d %s", pInfo->loadInfo.totalRows, pInfo->pRes->info.rows, - GET_TASKID(pTaskInfo)); + qDebug("get cols success, total rows:%" PRIu64 ", current:%" PRId64 " %s", pInfo->loadInfo.totalRows, + pInfo->pRes->info.rows, GET_TASKID(pTaskInfo)); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } From e9b115d9bc629e1c018ef6a32073961befadac51 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Mar 2024 18:57:35 +0800 Subject: [PATCH 4/6] fix(test): update test cases. --- tests/pytest/util/dnodes.py | 1 - tests/system-test/0-others/information_schema.py | 2 ++ tests/system-test/2-query/qnodeCluster.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 789e5866ee..67c3d37960 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -130,7 +130,6 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "DebugFlag": "131", "mDebugFlag": "143", "dDebugFlag": "143", "vDebugFlag": "143", diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index a7995762d8..21ad221ac5 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -19,6 +19,8 @@ from util.common import * from util.sqlset import * class TDTestCase: + updatecfgDict = {'qDebugFlag':135 , 'mDebugFlag':135} + def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index 851b5bb1d1..fbddcd299c 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -20,9 +20,9 @@ import threading class TDTestCase: clientCfgDict = {'queryproxy': '1','debugFlag': 135} - clientCfgDict["debugFlag"] = 131 + clientCfgDict["debugFlag"] = 135 updatecfgDict = {'clientCfg': {}} - updatecfgDict = {'debugFlag': 131} + updatecfgDict = {'debugFlag': 135} updatecfgDict = {'keepColumnName': 1} updatecfgDict["clientCfg"] = clientCfgDict From fb2667d92d0b107a81802ab045b302583424fabf Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 22 Mar 2024 10:33:46 +0800 Subject: [PATCH 5/6] fix: sys table scan and meta query --- include/libs/executor/storageapi.h | 12 ++++++------ source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 10 ++++++---- source/libs/executor/src/sysscanoperator.c | 16 ++++------------ 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 04bc839d51..b47a162a1a 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -238,12 +238,12 @@ typedef struct SStoreSnapshotFn { } SStoreSnapshotFn; typedef struct SStoreMeta { - SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor - void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor - void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor - void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first); // metaResumeTbCursor - int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext - int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev + SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor + void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor + void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first, int8_t move); // metaResumeTbCursor + int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext + int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev int32_t (*getTableTags)(void* pVnode, uint64_t suid, SArray* uidList); int32_t (*getTableTagsByUid)(void* pVnode, int64_t suid, SArray* uidList); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9fdb4993bd..d564c5a36e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -136,7 +136,7 @@ typedef SVCreateTSmaReq SSmaCfg; SMTbCursor* metaOpenTbCursor(void* pVnode); void metaCloseTbCursor(SMTbCursor* pTbCur); void metaPauseTbCursor(SMTbCursor* pTbCur); -void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first); +void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first, int8_t move); int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType); int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index cc7ae03483..c52e42d945 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -248,7 +248,7 @@ SMTbCursor *metaOpenTbCursor(void *pVnode) { // tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); pTbCur->pMeta = pVnodeObj->pMeta; pTbCur->paused = 1; - metaResumeTbCursor(pTbCur, 1); + metaResumeTbCursor(pTbCur, 1, 0); return pTbCur; } @@ -273,7 +273,7 @@ void metaPauseTbCursor(SMTbCursor *pTbCur) { pTbCur->paused = 1; } } -void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first) { +void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first, int8_t move) { if (pTbCur->paused) { metaReaderDoInit(&pTbCur->mr, pTbCur->pMeta, META_READER_LOCK); @@ -282,9 +282,11 @@ void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first) { if (first) { tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); } else { - int c = 0; + int c = 1; tdbTbcMoveTo(pTbCur->pDbc, pTbCur->pKey, pTbCur->kLen, &c); - if (c < 0) { + if (c == 0) { + if (move) tdbTbcMoveToNext(pTbCur->pDbc); + } else if (c < 0) { tdbTbcMoveToPrev(pTbCur->pDbc); } else { tdbTbcMoveToNext(pTbCur->pDbc); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 76103a8fb7..589c23ed3a 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -539,7 +539,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } else { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 0); } if (pInfo->pSchema == NULL) { @@ -555,14 +555,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { return NULL; } - int32_t restore = pInfo->restore; - pInfo->restore = false; - - while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { - if (restore) { - restore = false; - } + while (((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -616,7 +610,6 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, pDataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; - pInfo->restore = true; if (pInfo->pRes->info.rows > 0) { pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); @@ -724,7 +717,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } else { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 0); } bool blockFull = false; @@ -750,7 +743,6 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) { - pAPI->metaFn.cursorPrev(pInfo->pCur, TSDB_TABLE_MAX); blockFull = true; } else { sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, @@ -1354,7 +1346,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { firstMetaCursor = 1; } if (!firstMetaCursor) { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 1); } blockDataCleanup(pInfo->pRes); From de8670e79af53c19170e420e6780e457045c5a4c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Mar 2024 15:17:20 +0800 Subject: [PATCH 6/6] test: restore debugflag --- tests/pytest/util/dnodes.py | 1 + tests/system-test/2-query/qnodeCluster.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 67c3d37960..789e5866ee 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -130,6 +130,7 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", + "DebugFlag": "131", "mDebugFlag": "143", "dDebugFlag": "143", "vDebugFlag": "143", diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index fbddcd299c..851b5bb1d1 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -20,9 +20,9 @@ import threading class TDTestCase: clientCfgDict = {'queryproxy': '1','debugFlag': 135} - clientCfgDict["debugFlag"] = 135 + clientCfgDict["debugFlag"] = 131 updatecfgDict = {'clientCfg': {}} - updatecfgDict = {'debugFlag': 135} + updatecfgDict = {'debugFlag': 131} updatecfgDict = {'keepColumnName': 1} updatecfgDict["clientCfg"] = clientCfgDict