From ae54c4a50bf5c0eeb84c2f31795cea5638019c7a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Sep 2022 14:24:45 +0800 Subject: [PATCH 1/4] enh: let show vnodes works --- source/common/src/systable.c | 8 +++-- source/dnode/mnode/impl/src/mndVgroup.c | 39 ++++++++++++++++--------- tests/script/tsim/db/basic1.sim | 31 ++++++++++++++++++++ 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index f107e78478..3edf26cca7 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -257,10 +257,12 @@ static const SSysDbTableSchema subscriptionSchema[] = { }; static const SSysDbTableSchema vnodesSchema[] = { - {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "dnode_endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = true}, + {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "dnode_ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysTableMeta infosMeta[] = { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 09eed7fb32..6ab979213d 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -791,32 +791,43 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pShow->pIter == NULL) break; for (int32_t i = 0; i < pVgroup->replica && numOfRows < rows; ++i) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SColumnInfoData *pColInfo = NULL; cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false); - SName name = {0}; - char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&name, varDataVal(db)); - varDataSetLen(db, strlen(varDataVal(db))); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)db, false); - - uint32_t val = 0; - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&val, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->replica, false); char buf[20] = {0}; STR_TO_VARSTR(buf, syncStr(pVgid->role)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf, false); + const char *dbname = mndGetDbStr(pVgroup->dbName); + char b1[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + if (dbname != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(b1, dbname, TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE); + } else { + STR_WITH_MAXSIZE_TO_VARSTR(b1, "NULL", TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE); + } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->replica, false); // onlines + colDataAppend(pColInfo, numOfRows, (const char *)b1, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgid->dnodeId, false); + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + char b2[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; + if (pDnode != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(b2, pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE); + } else { + STR_WITH_MAXSIZE_TO_VARSTR(b2, "NULL", TSDB_EP_LEN + VARSTR_HEADER_SIZE); + } + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)b2, false); numOfRows++; } diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim index 69eeb9347b..3f6bf05398 100644 --- a/tests/script/tsim/db/basic1.sim +++ b/tests/script/tsim/db/basic1.sim @@ -107,6 +107,37 @@ if $data30 != 12 then return -1 endi +print =============== show vnodes +sql show vnodes 1 +if $rows != 9 then + return -1 +endi + +if $data(4)[1] != 1 then + return -1 +endi + +if $data(4)[2] != leader then + return -1 +endi + +if $data(4)[3] != d2 then + return -1 +endi + +if $data(4)[4] != 1 then + return -1 +endi + +if $data(4)[5] != localhost:7100 then + return -1 +endi + +#sql show vnodes 'localhost:7100' +#if $rows != 9 then +# return -1 +#endi + print =============== drop database sql drop database d2 sql drop database d3 From a170796f663e1dc217c11e49f3051779e3a4aae3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Sep 2022 14:34:30 +0800 Subject: [PATCH 2/4] fix(query): fix invalid read/write. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 ++++++------- source/libs/executor/inc/executil.h | 4 +--- source/libs/executor/src/timewindowoperator.c | 11 +++++++++-- tests/script/tsim/parser/slimit_alter_tags.sim | 1 + 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d76787915a..7f492f73b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2244,9 +2244,12 @@ static void extractOrderedTableUidList(SUidOrderCheckInfo* pOrderCheckInfo, SRea } static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) { - if (pOrderCheckInfo->tableUidList == NULL) { - int32_t total = taosHashGetSize(pStatus->pTableMap); + int32_t total = taosHashGetSize(pStatus->pTableMap); + if (total == 0) { + return TSDB_CODE_SUCCESS; + } + if (pOrderCheckInfo->tableUidList == NULL) { pOrderCheckInfo->currentIndex = 0; pOrderCheckInfo->tableUidList = taosMemoryMalloc(total * sizeof(uint64_t)); if (pOrderCheckInfo->tableUidList == NULL) { @@ -2254,21 +2257,17 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt } extractOrderedTableUidList(pOrderCheckInfo, pStatus); - uint64_t uid = pOrderCheckInfo->tableUidList[0]; pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); } else { if (pStatus->pTableIter == NULL) { // it is the last block of a new file - // ASSERT(pOrderCheckInfo->currentIndex == taosHashGetSize(pStatus->pTableMap)); - pOrderCheckInfo->currentIndex = 0; uint64_t uid = pOrderCheckInfo->tableUidList[pOrderCheckInfo->currentIndex]; pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); // the tableMap has already updated if (pStatus->pTableIter == NULL) { - void* p = - taosMemoryRealloc(pOrderCheckInfo->tableUidList, taosHashGetSize(pStatus->pTableMap) * sizeof(uint64_t)); + void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, total * sizeof(uint64_t)); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 9e7fcc2227..0722c2b306 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -87,9 +87,7 @@ struct SqlFunctionCtx; size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); - -void initResultRow(SResultRow* pResultRow); -void closeResultRow(SResultRow* pResultRow); +void closeResultRow(SResultRow* pResultRow); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 26667cd64f..0304dbdbc8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -609,6 +609,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num while (1) { SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); SOpenWindowInfo* pOpenWin = (SOpenWindowInfo *)pn->data; + uint64_t groupId = pOpenWin->groupId; SResultRowPosition* p1 = &pOpenWin->pos; if (p->pageId == p1->pageId && p->offset == p1->offset) { @@ -621,7 +622,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num if (pr->closed) { ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); - tdListPopHead(pResultRowInfo->openWindow); + SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow); + taosMemoryFree(pNode); continue; } @@ -650,7 +652,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { closeResultRow(pr); - tdListPopHead(pResultRowInfo->openWindow); + SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow); + taosMemoryFree(pNode); } else { // the remains are can not be closed yet. break; } @@ -1730,6 +1733,10 @@ void destroyIntervalOperatorInfo(void* param) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); + cleanupExprSupp(&pInfo->scalarSupp); + + tdListFree(pInfo->binfo.resultRowInfo.openWindow); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); taosArrayDestroyEx(pInfo->pPrevValues, freeItem); diff --git a/tests/script/tsim/parser/slimit_alter_tags.sim b/tests/script/tsim/parser/slimit_alter_tags.sim index 3827b14b45..b5afbfa56e 100644 --- a/tests/script/tsim/parser/slimit_alter_tags.sim +++ b/tests/script/tsim/parser/slimit_alter_tags.sim @@ -128,6 +128,7 @@ if $rows != 5 then return -1 endi if $data00 != $rowNum then + print expect $rowNum , actual: $data00 return -1 endi if $data10 != $rowNum then From eafad954a1a7057d3fb9db4213ead327d7040ac2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 14:56:57 +0800 Subject: [PATCH 3/4] fix: one little bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 42ab6ce3b2..acbdc65f12 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1308,7 +1308,7 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { SBlockData *pBDatal = &pCommitter->dWriter.bDatal; if (pBDatal->suid || pBDatal->uid) { - if (pBDatal->suid != id.suid || id.uid == 0) { + if ((pBDatal->suid != id.suid) || (id.suid == 0)) { if (pBDatal->nRow) { code = tsdbCommitLastBlock(pCommitter); if (code) goto _exit; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 37bd1ccbf6..40841ac85e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -311,17 +311,6 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { if (code) goto _err; } - // sst =========== - tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[0], fname); - if (taosStatFile(fname, &size, NULL)) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (size != pSet->aSstF[0]->size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - // sma ============= tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); if (taosStatFile(fname, &size, NULL)) { @@ -335,6 +324,19 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE); if (code) goto _err; } + + // sst =========== + for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) { + tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname); + if (taosStatFile(fname, &size, NULL)) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (size != pSet->aSstF[iSst]->size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + } } { From a71be005aea4d8100e7f3bc4e0de936461b00e14 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Sep 2022 15:58:55 +0800 Subject: [PATCH 4/4] fix: another fail CI case --- source/dnode/vnode/src/tsdb/tsdbFS.c | 31 ++++++++++++++++------------ 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 40841ac85e..e6bc9d9936 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -899,21 +899,26 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->aSstF[0]->nRef = 1; } else { for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) { - SSstFile *pSstFile = pSetOld->aSstF[iSst]; - nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); - if (nRef == 0) { - tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pSstFile); - } + if (pSetOld->aSstF[iSst]->commitID != pSetNew->aSstF[iSst]->commitID) { + SSstFile *pSstFile = pSetOld->aSstF[iSst]; + nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1); + if (nRef == 0) { + tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSstFile); + } - pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); - if (pSetOld->aSstF[iSst] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile)); + if (pSetOld->aSstF[iSst] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst]; + pSetOld->aSstF[iSst]->nRef = 1; + } else { + ASSERT(pSetOld->aSstF[iSst]->size == pSetOld->aSstF[iSst]->size); + ASSERT(pSetOld->aSstF[iSst]->offset == pSetOld->aSstF[iSst]->offset); } - *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst]; - pSetOld->aSstF[iSst]->nRef = 1; } } } else {