From f7d4c274a8ecc94710378e9729bb907eafbe170b Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 16 Aug 2024 18:03:16 +0800 Subject: [PATCH 001/100] fix exchange operator blocked --- source/libs/executor/src/exchangeoperator.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index bdc1e42b28..fb085c39c8 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -225,7 +225,10 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { } else { concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); } - + if (TSDB_CODE_SUCCESS != pOperator->pTaskInfo->code) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + T_LONG_JMP(pTaskInfo->env, pOperator->pTaskInfo->code); + } if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { return NULL; } else { From ee36bd741f4d511ea31c776f6cc7fa31c3cab8da Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Aug 2024 09:50:38 +0800 Subject: [PATCH 002/100] fix(query): return error code. --- source/libs/executor/src/executor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b1c9207ab7..a034e011f8 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -663,7 +663,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo if (isTaskKilled(pTaskInfo)) { atomic_store_64(&pTaskInfo->owner, 0); qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); - return TSDB_CODE_SUCCESS; + return pTaskInfo->code; } // error occurs, record the error code and return to client @@ -785,7 +785,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); taosRUnLockLatch(&pTaskInfo->lock); - return TSDB_CODE_SUCCESS; + return pTaskInfo->code; } if (pTaskInfo->owner != 0) { From 2f92b80cd694c87fdaee04e03cb5bdcc534cdad0 Mon Sep 17 00:00:00 2001 From: sima Date: Mon, 19 Aug 2024 10:10:47 +0800 Subject: [PATCH 003/100] fix:[TD-31511] Fix memory leak when error occurs. --- source/libs/scalar/src/filter.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 00487b140d..cc9cc9ed76 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2372,6 +2372,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t } gRes[gResIdx]->colInfo = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); if (gRes[gResIdx]->colInfo == NULL) { + filterFreeGroupCtx(gRes[gResIdx]); FLT_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } colIdxi = 0; @@ -2384,6 +2385,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t if (gRes[gResIdx]->colInfo[cidx].info == NULL) { gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); if (gRes[gResIdx]->colInfo[cidx].info == NULL) { + filterFreeGroupCtx(gRes[gResIdx]); FLT_ERR_JRET(terrno); } colIdx[colIdxi++] = cidx; @@ -2408,7 +2410,11 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t continue; } - FLT_ERR_JRET(filterMergeUnits(info, gRes[gResIdx], colIdx[l], &empty)); + code = filterMergeUnits(info, gRes[gResIdx], colIdx[l], &empty); + if (TSDB_CODE_SUCCESS != code) { + filterFreeGroupCtx(gRes[gResIdx]); + SCL_ERR_JRET(code); + } if (empty) { break; @@ -2426,10 +2432,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gRes[gResIdx]->colNum = colIdxi; FILTER_COPY_IDX(&gRes[gResIdx]->colIdx, colIdx, colIdxi); ++gResIdx; + *gResNum = gResIdx; } - *gResNum = gResIdx; - if (gResIdx == 0) { FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); } From d0e31f711fb82b1f3274365a12013a51d247f6f2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Aug 2024 11:01:54 +0800 Subject: [PATCH 004/100] fix(tsdb): return code for tMergeTreeNext --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 9 +-- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 21 +++++-- source/dnode/vnode/src/tsdb/tsdbRead2.c | 64 ++++++++++++--------- 4 files changed, 58 insertions(+), 38 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ab5b07581a..85084a0b81 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -898,7 +898,7 @@ typedef struct SSttDataInfoForTable { int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable *pTableInfo); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); -bool tMergeTreeNext(SMergeTree *pMTree); +int32_t tMergeTreeNext(SMergeTree *pMTree, bool* pHasNext); void tMergeTreePinSttBlock(SMergeTree *pMTree); void tMergeTreeUnpinSttBlock(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index aa92597211..70e6e1ee2a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2245,17 +2245,18 @@ static int32_t lastIterClose(SFSLastIter **iter) { } static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow) { - int32_t code = 0; + bool hasVal = false; + int32_t code = tMergeTreeNext(iter->pMergeTree, &hasVal); + if (code != 0) { + return code; + } - bool hasVal = tMergeTreeNext(iter->pMergeTree); if (!hasVal) { *ppRow = NULL; - TAOS_RETURN(code); } *ppRow = tMergeTreeGetRow(iter->pMergeTree); - TAOS_RETURN(code); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 4729b912a7..8bfc066731 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -1102,13 +1102,21 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree) { tLDataIterUnpinSttBlock(pIter, pMTree->idStr); } -bool tMergeTreeNext(SMergeTree *pMTree) { +int32_t tMergeTreeNext(SMergeTree *pMTree, bool *pHasNext) { + int32_t code = 0; + if (pHasNext == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (pMTree->pIter) { SLDataIter *pIter = pMTree->pIter; - - bool hasVal = false; - int32_t code = tLDataIterNextRow(pIter, pMTree->idStr, &hasVal); + bool hasVal = false; + code = tLDataIterNextRow(pIter, pMTree->idStr, &hasVal); if (!hasVal || (code != 0)) { + if (code == TSDB_CODE_FILE_CORRUPTED) { + code = 0; // suppress the file corrupt error to enable all queries within this cluster can run without failed. + } + pMTree->pIter = NULL; } @@ -1117,7 +1125,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { if (pMTree->pIter && pIter) { int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->node, &pIter->node); if (c > 0) { - (void) tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); + (void)tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; } else { ASSERT(c); @@ -1132,7 +1140,8 @@ bool tMergeTreeNext(SMergeTree *pMTree) { } } - return pMTree->pIter != NULL; + *pHasNext = (pMTree->pIter != NULL); + return code; } void tMergeTreeClose(SMergeTree *pMTree) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 9a9c74a3a0..607d96bcbc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1759,14 +1759,22 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return code; } -static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int32_t pkSrcSlot, - SVersionRange* pVerRange) { +static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int32_t pkSrcSlot, + SVersionRange* pVerRange) { + int32_t code = 0; int32_t order = pSttBlockReader->order; int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; SRowKey* pNextProc = &pScanInfo->sttKeyInfo.nextProcKey; while (1) { - bool hasVal = tMergeTreeNext(&pSttBlockReader->mergeTree); + bool hasVal = false; + code = tMergeTreeNext(&pSttBlockReader->mergeTree, &hasVal); + if (code) { + tsdbError("failed to iter the next row in stt-file merge tree, code:%s, %s", tstrerror(code), + pSttBlockReader->mergeTree.idStr); + return code; + } + if (!hasVal) { // the next value will be the accessed key in stt pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; @@ -1779,7 +1787,6 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData); } } - return false; } TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); @@ -1798,13 +1805,13 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange, pSttBlockReader->numOfPks > 0)) { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; - return true; } } else { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; - return true; } } + + return code; } static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); } @@ -2380,14 +2387,13 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STable return true; } -static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { - bool hasData = true; +static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { int32_t order = pReader->info.order; bool asc = ASCENDING_TRAVERSE(order); // the stt block reader has been initialized for this table. if (pSttBlockReader->uid == pScanInfo->uid) { - return hasDataInSttBlock(pScanInfo); + return; } if (pSttBlockReader->uid != 0) { @@ -2396,9 +2402,14 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pSttBlockReader->uid = pScanInfo->uid; - // second time init stt block reader + // second or third time init stt block reader if (pScanInfo->cleanSttBlocks && (pReader->info.execMode == READER_EXEC_ROWS)) { - return !pScanInfo->sttBlockReturned; + // only allowed to retrieve clean stt blocks for count once + if (pScanInfo->sttBlockReturned) { + pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; + tsdbDebug("uid:%" PRIu64 " set no stt-file data after stt-block retrieved", pScanInfo->uid, pReader->idStr); + } + return; } STimeWindow w = pSttBlockReader->window; @@ -2435,28 +2446,28 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))}; if (info.pKeyRangeList == NULL) { pReader->code = terrno; - return false; + return; } int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(info.pKeyRangeList); pReader->code = code; - return false; + return; } code = initMemDataIterator(pScanInfo, pReader); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(info.pKeyRangeList); pReader->code = code; - return false; + return; } code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(info.pKeyRangeList); pReader->code = code; - return false; + return; } if (conf.rspRows) { @@ -2484,27 +2495,26 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan SRowKey* p = asc ? &pScanInfo->sttRange.skey : &pScanInfo->sttRange.ekey; tRowKeyAssign(&pScanInfo->sttKeyInfo.nextProcKey, p); - - hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA); } else { // not clean stt blocks INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window - pScanInfo->sttBlockReturned = false; - hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); + code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); } } else { pScanInfo->cleanSttBlocks = false; INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window - pScanInfo->sttBlockReturned = false; - hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); + code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); } + pScanInfo->sttBlockReturned = false; taosArrayDestroy(info.pKeyRangeList); int64_t el = taosGetTimestampUs() - st; pReader->cost.initSttBlockReader += (el / 1000.0); tsdbDebug("init stt block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr); - return hasData; + if (code != 0) { + pReader->code = code; + } } static bool hasDataInSttBlock(STableBlockScanInfo* pInfo) { return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; } @@ -2772,7 +2782,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SBlockData* pBlockData = &pReader->status.fileBlockData; - (void) initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); + initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); if (pReader->code != 0) { code = pReader->code; goto _end; @@ -3190,12 +3200,12 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { continue; } - bool hasDataInSttFile = initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + initSttBlockReader(pSttBlockReader, pScanInfo, pReader); if (pReader->code != TSDB_CODE_SUCCESS) { return pReader->code; } - if (!hasDataInSttFile) { + if (!hasDataInSttBlock(pScanInfo)) { bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; @@ -3287,7 +3297,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { - (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + initSttBlockReader(pSttBlockReader, pScanInfo, pReader); if (pReader->code != 0) { return pReader->code; } @@ -3331,7 +3341,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int64_t st = taosGetTimestampUs(); // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files - (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + initSttBlockReader(pSttBlockReader, pScanInfo, pReader); if (pReader->code != 0) { return pReader->code; } From f2f0bad021205290ba959c6037c01b1800352cea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Aug 2024 11:06:43 +0800 Subject: [PATCH 005/100] fix(tsdb): return code for tMergeTreeNext --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 607d96bcbc..639cab9f52 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1826,9 +1826,14 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttB // avoid the fetch next row replace the referenced stt block in buffer doPinSttBlock(pSttBlockReader); - bool hasVal = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); + code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); doUnpinSttBlock(pSttBlockReader); - if (hasVal) { + + if (code) { + return code; + } + + if (hasDataInSttBlock(pScanInfo)) { SRowKey* pNext = getCurrentKeyInSttBlock(pSttBlockReader); if (pkCompEx(pSttKey, pNext) != 0) { code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); @@ -4097,7 +4102,11 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI SRowKey* pRowKey = &pScanInfo->lastProcKey; int32_t code = TSDB_CODE_SUCCESS; - while (nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange)) { + while (1) { + code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange); + if (code) { + + } SRowKey* pNextKey = getCurrentKeyInSttBlock(pSttBlockReader); int32_t ret = pkCompEx(pRowKey, pNextKey); From bc2f648cf79d5e13c83276a4db0aefb6c7b9e94b Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 19 Aug 2024 13:33:34 +0800 Subject: [PATCH 006/100] fix memory leak in tsdbreader --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 4729b912a7..cfb9a2f215 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -115,16 +115,14 @@ void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoad SArray *pList = taosArrayGetP(pLDataIterArray, i); for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) { SLDataIter *pIter = taosArrayGetP(pList, j); - if (pIter->pBlockLoadInfo == NULL) { - continue; - } - - SSttBlockLoadCostInfo *pCost = &pIter->pBlockLoadInfo->cost; - if (pLoadCost != NULL) { - pLoadCost->loadBlocks += pCost->loadBlocks; - pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks; - pLoadCost->blockElapsedTime += pCost->blockElapsedTime; - pLoadCost->statisElapsedTime += pCost->statisElapsedTime; + if (pIter->pBlockLoadInfo != NULL) { + SSttBlockLoadCostInfo *pCost = &pIter->pBlockLoadInfo->cost; + if (pLoadCost != NULL) { + pLoadCost->loadBlocks += pCost->loadBlocks; + pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks; + pLoadCost->blockElapsedTime += pCost->blockElapsedTime; + pLoadCost->statisElapsedTime += pCost->statisElapsedTime; + } } destroyLDataIter(pIter); From d4dc632d6f43aab9d5e22c5b41aa5ca1c7ab52d5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Aug 2024 13:39:30 +0800 Subject: [PATCH 007/100] fix(tsdb): fix syntax error. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- source/libs/executor/src/scanoperator.c | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 639cab9f52..5987b673c3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2412,7 +2412,7 @@ static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan // only allowed to retrieve clean stt blocks for count once if (pScanInfo->sttBlockReturned) { pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; - tsdbDebug("uid:%" PRIu64 " set no stt-file data after stt-block retrieved", pScanInfo->uid, pReader->idStr); + tsdbDebug("uid:%" PRIu64 " set no stt-file data after stt-block retrieved, %s", pScanInfo->uid, pReader->idStr); } return; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1935f2b0b6..5984f75c05 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -465,14 +465,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca SSDataBlock* p = NULL; code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL); - if (p == NULL || code != TSDB_CODE_SUCCESS) { + if (p == NULL || code != TSDB_CODE_SUCCESS || p != pBlock) { return code; } - if(p != pBlock) { - qError("[loadDataBlock] p != pBlock"); - return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; - } doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value From c6350794fee265cffe8b4a8016199da35df0a285 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 19 Aug 2024 15:53:55 +0800 Subject: [PATCH 008/100] fix(scheduler/exec cb): remove schedulerFreeJob from cb --- source/client/src/clientImpl.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 664de5619f..e12c761fcc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1135,8 +1135,6 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { (void)atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows); } } - - schedulerFreeJob(&pRequest->body.queryJob, 0); } taosMemoryFree(pResult); From bf471f8aee9731e31dfc20fef30bd1015a0568ae Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 19 Aug 2024 16:47:18 +0800 Subject: [PATCH 009/100] fix: unlock --- source/client/src/clientMonitor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 4bb29f8d97..8f28cbbce4 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -315,7 +315,7 @@ void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* void monitorCounterInc(int64_t clusterId, const char* counterName, const char** label_values) { taosWLockLatch(&monitorLock); if (atomic_load_32(&monitorFlag) == 1) { - taosRUnLockLatch(&monitorLock); + taosWUnLockLatch(&monitorLock); return; } From 5fb431aff4e172d50c76391a17b4f0dfda91d7dc Mon Sep 17 00:00:00 2001 From: sima Date: Mon, 19 Aug 2024 15:24:05 +0800 Subject: [PATCH 010/100] enh:[TD-31525] Remove ASSERT in libs/function. --- include/util/taoserror.h | 2 + source/libs/function/inc/thistogram.h | 2 +- source/libs/function/inc/tpercentile.h | 2 +- source/libs/function/src/builtinsimpl.c | 4 +- source/libs/function/src/thistogram.c | 89 +++++++++-------- source/libs/function/src/tpercentile.c | 126 ++++++++++++++++-------- source/util/src/terror.c | 2 + 7 files changed, 142 insertions(+), 85 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 1911c48d26..94b6d29cd9 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -878,6 +878,8 @@ int32_t taosGetErrSize(); #define TSDB_CODE_FUNC_INVALID_VALUE_RANGE TAOS_DEF_ERROR_CODE(0, 0x280C) #define TSDB_CODE_FUNC_SETUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x280D) #define TSDB_CODE_FUNC_INVALID_RES_LENGTH TAOS_DEF_ERROR_CODE(0, 0x280E) +#define TSDB_CODE_FUNC_HISTOGRAM_ERROR TAOS_DEF_ERROR_CODE(0, 0x280F) +#define TSDB_CODE_FUNC_PERCENTILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x2810) //udf diff --git a/source/libs/function/inc/thistogram.h b/source/libs/function/inc/thistogram.h index 5bc6a87c70..08bff7117e 100644 --- a/source/libs/function/inc/thistogram.h +++ b/source/libs/function/inc/thistogram.h @@ -59,7 +59,7 @@ int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto); SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins); int32_t tHistogramAdd(SHistogramInfo** pHisto, double val); -int64_t tHistogramSum(SHistogramInfo* pHisto, double v); +int32_t tHistogramSum(SHistogramInfo* pHisto, double v, int64_t *res); int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal); int32_t tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries, diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 118571c8aa..1b80c2b1da 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -47,7 +47,7 @@ typedef struct tMemBucketSlot { } tMemBucketSlot; struct tMemBucket; -typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value); +typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value, int32_t *index); typedef struct tMemBucket { int16_t numOfSlots; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 2d664e5d31..08abf41973 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4018,7 +4018,9 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* if (NULL == pColInfo) { return TSDB_CODE_OUT_OF_RANGE; } - ASSERT(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); + if (pColInfo->info.type != TSDB_DATA_TYPE_TIMESTAMP) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } key.groupId = pSrcBlock->info.id.groupId; key.ts = *(int64_t*)colDataGetData(pColInfo, rowIndex); } diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index f57f6aa118..8594b0584c 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -14,6 +14,7 @@ */ #include "os.h" +#include "query.h" #include "taosdef.h" #include "thistogram.h" #include "tlosertree.h" @@ -81,9 +82,9 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { #if defined(USE_ARRAYLIST) int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - if (ASSERTS(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL, "tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p", - idx, (*pHisto)->maxEntries, (*pHisto)->elems)) { - return TSDB_CODE_FAILED; + if (idx < 0 || idx > (*pHisto)->maxEntries || (*pHisto)->elems == NULL) { + qError("tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p", idx, (*pHisto)->maxEntries, (*pHisto)->elems); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } if ((*pHisto)->elems[idx].val == val && idx >= 0) { @@ -95,21 +96,19 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } else { /* insert a new slot */ if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { if (idx > 0) { - if (ASSERTS((*pHisto)->elems[idx - 1].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", - idx - 1, (*pHisto)->elems[idx - 1].val, val)) { - return TSDB_CODE_FAILED; + if ((*pHisto)->elems[idx - 1].val > val) { + qError("tHistogramAdd Error, elems[%d].val:%lf, val:%lf", idx - 1, (*pHisto)->elems[idx - 1].val, val); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } } else { - if (ASSERTS((*pHisto)->elems[idx].val > val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", - idx, (*pHisto)->elems[idx].val, val)) { - return TSDB_CODE_FAILED; + if ((*pHisto)->elems[idx].val <= val) { + qError("tHistogramAdd Error, elems[%d].val:%lf, val:%lf", idx, (*pHisto)->elems[idx].val, val); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } } - } else if ((*pHisto)->numOfElems > 0) { - if (ASSERTS((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", - (*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val)) { - return TSDB_CODE_FAILED; - } + } else if ((*pHisto)->numOfElems > 0 && (*pHisto)->elems[(*pHisto)->numOfEntries].val > val) { + qError("tHistogramAdd Error, elems[%d].val:%lf, val:%lf", (*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } code = histogramCreateBin(*pHisto, idx, val); @@ -225,9 +224,9 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { tSkipListNode* pNext = pNode->pForward[0]; SHistBin* pNextEntry = (SHistBin*)pNext->pData; - if (ASSERTS(pNextEntry->val - pEntry->val == pEntry->delta, "tHistogramAdd Error, pNextEntry->val:%lf, pEntry->val:%lf, pEntry->delta:%lf", - pNextEntry->val, pEntry->val, pEntry->delta)) { - return -1; + if (pNextEntry->val - pEntry->val != pEntry->delta) { + qError("tHistogramAdd Error, pNextEntry->val:%lf, pEntry->val:%lf, pEntry->delta:%lf", pNextEntry->val, pEntry->val, pEntry->delta); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } double newVal = (pEntry->val * pEntry->num + pNextEntry->val * pNextEntry->num) / (pEntry->num + pNextEntry->num); @@ -278,8 +277,9 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } else { SHistBin* pEntry = (SHistBin*)pResNode->pData; - if (ASSERTS(pEntry->val == val, "tHistogramAdd Error, pEntry->val:%lf, val:%lf")) { - return -1; + if (pEntry->val != val) { + qError("tHistogramAdd Error, pEntry->val:%lf, val:%lf", pEntry->val, val); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } pEntry->num += 1; } @@ -356,9 +356,9 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { (void)memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); } - if (ASSERTS(index >= 0 && index <= pHisto->maxEntries, "histogramCreateBin Error, index:%d, maxEntries:%d", - index, pHisto->maxEntries)) { - return TSDB_CODE_FAILED; + if (index < 0 || index > pHisto->maxEntries) { + qError("histogramCreateBin Error, index:%d, maxEntries:%d", index, pHisto->maxEntries); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } pHisto->elems[index].num = 1; @@ -373,9 +373,9 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { pHisto->elems[pHisto->maxEntries].num = 0; } #endif - if (ASSERTS(pHisto->numOfEntries <= pHisto->maxEntries, "histogramCreateBin Error, numOfEntries:%d, maxEntries:%d", - pHisto->numOfEntries, pHisto->maxEntries)) { - return TSDB_CODE_FAILED; + if (pHisto->numOfEntries > pHisto->maxEntries) { + qError("histogramCreateBin Error, numOfEntries:%d, maxEntries:%d", pHisto->numOfEntries, pHisto->maxEntries); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } return TSDB_CODE_SUCCESS; @@ -411,8 +411,9 @@ void tHistogramPrint(SHistogramInfo* pHisto) { * Estimated number of points in the interval (−inf,b]. * @param pHisto * @param v + * @param res */ -int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { +int32_t tHistogramSum(SHistogramInfo* pHisto, double v, int64_t *res) { #if defined(USE_ARRAYLIST) int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v); if (pHisto->elems[slotIdx].val != v) { @@ -420,14 +421,18 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { if (slotIdx < 0) { slotIdx = 0; - ASSERTS(v <= pHisto->elems[slotIdx].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", - slotIdx, pHisto->elems[slotIdx].val, v); + if (v > pHisto->elems[slotIdx].val) { + qError("tHistogramSum Error, elems[%d].val:%lf, v:%lf", slotIdx, pHisto->elems[slotIdx].val, v); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; + } } else { - ASSERTS(v >= pHisto->elems[slotIdx].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", - slotIdx, pHisto->elems[slotIdx].val, v); - if (slotIdx + 1 < pHisto->numOfEntries) { - ASSERTS(v < pHisto->elems[slotIdx + 1].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", - slotIdx + 1, pHisto->elems[slotIdx + 1].val, v); + if (v < pHisto->elems[slotIdx].val) { + qError("tHistogramSum Error, elems[%d].val:%lf, v:%lf", slotIdx, pHisto->elems[slotIdx].val, v); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; + } + if (slotIdx + 1 < pHisto->numOfEntries && v >= pHisto->elems[slotIdx + 1].val) { + qError("tHistogramSum Error, elems[%d].val:%lf, v:%lf", slotIdx + 1, pHisto->elems[slotIdx + 1].val, v); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; } } } @@ -447,8 +452,9 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { s1 = s1 + m1 / 2; - return (int64_t)s1; + *res = (int64_t)s1; #endif + return TSDB_CODE_SUCCESS; } int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal) { @@ -484,9 +490,11 @@ int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, do j += 1; } - ASSERTS(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem, - "tHistogramUniform Error, total:%ld, numOfElem:%ld, elems[%d].num:%ld", - total, (int64_t)numOfElem, j + 1, pHisto->elems[j + 1].num); + if (total > numOfElem || total + pHisto->elems[j + 1].num <= numOfElem) { + qError("tHistogramUniform Error, total:%d, numOfElem:%d, elems[%d].num:%d", + (int32_t)total, (int32_t)numOfElem, j + 1, (int32_t)pHisto->elems[j + 1].num); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; + } double delta = numOfElem - total; if (fabs(delta) < FLT_EPSILON) { @@ -545,9 +553,10 @@ int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, do j += 1; } - ASSERTS(total <= numOfElem && total + pEntry->num > numOfElem, - "tHistogramUniform Error, total:%d, numOfElem:%d, pEntry->num:%d", - total, numOfElem, pEntry->num); + if (total > numOfElem || total + pEntry->num <= numOfElem) { + qError("tHistogramUniform Error, total:%d, numOfElem:%d, pEntry->num:%d", (int32_t)total, (int32_t)numOfElem, (int32_t)pEntry->num); + return TSDB_CODE_FUNC_HISTOGRAM_ERROR; + } double delta = numOfElem - total; if (fabs(delta) < FLT_EPSILON) { diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 4eefd150f3..ae0459427e 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "query.h" #include "taoserror.h" #include "tcompare.h" #include "tglobal.h" @@ -107,7 +108,10 @@ static void resetPosInfo(SSlotInfo *pInfo) { } int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { - ASSERT(pMemBucket->total == 1); + if (pMemBucket->total != 1) { + qError("MemBucket:%p, total:%d, but only one element is expected", pMemBucket, pMemBucket->total); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } terrno = 0; for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { @@ -120,7 +124,10 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { SArray **pList = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); if (pList != NULL) { SArray *list = *pList; - ASSERT(list->size == 1); + if (list->size != 1) { + qError("list:%p, total list size:%zu, but only one element is expected", list, list->size); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } int32_t *pageId = taosArrayGet(list, 0); if (NULL == pageId) { @@ -130,7 +137,11 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { if (pPage == NULL) { return terrno; } - ASSERT(pPage->num == 1); + if (pPage->num != 1) { + qError("page:%p, total num:%d, but only one element is expected", pPage, pPage->num); + releaseBufPage(pMemBucket->pBuffer, pPage); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } GET_TYPED_DATA(*result, double, pMemBucket->type, pPage->data); return TSDB_CODE_SUCCESS; @@ -141,64 +152,69 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { return TSDB_CODE_SUCCESS; } -int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { +int32_t tBucketIntHash(tMemBucket *pBucket, const void *value, int32_t *index) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pBucket->type, value); - int32_t index = -1; + *index = -1; if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { - return index; + return TSDB_CODE_SUCCESS; } // divide the value range into 1024 buckets uint64_t span = pBucket->range.dMaxVal - pBucket->range.dMinVal; if (span < pBucket->numOfSlots) { int64_t delta = v - pBucket->range.dMinVal; - index = (delta % pBucket->numOfSlots); + *index = (delta % pBucket->numOfSlots); } else { double slotSpan = ((double)span) / pBucket->numOfSlots; uint64_t delta = (uint64_t)(v - pBucket->range.dMinVal); - index = delta / slotSpan; - if (v == pBucket->range.dMaxVal || index == pBucket->numOfSlots) { - index -= 1; + *index = delta / slotSpan; + if (v == pBucket->range.dMaxVal || *index == pBucket->numOfSlots) { + *index -= 1; } } - ASSERTS(index >= 0 && index < pBucket->numOfSlots, "tBucketIntHash Error, index:%d, numOfSlots:%d", - index, pBucket->numOfSlots); - return index; + if (*index < 0 || *index >= pBucket->numOfSlots) { + qError("tBucketIntHash Error, index:%d, numOfSlots:%d", *index, pBucket->numOfSlots); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } + return TSDB_CODE_SUCCESS; } -int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { +int32_t tBucketUintHash(tMemBucket *pBucket, const void *value, int32_t *index) { int64_t v = 0; GET_TYPED_DATA(v, uint64_t, pBucket->type, value); - int32_t index = -1; + *index = -1; if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) { - return index; + return TSDB_CODE_SUCCESS; } // divide the value range into 1024 buckets uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal; if (span < pBucket->numOfSlots) { int64_t delta = v - pBucket->range.u64MinVal; - index = (int32_t)(delta % pBucket->numOfSlots); + *index = (int32_t)(delta % pBucket->numOfSlots); } else { double slotSpan = (double)span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan); + *index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan); if (v == pBucket->range.u64MaxVal) { - index -= 1; + *index -= 1; } } - ASSERT(index >= 0 && index < pBucket->numOfSlots); - return index; + if (*index < 0 || *index >= pBucket->numOfSlots) { + qError("tBucketUintHash Error, index:%d, numOfSlots:%d", *index, pBucket->numOfSlots); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } + return TSDB_CODE_SUCCESS; } -int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { +int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value, int32_t *index) { double v = 0; if (pBucket->type == TSDB_DATA_TYPE_FLOAT) { v = GET_FLOAT_VAL(value); @@ -206,27 +222,30 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { v = GET_DOUBLE_VAL(value); } - int32_t index = -1; + *index = -1; if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { - return index; + return TSDB_CODE_SUCCESS; } // divide a range of [dMinVal, dMaxVal] into 1024 buckets double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; if (span < pBucket->numOfSlots) { int32_t delta = (int32_t)(v - pBucket->range.dMinVal); - index = (delta % pBucket->numOfSlots); + *index = (delta % pBucket->numOfSlots); } else { double slotSpan = span / pBucket->numOfSlots; - index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); + *index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); if (v == pBucket->range.dMaxVal) { - index -= 1; + *index -= 1; } } - ASSERT(index >= 0 && index < pBucket->numOfSlots); - return index; + if (*index < 0 || *index >= pBucket->numOfSlots) { + qError("tBucketDoubleHash Error, index:%d, numOfSlots:%d", *index, pBucket->numOfSlots); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } + return TSDB_CODE_SUCCESS; } static __perc_hash_func_t getHashFunc(int32_t type) { @@ -333,7 +352,7 @@ void tMemBucketDestroy(tMemBucket *pBucket) { taosMemoryFreeClear(pBucket); } -void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { +int32_t tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { if (IS_SIGNED_NUMERIC_TYPE(dataType)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, dataType, data); @@ -368,8 +387,10 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT r->dMaxVal = v; } } else { - ASSERT(0); + qError("tMemBucketUpdateBoundingBox Error, invalid data type:%d", dataType); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; } + return TSDB_CODE_SUCCESS; } /* @@ -378,9 +399,14 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { int32_t count = 0; int32_t bytes = pBucket->bytes; + int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < size; ++i) { char *d = (char *)data + i * bytes; - int32_t index = (pBucket->hashFunc)(pBucket, d); + int32_t index = -1; + code = (pBucket->hashFunc)(pBucket, d, &index); + if (TSDB_CODE_SUCCESS != code) { + return code; + } if (index < 0) { continue; } @@ -388,7 +414,10 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { count += 1; tMemBucketSlot *pSlot = &pBucket->pSlots[index]; - tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); + code = tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); + if (TSDB_CODE_SUCCESS != code) { + return code; + } // ensure available memory pages to allocate int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times); @@ -396,7 +425,11 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { if (pSlot->info.data != NULL) { - ASSERT(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); + if (pSlot->info.data->num < pBucket->elemPerPage || pSlot->info.size <= 0) { + qError("tMemBucketPut failed since wrong pSLot info dataNum : %d, size : %d", + pSlot->info.data->num, pSlot->info.size); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } // keep the pointer in memory setBufPageDirty(pSlot->info.data, true); @@ -411,7 +444,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { if (NULL == pPageIdList) { return terrno; } - int32_t code = taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); + code = taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); if (TSDB_CODE_SUCCESS != code) { taosArrayDestroy(pPageIdList); return code; @@ -449,21 +482,23 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { * j is the last slot of current segment, we need to get the first * slot of the next segment. */ -static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t slotIdx) { +static int32_t getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t slotIdx, MinMaxEntry *next) { int32_t j = slotIdx + 1; while (j < pMemBucket->numOfSlots && (pMemBucket->pSlots[j].info.size == 0)) { ++j; } - ASSERT(j < pMemBucket->numOfSlots); - return pMemBucket->pSlots[j].range; + if (j >= pMemBucket->numOfSlots) { + qError("getMinMaxEntryOfNextSlotWithData can not get valid slot, start with slotIdx:%d", slotIdx); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } + *next = pMemBucket->pSlots[j].range; + return TSDB_CODE_SUCCESS; } static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index); static double getIdenticalDataVal(tMemBucket *pMemBucket, int32_t slotIndex) { - ASSERT(isIdenticalData(pMemBucket, slotIndex)); - tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex]; double finalResult = 0.0; @@ -494,7 +529,11 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction * now, we need to find the minimum value of the next slot for interpolating the percentile value * j is the last slot of current segment, we need to get the first slot of the next segment. */ - MinMaxEntry next = getMinMaxEntryOfNextSlotWithData(pMemBucket, i); + MinMaxEntry next; + int32_t code = getMinMaxEntryOfNextSlotWithData(pMemBucket, i, &next); + if (TSDB_CODE_SUCCESS != code) { + return code; + } double maxOfThisSlot = 0; double minOfNextSlot = 0; @@ -509,7 +548,10 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction minOfNextSlot = (double)next.dMinVal; } - ASSERT(minOfNextSlot > maxOfThisSlot); + if (minOfNextSlot <= maxOfThisSlot) { + qError("getPercentileImpl get minOfNextSlot : %f less equal than maxOfThisSlot : %f", minOfNextSlot, maxOfThisSlot); + return TSDB_CODE_FUNC_PERCENTILE_ERROR; + } *result = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot; return TSDB_CODE_SUCCESS; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 396abf21a7..f2d15abb80 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -724,6 +724,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL, "Function time unit c TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_INVALID_VALUE_RANGE, "Function got invalid value range") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_SETUP_ERROR, "Function set up failed") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_INVALID_RES_LENGTH, "Function result exceed max length") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_HISTOGRAM_ERROR, "Function failed to calculate histogram") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_PERCENTILE_ERROR, "Function failed to calculate percentile") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") From 15a4fe9c845642d377881e6e96c4c737c02eb3ae Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 19 Aug 2024 17:06:58 +0800 Subject: [PATCH 011/100] fix: close file --- source/client/src/clientMonitor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 8f28cbbce4..ddd2595be5 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -681,6 +681,7 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) { } char* tmp = taosStrdup(filename); monitorSendSlowLogAtBeginning(clusterId, &tmp, pFile, 0); + (void)taosCloseFile(&pFile); taosMemoryFree(tmp); } From 5035b3a624e970b805e7d66facd1e0aa66a484c5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 19 Aug 2024 17:13:27 +0800 Subject: [PATCH 012/100] fix: merge join destroy table issue --- source/libs/executor/src/mergejoinoperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 946a1d2aa5..52b0da7c92 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -1746,6 +1746,9 @@ void destroyGrpArray(void* ppArray) { } void destroyMergeJoinTableCtx(SMJoinTableCtx* pTable) { + if (NULL == pTable) { + return; + } mJoinDestroyCreatedBlks(pTable->createdBlks); taosArrayDestroy(pTable->createdBlks); tSimpleHashCleanup(pTable->pGrpHash); From 800e7c4e7acd19ec33c2524d1fdde61e8ce64495 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 19 Aug 2024 17:27:15 +0800 Subject: [PATCH 013/100] fix: memory leak of geos --- source/client/src/clientMain.c | 2 + source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 2 + source/util/src/tgeosctx.c | 92 +++++++++++++++++------- source/util/src/tsched.c | 2 +- source/util/src/tworker.c | 4 +- 5 files changed, 75 insertions(+), 27 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 12702a93f3..a403f9d1c2 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -23,6 +23,7 @@ #include "query.h" #include "scheduler.h" #include "tdatablock.h" +#include "tgeosctx.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" @@ -94,6 +95,7 @@ void taos_cleanup(void) { tmqMgmtClose(); DestroyRegexCache(); + destroyThreadLocalGeosCtx(); tscInfo("all local resources released"); taosCleanupCfg(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index fdce9fd4c9..1d62d4bd90 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -19,6 +19,7 @@ #include "index.h" #include "qworker.h" #include "tcompression.h" +#include "tgeosctx.h" #include "tglobal.h" #include "tgrant.h" #include "tstream.h" @@ -121,6 +122,7 @@ void dmCleanupDnode(SDnode *pDnode) { streamMetaCleanup(); indexCleanup(); taosConvDestroy(); + destroyThreadLocalGeosCtx(); // compress destroy tsCompressExit(); diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index a05734c911..47d5cc992b 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -15,38 +15,82 @@ #include "tgeosctx.h" #include "tdef.h" +#include "tlockfree.h" +#include "tlog.h" -static threadlocal SGeosContext tlGeosCtx = {0}; +typedef struct { + SGeosContext *pool; + int32_t capacity; + int32_t size; + SRWLatch lock; +} SGeosContextPool; -SGeosContext* getThreadLocalGeosCtx() { return &tlGeosCtx; } +static SGeosContextPool sGeosPool = {0}; -void destroyThreadLocalGeosCtx() { - if (tlGeosCtx.WKTReader) { - GEOSWKTReader_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKTReader); - tlGeosCtx.WKTReader = NULL; +static threadlocal SGeosContext *tlGeosCtx = NULL; + +SGeosContext *getThreadLocalGeosCtx() { + if (tlGeosCtx) return tlGeosCtx; + + taosWLockLatch(&sGeosPool.lock); + if (sGeosPool.size >= sGeosPool.capacity) { + sGeosPool.capacity += 64; + void *tmp = taosMemoryRealloc(sGeosPool.pool, sGeosPool.capacity * sizeof(SGeosContext)); + if (!tmp) { + taosWUnLockLatch(&sGeosPool.lock); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + sGeosPool.pool = tmp; + TAOS_MEMSET(sGeosPool.pool + sGeosPool.size, 0, (sGeosPool.capacity - sGeosPool.size) * sizeof(SGeosContext)); } + tlGeosCtx = sGeosPool.pool + sGeosPool.size; + ++sGeosPool.size; + taosWUnLockLatch(&sGeosPool.lock); - if (tlGeosCtx.WKTWriter) { - GEOSWKTWriter_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKTWriter); - tlGeosCtx.WKTWriter = NULL; - } + return tlGeosCtx; +} - if (tlGeosCtx.WKBReader) { - GEOSWKBReader_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKBReader); - tlGeosCtx.WKBReader = NULL; - } +static void destroyGeosCtx(SGeosContext *pCtx) { + if (pCtx) { + if (pCtx->WKTReader) { + GEOSWKTReader_destroy_r(pCtx->handle, pCtx->WKTReader); + pCtx->WKTReader = NULL; + } - if (tlGeosCtx.WKBWriter) { - GEOSWKBWriter_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKBWriter); - tlGeosCtx.WKBWriter = NULL; - } + if (pCtx->WKTWriter) { + GEOSWKTWriter_destroy_r(pCtx->handle, pCtx->WKTWriter); + pCtx->WKTWriter = NULL; + } - if (tlGeosCtx.WKTRegex) { - destroyRegexes(tlGeosCtx.WKTRegex, tlGeosCtx.WKTMatchData); - } + if (pCtx->WKBReader) { + GEOSWKBReader_destroy_r(pCtx->handle, pCtx->WKBReader); + pCtx->WKBReader = NULL; + } - if (tlGeosCtx.handle) { - GEOS_finish_r(tlGeosCtx.handle); - tlGeosCtx.handle = NULL; + if (pCtx->WKBWriter) { + GEOSWKBWriter_destroy_r(pCtx->handle, pCtx->WKBWriter); + pCtx->WKBWriter = NULL; + } + + if (pCtx->WKTRegex) { + destroyRegexes(pCtx->WKTRegex, pCtx->WKTMatchData); + pCtx->WKTRegex = NULL; + pCtx->WKTMatchData = NULL; + } + + if (pCtx->handle) { + GEOS_finish_r(pCtx->handle); + pCtx->handle = NULL; + } } } + +void destroyThreadLocalGeosCtx() { + uInfo("geos ctx is cleaned up"); + if (!sGeosPool.pool) return; + for (int32_t i = 0; i < sGeosPool.size; ++i) { + destroyGeosCtx(sGeosPool.pool + i); + } + taosMemoryFreeClear(sGeosPool.pool); +} \ No newline at end of file diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 6779e8dee5..509dba0890 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -178,7 +178,7 @@ void *taosProcessSchedQueue(void *scheduler) { (*(msg.tfp))(msg.ahandle, msg.thandle); } - destroyThreadLocalGeosCtx(); + // destroyThreadLocalGeosCtx(); return NULL; } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index b2064d6787..2da1abed78 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -105,7 +105,7 @@ static void *tQWorkerThreadFp(SQueueWorker *worker) { taosUpdateItemSize(qinfo.queue, 1); } - destroyThreadLocalGeosCtx(); + // destroyThreadLocalGeosCtx(); DestoryThreadLocalRegComp(); return NULL; @@ -665,7 +665,7 @@ static void *tQueryAutoQWorkerThreadFp(SQueryAutoQWorker *worker) { } } - destroyThreadLocalGeosCtx(); + // destroyThreadLocalGeosCtx(); DestoryThreadLocalRegComp(); return NULL; From 5ce7bcad1e42a37bd3e2fcf8ffb4252120aad5b0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 19 Aug 2024 17:30:10 +0800 Subject: [PATCH 014/100] refact: remove the return value of function taosCloseRef --- .gitignore | 1 + include/util/tref.h | 3 +-- source/client/src/clientMain.c | 8 ++------ source/client/src/clientTmq.c | 2 +- source/dnode/vnode/src/sma/smaEnv.c | 4 ++-- source/libs/executor/src/executor.c | 2 +- source/libs/index/src/index.c | 2 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/qworker/src/qwUtil.c | 2 +- source/libs/scheduler/src/schUtil.c | 2 +- source/libs/stream/src/streamMeta.c | 6 +++--- source/libs/transport/src/transComm.c | 2 +- source/libs/wal/src/walMgmt.c | 2 +- source/util/src/tref.c | 6 ++---- 14 files changed, 19 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index f8b42f9176..334947a64c 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,4 @@ tags .clangd *CMakeCache* *CMakeFiles* +.history/ diff --git a/include/util/tref.h b/include/util/tref.h index c4b2ec8fa7..1520ced14e 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -29,8 +29,7 @@ typedef void (*RefFp)(void *); int32_t taosOpenRef(int32_t max, RefFp fp); // close the reference set, refId is the return value by taosOpenRef -// return 0 if success. On error, -1 is returned, and terrno is set appropriately -int32_t taosCloseRef(int32_t rsetId); +void taosCloseRef(int32_t rsetId); // add ref, p is the pointer to resource or pointer ID // return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 12702a93f3..d007dae7f7 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -74,15 +74,11 @@ void taos_cleanup(void) { int32_t id = clientReqRefPool; clientReqRefPool = -1; - if (TSDB_CODE_SUCCESS != taosCloseRef(id)) { - tscWarn("failed to close clientReqRefPool"); - } + taosCloseRef(id); id = clientConnRefPool; clientConnRefPool = -1; - if (TSDB_CODE_SUCCESS != taosCloseRef(id)) { - tscWarn("failed to close clientReqRefPool"); - } + taosCloseRef(id); nodesDestroyAllocatorSet(); cleanupAppInfo(); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 91883869e9..c9f166e565 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1184,7 +1184,7 @@ void tmqMgmtClose(void) { } if (tmqMgmt.rsetId >= 0) { - (void)taosCloseRef(tmqMgmt.rsetId); + taosCloseRef(tmqMgmt.rsetId); tmqMgmt.rsetId = -1; } } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index a219da33db..d90e869bd4 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -69,7 +69,7 @@ int32_t smaInit() { if (!smaMgmt.refHash || !smaMgmt.tmrHandle) { code = terrno; - (void)taosCloseRef(smaMgmt.rsetId); + taosCloseRef(smaMgmt.rsetId); if (smaMgmt.refHash) { taosHashCleanup(smaMgmt.refHash); smaMgmt.refHash = NULL; @@ -103,7 +103,7 @@ void smaCleanUp() { } if (old == 1) { - (void)taosCloseRef(smaMgmt.rsetId); + taosCloseRef(smaMgmt.rsetId); taosHashCleanup(smaMgmt.refHash); smaMgmt.refHash = NULL; taosTmrCleanUp(smaMgmt.tmrHandle); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b1c9207ab7..eb249575fc 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -31,7 +31,7 @@ int32_t exchangeObjRefPool = -1; static void cleanupRefPool() { int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0); - (void)taosCloseRef(ref); + taosCloseRef(ref); } static void initRefPool() { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 986693ab00..b881d2cac8 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -74,7 +74,7 @@ void indexCleanup() { // refacto later taosCleanUpScheduler(indexQhandle); taosMemoryFreeClear(indexQhandle); - (void)taosCloseRef(indexRefMgt); + taosCloseRef(indexRefMgt); } typedef struct SIdxColInfo { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 6b06530b3e..c1afc4afb3 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -238,7 +238,7 @@ void nodesDestroyAllocatorSet() { (void)taosRemoveRef(g_allocatorReqRefPool, refId); pAllocator = taosIterateRef(g_allocatorReqRefPool, refId); } - (void)taosCloseRef(g_allocatorReqRefPool); + taosCloseRef(g_allocatorReqRefPool); } } diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 441714313c..4b9067a191 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -564,7 +564,7 @@ int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { void qwCloseRef(void) { taosWLockLatch(&gQwMgmt.lock); if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { - (void)taosCloseRef(gQwMgmt.qwRef); // ignore error + taosCloseRef(gQwMgmt.qwRef); // ignore error gQwMgmt.qwRef = -1; } taosWUnLockLatch(&gQwMgmt.lock); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 811890dde5..01249dbb98 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -273,7 +273,7 @@ void schCloseJobRef(void) { } if (schMgmt.jobRef >= 0) { - (void)taosCloseRef(schMgmt.jobRef); + taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = -1; } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a9976760b6..bc12f4307b 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -65,9 +65,9 @@ static void streamMetaEnvInit() { void streamMetaInit() { (void)taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaCleanup() { - (void)taosCloseRef(streamBackendId); - (void)taosCloseRef(streamBackendCfWrapperId); - (void)taosCloseRef(streamMetaId); + taosCloseRef(streamBackendId); + taosCloseRef(streamBackendCfWrapperId); + taosCloseRef(streamMetaId); metaRefMgtCleanup(); streamTimerCleanUp(); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index b940c494d8..5d82e157b3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -736,7 +736,7 @@ int32_t transOpenRefMgt(int size, void (*func)(void*)) { } void transCloseRefMgt(int32_t mgt) { // close ref - (void)taosCloseRef(mgt); + taosCloseRef(mgt); } int64_t transAddExHandle(int32_t refMgt, void* p) { // acquire extern handle diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 581a63671c..e1d31ce113 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -75,7 +75,7 @@ void walCleanUp() { if (old == 1) { walStopThread(); - TAOS_UNUSED(taosCloseRef(tsWal.refSetId)); + taosCloseRef(tsWal.refSetId); wInfo("wal module is cleaned up"); atomic_store_8(&tsWal.inited, 0); } diff --git a/source/util/src/tref.c b/source/util/src/tref.c index f1d9a24757..4b1b477e06 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -110,13 +110,13 @@ int32_t taosOpenRef(int32_t max, RefFp fp) { return rsetId; } -int32_t taosCloseRef(int32_t rsetId) { +void taosCloseRef(int32_t rsetId) { SRefSet *pSet; int32_t deleted = 0; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { uTrace("rsetId:%d is invalid, out of range", rsetId); - return terrno = TSDB_CODE_REF_INVALID_ID; + return; } pSet = tsRefSetList + rsetId; @@ -134,8 +134,6 @@ int32_t taosCloseRef(int32_t rsetId) { (void)taosThreadMutexUnlock(&tsRefMutex); if (deleted) taosDecRsetCount(pSet); - - return 0; } int64_t taosAddRef(int32_t rsetId, void *p) { From c7f9c82950c6197bba0c0afcd1cf71898ca7c3ef Mon Sep 17 00:00:00 2001 From: menshibin Date: Mon, 19 Aug 2024 18:02:50 +0800 Subject: [PATCH 015/100] add kafka config propose --- .../20-third-party/01-collection/11-kafka.md | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/docs/zh/20-third-party/01-collection/11-kafka.md b/docs/zh/20-third-party/01-collection/11-kafka.md index 651ef860cb..2e4677ca31 100644 --- a/docs/zh/20-third-party/01-collection/11-kafka.md +++ b/docs/zh/20-third-party/01-collection/11-kafka.md @@ -27,22 +27,35 @@ TDengine Source Connector 用于把数据实时地从 TDengine 读出来发送 ## 安装 Kafka -在任意目录下执行： +- 在任意目录下执行： -```shell -curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz -tar xzf kafka_2.13-3.4.0.tgz -C /opt/ -ln -s /opt/kafka_2.13-3.4.0 /opt/kafka -``` + ```shell + curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz + tar xzf kafka_2.13-3.4.0.tgz -C /opt/ + ln -s /opt/kafka_2.13-3.4.0 /opt/kafka + ``` -然后需要把 `$KAFKA_HOME/bin` 目录加入 PATH。 +- 然后需要把 `$KAFKA_HOME/bin` 目录加入 PATH。 -```title=".profile" -export KAFKA_HOME=/opt/kafka -export PATH=$PATH:$KAFKA_HOME/bin -``` + ```title=".profile" + export KAFKA_HOME=/opt/kafka + export PATH=$PATH:$KAFKA_HOME/bin + ``` + 以上脚本可以追加到当前用户的 profile 文件（~/.profile 或 ~/.bash_profile） -以上脚本可以追加到当前用户的 profile 文件（~/.profile 或 ~/.bash_profile） +- 提升 Kafka 吞吐率的建议配置 + + 1. 打开 KAFKA_HOME/config/producer.properties 配置文件。 + 2. 参数说明及建议如下： + + | **参数** | **参数说明** | **设置建议** | + | --------| --------------------------------- | -------------- | + | producer.type | 此参数用于设置消息的发送方式，默认值为 `sync` 表示同步发送，`async` 表示异步发送。采用异步发送能够提升消息发送的吞吐量。 | async | + | request.required.acks | 参数用于配置生产者发送消息后需要等待的确认数量。当设置为1时，表示只要领导者副本成功写入消息就会给生产者发送确认，而无需等待集群中的其他副本写入成功。这种设置可以在一定程度上保证消息的可靠性，同时也能保证一定的吞吐量。因为不需要等待所有副本都写入成功，所以可以减少生产者的等待时间，提高发送消息的效率。|1| + | max.request.size| 该参数决定了生产者在一次请求中可以发送的最大数据量。其默认值为 1048576，也就是 1M。如果设置得太小，可能会导致频繁的网络请求，降低吞吐量。如果设置得太大，可能会导致内存占用过高，或者在网络状况不佳时增加请求失败的概率。建议设置为 100M。|104857600| + |batch.size| 此参数用于设定 batch 的大小，默认值为 16384，即 16KB。在消息发送过程中，发送到 Kafka 缓冲区中的消息会被划分成一个个的 batch。故而减小 batch 大小有助于降低消息延迟，而增大 batch 大小则有利于提升吞吐量，可根据实际的数据量大小进行合理配置。可根据实际情况进行调整，建议设置为 512K。|524288| + | buffer.memory| 此参数用于设置生产者缓冲待发送消息的内存总量。较大的缓冲区可以允许生产者积累更多的消息后批量发送，提高吞吐量，但也会增加延迟和内存使用。可根据机器资源来配置，建议配置为 1G。|1073741824| + ## 安装 TDengine Connector 插件 From e296a2a076e2b443ca5e0a03190a7bad2c02e0f5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 19 Aug 2024 18:04:29 +0800 Subject: [PATCH 016/100] fix: init refId before setting env start --- source/libs/sync/inc/syncEnv.h | 4 --- source/libs/sync/src/syncEnv.c | 58 ++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 4dc5f58cfe..41ad915333 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -42,10 +42,6 @@ typedef struct SSyncEnv { // timer manager tmr_h pTimerManager; - - // other resources shared by SyncNodes - // ... - } SSyncEnv; SSyncEnv* syncEnv(); diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 8d1e2cfebd..6d16a76d98 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -42,23 +42,25 @@ int32_t syncInit() { // start tmr thread gSyncEnv.pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); - atomic_store_8(&gSyncEnv.isStart, 1); gNodeRefId = taosOpenRef(200, (RefFp)syncNodeClose); if (gNodeRefId < 0) { - sError("failed to init node ref"); + sError("failed to init node rset"); syncCleanUp(); return TSDB_CODE_SYN_WRONG_REF; } + sDebug("sync node rset is open, rsetId:%d", gNodeRefId); gHbDataRefId = taosOpenRef(200, (RefFp)syncHbTimerDataFree); if (gHbDataRefId < 0) { - sError("failed to init hb-data ref"); + sError("failed to init hbdata rset"); syncCleanUp(); return TSDB_CODE_SYN_WRONG_REF; } - sDebug("sync rsetId:%d is open", gNodeRefId); + sDebug("sync hbdata rset is open, rsetId:%d", gHbDataRefId); + + atomic_store_8(&gSyncEnv.isStart, 1); return 0; } @@ -68,32 +70,40 @@ void syncCleanUp() { memset(&gSyncEnv, 0, sizeof(SSyncEnv)); if (gNodeRefId != -1) { - sDebug("sync rsetId:%d is closed", gNodeRefId); - (void)taosCloseRef(gNodeRefId); + sDebug("sync node rset is closed, rsetId:%d", gNodeRefId); + taosCloseRef(gNodeRefId); gNodeRefId = -1; } if (gHbDataRefId != -1) { - sDebug("sync rsetId:%d is closed", gHbDataRefId); - (void)taosCloseRef(gHbDataRefId); + sDebug("sync hbdata rset is closed, rsetId:%d", gHbDataRefId); + taosCloseRef(gHbDataRefId); gHbDataRefId = -1; } } int64_t syncNodeAdd(SSyncNode *pNode) { pNode->rid = taosAddRef(gNodeRefId, pNode); - if (pNode->rid < 0) return -1; + if (pNode->rid < 0) { + terrno = TSDB_CODE_SYN_WRONG_REF; + return -1; + } - sDebug("vgId:%d, sync rid:%" PRId64 " is added to rsetId:%d", pNode->vgId, pNode->rid, gNodeRefId); + sDebug("vgId:%d, sync node refId:%" PRId64 " is added to rsetId:%d", pNode->vgId, pNode->rid, gNodeRefId); return pNode->rid; } -void syncNodeRemove(int64_t rid) { (void)taosRemoveRef(gNodeRefId, rid); } +void syncNodeRemove(int64_t rid) { + sDebug("sync node refId:%" PRId64 " is removed from rsetId:%d", rid, gNodeRefId); + if (rid > 0) { + (void)taosRemoveRef(gNodeRefId, rid); + } +} SSyncNode *syncNodeAcquire(int64_t rid) { SSyncNode *pNode = taosAcquireRef(gNodeRefId, rid); if (pNode == NULL) { - sError("failed to acquire node from refId:%" PRId64, rid); + sError("failed to acquire sync node from refId:%" PRId64 ", rsetId:%d", rid, gNodeRefId); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; } @@ -101,28 +111,42 @@ SSyncNode *syncNodeAcquire(int64_t rid) { } void syncNodeRelease(SSyncNode *pNode) { - if (pNode) (void)taosReleaseRef(gNodeRefId, pNode->rid); + if (pNode) { + (void)taosReleaseRef(gNodeRefId, pNode->rid); + } } int64_t syncHbTimerDataAdd(SSyncHbTimerData *pData) { pData->rid = taosAddRef(gHbDataRefId, pData); - if (pData->rid < 0) return TSDB_CODE_SYN_WRONG_REF; + if (pData->rid < 0) { + terrno = TSDB_CODE_SYN_WRONG_REF; + return -1; + } + return pData->rid; } -void syncHbTimerDataRemove(int64_t rid) { (void)taosRemoveRef(gHbDataRefId, rid); } +void syncHbTimerDataRemove(int64_t rid) { + if (rid > 0) { + (void)taosRemoveRef(gHbDataRefId, rid); + } +} SSyncHbTimerData *syncHbTimerDataAcquire(int64_t rid) { SSyncHbTimerData *pData = taosAcquireRef(gHbDataRefId, rid); if (pData == NULL && rid > 0) { - sInfo("failed to acquire hb-timer-data from refId:%" PRId64, rid); + sInfo("failed to acquire hbdata from refId:%" PRId64 ", rsetId:%d", rid, gHbDataRefId); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; } return pData; } -void syncHbTimerDataRelease(SSyncHbTimerData *pData) { (void)taosReleaseRef(gHbDataRefId, pData->rid); } +void syncHbTimerDataRelease(SSyncHbTimerData *pData) { + if (pData) { + (void)taosReleaseRef(gHbDataRefId, pData->rid); + } +} #if 0 void syncEnvStartTimer() { From aef00cfb8338e0cebe43f7d77cb55c0407fcd28c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 19 Aug 2024 18:04:50 +0800 Subject: [PATCH 017/100] refact: remove unused variables --- source/libs/sync/inc/syncEnv.h | 15 +----------- source/libs/sync/src/syncEnv.c | 45 ---------------------------------- 2 files changed, 1 insertion(+), 59 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 41ad915333..0376920e8a 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -23,25 +23,12 @@ extern "C" { #include "syncInt.h" #define TIMER_MAX_MS 0x7FFFFFFF -#define ENV_TICK_TIMER_MS 1000 #define PING_TIMER_MS 5000 -#define ELECT_TIMER_MS_MIN 2500 -#define HEARTBEAT_TIMER_MS 1000 #define HEARTBEAT_TICK_NUM 20 typedef struct SSyncEnv { uint8_t isStart; - - // tick timer - tmr_h pEnvTickTimer; - int32_t envTickTimerMS; - uint64_t envTickTimerLogicClock; // if use queue, should pass logic clock into queue item - uint64_t envTickTimerLogicClockUser; - TAOS_TMR_CALLBACK FpEnvTickTimer; // Timer Fp - uint64_t envTickTimerCounter; - - // timer manager - tmr_h pTimerManager; + tmr_h pTimerManager; } SSyncEnv; SSyncEnv* syncEnv(); diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 6d16a76d98..7474cea83d 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -21,7 +21,6 @@ static SSyncEnv gSyncEnv = {0}; static int32_t gNodeRefId = -1; static int32_t gHbDataRefId = -1; -static void syncEnvTick(void *param, void *tmrId); SSyncEnv *syncEnv() { return &gSyncEnv; } @@ -34,13 +33,6 @@ int32_t syncInit() { taosSeedRand(seed); memset(&gSyncEnv, 0, sizeof(SSyncEnv)); - gSyncEnv.envTickTimerCounter = 0; - gSyncEnv.envTickTimerMS = ENV_TICK_TIMER_MS; - gSyncEnv.FpEnvTickTimer = syncEnvTick; - atomic_store_64(&gSyncEnv.envTickTimerLogicClock, 0); - atomic_store_64(&gSyncEnv.envTickTimerLogicClockUser, 0); - - // start tmr thread gSyncEnv.pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); gNodeRefId = taosOpenRef(200, (RefFp)syncNodeClose); @@ -147,40 +139,3 @@ void syncHbTimerDataRelease(SSyncHbTimerData *pData) { (void)taosReleaseRef(gHbDataRefId, pData->rid); } } - -#if 0 -void syncEnvStartTimer() { - taosTmrReset(gSyncEnv.FpEnvTickTimer, gSyncEnv.envTickTimerMS, &gSyncEnv, gSyncEnv.pTimerManager, - &gSyncEnv.pEnvTickTimer); - atomic_store_64(&gSyncEnv.envTickTimerLogicClock, gSyncEnv.envTickTimerLogicClockUser); -} - -void syncEnvStopTimer() { - int32_t ret = 0; - atomic_add_fetch_64(&gSyncEnv.envTickTimerLogicClockUser, 1); - taosTmrStop(gSyncEnv.pEnvTickTimer); - gSyncEnv.pEnvTickTimer = NULL; - return ret; -} -#endif - -static void syncEnvTick(void *param, void *tmrId) { -#if 0 - SSyncEnv *pSyncEnv = param; - if (atomic_load_64(&gSyncEnv.envTickTimerLogicClockUser) <= atomic_load_64(&gSyncEnv.envTickTimerLogicClock)) { - gSyncEnv.envTickTimerCounter++; - sTrace("syncEnvTick do ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 - ", envTickTimerCounter:%" PRIu64 ", envTickTimerMS:%d, tmrId:%p", - gSyncEnv.envTickTimerLogicClockUser, gSyncEnv.envTickTimerLogicClock, gSyncEnv.envTickTimerCounter, - gSyncEnv.envTickTimerMS, tmrId); - - // do something, tick ... - taosTmrReset(syncEnvTick, gSyncEnv.envTickTimerMS, pSyncEnv, gSyncEnv.pTimerManager, &gSyncEnv.pEnvTickTimer); - } else { - sTrace("syncEnvTick pass ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 - ", envTickTimerCounter:%" PRIu64 ", envTickTimerMS:%d, tmrId:%p", - gSyncEnv.envTickTimerLogicClockUser, gSyncEnv.envTickTimerLogicClock, gSyncEnv.envTickTimerCounter, - gSyncEnv.envTickTimerMS, tmrId); - } -#endif -} From 12dbcf561201937c464cbc0ef1054138a52e05a3 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 19 Aug 2024 18:26:34 +0800 Subject: [PATCH 018/100] docs: add privilege control --- .../03-taos-sql/{25-grant.md => 25-user.md} | 0 docs/zh/14-reference/03-taos-sql/26-grant.md | 168 ++++++++++++++++++ .../03-taos-sql/{26-udf.md => 27-udf.md} | 0 .../{27-indexing.md => 28-index.md} | 0 .../{28-recovery.md => 29-recovery.md} | 0 .../{29-changes.md => 30-changes.md} | 0 .../03-taos-sql/{30-join.md => 31-join.md} | 0 .../{31-compress.md => 32-compress.md} | 0 .../03-taos-sql/{32-view.md => 33-view.md} | 0 9 files changed, 168 insertions(+) rename docs/zh/14-reference/03-taos-sql/{25-grant.md => 25-user.md} (100%) create mode 100644 docs/zh/14-reference/03-taos-sql/26-grant.md rename docs/zh/14-reference/03-taos-sql/{26-udf.md => 27-udf.md} (100%) rename docs/zh/14-reference/03-taos-sql/{27-indexing.md => 28-index.md} (100%) rename docs/zh/14-reference/03-taos-sql/{28-recovery.md => 29-recovery.md} (100%) rename docs/zh/14-reference/03-taos-sql/{29-changes.md => 30-changes.md} (100%) rename docs/zh/14-reference/03-taos-sql/{30-join.md => 31-join.md} (100%) rename docs/zh/14-reference/03-taos-sql/{31-compress.md => 32-compress.md} (100%) rename docs/zh/14-reference/03-taos-sql/{32-view.md => 33-view.md} (100%) diff --git a/docs/zh/14-reference/03-taos-sql/25-grant.md b/docs/zh/14-reference/03-taos-sql/25-user.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/25-grant.md rename to docs/zh/14-reference/03-taos-sql/25-user.md diff --git a/docs/zh/14-reference/03-taos-sql/26-grant.md b/docs/zh/14-reference/03-taos-sql/26-grant.md new file mode 100644 index 0000000000..c3fd1790d0 --- /dev/null +++ b/docs/zh/14-reference/03-taos-sql/26-grant.md @@ -0,0 +1,168 @@ +--- +toc_max_heading_level: 4 +title: 权限管理 +--- + +TDengine 中的权限管理分为[用户管理](../user)、数据库授权管理以及消息订阅授权管理，本节重点说明数据库授权和订阅授权。 + +## 数据库访问授权 + +系统管理员可以根据业务需要对系统中的每个用户针对每个数据库进行特定的授权，以防止业务数据被不恰当的用户读取或修改。对某个用户进行数据库访问授权的语法如下： + +```sql +GRANT privileges ON priv_level TO user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + READ + | WRITE +} + +priv_level : { + dbname.tbname + | dbname.* + | *.* +} +``` + +对数据库的访问权限包含读和写两种权限，它们可以被分别授予，也可以被同时授予。 + +说明 + +- priv_level 格式中 "." 之前为数据库名称， "." 之后为表名称，意思为表级别的授权控制。如果 "." 之后为 "\*" ，意为 "." 前所指定的数据库中的所有表 +- "dbname.\*" 意思是名为 "dbname" 的数据库中的所有表 +- "\*.\*" 意思是所有数据库名中的所有表 + +### 数据库权限说明 + +对 root 用户和普通用户的权限的说明如下表 + +| 用户 | 描述 | 权限说明 | +| -------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 超级用户 | 只有 root 是超级用户 | DB 外部 所有操作权限，例如user、dnode、udf、qnode等的CRUD DB 权限，包括 创建 删除 更新，例如修改 Option，移动 Vgruop等 读 写 Enable/Disable 用户 | +| 普通用户 | 除 root 以外的其它用户均为普通用户 | 在可读的 DB 中，普通用户可以进行读操作 select describe show subscribe 在可写 DB 的内部，用户可以进行写操作： 创建、删除、修改 超级表 创建、删除、修改 子表 创建、删除、修改 topic 写入数据 被限制系统信息时，不可进行如下操作 show dnode、mnode、vgroups、qnode、snode 修改用户包括自身密码 show db时只能看到自己的db，并且不能看到vgroups、副本、cache等信息 无论是否被限制系统信息，都可以 管理 udf 可以创建 DB 自己创建的 DB 具备所有权限 非自己创建的 DB ，参照读、写列表中的权限 | + +## 消息订阅授权 + +任意用户都可以在自己拥有读权限的数据库上创建 topic。超级用户 root 可以在任意数据库上创建 topic。每个 topic 的订阅权限都可以被独立授权给任何用户，不管该用户是否拥有该数据库的访问权限。删除 topic 只能由 root 用户或者该 topic 的创建者进行。topic 只能由超级用户、topic的创建者或者被显式授予 subscribe 权限的用户订阅。 + +具体的 SQL 语法如下： + +```sql +GRANT SUBSCRIBE ON topic_name TO user_name + +REVOKE SUBSCRIBE ON topic_name FROM user_name +``` + +## 基于标签的授权（表级授权） + +从 TDengine 3.0.5.0 开始，我们支持按标签授权某个超级表中部分特定的子表。具体的 SQL 语法如下。 + +```sql +GRANT privileges ON priv_level [WITH tag_condition] TO user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + READ + | WRITE +} + +priv_level : { + dbname.tbname + | dbname.* + | *.* +} + +REVOKE privileges ON priv_level [WITH tag_condition] FROM user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + READ + | WRITE +} + +priv_level : { + dbname.tbname + | dbname.* + | *.* +} +``` + +上面 SQL 的语义为： + +- 用户可以通过 dbname.tbname 来为指定的表（包括超级表和普通表）授予或回收其读写权限，不支持直接对子表授予或回收权限。 +- 用户可以通过 dbname.tbname 和 WITH 子句来为符合条件的所有子表授予或回收其读写权限。使用 WITH 子句时，权限级别必须为超级表。 + +## 表级权限和数据库权限的关系 + +下表列出了在不同的数据库授权和表级授权的组合下产生的实际权限。 + +| | **表无授权** | **表读授权** | **表读授权有标签条件** | **表写授权** | **表写授权有标签条件** | +| ---------------- | ---------------- | ---------------------------------------- | ------------------------------------------------------------ | ---------------------------------------- | ---------------------------------------------------------- | +| **数据库无授权** | 无授权 | 对此表有读权限，对数据库下的其他表无权限 | 对此表符合标签权限的子表有读权限，对数据库下的其他表无权限 | 对此表有写权限，对数据库下的其他表无权限 | 对此表符合标签权限的子表有写权限，对数据库下的其他表无权限 | +| **数据库读授权** | 对所有表有读权限 | 对所有表有读权限 | 对此表符合标签权限的子表有读权限，对数据库下的其他表有读权限 | 对此表有写权限，对所有表有读权限 | 对此表符合标签权限的子表有写权限，所有表有读权限 | +| **数据库写授权** | 对所有表有写权限 | 对此表有读权限，对所有表有写权限 | 对此表符合标签权限的子表有读权限，对所有表有写权限 | 对所有表有写权限 | 对此表符合标签权限的子表有写权限，数据库下的其他表有写权限 | + + +## 查看用户授权 + +使用下面的命令可以显示一个用户所拥有的授权： + +```sql +show user privileges +``` + +## 撤销授权 + +1. 撤销数据库访问的授权 + +```sql +REVOKE privileges ON priv_level FROM user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + READ + | WRITE +} + +priv_level : { + dbname.tbname + | dbname.* + | *.* +} +``` + +2. 撤销数据订阅的授权 + +```sql +REVOKE privileges ON priv_level FROM user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + SUBSCRIBE +} + +priv_level : { + topic_name +} +``` diff --git a/docs/zh/14-reference/03-taos-sql/26-udf.md b/docs/zh/14-reference/03-taos-sql/27-udf.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/26-udf.md rename to docs/zh/14-reference/03-taos-sql/27-udf.md diff --git a/docs/zh/14-reference/03-taos-sql/27-indexing.md b/docs/zh/14-reference/03-taos-sql/28-index.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/27-indexing.md rename to docs/zh/14-reference/03-taos-sql/28-index.md diff --git a/docs/zh/14-reference/03-taos-sql/28-recovery.md b/docs/zh/14-reference/03-taos-sql/29-recovery.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/28-recovery.md rename to docs/zh/14-reference/03-taos-sql/29-recovery.md diff --git a/docs/zh/14-reference/03-taos-sql/29-changes.md b/docs/zh/14-reference/03-taos-sql/30-changes.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/29-changes.md rename to docs/zh/14-reference/03-taos-sql/30-changes.md diff --git a/docs/zh/14-reference/03-taos-sql/30-join.md b/docs/zh/14-reference/03-taos-sql/31-join.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/30-join.md rename to docs/zh/14-reference/03-taos-sql/31-join.md diff --git a/docs/zh/14-reference/03-taos-sql/31-compress.md b/docs/zh/14-reference/03-taos-sql/32-compress.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/31-compress.md rename to docs/zh/14-reference/03-taos-sql/32-compress.md diff --git a/docs/zh/14-reference/03-taos-sql/32-view.md b/docs/zh/14-reference/03-taos-sql/33-view.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/32-view.md rename to docs/zh/14-reference/03-taos-sql/33-view.md From 128adaa3b4adfa23e82b1f0b8525c229282a87e6 Mon Sep 17 00:00:00 2001 From: menshibin Date: Mon, 19 Aug 2024 18:28:53 +0800 Subject: [PATCH 019/100] add kafka config propose --- .../20-third-party/01-collection/11-kafka.md | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/zh/20-third-party/01-collection/11-kafka.md b/docs/zh/20-third-party/01-collection/11-kafka.md index 2e4677ca31..e9ac68251c 100644 --- a/docs/zh/20-third-party/01-collection/11-kafka.md +++ b/docs/zh/20-third-party/01-collection/11-kafka.md @@ -43,20 +43,6 @@ TDengine Source Connector 用于把数据实时地从 TDengine 读出来发送 ``` 以上脚本可以追加到当前用户的 profile 文件（~/.profile 或 ~/.bash_profile） -- 提升 Kafka 吞吐率的建议配置 - - 1. 打开 KAFKA_HOME/config/producer.properties 配置文件。 - 2. 参数说明及建议如下： - - | **参数** | **参数说明** | **设置建议** | - | --------| --------------------------------- | -------------- | - | producer.type | 此参数用于设置消息的发送方式，默认值为 `sync` 表示同步发送，`async` 表示异步发送。采用异步发送能够提升消息发送的吞吐量。 | async | - | request.required.acks | 参数用于配置生产者发送消息后需要等待的确认数量。当设置为1时，表示只要领导者副本成功写入消息就会给生产者发送确认，而无需等待集群中的其他副本写入成功。这种设置可以在一定程度上保证消息的可靠性，同时也能保证一定的吞吐量。因为不需要等待所有副本都写入成功，所以可以减少生产者的等待时间，提高发送消息的效率。|1| - | max.request.size| 该参数决定了生产者在一次请求中可以发送的最大数据量。其默认值为 1048576，也就是 1M。如果设置得太小，可能会导致频繁的网络请求，降低吞吐量。如果设置得太大，可能会导致内存占用过高，或者在网络状况不佳时增加请求失败的概率。建议设置为 100M。|104857600| - |batch.size| 此参数用于设定 batch 的大小，默认值为 16384，即 16KB。在消息发送过程中，发送到 Kafka 缓冲区中的消息会被划分成一个个的 batch。故而减小 batch 大小有助于降低消息延迟，而增大 batch 大小则有利于提升吞吐量，可根据实际的数据量大小进行合理配置。可根据实际情况进行调整，建议设置为 512K。|524288| - | buffer.memory| 此参数用于设置生产者缓冲待发送消息的内存总量。较大的缓冲区可以允许生产者积累更多的消息后批量发送，提高吞吐量，但也会增加延迟和内存使用。可根据机器资源来配置，建议配置为 1G。|1073741824| - - ## 安装 TDengine Connector 插件 ### 编译插件 @@ -338,6 +324,21 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSinkConnector curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector ``` +### 性能调优 + +如果在从 TDengine 同步数据到 Kafka 的过程中发现性能不达预期，可以尝试使用如下参数提升 Kafka 的写入吞吐量。 + +1. 打开 KAFKA_HOME/config/producer.properties 配置文件。 +2. 参数说明及配置建议如下： + | **参数** | **参数说明** | **设置建议** | + | --------| --------------------------------- | -------------- | + | producer.type | 此参数用于设置消息的发送方式，默认值为 `sync` 表示同步发送，`async` 表示异步发送。采用异步发送能够提升消息发送的吞吐量。 | async | + | request.required.acks | 参数用于配置生产者发送消息后需要等待的确认数量。当设置为1时，表示只要领导者副本成功写入消息就会给生产者发送确认，而无需等待集群中的其他副本写入成功。这种设置可以在一定程度上保证消息的可靠性，同时也能保证一定的吞吐量。因为不需要等待所有副本都写入成功，所以可以减少生产者的等待时间，提高发送消息的效率。|1| + | max.request.size| 该参数决定了生产者在一次请求中可以发送的最大数据量。其默认值为 1048576，也就是 1M。如果设置得太小，可能会导致频繁的网络请求，降低吞吐量。如果设置得太大，可能会导致内存占用过高，或者在网络状况不佳时增加请求失败的概率。建议设置为 100M。|104857600| + |batch.size| 此参数用于设定 batch 的大小，默认值为 16384，即 16KB。在消息发送过程中，发送到 Kafka 缓冲区中的消息会被划分成一个个的 batch。故而减小 batch 大小有助于降低消息延迟，而增大 batch 大小则有利于提升吞吐量，可根据实际的数据量大小进行合理配置。可根据实际情况进行调整，建议设置为 512K。|524288| + | buffer.memory| 此参数用于设置生产者缓冲待发送消息的内存总量。较大的缓冲区可以允许生产者积累更多的消息后批量发送，提高吞吐量，但也会增加延迟和内存使用。可根据机器资源来配置，建议配置为 1G。|1073741824| + + ## 配置参考 ### 通用配置 From 28b9611f380062aeea0a6120fd62b12c2d27fbfa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Aug 2024 18:33:39 +0800 Subject: [PATCH 020/100] fix(tsdb): return if no data. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 5987b673c3..355bcca469 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1383,7 +1383,6 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) { ASSERT(pReader->info.pSchema == NULL); - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema); if (code != TSDB_CODE_SUCCESS || pReader->info.pSchema == NULL) { terrno = code; @@ -1787,6 +1786,7 @@ static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData); } } + return code; } TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); From 0531a4f4bd7a26468232cbedfd1ae4b62c33aa0f Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 19 Aug 2024 18:36:39 +0800 Subject: [PATCH 021/100] fix: memory leak of geos --- include/util/tgeosctx.h | 8 ++-- source/client/src/clientMain.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 2 +- source/libs/executor/src/sysscanoperator.c | 2 +- source/libs/geometry/src/geosWrapper.c | 45 +++++++++++++++++++++- source/libs/parser/src/parInsertSql.c | 4 +- source/libs/scalar/src/sclvector.c | 8 ++-- source/util/src/tgeosctx.c | 17 +++++--- source/util/src/tsched.c | 2 - source/util/src/tworker.c | 2 - tools/shell/src/shellEngine.c | 5 +-- 11 files changed, 71 insertions(+), 26 deletions(-) diff --git a/include/util/tgeosctx.h b/include/util/tgeosctx.h index 267ba9e049..a4355db29a 100644 --- a/include/util/tgeosctx.h +++ b/include/util/tgeosctx.h @@ -32,14 +32,16 @@ typedef struct SGeosContext { GEOSWKBReader *WKBReader; GEOSWKBWriter *WKBWriter; - pcre2_code *WKTRegex; + pcre2_code *WKTRegex; pcre2_match_data *WKTMatchData; char errMsg[512]; } SGeosContext; -SGeosContext* getThreadLocalGeosCtx(); -void destroyThreadLocalGeosCtx(); +SGeosContext *acquireThreadLocalGeosCtx(); +SGeosContext *getThreadLocalGeosCtx(); +const char *getGeosErrMsg(int32_t code); +void taosGeosDestroy(); #ifdef __cplusplus } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index a403f9d1c2..ec3147f49a 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -91,11 +91,11 @@ void taos_cleanup(void) { tscDebug("rpc cleanup"); taosConvDestroy(); + taosGeosDestroy(); tmqMgmtClose(); DestroyRegexCache(); - destroyThreadLocalGeosCtx(); tscInfo("all local resources released"); taosCleanupCfg(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 1d62d4bd90..ae6efa7af4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -122,7 +122,7 @@ void dmCleanupDnode(SDnode *pDnode) { streamMetaCleanup(); indexCleanup(); taosConvDestroy(); - destroyThreadLocalGeosCtx(); + taosGeosDestroy(); // compress destroy tsCompressExit(); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index d8a2331980..082d4e7789 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -979,7 +979,7 @@ static int32_t sysTableGetGeomText(char* iGeom, int32_t nGeom, char** output, in if (TSDB_CODE_SUCCESS != (code = initCtxAsText()) || TSDB_CODE_SUCCESS != (code = doAsText(iGeom, nGeom, &outputWKT))) { - qError("geo text for systable failed:%s", getThreadLocalGeosCtx()->errMsg); + qError("geo text for systable failed:%s", getGeosErrMsg(code)); *output = NULL; *nOutput = 0; return code; diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index dde34edc91..4f3f7d75c2 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -23,7 +23,8 @@ typedef char (*_geosPreparedRelationFunc_t)(GEOSContextHandle_t handle, const GE void geosFreeBuffer(void *buffer) { if (buffer) { - GEOSFree_r(getThreadLocalGeosCtx()->handle, buffer); + SGeosContext *pCtx = acquireThreadLocalGeosCtx(); + if (pCtx) GEOSFree_r(pCtx->handle, buffer); } } @@ -36,6 +37,11 @@ int32_t initCtxMakePoint() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); if (geosCtx->handle == NULL) { @@ -61,6 +67,11 @@ int32_t doMakePoint(double x, double y, unsigned char **outputGeom, size_t *size int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + GEOSGeometry *geom = NULL; unsigned char *wkb = NULL; @@ -166,6 +177,11 @@ int32_t initCtxGeomFromText() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); if (geosCtx->handle == NULL) { @@ -202,6 +218,11 @@ int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + GEOSGeometry *geom = NULL; unsigned char *wkb = NULL; @@ -237,6 +258,11 @@ int32_t initCtxAsText() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); if (geosCtx->handle == NULL) { @@ -273,6 +299,11 @@ int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT) int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + GEOSGeometry *geom = NULL; char *wkt = NULL; @@ -304,6 +335,11 @@ int32_t initCtxRelationFunc() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); if (geosCtx->handle == NULL) { @@ -329,6 +365,10 @@ int32_t doGeosRelation(const GEOSGeometry *geom1, const GEOSPreparedGeometry *pr _geosPreparedRelationFunc_t swappedPreparedRelationFn) { SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (!preparedGeom1) { if (!swapped) { ASSERT(relationFn); @@ -390,6 +430,9 @@ int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom) { SGeosContext *geosCtx = getThreadLocalGeosCtx(); + if (!geosCtx) { + return TSDB_CODE_OUT_OF_MEMORY; + } ASSERT(outputGeom); // it is not allowed if outputGeom is NULL *outputGeom = NULL; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index cb94cd42f7..aa6116287e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -655,7 +655,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, code = parseGeometry(pToken, &output, &size); if (code != TSDB_CODE_SUCCESS) { - code = buildSyntaxErrMsg(pMsgBuf, getThreadLocalGeosCtx()->errMsg, pToken->z); + code = buildSyntaxErrMsg(pMsgBuf, getGeosErrMsg(code), pToken->z); } else if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { // Too long values will raise the invalid sql error message code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); @@ -1646,7 +1646,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, code = parseGeometry(pToken, &output, &size); if (code != TSDB_CODE_SUCCESS) { - code = buildSyntaxErrMsg(&pCxt->msg, getThreadLocalGeosCtx()->errMsg, pToken->z); + code = buildSyntaxErrMsg(&pCxt->msg, getGeosErrMsg(code), pToken->z); } // Too long values will raise the invalid sql error message else if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 71773ced57..daf44ec527 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -441,12 +441,12 @@ static FORCE_INLINE int32_t varToGeometry(char *buf, SScalarParam *pOut, int32_t unsigned char *t = NULL; char *output = NULL; - if (initCtxGeomFromText()) { - sclError("failed to init geometry ctx, %s", getThreadLocalGeosCtx()->errMsg); + if ((code = initCtxGeomFromText()) != 0) { + sclError("failed to init geometry ctx, %s", getGeosErrMsg(code)); SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } - if (doGeomFromText(buf, &t, &len)) { - sclInfo("failed to convert text to geometry, %s", getThreadLocalGeosCtx()->errMsg); + if ((code = doGeomFromText(buf, &t, &len))) { + sclInfo("failed to convert text to geometry, %s", getGeosErrMsg(code)); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); } diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index 47d5cc992b..473b7539fc 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -25,16 +25,19 @@ typedef struct { SRWLatch lock; } SGeosContextPool; -static SGeosContextPool sGeosPool = {0}; - +static SGeosContextPool sGeosPool = {0}; static threadlocal SGeosContext *tlGeosCtx = NULL; +SGeosContext *acquireThreadLocalGeosCtx() { return tlGeosCtx; } + SGeosContext *getThreadLocalGeosCtx() { - if (tlGeosCtx) return tlGeosCtx; + if (tlGeosCtx) { + return tlGeosCtx; + } taosWLockLatch(&sGeosPool.lock); if (sGeosPool.size >= sGeosPool.capacity) { - sGeosPool.capacity += 64; + sGeosPool.capacity += 128; void *tmp = taosMemoryRealloc(sGeosPool.pool, sGeosPool.capacity * sizeof(SGeosContext)); if (!tmp) { taosWUnLockLatch(&sGeosPool.lock); @@ -51,6 +54,8 @@ SGeosContext *getThreadLocalGeosCtx() { return tlGeosCtx; } +const char *getGeosErrMsg(int32_t code) { return tlGeosCtx ? tlGeosCtx->errMsg : (code != 0 ? tstrerror(code) : ""); } + static void destroyGeosCtx(SGeosContext *pCtx) { if (pCtx) { if (pCtx->WKTReader) { @@ -86,8 +91,8 @@ static void destroyGeosCtx(SGeosContext *pCtx) { } } -void destroyThreadLocalGeosCtx() { - uInfo("geos ctx is cleaned up"); +void taosGeosDestroy() { + uInfo("geos is cleaned up"); if (!sGeosPool.pool) return; for (int32_t i = 0; i < sGeosPool.size; ++i) { destroyGeosCtx(sGeosPool.pool + i); diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 509dba0890..55a927f340 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -178,8 +178,6 @@ void *taosProcessSchedQueue(void *scheduler) { (*(msg.tfp))(msg.ahandle, msg.thandle); } - // destroyThreadLocalGeosCtx(); - return NULL; } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 2da1abed78..ebec134c91 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -105,7 +105,6 @@ static void *tQWorkerThreadFp(SQueueWorker *worker) { taosUpdateItemSize(qinfo.queue, 1); } - // destroyThreadLocalGeosCtx(); DestoryThreadLocalRegComp(); return NULL; @@ -665,7 +664,6 @@ static void *tQueryAutoQWorkerThreadFp(SQueryAutoQWorker *worker) { } } - // destroyThreadLocalGeosCtx(); DestoryThreadLocalRegComp(); return NULL; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 0ccbd683dc..2c8330c433 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -611,14 +611,14 @@ void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width) code = initCtxAsText(); if (code != TSDB_CODE_SUCCESS) { - shellPrintString(getThreadLocalGeosCtx()->errMsg, width); + shellPrintString(getGeosErrMsg(code), width); return; } char *outputWKT = NULL; code = doAsText(val, length, &outputWKT); if (code != TSDB_CODE_SUCCESS) { - shellPrintString(getThreadLocalGeosCtx()->errMsg, width); // should NOT happen + shellPrintString(getGeosErrMsg(code), width); // should NOT happen return; } @@ -1282,7 +1282,6 @@ void *shellThreadLoop(void *arg) { taosResetTerminalMode(); } while (shellRunCommand(command, true) == 0); - destroyThreadLocalGeosCtx(); taosMemoryFreeClear(command); shellWriteHistory(); shellExit(); From e298b5acb320cc379660885e4c6288852d162f04 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 19 Aug 2024 18:39:08 +0800 Subject: [PATCH 022/100] fix: s3 support multi proto --- source/common/src/cos.c | 44 +++++++++++++++++++------------------ source/common/src/tglobal.c | 9 ++++---- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 8392b0564a..a5a278e82e 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -12,7 +12,7 @@ extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; extern char tsS3BucketName[TSDB_FQDN_LEN]; extern char tsS3AppId[][TSDB_FQDN_LEN]; extern char tsS3Hostname[][TSDB_FQDN_LEN]; -extern int8_t tsS3Https; +extern int8_t tsS3Https[]; static int32_t s3ListBucketByEp(char const *bucketname, int8_t epIndex); static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *object_name, int64_t offset, int64_t size, @@ -33,13 +33,13 @@ static int verifyPeerG = 0; static const char *awsRegionG = NULL; static int forceG = 0; static int showResponsePropertiesG = 0; -static S3Protocol protocolG = S3ProtocolHTTPS; +static S3Protocol protocolG[TSDB_MAX_EP_NUM] = {S3ProtocolHTTPS}; // static S3Protocol protocolG = S3ProtocolHTTP; -static S3UriStyle uriStyleG = S3UriStylePath; +static S3UriStyle uriStyleG[TSDB_MAX_EP_NUM] = {S3UriStylePath}; static int retriesG = 5; static int timeoutMsG = 0; -extern int8_t tsS3Oss; +extern int8_t tsS3Oss[]; int32_t s3Begin() { S3Status status; @@ -55,9 +55,11 @@ int32_t s3Begin() { TAOS_RETURN(TSDB_CODE_FAILED); } - protocolG = !tsS3Https; - if (tsS3Oss) { - uriStyleG = S3UriStyleVirtualHost; + for (int i = 0; i < tsS3EpNum; i++) { + protocolG[i] = !tsS3Https[i]; + if (tsS3Oss[i]) { + uriStyleG[i] = S3UriStyleVirtualHost; + } } TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -976,8 +978,8 @@ int32_t s3PutObjectFromFile2ByEp(const char *file, const char *object_name, int8 S3BucketContext bucketContext = {tsS3Hostname[epIndex], tsS3BucketName, - protocolG, - uriStyleG, + protocolG[epIndex], + uriStyleG[epIndex], tsS3AccessKeyId[epIndex], tsS3AccessKeySecret[epIndex], 0, @@ -1059,8 +1061,8 @@ static int32_t s3PutObjectFromFileOffsetByEp(const char *file, const char *objec S3BucketContext bucketContext = {tsS3Hostname[epIndex], tsS3BucketName, - protocolG, - uriStyleG, + protocolG[epIndex], + uriStyleG[epIndex], tsS3AccessKeyId[epIndex], tsS3AccessKeySecret[epIndex], 0, @@ -1155,8 +1157,8 @@ static void s3FreeObjectKey(void *pItem) { static SArray *getListByPrefixByEp(const char *prefix, int8_t epIndex) { S3BucketContext bucketContext = {tsS3Hostname[epIndex], tsS3BucketName, - protocolG, - uriStyleG, + protocolG[epIndex], + uriStyleG[epIndex], tsS3AccessKeyId[epIndex], tsS3AccessKeySecret[epIndex], 0, @@ -1223,8 +1225,8 @@ static int32_t s3DeleteObjectsByEp(const char *object_name[], int nobject, int8_ S3BucketContext bucketContext = {tsS3Hostname[epIndex], tsS3BucketName, - protocolG, - uriStyleG, + protocolG[epIndex], + uriStyleG[epIndex], tsS3AccessKeyId[epIndex], tsS3AccessKeySecret[epIndex], 0, @@ -1299,8 +1301,8 @@ static int32_t s3GetObjectBlockByEp(const char *object_name, int64_t offset, int S3BucketContext bucketContext = {tsS3Hostname[epIndex], tsS3BucketName, - protocolG, - uriStyleG, + protocolG[epIndex], + uriStyleG[epIndex], tsS3AccessKeyId[epIndex], tsS3AccessKeySecret[epIndex], 0, @@ -1372,8 +1374,8 @@ static int32_t s3GetObjectToFileByEp(const char *object_name, const char *fileNa S3BucketContext bucketContext = {tsS3Hostname[epIndex], tsS3BucketName, - protocolG, - uriStyleG, + protocolG[epIndex], + uriStyleG[epIndex], tsS3AccessKeyId[epIndex], tsS3AccessKeySecret[epIndex], 0, @@ -1449,8 +1451,8 @@ static long s3SizeByEp(const char *object_name, int8_t epIndex) { S3BucketContext bucketContext = {tsS3Hostname[epIndex], tsS3BucketName, - protocolG, - uriStyleG, + protocolG[epIndex], + uriStyleG[epIndex], tsS3AccessKeyId[epIndex], tsS3AccessKeySecret[epIndex], 0, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 40ace11d4f..cf0a4725c1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -303,10 +303,10 @@ char tsS3BucketName[TSDB_FQDN_LEN] = ""; char tsS3AppId[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; int8_t tsS3Enabled = false; int8_t tsS3EnabledCfg = false; -int8_t tsS3Oss = false; +int8_t tsS3Oss[TSDB_MAX_EP_NUM] = {false}; int8_t tsS3StreamEnabled = false; -int8_t tsS3Https = true; +int8_t tsS3Https[TSDB_MAX_EP_NUM] = {true}; char tsS3Hostname[TSDB_MAX_EP_NUM][TSDB_FQDN_LEN] = {""}; int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) @@ -431,11 +431,10 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { tstrncpy(tsS3AppId[i], appid + 1, TSDB_FQDN_LEN); } } + tsS3Https[i] = (strstr(tsS3Endpoint[i], "https://") != NULL); + tsS3Oss[i] = (strstr(tsS3Endpoint[i], "aliyuncs.") != NULL); } - tsS3Https = (strstr(tsS3Endpoint[0], "https://") != NULL); - tsS3Oss = (strstr(tsS3Endpoint[0], "aliyuncs.") != NULL); - if (tsS3BucketName[0] != '<') { #if defined(USE_COS) || defined(USE_S3) #ifdef TD_ENTERPRISE From 21c266d1231c476ee76c1778d1c93debbb5260f2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 19 Aug 2024 18:52:14 +0800 Subject: [PATCH 023/100] fix(vnode/cfg): use default value if loading 0 --- source/dnode/vnode/src/vnd/vnodeCfg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 1f2cf707f3..e2791d8a00 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -375,11 +375,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { } tjsonGetNumberValue(pJson, "s3ChunkSize", pCfg->s3ChunkSize, code); - if (code < 0) { + if (code < 0 || pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE) { pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; } tjsonGetNumberValue(pJson, "s3KeepLocal", pCfg->s3KeepLocal, code); - if (code < 0) { + if (code < 0 || pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL) { pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; } tjsonGetNumberValue(pJson, "s3Compact", pCfg->s3Compact, code); From 1b59d4a8d7443c41c971ad39bdc5de467d0c6c9b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Aug 2024 18:54:42 +0800 Subject: [PATCH 024/100] fix(tsdb): pass the error code out. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 355bcca469..24476e8df1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1413,7 +1413,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI if (pReader->info.pSchema == NULL) { pSchema = getTableSchemaImpl(pReader, uid); if (pSchema == NULL) { - tsdbDebug("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr); + code = terrno; + tsdbError("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr); return code; } } @@ -1448,7 +1449,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI pReader->cost.blockLoadTime += elapsedTime; pDumpInfo->allDumped = false; - return TSDB_CODE_SUCCESS; + return code; } /** @@ -2137,7 +2138,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (piRow->type == TSDBROW_ROW_FMT) { piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); if (piSchema == NULL) { - return code; + return terrno; } } From 2b68e110e51728c0ecd9ea2346eab4641aecf4af Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Aug 2024 19:01:26 +0800 Subject: [PATCH 025/100] fix(tsdb): check return value. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +++- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 1 + source/dnode/vnode/src/tsdb/tsdbRead2.c | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 70e6e1ee2a..fb72784229 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2245,7 +2245,9 @@ static int32_t lastIterClose(SFSLastIter **iter) { } static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow) { - bool hasVal = false; + bool hasVal = false; + *ppRow = NULL; + int32_t code = tMergeTreeNext(iter->pMergeTree, &hasVal); if (code != 0) { return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 8bfc066731..160ff2e13c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -1108,6 +1108,7 @@ int32_t tMergeTreeNext(SMergeTree *pMTree, bool *pHasNext) { return TSDB_CODE_INVALID_PARA; } + *pHasNext = false; if (pMTree->pIter) { SLDataIter *pIter = pMTree->pIter; bool hasVal = false; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 24476e8df1..cc369ba3b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4105,9 +4105,10 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI while (1) { code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange); - if (code) { - + if (code || (!hasDataInSttBlock(pScanInfo))) { + return code; } + SRowKey* pNextKey = getCurrentKeyInSttBlock(pSttBlockReader); int32_t ret = pkCompEx(pRowKey, pNextKey); From a760ede4ae473a52cc7284706362ab44c25cc1f5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Aug 2024 19:16:40 +0800 Subject: [PATCH 026/100] refactor: update logs. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index cc369ba3b0..74992c40d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1414,7 +1414,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI pSchema = getTableSchemaImpl(pReader, uid); if (pSchema == NULL) { code = terrno; - tsdbError("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr); + tsdbError("%p table uid:%" PRIu64 " failed to get tableschema, code:%s, %s", pReader, uid, tstrerror(code), + pReader->idStr); return code; } } From 660bfde59370e531dd290c2d1ab61c65235a650f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 19 Aug 2024 19:16:37 +0800 Subject: [PATCH 027/100] refact: adjust util log --- source/libs/sync/inc/syncUtil.h | 2 -- source/libs/sync/src/syncEnv.c | 10 ++++---- source/libs/sync/src/syncUtil.c | 41 +++++++++++++++++---------------- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index a550ae8fbb..555607d40c 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -75,8 +75,6 @@ int32_t syncUtilElectRandomMS(int32_t min, int32_t max); int32_t syncUtilQuorum(int32_t replicaNum); const char* syncStr(ESyncState state); void syncUtilMsgHtoN(void* msg); -bool syncUtilUserPreCommit(tmsg_t msgType); -bool syncUtilUserRollback(tmsg_t msgType); void syncUtilGenerateArbToken(int32_t nodeId, int32_t groupId, char* buf); diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 7474cea83d..1ebf47403a 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -32,7 +32,7 @@ int32_t syncInit() { uint32_t seed = (uint32_t)(taosGetTimestampNs() & 0x00000000FFFFFFFF); taosSeedRand(seed); - memset(&gSyncEnv, 0, sizeof(SSyncEnv)); + (void)memset(&gSyncEnv, 0, sizeof(SSyncEnv)); gSyncEnv.pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); gNodeRefId = taosOpenRef(200, (RefFp)syncNodeClose); @@ -59,7 +59,7 @@ int32_t syncInit() { void syncCleanUp() { atomic_store_8(&gSyncEnv.isStart, 0); taosTmrCleanUp(gSyncEnv.pTimerManager); - memset(&gSyncEnv, 0, sizeof(SSyncEnv)); + (void)memset(&gSyncEnv, 0, sizeof(SSyncEnv)); if (gNodeRefId != -1) { sDebug("sync node rset is closed, rsetId:%d", gNodeRefId); @@ -77,8 +77,7 @@ void syncCleanUp() { int64_t syncNodeAdd(SSyncNode *pNode) { pNode->rid = taosAddRef(gNodeRefId, pNode); if (pNode->rid < 0) { - terrno = TSDB_CODE_SYN_WRONG_REF; - return -1; + return terrno = TSDB_CODE_SYN_WRONG_REF; } sDebug("vgId:%d, sync node refId:%" PRId64 " is added to rsetId:%d", pNode->vgId, pNode->rid, gNodeRefId); @@ -111,8 +110,7 @@ void syncNodeRelease(SSyncNode *pNode) { int64_t syncHbTimerDataAdd(SSyncHbTimerData *pData) { pData->rid = taosAddRef(gHbDataRefId, pData); if (pData->rid < 0) { - terrno = TSDB_CODE_SYN_WRONG_REF; - return -1; + return terrno = TSDB_CODE_SYN_WRONG_REF; } return pData->rid; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 49737b9045..ca879f70d9 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -23,7 +23,7 @@ #include "syncSnapshot.h" #include "tglobal.h" -void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { +static void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { int32_t len = snprintf(buf, bufLen, "{num:%d, as:%d, [", pCfg->replicaNum, pCfg->myIndex); for (int32_t i = 0; i < pCfg->replicaNum; ++i) { len += snprintf(buf + len, bufLen - len, "%s:%d", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort); @@ -43,14 +43,13 @@ void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = 0xFFFFFFFF; - sDebug( - "vgId:%d, start to resolve sync addr fqdn in %d seconds, " - "dnode:%d cluster:%" PRId64 " fqdn:%s port:%u ", - vgId, tsResolveFQDNRetryTime, pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort); - for (int i = 0; i < tsResolveFQDNRetryTime; i++) { + sDebug("vgId:%d, resolve sync addr from fqdn, dnode:%d cluster:%" PRId64 " fqdn:%s port:%u", vgId, pInfo->nodeId, + pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort); + for (int32_t i = 0; i < tsResolveFQDNRetryTime; i++) { int32_t code = taosGetIpv4FromFqdn(pInfo->nodeFqdn, &ipv4); if (code) { - sError("failed to resolve ipv4 addr, fqdn:%s, wait one second", pInfo->nodeFqdn); + sError("vgId:%d, failed to resolve sync addr, dnode:%d fqdn:%s, wait one second", vgId, pInfo->nodeId, + pInfo->nodeFqdn); taosSsleep(1); } else { break; @@ -58,7 +57,7 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* } if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { - sError("failed to resolve ipv4 addr, fqdn:%s", pInfo->nodeFqdn); + sError("vgId:%d, failed to resolve sync addr, fqdn:%s", vgId, pInfo->nodeFqdn); terrno = TSDB_CODE_TSC_INVALID_FQDN; return false; } @@ -68,14 +67,20 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId->addr = SYNC_ADDR(pInfo); raftId->vgId = vgId; - sInfo("vgId:%d, sync addr:%" PRIu64 ", dnode:%d cluster:%" PRId64 " fqdn:%s ip:%s port:%u ipv4:%u", vgId, - raftId->addr, pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, ipbuf, pInfo->nodePort, ipv4); + sInfo("vgId:%d, sync addr:%" PRIu64 " is resolved, dnode:%d cluster:%" PRId64 " fqdn:%s port:%u ip:%s ipv4:%u", vgId, + raftId->addr, pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort, ipbuf, ipv4); return true; } bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { - if (pId1->addr == pId2->addr && pId1->vgId == pId2->vgId) return true; - if ((CID(pId1) == 0 || CID(pId2) == 0) && (DID(pId1) == DID(pId2)) && pId1->vgId == pId2->vgId) return true; + if (pId1->addr == pId2->addr && pId1->vgId == pId2->vgId) { + return true; + } + + if ((CID(pId1) == 0 || CID(pId2) == 0) && (DID(pId1) == DID(pId2)) && pId1->vgId == pId2->vgId) { + return true; + } + return false; } @@ -98,10 +103,6 @@ void syncUtilMsgHtoN(void* msg) { pHead->vgId = htonl(pHead->vgId); } -bool syncUtilUserPreCommit(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } - -bool syncUtilUserRollback(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } - void syncUtilGenerateArbToken(int32_t nodeId, int32_t groupId, char* buf) { (void)memset(buf, 0, TSDB_ARB_TOKEN_SIZE); int32_t randVal = taosSafeRand() % 1000; @@ -142,18 +143,18 @@ static void syncLogBufferStates2Str(SSyncNode* pSyncNode, char* buf, int32_t buf if (pBuf == NULL) { return; } - int len = 0; + int32_t len = 0; len += snprintf(buf + len, bufLen - len, "[%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); } static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { - int len = 0; + int32_t len = 0; len += snprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { SSyncLogReplMgr* pMgr = pSyncNode->logReplMgrs[i]; if (pMgr == NULL) break; - len += snprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 " %" PRId64 ", %" PRId64 ")", i, pMgr->restored, + len += snprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 " %" PRId64 ", %" PRId64 "]", i, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex); if (i + 1 < pSyncNode->replicaNum) { len += snprintf(buf + len, bufLen - len, "%s", ", "); @@ -280,7 +281,7 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRId64 " last-cfg:%" PRId64 ", seq:%d, ack:%d, " " buf:[%" PRId64 " %" PRId64 ", %" PRId64 - "), finish:%d, as:%d, to-dnode:%d}" + "], finish:%d, as:%d, to-dnode:%d}" ", term:%" PRIu64 ", commit-index:%" PRId64 ", firstver:%" PRId64 ", lastver:%" PRId64 ", min-match:%" PRId64 ", snap:{last-index:%" PRId64 ", term:%" PRIu64 "}, standby:%d, batch-sz:%d, replicas:%d, last-cfg:%" PRId64 From 50672cd1a875f81dcb83db05d2b50e6e9fd1ad86 Mon Sep 17 00:00:00 2001 From: sima Date: Mon, 19 Aug 2024 16:52:30 +0800 Subject: [PATCH 028/100] enh:[TD-31529] Remove ASSERT in libs/scalar. --- include/util/taoserror.h | 3 + source/libs/geometry/src/geomFunc.c | 6 +- source/libs/scalar/inc/sclvector.h | 37 ++-- source/libs/scalar/src/filter.c | 172 ++++++++++++++---- source/libs/scalar/src/sclfunc.c | 7 +- source/libs/scalar/src/sclvector.c | 168 ++++++++++------- source/libs/scalar/test/filter/CMakeLists.txt | 2 +- .../libs/scalar/test/filter/filterTests.cpp | 3 +- .../libs/scalar/test/scalar/scalarTests.cpp | 14 +- source/util/src/terror.c | 3 + 10 files changed, 281 insertions(+), 134 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 1911c48d26..cc31cde61a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -604,6 +604,9 @@ int32_t taosGetErrSize(); #define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732) #define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733) #define TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE TAOS_DEF_ERROR_CODE(0, 0x0734) +#define TSDB_CODE_QRY_FILTER_WRONG_OPTR_TYPE TAOS_DEF_ERROR_CODE(0, 0x0735) +#define TSDB_CODE_QRY_FILTER_RANGE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0736) +#define TSDB_CODE_QRY_FILTER_INVALID_TYPE TAOS_DEF_ERROR_CODE(0, 0x0737) // grant #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 4426427bf5..1752493dff 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -131,12 +131,12 @@ _exit: int32_t executeMakePointFunc(SColumnInfoData *pInputData[], int32_t iLeft, int32_t iRight, SColumnInfoData *pOutputData) { int32_t code = TSDB_CODE_FAILED; + unsigned char *output = NULL; _getDoubleValue_fn_t getDoubleValueFn[2]; - getDoubleValueFn[0] = getVectorDoubleValueFn(pInputData[0]->info.type); - getDoubleValueFn[1] = getVectorDoubleValueFn(pInputData[1]->info.type); + TAOS_CHECK_GOTO(getVectorDoubleValueFn(pInputData[0]->info.type, &getDoubleValueFn[0]), NULL, _exit); + TAOS_CHECK_GOTO(getVectorDoubleValueFn(pInputData[1]->info.type, &getDoubleValueFn[1]), NULL, _exit); - unsigned char *output = NULL; double leftRes = 0; double rightRes = 0; diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index c2eb13dc75..fdd3e92501 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -78,40 +78,41 @@ static FORCE_INLINE int32_t getVectorDoubleValue_BOOL(void *src, int32_t index, int32_t getVectorDoubleValue_JSON(void *src, int32_t index, double *out); -static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { - _getDoubleValue_fn_t p = NULL; +static FORCE_INLINE int32_t getVectorDoubleValueFn(int32_t srcType, _getDoubleValue_fn_t *p) { + *p = NULL; if (srcType == TSDB_DATA_TYPE_TINYINT) { - p = getVectorDoubleValue_TINYINT; + *p = getVectorDoubleValue_TINYINT; } else if (srcType == TSDB_DATA_TYPE_UTINYINT) { - p = getVectorDoubleValue_UTINYINT; + *p = getVectorDoubleValue_UTINYINT; } else if (srcType == TSDB_DATA_TYPE_SMALLINT) { - p = getVectorDoubleValue_SMALLINT; + *p = getVectorDoubleValue_SMALLINT; } else if (srcType == TSDB_DATA_TYPE_USMALLINT) { - p = getVectorDoubleValue_USMALLINT; + *p = getVectorDoubleValue_USMALLINT; } else if (srcType == TSDB_DATA_TYPE_INT) { - p = getVectorDoubleValue_INT; + *p = getVectorDoubleValue_INT; } else if (srcType == TSDB_DATA_TYPE_UINT) { - p = getVectorDoubleValue_UINT; + *p = getVectorDoubleValue_UINT; } else if (srcType == TSDB_DATA_TYPE_BIGINT) { - p = getVectorDoubleValue_BIGINT; + *p = getVectorDoubleValue_BIGINT; } else if (srcType == TSDB_DATA_TYPE_UBIGINT) { - p = getVectorDoubleValue_UBIGINT; + *p = getVectorDoubleValue_UBIGINT; } else if (srcType == TSDB_DATA_TYPE_FLOAT) { - p = getVectorDoubleValue_FLOAT; + *p = getVectorDoubleValue_FLOAT; } else if (srcType == TSDB_DATA_TYPE_DOUBLE) { - p = getVectorDoubleValue_DOUBLE; + *p = getVectorDoubleValue_DOUBLE; } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) { - p = getVectorDoubleValue_BIGINT; + *p = getVectorDoubleValue_BIGINT; } else if (srcType == TSDB_DATA_TYPE_JSON) { - p = getVectorDoubleValue_JSON; + *p = getVectorDoubleValue_JSON; } else if (srcType == TSDB_DATA_TYPE_BOOL) { - p = getVectorDoubleValue_BOOL; + *p = getVectorDoubleValue_BOOL; } else if (srcType == TSDB_DATA_TYPE_NULL) { - p = NULL; + *p = NULL; } else { - ASSERT(0); + *p = NULL; + return TSDB_CODE_SCALAR_CONVERT_ERROR; } - return p; + return TSDB_CODE_SUCCESS; } typedef int32_t (*_bufConverteFunc)(char *buf, SScalarParam *pOut, int32_t outType, int32_t *overflow); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index cc9cc9ed76..988aeba46f 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -92,7 +92,9 @@ rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRan int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { if (optr2) { - ASSERT(optr2 == OP_TYPE_LOWER_THAN || optr2 == OP_TYPE_LOWER_EQUAL); + if (optr2 != OP_TYPE_LOWER_THAN && optr2 != OP_TYPE_LOWER_EQUAL) { + return -1; + } if (optr == OP_TYPE_GREATER_THAN) { if (optr2 == OP_TYPE_LOWER_THAN) { @@ -763,7 +765,10 @@ int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - ASSERT(optr == LOGIC_COND_TYPE_OR); + if (optr != LOGIC_COND_TYPE_OR) { + fltError("filterAddRangeCtx get invalid optr:%d", optr); + return TSDB_CODE_QRY_FILTER_WRONG_OPTR_TYPE; + } if (sctx->rs == NULL) { return TSDB_CODE_SUCCESS; @@ -1204,7 +1209,10 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, if (u->right.type == FLD_TYPE_VALUE) { SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); - ASSERT(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); + if (!FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)) { + fltError("filterAddUnitImpl get invalid flag : %d in val", val->flag); + return TSDB_CODE_APP_ERROR; + } } else { int32_t paramNum = scalarGetOperatorParamNum(optr); if (1 != paramNum) { @@ -1214,7 +1222,10 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, } SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); - ASSERT(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); + if (!FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)) { + fltError("filterAddUnitImpl get invalid flag : %d in col", col->flag); + return TSDB_CODE_APP_ERROR; + } info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); info->units[info->unitNum].compare.precision = FILTER_GET_COL_FIELD_PRECISION(col); @@ -1398,29 +1409,48 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (optr == LOGIC_COND_TYPE_AND) { if (ctx->isnull) { - ASSERT(ctx->notnull == false && ctx->isrange == false); + if (ctx->notnull || ctx->isrange) { + fltError("filterAddGroupUnitFromCtx get invalid ctx : isnull %d, notnull %d, isrange %d", + ctx->isnull, ctx->notnull, ctx->isrange); + FLT_ERR_RET(TSDB_CODE_QRY_FILTER_RANGE_ERROR); + } FLT_ERR_RET(filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx)); FLT_ERR_RET(filterAddUnitToGroup(g, uidx)); return TSDB_CODE_SUCCESS; } if (ctx->notnull) { - ASSERT(ctx->isnull == false && ctx->isrange == false); + if (ctx->isnull || ctx->isrange) { + fltError("filterAddGroupUnitFromCtx get invalid ctx : isnull %d, notnull %d, isrange %d", + ctx->isnull, ctx->notnull, ctx->isrange); + FLT_ERR_RET(TSDB_CODE_QRY_FILTER_RANGE_ERROR); + } FLT_ERR_RET(filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx)); FLT_ERR_RET(filterAddUnitToGroup(g, uidx)); return TSDB_CODE_SUCCESS; } if (!ctx->isrange) { - ASSERT(ctx->isnull || ctx->notnull); + if (!ctx->isnull && !ctx->notnull) { + fltError("filterAddGroupUnitFromCtx get invalid ctx : isnull %d, notnull %d, isrange %d", + ctx->isnull, ctx->notnull, ctx->isrange); + FLT_ERR_RET(TSDB_CODE_QRY_FILTER_RANGE_ERROR); + } return TSDB_CODE_SUCCESS; } - ASSERT(ctx->rs && ctx->rs->next == NULL); + if (!ctx->rs || ctx->rs->next != NULL) { + fltError("filterAddGroupUnitFromCtx get invalid range node with rs:%p", ctx->rs); + FLT_ERR_RET(TSDB_CODE_QRY_FILTER_RANGE_ERROR); + } SFilterRange *ra = &ctx->rs->ra; - ASSERT(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); + if (((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))) { + fltError("filterAddGroupUnitFromCtx get invalid range with sflag:%d, eflag:%d", + FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL), FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)); + FLT_ERR_RET(TSDB_CODE_QRY_FILTER_RANGE_ERROR); + } if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); @@ -1489,7 +1519,11 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SFilterGroup ng = {0}; g = &ng; - ASSERT(ctx->isnull || ctx->notnull || ctx->isrange); + if (!ctx->isnull && !ctx->notnull && !ctx->isrange) { + fltError("filterAddGroupUnitFromCtx get invalid ctx : isnull %d, notnull %d, isrange %d", + ctx->isnull, ctx->notnull, ctx->isrange); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); + } if (ctx->isnull) { FLT_ERR_RET(filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx)); @@ -1500,7 +1534,11 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (ctx->notnull) { - ASSERT(!ctx->isrange); + if (ctx->isrange) { + fltError("filterAddGroupUnitFromCtx get invalid ctx : isnull %d, notnull %d, isrange %d", + ctx->isnull, ctx->notnull, ctx->isrange); + FLT_ERR_RET(TSDB_CODE_QRY_FILTER_RANGE_ERROR); + } (void)memset(g, 0, sizeof(*g)); FLT_ERR_RET(filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx)); @@ -1511,7 +1549,11 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!ctx->isrange) { - ASSERT(ctx->isnull || ctx->notnull); + if (!ctx->isnull && !ctx->notnull) { + fltError("filterAddGroupUnitFromCtx get invalid ctx : isnull %d, notnull %d, isrange %d", + ctx->isnull, ctx->notnull, ctx->isrange); + FLT_ERR_RET(TSDB_CODE_QRY_FILTER_RANGE_ERROR); + } g->unitNum = 0; return TSDB_CODE_SUCCESS; } @@ -1586,7 +1628,10 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan FLT_ERR_RET(filterAddUnitToGroup(g, uidx)); } - ASSERT(g->unitNum > 0); + if (g->unitNum <= 0) { + fltError("filterAddGroupUnitFromCtx get invalid filter group unit num %d", g->unitNum); + FLT_ERR_RET(TSDB_CODE_QRY_FILTER_RANGE_ERROR); + } if (NULL == taosArrayPush(res,g)) { FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -2066,7 +2111,10 @@ void filterFreeInfo(SFilterInfo *info) { } int32_t filterHandleValueExtInfo(SFilterUnit *unit, char extInfo) { - ASSERT(extInfo > 0 || extInfo < 0); + if (extInfo == 0) { + fltError("filterHandleValueExtInfo get invalid extInfo : %d", extInfo); + return TSDB_CODE_APP_ERROR; + } uint8_t optr = FILTER_UNIT_OPTR(unit); switch (optr) { @@ -2093,13 +2141,20 @@ int32_t fltInitValFieldData(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { - ASSERT(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1); + if (unit->compare.optr != FILTER_DUMMY_EMPTY_OPTR && scalarGetOperatorParamNum(unit->compare.optr) != 1) { + fltError("filterInitValFieldData get invalid operator param num : %d and invalid compare optr %d", + scalarGetOperatorParamNum(unit->compare.optr), unit->compare.optr); + return TSDB_CODE_APP_ERROR; + } continue; } SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - ASSERT(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); + if (!FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)) { + fltError("filterInitValFieldData get invalid field flag : %d", right->flag); + return TSDB_CODE_APP_ERROR; + } uint32_t type = FILTER_UNIT_DATA_TYPE(unit); int8_t precision = FILTER_UNIT_DATA_PRECISION(unit); @@ -2107,7 +2162,10 @@ int32_t fltInitValFieldData(SFilterInfo *info) { SValueNode *var = (SValueNode *)fi->desc; if (var == NULL) { - ASSERT(fi->data != NULL); + if (!fi->data) { + fltError("filterInitValFieldData get invalid field data : NULL"); + return TSDB_CODE_APP_ERROR; + } continue; } @@ -2257,7 +2315,10 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit *u, SFilterRangeCtx *c FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; case OP_TYPE_NOT_EQUAL: - ASSERT(type == TSDB_DATA_TYPE_BOOL); + if (type != TSDB_DATA_TYPE_BOOL) { + fltError("filterAddUnitRange get invalid type : %d", type); + return TSDB_CODE_QRY_FILTER_INVALID_TYPE; + } if (GET_INT8_VAL(val)) { SIMPLE_COPY_VALUES(&ra.s, &tmp); SIMPLE_COPY_VALUES(&ra.e, &tmp); @@ -2273,7 +2334,7 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit *u, SFilterRangeCtx *c break; default: fltError("unsupported operator type"); - return TSDB_CODE_APP_ERROR; + return TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE; } FLT_ERR_RET(filterAddRange(ctx, &ra, optr)); @@ -2547,8 +2608,11 @@ int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32 FLT_ERR_RET(filterReuseRangeCtx(*ctx, type, 0)); } - ASSERT(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); - ASSERT(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + if (gRes2->colInfo[cidx].type != RANGE_TYPE_MR_CTX || gRes1->colInfo[cidx].type != RANGE_TYPE_MR_CTX) { + fltError("filterMergeTwoGroupsImpl get invalid col type : %d and %d", + gRes2->colInfo[cidx].type, gRes1->colInfo[cidx].type); + return TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE; + } FLT_ERR_RET(filterCopyRangeCtx(*ctx, gRes2->colInfo[cidx].info)); FLT_ERR_RET(filterSourceRangeFromCtx(*ctx, gRes1->colInfo[cidx].info, optr, empty, all)); @@ -2588,7 +2652,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx **gRes1, SFilter continue; } - ASSERT(idx1 == idx2); + if (idx1 != idx2) { + fltError("filterMergeTwoGroups get invalid idx : %d and %d", idx1, idx2); + FLT_ERR_JRET(TSDB_CODE_APP_ERROR); + } ++merNum; @@ -2644,16 +2711,19 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx **gRes1, SFilter } } - ASSERT(merNum > 0); - - SFilterColInfo *colInfo = NULL; - ASSERT(merNum == equal1 || merNum == equal2); + if (merNum == 0 || (equal1 != merNum && equal2 != merNum)) { + fltError("filterMergeTwoGroups get invalid merge num : %d, equal1 : %d, equal2 : %d", merNum, equal1, equal2); + FLT_ERR_JRET(TSDB_CODE_APP_ERROR); + } filterFreeGroupCtx(*gRes2); *gRes2 = NULL; - ASSERT(colCtxs && taosArrayGetSize(colCtxs) > 0); - + if (!colCtxs || taosArrayGetSize(colCtxs) <= 0) { + fltError("filterMergeTwoGroups get invalid colCtxs with size %zu", taosArrayGetSize(colCtxs)); + FLT_ERR_JRET(TSDB_CODE_APP_ERROR); + } + SFilterColInfo *colInfo = NULL; int32_t ctxSize = (int32_t)taosArrayGetSize(colCtxs); SFilterColCtx *pctx = NULL; @@ -2713,7 +2783,10 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t *gR if (pColNum > 0) { for (int32_t m = 0; m <= pEnd; ++m) { for (int32_t n = cStart; n <= cEnd; ++n) { - ASSERT(m < n); + if (m >= n) { + fltError("filterMergeGroups get invalid m : %d and n : %d", m, n); + FLT_ERR_JRET(TSDB_CODE_APP_ERROR); + } FLT_ERR_JRET(filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all)); FLT_CHK_JMP(all); @@ -2734,7 +2807,10 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t *gR for (int32_t m = cStart; m < cEnd; ++m) { for (int32_t n = m + 1; n <= cEnd; ++n) { - ASSERT(m < n); + if (m >= n) { + fltError("filterMergeGroups get invalid m : %d and n : %d", m, n); + FLT_ERR_JRET(TSDB_CODE_APP_ERROR); + } FLT_ERR_JRET(filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all)); FLT_CHK_JMP(all); @@ -2844,7 +2920,10 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum for (uint32_t m = 0; m < res->colNum; ++m) { colInfo = &res->colInfo[res->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { - ASSERT(colInfo->type == RANGE_TYPE_UNIT); + if (colInfo->type != RANGE_TYPE_UNIT) { + fltError("filterRewrite get invalid col type : %d", colInfo->type); + FLT_ERR_JRET(TSDB_CODE_QRY_FILTER_INVALID_TYPE); + } int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); for (int32_t n = 0; n < usize; ++n) { @@ -2859,7 +2938,10 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum continue; } - ASSERT(colInfo->type == RANGE_TYPE_MR_CTX); + if (colInfo->type != RANGE_TYPE_MR_CTX) { + fltError("filterRewrite get invalid col type : %d", colInfo->type); + FLT_ERR_JRET(TSDB_CODE_QRY_FILTER_INVALID_TYPE); + } FLT_ERR_JRET(filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group)); } @@ -2905,7 +2987,10 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ continue; } - ASSERT(idxNum[i] == gResNum); + if (idxNum[i] != gResNum) { + fltError("filterGenerateColRange get invalid idxNum : %d and gResNum : %d", idxNum[i], gResNum); + FLT_ERR_JRET(TSDB_CODE_APP_ERROR); + } if (idxs == NULL) { idxs = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); @@ -2936,7 +3021,10 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ continue; } - ASSERT(res->colIdx[n] == idxs[m]); + if (res->colIdx[n] != idxs[m]) { + fltError("filterGenerateColRange get invalid colIdx : %d and idxs : %d", res->colIdx[n], idxs[m]); + SCL_ERR_JRET(TSDB_CODE_APP_ERROR); + } SFilterColInfo *colInfo = &res->colInfo[res->colIdx[n]]; if (info->colRange[m] == NULL) { @@ -2945,7 +3033,10 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ info->colRange[m]->colId = FILTER_GET_COL_FIELD_ID(fi); } - ASSERT(colInfo->type == RANGE_TYPE_MR_CTX); + if (colInfo->type != RANGE_TYPE_MR_CTX) { + fltError("filterGenerateColRange get invalid col type : %d", colInfo->type); + FLT_ERR_JRET(TSDB_CODE_QRY_FILTER_INVALID_TYPE); + } bool all = false; FLT_ERR_JRET(filterSourceRangeFromCtx(info->colRange[m], colInfo->info, LOGIC_COND_TYPE_OR, NULL, &all)); @@ -3195,7 +3286,10 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 unitIdx = pGroupIdx; --info->blkGroupNum; - ASSERT(empty || all); + if (!empty && !all) { + fltError("filterRmUnitByRange get invalid empty and all : %d and %d", empty, all); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); + } if (empty) { FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); @@ -3303,7 +3397,10 @@ int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, SColumn goto _return; } - ASSERT(info->unitNum > 1); + if (info->unitNum <= 1) { + fltError("filterExecuteBasedOnStatis get invalid unit num : %d", info->unitNum); + FLT_ERR_JRET(TSDB_CODE_APP_ERROR); + } *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); goto _return; @@ -5161,7 +5258,6 @@ int32_t filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, return TSDB_CODE_SUCCESS; } - ASSERT(false == info->scalarMode); *p = output.columnData; output.numOfRows = pSrc->info.rows; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index f81205df7a..37daff1d63 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -297,7 +297,8 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - _getDoubleValue_fn_t getValueFn = getVectorDoubleValueFn(type); + _getDoubleValue_fn_t getValueFn; + SCL_ERR_RET(getVectorDoubleValueFn(type, &getValueFn)); double *out = (double *)pOutputData->pData; @@ -328,7 +329,7 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S for (int32_t i = 0; i < inputNum; ++i) { pInputData[i] = pInput[i].columnData; - getValueFn[i] = getVectorDoubleValueFn(GET_PARAM_TYPE(&pInput[i])); + SCL_ERR_RET(getVectorDoubleValueFn(GET_PARAM_TYPE(&pInput[i]), &getValueFn[i])); } double *out = (double *)pOutputData->pData; @@ -2918,7 +2919,7 @@ static int32_t doScalarFunction2(SScalarParam *pInput, int32_t inputNum, SScalar for (int32_t i = 0; i < inputNum; ++i) { pInputData[i] = pInput[i].columnData; - getValueFn[i] = getVectorDoubleValueFn(GET_PARAM_TYPE(&pInput[i])); + SCL_ERR_RET(getVectorDoubleValueFn(GET_PARAM_TYPE(&pInput[i]), &getValueFn[i])); } bool hasNullType = (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 71773ced57..5556108a52 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -43,7 +43,7 @@ bool noConvertBeforeCompare(int32_t leftType, int32_t rightType, int32_t optr) { (optr >= OP_TYPE_GREATER_THAN && optr <= OP_TYPE_NOT_EQUAL); } -void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType) { +int32_t convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType) { switch (outType) { case TSDB_DATA_TYPE_BOOL: { GET_TYPED_DATA(*((bool *)outData), bool, inType, inData); @@ -91,9 +91,10 @@ void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int break; } default: { - ASSERT(0); + return TSDB_CODE_SCALAR_CONVERT_ERROR; } } + return TSDB_CODE_SUCCESS; } int32_t convertNcharToDouble(const void *inData, void *outData) { @@ -180,7 +181,10 @@ int32_t getVectorBigintValue_BOOL(void *src, int32_t index, int64_t *res) { } int32_t getVectorBigintValue_JSON(void *src, int32_t index, int64_t *res) { - ASSERT(!colDataIsNull_var(((SColumnInfoData *)src), index)); + if (colDataIsNull_var(((SColumnInfoData *)src), index)) { + sclError("getVectorBigintValue_JSON get json data null with index %d", index); + SCL_ERR_RET(TSDB_CODE_SCALAR_CONVERT_ERROR); + } char *data = colDataGetVarData((SColumnInfoData *)src, index); double out = 0; if (*data == TSDB_DATA_TYPE_NULL) { @@ -192,46 +196,47 @@ int32_t getVectorBigintValue_JSON(void *src, int32_t index, int64_t *res) { *res = 0; SCL_ERR_RET(TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR); } else { - convertNumberToNumber(data + CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + SCL_ERR_RET(convertNumberToNumber(data + CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE)); } *res = (int64_t)out; SCL_RET(TSDB_CODE_SUCCESS); } -_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { - _getBigintValue_fn_t p = NULL; +int32_t getVectorBigintValueFn(int32_t srcType, _getBigintValue_fn_t *p) { + *p = NULL; if (srcType == TSDB_DATA_TYPE_TINYINT) { - p = getVectorBigintValue_TINYINT; + *p = getVectorBigintValue_TINYINT; } else if (srcType == TSDB_DATA_TYPE_UTINYINT) { - p = getVectorBigintValue_UTINYINT; + *p = getVectorBigintValue_UTINYINT; } else if (srcType == TSDB_DATA_TYPE_SMALLINT) { - p = getVectorBigintValue_SMALLINT; + *p = getVectorBigintValue_SMALLINT; } else if (srcType == TSDB_DATA_TYPE_USMALLINT) { - p = getVectorBigintValue_USMALLINT; + *p = getVectorBigintValue_USMALLINT; } else if (srcType == TSDB_DATA_TYPE_INT) { - p = getVectorBigintValue_INT; + *p = getVectorBigintValue_INT; } else if (srcType == TSDB_DATA_TYPE_UINT) { - p = getVectorBigintValue_UINT; + *p = getVectorBigintValue_UINT; } else if (srcType == TSDB_DATA_TYPE_BIGINT) { - p = getVectorBigintValue_BIGINT; + *p = getVectorBigintValue_BIGINT; } else if (srcType == TSDB_DATA_TYPE_UBIGINT) { - p = getVectorBigintValue_UBIGINT; + *p = getVectorBigintValue_UBIGINT; } else if (srcType == TSDB_DATA_TYPE_FLOAT) { - p = getVectorBigintValue_FLOAT; + *p = getVectorBigintValue_FLOAT; } else if (srcType == TSDB_DATA_TYPE_DOUBLE) { - p = getVectorBigintValue_DOUBLE; + *p = getVectorBigintValue_DOUBLE; } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) { - p = getVectorBigintValue_BIGINT; + *p = getVectorBigintValue_BIGINT; } else if (srcType == TSDB_DATA_TYPE_BOOL) { - p = getVectorBigintValue_BOOL; + *p = getVectorBigintValue_BOOL; } else if (srcType == TSDB_DATA_TYPE_JSON) { - p = getVectorBigintValue_JSON; + *p = getVectorBigintValue_JSON; } else if (srcType == TSDB_DATA_TYPE_NULL) { - p = NULL; + *p = NULL; } else { - ASSERT(0); + sclError("getVectorBigintValueFn invalid srcType : %d", srcType); + return TSDB_CODE_SCALAR_CONVERT_ERROR; } - return p; + return TSDB_CODE_SUCCESS; } static FORCE_INLINE int32_t varToTimestamp(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { @@ -467,7 +472,7 @@ static FORCE_INLINE int32_t varToGeometry(char *buf, SScalarParam *pOut, int32_t _return: taosMemoryFree(output); geosFreeBuffer(t); - ASSERT(t == NULL && len == 0); + t = NULL; VarDataLenT dummyHeader = 0; SCL_ERR_RET(colDataSetVal(pOut->columnData, rowIndex, (const char *)&dummyHeader, false)); SCL_RET(code); @@ -525,7 +530,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { } else if (tTagIsJson(data) || *data == TSDB_DATA_TYPE_NULL) { SCL_ERR_JRET(TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR); } else { - convertNumberToNumber(data + CHAR_BYTES, colDataGetNumData(pCtx->pOut->columnData, i), *data, pCtx->outType); + SCL_ERR_JRET(convertNumberToNumber(data + CHAR_BYTES, colDataGetNumData(pCtx->pOut->columnData, i), *data, pCtx->outType)); continue; } } @@ -582,7 +587,7 @@ int32_t getVectorDoubleValue_JSON(void *src, int32_t index, double *out) { } else if (tTagIsJson(data)) { SCL_ERR_RET(TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR); } else { - convertNumberToNumber(data + CHAR_BYTES, out, *data, TSDB_DATA_TYPE_DOUBLE); + SCL_ERR_RET(convertNumberToNumber(data + CHAR_BYTES, out, *data, TSDB_DATA_TYPE_DOUBLE)); } SCL_RET(TSDB_CODE_SUCCESS); } @@ -673,7 +678,7 @@ int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_ *result = false; return TSDB_CODE_SUCCESS; } else if (typeLeft != type) { - convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); + SCL_ERR_RET(convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type)); *pLeftData = pLeftOut; } @@ -683,7 +688,7 @@ int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_ *result = false; return TSDB_CODE_SUCCESS; } else if (typeRight != type) { - convertNumberToNumber(*pRightData, pRightOut, typeRight, type); + SCL_ERR_RET(convertNumberToNumber(*pRightData, pRightOut, typeRight, type)); *pRightData = pRightOut; } } else if (type == TSDB_DATA_TYPE_BINARY || @@ -1130,8 +1135,10 @@ enum { // TODO not correct for descending order scan static int32_t vectorMathAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_RET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_RET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); double *output = (double *)pOutputCol->pData; @@ -1155,9 +1162,10 @@ static int32_t vectorMathAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *p static int32_t vectorMathTsAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); - + _getBigintValue_fn_t getVectorBigintValueFnLeft; + _getBigintValue_fn_t getVectorBigintValueFnRight; + SCL_ERR_RET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); + SCL_ERR_RET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); int64_t *output = (int64_t *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value @@ -1230,8 +1238,10 @@ int32_t vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BOOL)) { // timestamp plus duration int64_t *output = (int64_t *)pOutputCol->pData; - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnLeft; + _getBigintValue_fn_t getVectorBigintValueFnRight; + SCL_ERR_JRET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); + SCL_ERR_JRET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); if (pLeft->numOfRows == 1 && pRight->numOfRows == 1) { if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP) { @@ -1258,9 +1268,10 @@ int32_t vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p } } else { double *output = (double *)pOutputCol->pData; - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); - + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { @@ -1289,8 +1300,10 @@ _return: // TODO not correct for descending order scan static int32_t vectorMathSubHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t factor, int32_t i) { - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_RET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_RET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); double *output = (double *)pOutputCol->pData; @@ -1314,8 +1327,10 @@ static int32_t vectorMathSubHelper(SColumnInfoData *pLeftCol, SColumnInfoData *p static int32_t vectorMathTsSubHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t factor, int32_t i) { - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnLeft; + _getBigintValue_fn_t getVectorBigintValueFnRight; + SCL_ERR_RET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); + SCL_ERR_RET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); int64_t *output = (int64_t *)pOutputCol->pData; @@ -1357,8 +1372,10 @@ int32_t vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BIGINT)) { // timestamp minus duration int64_t *output = (int64_t *)pOutputCol->pData; - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnLeft; + _getBigintValue_fn_t getVectorBigintValueFnRight; + SCL_ERR_JRET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); + SCL_ERR_JRET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); if (pLeft->numOfRows == 1 && pRight->numOfRows == 1) { SCL_ERR_JRET(vectorMathTsSubHelper(pLeftCol, pRightCol, pOutputCol, pLeft->numOfRows, step, 1, i)); @@ -1381,8 +1398,10 @@ int32_t vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p } } else { double *output = (double *)pOutputCol->pData; - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { @@ -1412,8 +1431,10 @@ _return: // TODO not correct for descending order scan static int32_t vectorMathMultiplyHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_RET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_RET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); double *output = (double *)pOutputCol->pData; @@ -1449,8 +1470,10 @@ int32_t vectorMathMultiply(SScalarParam *pLeft, SScalarParam *pRight, SScalarPar SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { @@ -1491,8 +1514,10 @@ int32_t vectorMathDivide(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { @@ -1573,8 +1598,10 @@ int32_t vectorMathRemainder(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + _getDoubleValue_fn_t getVectorDoubleValueFnRight; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); + SCL_ERR_JRET(getVectorDoubleValueFn(pRightCol->info.type, &getVectorDoubleValueFnRight)); double *output = (double *)pOutputCol->pData; @@ -1661,7 +1688,8 @@ int32_t vectorMathMinus(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam SColumnInfoData *pLeftCol = NULL; SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft; + SCL_ERR_JRET(getVectorDoubleValueFn(pLeftCol->info.type, &getVectorDoubleValueFnLeft)); double *output = (double *)pOutputCol->pData; for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { @@ -1692,15 +1720,20 @@ int32_t vectorAssign(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO } } - ASSERT(pRight->numOfQualified == 1 || pRight->numOfQualified == 0); + if (pRight->numOfQualified != 1 && pRight->numOfQualified != 0) { + sclError("vectorAssign: invalid qualified number %d", pRight->numOfQualified); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); + } pOut->numOfQualified = pRight->numOfQualified * pOut->numOfRows; return TSDB_CODE_SUCCESS; } static int32_t vectorBitAndHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnLeft; + _getBigintValue_fn_t getVectorBigintValueFnRight; + SCL_ERR_RET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); + SCL_ERR_RET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); int64_t *output = (int64_t *)pOutputCol->pData; @@ -1736,8 +1769,10 @@ int32_t vectorBitAnd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnLeft; + _getBigintValue_fn_t getVectorBigintValueFnRight; + SCL_ERR_JRET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); + SCL_ERR_JRET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { @@ -1766,8 +1801,10 @@ _return: static int32_t vectorBitOrHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnLeft; + _getBigintValue_fn_t getVectorBigintValueFnRight; + SCL_ERR_RET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); + SCL_ERR_RET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); int64_t *output = (int64_t *)pOutputCol->pData; @@ -1803,8 +1840,10 @@ int32_t vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOu SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnLeft; + _getBigintValue_fn_t getVectorBigintValueFnRight; + SCL_ERR_JRET(getVectorBigintValueFn(pLeftCol->info.type, &getVectorBigintValueFnLeft)); + SCL_ERR_JRET(getVectorBigintValueFn(pRightCol->info.type, &getVectorBigintValueFnRight)); int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { @@ -1892,7 +1931,8 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result)); if (isJsonnull) { - ASSERT(0); + sclError("doVectorCompareImpl: invalid json null value"); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); } if (!pLeftData || !pRightData) { diff --git a/source/libs/scalar/test/filter/CMakeLists.txt b/source/libs/scalar/test/filter/CMakeLists.txt index 94af1eb6f0..44a0395286 100644 --- a/source/libs/scalar/test/filter/CMakeLists.txt +++ b/source/libs/scalar/test/filter/CMakeLists.txt @@ -1,7 +1,7 @@ MESSAGE(STATUS "build filter unit test") -IF(NOT TD_DARWIN) +IF(TD_DARWIN) # GoogleTest requires at least C++11 SET(CMAKE_CXX_STANDARD 11) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 33d2c1e2ef..b970bf5297 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -113,8 +113,7 @@ int32_t flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, if (NULL == *block) { SSDataBlock *res = NULL; - int32_t code = createDataBlock(&res); - ASSERT(code == 0); + FLT_ERR_RET(createDataBlock(&res)); for (int32_t i = 0; i < 2; ++i) { SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1 + i); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 032b197046..c29f7b8a5b 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -115,7 +115,10 @@ int32_t scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t res->info.capacity = rows; res->info.rows = rows; SColumnInfoData *p = static_cast(taosArrayGet(res->pDataBlock, 0)); - ASSERT(p->pData != NULL && p->nullbitmap != NULL); + if (p->pData == NULL || p->nullbitmap == NULL) { + sclError("data block is not initialized since pData or nullbitmap is NULL"); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); + } (void)taosArrayPush(pBlockList, &res); *dataBlockId = taosArrayGetSize(pBlockList) - 1; @@ -189,8 +192,7 @@ int32_t scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, if (NULL == *block) { SSDataBlock *res = NULL; - int32_t code = createDataBlock(&res); - ASSERT(code == 0); + SCL_ERR_RET(createDataBlock(&res)); for (int32_t i = 0; i < 2; ++i) { SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1); @@ -1422,8 +1424,10 @@ int32_t makeCalculate(void *json, void *key, int32_t rightType, void *rightData, opType == OP_TYPE_NMATCH) { (void)printf("op:%s,3result:%d,except:%f\n", operatorTypeStr(opType), *((bool *)colDataGetData(column, 0)), exceptValue); - assert(*(bool *)colDataGetData(column, 0) == exceptValue); -// ASSERT_EQ((int) *((bool *)colDataGetData(column, 0)), (int)exceptValue); + if(*(bool *)colDataGetData(column, 0) != exceptValue) { + (void)printf("expect value %d, but got %d\n", *((bool *)colDataGetData(column, 0)), exceptValue); + SCL_ERR_RET(TSDB_CODE_FAILED); + } } taosArrayDestroyEx(blockList, scltFreeDataBlock); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 396abf21a7..c04e34b443 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -478,6 +478,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_WINDOW_CONDITION, "The time pseudo colum TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR, "Executor internal error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_JOIN_CONDITION, "Not supported join on condition") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE, "Not supported range type") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_WRONG_OPTR_TYPE, "Wrong operator type") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_RANGE_ERROR, "Wrong filter range") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_FILTER_INVALID_TYPE, "Invalid filter type") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired") From 6055b9172ec4e1192c077b7de1f57438ef444388 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Aug 2024 09:03:57 +0800 Subject: [PATCH 029/100] fix: memory leak of geos --- source/libs/geometry/src/geosWrapper.c | 51 +++++++------------------- source/util/src/tgeosctx.c | 4 +- 2 files changed, 15 insertions(+), 40 deletions(-) diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 4f3f7d75c2..7372521276 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -37,10 +37,7 @@ int32_t initCtxMakePoint() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); @@ -67,10 +64,7 @@ int32_t doMakePoint(double x, double y, unsigned char **outputGeom, size_t *size int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; GEOSGeometry *geom = NULL; unsigned char *wkb = NULL; @@ -177,10 +171,7 @@ int32_t initCtxGeomFromText() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); @@ -218,10 +209,7 @@ int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; GEOSGeometry *geom = NULL; unsigned char *wkb = NULL; @@ -258,10 +246,7 @@ int32_t initCtxAsText() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); @@ -299,10 +284,7 @@ int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT) int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; GEOSGeometry *geom = NULL; char *wkt = NULL; @@ -335,10 +317,7 @@ int32_t initCtxRelationFunc() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); @@ -365,9 +344,7 @@ int32_t doGeosRelation(const GEOSGeometry *geom1, const GEOSPreparedGeometry *pr _geosPreparedRelationFunc_t swappedPreparedRelationFn) { SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - return TSDB_CODE_OUT_OF_MEMORY; - } + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; if (!preparedGeom1) { if (!swapped) { @@ -429,11 +406,6 @@ int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry // need to call destroyGeometry(outputGeom, outputPreparedGeom) later int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom) { - SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) { - return TSDB_CODE_OUT_OF_MEMORY; - } - ASSERT(outputGeom); // it is not allowed if outputGeom is NULL *outputGeom = NULL; @@ -445,6 +417,10 @@ int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, return TSDB_CODE_SUCCESS; } + SGeosContext *geosCtx = getThreadLocalGeosCtx(); + + if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; + *outputGeom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, varDataVal(input), varDataLen(input)); if (*outputGeom == NULL) { return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; @@ -461,7 +437,8 @@ int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, } void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom) { - SGeosContext *geosCtx = getThreadLocalGeosCtx(); + SGeosContext *geosCtx = acquireThreadLocalGeosCtx(); + if (!geosCtx) return; if (preparedGeom && *preparedGeom) { GEOSPreparedGeom_destroy_r(geosCtx->handle, *preparedGeom); diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index 473b7539fc..5d47452fda 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -31,9 +31,7 @@ static threadlocal SGeosContext *tlGeosCtx = NULL; SGeosContext *acquireThreadLocalGeosCtx() { return tlGeosCtx; } SGeosContext *getThreadLocalGeosCtx() { - if (tlGeosCtx) { - return tlGeosCtx; - } + if (tlGeosCtx) return tlGeosCtx; taosWLockLatch(&sGeosPool.lock); if (sGeosPool.size >= sGeosPool.capacity) { From a2217973aeb70a8d7651263c7f11c6dd6e0c7a99 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Aug 2024 09:07:29 +0800 Subject: [PATCH 030/100] fix: memory leak of geos --- source/libs/scalar/src/sclvector.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index daf44ec527..e20b0cb6fc 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -445,7 +445,7 @@ static FORCE_INLINE int32_t varToGeometry(char *buf, SScalarParam *pOut, int32_t sclError("failed to init geometry ctx, %s", getGeosErrMsg(code)); SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } - if ((code = doGeomFromText(buf, &t, &len))) { + if ((code = doGeomFromText(buf, &t, &len)) != 0) { sclInfo("failed to convert text to geometry, %s", getGeosErrMsg(code)); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); } From 2f4d0815920a46e45f38fb1e927528e4ff5ca07f Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Aug 2024 09:08:22 +0800 Subject: [PATCH 031/100] fix: memory leak of geos --- source/libs/scalar/src/sclvector.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e20b0cb6fc..9bac02e5f9 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -446,7 +446,7 @@ static FORCE_INLINE int32_t varToGeometry(char *buf, SScalarParam *pOut, int32_t SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } if ((code = doGeomFromText(buf, &t, &len)) != 0) { - sclInfo("failed to convert text to geometry, %s", getGeosErrMsg(code)); + sclError("failed to convert text to geometry, %s", getGeosErrMsg(code)); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); } From 051763e71f0e41a44fd38a93ab3ffe10bc2e1a88 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Aug 2024 09:14:50 +0800 Subject: [PATCH 032/100] fix(tsdb): return if get data. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 74992c40d3..9be2c3b3f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1807,9 +1807,11 @@ static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange, pSttBlockReader->numOfPks > 0)) { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; + return code; } } else { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; + return code; } } From 0e25059039a154f16e67cb15a6514fca8ec023a2 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 20 Aug 2024 09:17:50 +0800 Subject: [PATCH 033/100] fix: s3 protocolG/uriStyleG init --- source/common/src/cos.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index a5a278e82e..6e9c7dd50d 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -56,10 +56,8 @@ int32_t s3Begin() { } for (int i = 0; i < tsS3EpNum; i++) { - protocolG[i] = !tsS3Https[i]; - if (tsS3Oss[i]) { - uriStyleG[i] = S3UriStyleVirtualHost; - } + protocolG[i] = tsS3Https[i] ? S3ProtocolHTTPS : S3ProtocolHTTP; + uriStyleG[i] = tsS3Oss[i] ? S3UriStyleVirtualHost : S3UriStylePath; } TAOS_RETURN(TSDB_CODE_SUCCESS); From 1f1a2d9f6c7f464ad31d941817c4f369609eee05 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 20 Aug 2024 09:54:26 +0800 Subject: [PATCH 034/100] fix: handle error code --- include/libs/sync/sync.h | 30 ++++++++++----------- source/libs/sync/inc/syncRaftCfg.h | 1 - source/libs/sync/inc/syncUtil.h | 2 +- source/libs/sync/src/syncMain.c | 26 +++++++++++++----- source/libs/sync/src/syncRaftCfg.c | 42 +++++++++++++++++------------- 5 files changed, 60 insertions(+), 41 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 1fb077e3ca..07d56f9b07 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -156,11 +156,11 @@ typedef struct SSnapshotParam { typedef struct SSnapshot { int32_t type; - SSyncTLV* data; + SSyncTLV* data; ESyncFsmState state; - SyncIndex lastApplyIndex; - SyncTerm lastApplyTerm; - SyncIndex lastConfigIndex; + SyncIndex lastApplyIndex; + SyncTerm lastApplyTerm; + SyncIndex lastConfigIndex; } SSnapshot; typedef struct SSnapshotMeta { @@ -263,16 +263,16 @@ typedef struct SSyncState { int64_t startTimeMs; } SSyncState; -int32_t syncInit(); -void syncCleanUp(); -int64_t syncOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion); -int32_t syncStart(int64_t rid); -void syncStop(int64_t rid); -void syncPreStop(int64_t rid); -void syncPostStop(int64_t rid); -int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); -int32_t syncCheckMember(int64_t rid); -int32_t syncIsCatchUp(int64_t rid); +int32_t syncInit(); +void syncCleanUp(); +int64_t syncOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion); +int32_t syncStart(int64_t rid); +void syncStop(int64_t rid); +void syncPreStop(int64_t rid); +void syncPostStop(int64_t rid); +int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); +int32_t syncCheckMember(int64_t rid); +int32_t syncIsCatchUp(int64_t rid); ESyncRole syncGetRole(int64_t rid); int64_t syncGetTerm(int64_t rid); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); @@ -296,7 +296,7 @@ int32_t syncGetAssignedLogSynced(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); const char* syncStr(ESyncState state); -int32_t syncNodeGetConfig(int64_t rid, SSyncCfg *cfg); +int32_t syncNodeGetConfig(int64_t rid, SSyncCfg* cfg); // util int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 4f03a60fbc..2c1626c4e8 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -24,7 +24,6 @@ extern "C" { int32_t syncWriteCfgFile(SSyncNode *pNode); int32_t syncReadCfgFile(SSyncNode *pNode); -int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 555607d40c..2b5b818da3 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -107,7 +107,7 @@ void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, int32_t voteGranted, const char* s); -void syncLogSendRequestVote(SSyncNode* pNode, const SyncRequestVote* pMsg, const char* s); +void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5465007b18..8b8a9e1279 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -50,7 +50,7 @@ static int32_t syncHbTimerStart(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer); static int32_t syncHbTimerStop(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer); static int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg); static bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config); -static void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex); +static int32_t syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex); static bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg); static bool syncNodeCanChange(SSyncNode* pSyncNode); @@ -182,7 +182,12 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { } TAOS_CHECK_RETURN(syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg)); - syncNodeDoConfigChange(pSyncNode, pNewCfg, pNewCfg->lastIndex); + + if (syncNodeDoConfigChange(pSyncNode, pNewCfg, pNewCfg->lastIndex) != 0) { + code = TSDB_CODE_SYN_NEW_CONFIG_ERROR; + sError("vgId:%d, failed to reconfig since do change error", pSyncNode->vgId); + TAOS_RETURN(code); + } if (pSyncNode->state == TAOS_SYNC_STATE_LEADER || pSyncNode->state == TAOS_SYNC_STATE_ASSIGNED_LEADER) { // TODO check return value @@ -1015,7 +1020,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { if (!taosDirExist((char*)(pSyncInfo->path))) { if (taosMkDir(pSyncInfo->path) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr()); + sError("vgId:%d, failed to create dir:%s since %s", pSyncInfo->vgId, pSyncInfo->path, terrstr()); goto _error; } } @@ -1766,11 +1771,11 @@ static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg return false; } -void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex lastConfigChangeIndex) { +int32_t syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex lastConfigChangeIndex) { SSyncCfg oldConfig = pSyncNode->raftCfg.cfg; if (!syncIsConfigChanged(&oldConfig, pNewConfig)) { sInfo("vgId:1, sync not reconfig since not changed"); - return; + return 0; } pSyncNode->raftCfg.cfg = *pNewConfig; @@ -1809,7 +1814,15 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // add last config index - (void)syncAddCfgIndex(pSyncNode, lastConfigChangeIndex); + SRaftCfg* pCfg = &pSyncNode->raftCfg; + if (pCfg->configIndexCount >= MAX_CONFIG_INDEX_COUNT) { + sNError(pSyncNode, "failed to add cfg index:%d since out of range", pCfg->configIndexCount); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pCfg->configIndexArr[pCfg->configIndexCount] = lastConfigChangeIndex; + pCfg->configIndexCount++; if (IamInNew) { //----------------------------------------- @@ -1924,6 +1937,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde _END: // log end config change sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum); + return 0; } // raft state change -------------- diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index b0e6abffc6..82cc86ed86 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -18,7 +18,7 @@ #include "syncUtil.h" #include "tjson.h" -const char *syncRoleToStr(ESyncRole role) { +static const char *syncRoleToStr(ESyncRole role) { switch (role) { case TAOS_SYNC_ROLE_VOTER: return "true"; @@ -29,15 +29,14 @@ const char *syncRoleToStr(ESyncRole role) { } } -const ESyncRole syncStrToRole(char *str) { +static const ESyncRole syncStrToRole(char *str) { if (strcmp(str, "true") == 0) { return TAOS_SYNC_ROLE_VOTER; - } - if (strcmp(str, "false") == 0) { + } else if (strcmp(str, "false") == 0) { return TAOS_SYNC_ROLE_LEARNER; + } else { + return TAOS_SYNC_ROLE_ERROR; } - - return TAOS_SYNC_ROLE_ERROR; } static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { @@ -52,10 +51,12 @@ static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { if (nodeInfo == NULL) { TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } + if ((code = tjsonAddItemToObject(pJson, "nodeInfo", nodeInfo)) < 0) { tjsonDelete(nodeInfo); TAOS_CHECK_EXIT(code); } + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SJson *info = tjsonCreateObject(); if (info == NULL) { @@ -68,20 +69,25 @@ static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { TAOS_CHECK_GOTO(tjsonAddStringToObject(info, "isReplica", syncRoleToStr(pCfg->nodeInfo[i].nodeRole)), NULL, _err); TAOS_CHECK_GOTO(tjsonAddItemToArray(nodeInfo, info), NULL, _err); continue; + _err: tjsonDelete(info); break; } + _exit: if (code < 0) { sError("failed to encode sync cfg at line %d since %s", lino, tstrerror(code)); } + TAOS_RETURN(code); } static int32_t syncEncodeRaftCfg(const void *pObj, SJson *pJson) { SRaftCfg *pCfg = (SRaftCfg *)pObj; - int32_t code = 0, lino = 0; + int32_t code = 0; + int32_t lino = 0; + TAOS_CHECK_EXIT(tjsonAddObject(pJson, "SSyncCfg", syncEncodeSyncCfg, (void *)&pCfg->cfg)); TAOS_CHECK_EXIT(tjsonAddDoubleToObject(pJson, "isStandBy", pCfg->isStandBy)); TAOS_CHECK_EXIT(tjsonAddDoubleToObject(pJson, "snapshotStrategy", pCfg->snapshotStrategy)); @@ -93,10 +99,12 @@ static int32_t syncEncodeRaftCfg(const void *pObj, SJson *pJson) { if (configIndexArr == NULL) { TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } + if ((code = tjsonAddItemToObject(pJson, "configIndexArr", configIndexArr)) < 0) { tjsonDelete(configIndexArr); TAOS_CHECK_EXIT(code); } + for (int32_t i = 0; i < pCfg->configIndexCount; ++i) { SJson *configIndex = tjsonCreateObject(); if (configIndex == NULL) { @@ -105,14 +113,17 @@ static int32_t syncEncodeRaftCfg(const void *pObj, SJson *pJson) { TAOS_CHECK_EXIT(tjsonAddIntegerToObject(configIndex, "index", pCfg->configIndexArr[i])); TAOS_CHECK_EXIT(tjsonAddItemToArray(configIndexArr, configIndex)); continue; + _err: tjsonDelete(configIndex); break; } + _exit: if (code < 0) { sError("failed to encode raft cfg at line %d since %s", lino, tstrerror(code)); } + TAOS_RETURN(code); } @@ -124,11 +135,13 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { const char *realfile = pNode->configPath; SRaftCfg *pCfg = &pNode->raftCfg; char file[PATH_MAX] = {0}; + (void)snprintf(file, sizeof(file), "%s.bak", realfile); if ((pJson = tjsonCreateObject()) == NULL) { TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } + TAOS_CHECK_EXIT(tjsonAddObject(pJson, "RaftCfg", syncEncodeRaftCfg, pCfg)); buffer = tjsonToString(pJson); if (buffer == NULL) { @@ -145,6 +158,7 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { if (taosWriteFile(pFile, buffer, len) <= 0) { TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); } + if (taosFsyncFile(pFile) < 0) { TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(errno)); } @@ -165,6 +179,7 @@ _exit: if (code != 0) { sError("vgId:%d, failed to write sync cfg file:%s since %s", pNode->vgId, realfile, tstrerror(code)); } + TAOS_RETURN(code); } @@ -232,6 +247,7 @@ static int32_t syncDecodeRaftCfg(const SJson *pJson, void *pObj) { tjsonGetNumberValue(configIndex, "index", pCfg->configIndexArr[i], code); if (code < 0) return TSDB_CODE_INVALID_JSON_FORMAT; } + return 0; } @@ -292,16 +308,6 @@ _OVER: if (code != 0) { sError("vgId:%d, failed to read sync cfg file:%s since %s", pNode->vgId, file, tstrerror(code)); } + TAOS_RETURN(code); } - -int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex) { - SRaftCfg *pCfg = &pNode->raftCfg; - if (pCfg->configIndexCount >= MAX_CONFIG_INDEX_COUNT) { - return TSDB_CODE_OUT_OF_RANGE; - } - - pCfg->configIndexArr[pCfg->configIndexCount] = cfgIndex; - pCfg->configIndexCount++; - return 0; -} \ No newline at end of file From 66878fd040922b8881501d04768d879c9f5316b0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Aug 2024 10:10:42 +0800 Subject: [PATCH 035/100] fix: memory leak of geos --- source/util/src/tgeosctx.c | 45 ++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index 5d47452fda..42cde5b8c7 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -14,14 +14,16 @@ */ #include "tgeosctx.h" +#include "tarray.h" #include "tdef.h" #include "tlockfree.h" #include "tlog.h" +#define GEOS_POOL_CAPACITY 64 typedef struct { - SGeosContext *pool; - int32_t capacity; - int32_t size; + SArray *poolArray; // totalSize: (GEOS_POOL_CAPACITY * (taosArrayGetSize(poolArray) - 1)) + size + SGeosContext *pool; // current SGeosContext pool + int32_t size; // size of current SGeosContext pool, size <= GEOS_POOL_CAPACITY SRWLatch lock; } SGeosContextPool; @@ -34,16 +36,26 @@ SGeosContext *getThreadLocalGeosCtx() { if (tlGeosCtx) return tlGeosCtx; taosWLockLatch(&sGeosPool.lock); - if (sGeosPool.size >= sGeosPool.capacity) { - sGeosPool.capacity += 128; - void *tmp = taosMemoryRealloc(sGeosPool.pool, sGeosPool.capacity * sizeof(SGeosContext)); - if (!tmp) { + if (!sGeosPool.pool || sGeosPool.size >= GEOS_POOL_CAPACITY) { + if (!(sGeosPool.pool = (SGeosContext *)taosMemoryCalloc(GEOS_POOL_CAPACITY, sizeof(SGeosContext)))) { taosWUnLockLatch(&sGeosPool.lock); - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - sGeosPool.pool = tmp; - TAOS_MEMSET(sGeosPool.pool + sGeosPool.size, 0, (sGeosPool.capacity - sGeosPool.size) * sizeof(SGeosContext)); + if (!sGeosPool.poolArray) { + if (!(sGeosPool.poolArray = taosArrayInit(16, POINTER_BYTES))) { + taosMemoryFree(sGeosPool.pool); + sGeosPool.pool = NULL; + taosWUnLockLatch(&sGeosPool.lock); + return NULL; + } + } + if (!taosArrayPush(sGeosPool.poolArray, &sGeosPool.pool)) { + taosMemoryFree(sGeosPool.pool); + sGeosPool.pool = NULL; + taosWUnLockLatch(&sGeosPool.lock); + return NULL; + } + sGeosPool.size = 0; } tlGeosCtx = sGeosPool.pool + sGeosPool.size; ++sGeosPool.size; @@ -91,9 +103,14 @@ static void destroyGeosCtx(SGeosContext *pCtx) { void taosGeosDestroy() { uInfo("geos is cleaned up"); - if (!sGeosPool.pool) return; - for (int32_t i = 0; i < sGeosPool.size; ++i) { - destroyGeosCtx(sGeosPool.pool + i); + int32_t size = taosArrayGetSize(sGeosPool.poolArray); + for (int32_t i = 0; i < size; ++i) { + SGeosContext *pool = *(SGeosContext **)TARRAY_GET_ELEM(sGeosPool.poolArray, i); + for (int32_t j = 0; j < GEOS_POOL_CAPACITY; ++j) { + destroyGeosCtx(pool + j); + } + taosMemoryFree(pool); } - taosMemoryFreeClear(sGeosPool.pool); + taosArrayDestroy(sGeosPool.poolArray); + sGeosPool.poolArray = NULL; } \ No newline at end of file From 1bff0e89f4f3b7b12b4cca070a23a557c341b44c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 20 Aug 2024 10:15:52 +0800 Subject: [PATCH 036/100] fix issue --- source/libs/parser/src/parTranslater.c | 58 ++++++++++++++++++++++---- tests/script/tsim/show/showalive.sim | 44 ++++++++++++++++++- 2 files changed, 92 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a262acb2ec..05cceb656e 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -12628,6 +12628,27 @@ static int32_t createParOperatorNode(EOperatorType opType, const char* pLeftCol, return TSDB_CODE_SUCCESS; } +static int32_t createIsOperatorNode(EOperatorType opType, const char* pColName, SNode** pOp) { + SOperatorNode* pOper = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOper); + if (NULL == pOper) { + return code; + } + + pOper->opType = opType; + code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pOper->pLeft); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pOper); + return code; + } + pOper->pRight = NULL; + + snprintf(((SColumnNode*)pOper->pLeft)->colName, sizeof(((SColumnNode*)pOper->pLeft)->colName), "%s", pColName); + + *pOp = (SNode*)pOper; + return TSDB_CODE_SUCCESS; +} + static const char* getTbNameColName(ENodeType type) { const char* colName; switch (type) { @@ -15035,7 +15056,7 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) { // pWhenThenlist and pElse need to free - // case when (v1_status = "leader" or v2_status = "lead er" or v3_status = "leader" or v4_status = "leader") then 1 + // case when (v1_status = "leader" or v2_status = "leader" or v3_status = "leader" or v4_status = "leader") then 1 // else 0 end SNode* pCaseWhen = NULL; code = createParCaseWhenNode(NULL, pWhenThenlist, pElse, NULL, &pCaseWhen); @@ -15190,23 +15211,42 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) { return code; } - // pSubSelect, pTemp1, pTempVal need to free - - pThen = NULL; - code = nodesMakeValueNodeFromInt32(1, &pThen); + SNode* pCondIsNULL = NULL; + code = createIsOperatorNode(OP_TYPE_IS_NULL, pSumColAlias, &pCondIsNULL); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pSubSelect); nodesDestroyNode(pTemp1); nodesDestroyNode(pTempVal); return code; } - // pSubSelect, pTemp1, pThen, pTempVal need to free - pWhenThen = NULL; - code = createParWhenThenNode(pTemp1, pThen, &pWhenThen); + SNode* pCondFull1 = NULL; + code = createLogicCondNode(&pTemp1, &pCondIsNULL, &pCondFull1, LOGIC_COND_TYPE_OR); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pSubSelect); nodesDestroyNode(pTemp1); + nodesDestroyNode(pTempVal); + nodesDestroyNode(pCondIsNULL); + return code; + } + + // pSubSelect, pCondFull1, pTempVal need to free + + pThen = NULL; + code = nodesMakeValueNodeFromInt32(1, &pThen); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pSubSelect); + nodesDestroyNode(pCondFull1); + nodesDestroyNode(pTempVal); + return code; + } + // pSubSelect, pCondFull1, pThen, pTempVal need to free + + pWhenThen = NULL; + code = createParWhenThenNode(pCondFull1, pThen, &pWhenThen); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pSubSelect); + nodesDestroyNode(pCondFull1); nodesDestroyNode(pThen); nodesDestroyNode(pTempVal); return code; @@ -15289,7 +15329,7 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) { } // pSubSelect, pWhenThenlist need to free - // case when leader_col = count_col and count_col > 0 then 1 when leader_col < count_col and count_col > 0 then 2 else + // case when leader_col = count_col and leader_col > 0 then 1 when leader_col < count_col and leader_col > 0 then 2 else // 0 end as status pElse = NULL; code = nodesMakeValueNodeFromInt32(0, &pElse); diff --git a/tests/script/tsim/show/showalive.sim b/tests/script/tsim/show/showalive.sim index 4cad1da01d..72fad47f57 100644 --- a/tests/script/tsim/show/showalive.sim +++ b/tests/script/tsim/show/showalive.sim @@ -21,6 +21,30 @@ sql create dnode $hostname4 port 7500 sleep 1000 +$loop_count = 0 + +loop00: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +print 0 show cluster alive; +sql show cluster alive; + +print res------------------------ +print $data00 $data01 +print $data10 $data11 + +if $data00 != 1 then + print =====data00=$data00 + goto loop00 +endi + + print =============== create database, stable, table sql create database test vgroups 6; sql use test; @@ -46,6 +70,15 @@ endi print show cluster alive; sql show cluster alive; +print res------------------------ +print $data00 $data01 +print $data10 $data11 + +if $rows != 1 then + print =====rows=$rows + goto loop0 +endi + if $data00 != 1 then print =====data00=$data00 goto loop0 @@ -54,6 +87,15 @@ endi print show test.alive; sql show test.alive; +print res------------------------ +print $data00 $data01 +print $data10 $data11 + +if $rows != 1 then + print =====rows=$rows + goto loop0 +endi + if $data00 != 1 then print =====data00=$data00 goto loop0 @@ -164,4 +206,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file From 5e77f6f6ca873e0b1aa308e5721a383fdec1f7d6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Aug 2024 10:18:14 +0800 Subject: [PATCH 037/100] fix: memory leak of geos --- source/util/src/tgeosctx.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index 42cde5b8c7..99655ed7f7 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -43,15 +43,13 @@ SGeosContext *getThreadLocalGeosCtx() { } if (!sGeosPool.poolArray) { if (!(sGeosPool.poolArray = taosArrayInit(16, POINTER_BYTES))) { - taosMemoryFree(sGeosPool.pool); - sGeosPool.pool = NULL; + taosMemoryFreeClear(sGeosPool.pool); taosWUnLockLatch(&sGeosPool.lock); return NULL; } } if (!taosArrayPush(sGeosPool.poolArray, &sGeosPool.pool)) { - taosMemoryFree(sGeosPool.pool); - sGeosPool.pool = NULL; + taosMemoryFreeClear(sGeosPool.pool); taosWUnLockLatch(&sGeosPool.lock); return NULL; } From e15bd712740de4df8bf7e5778fbb69935e4a1859 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 20 Aug 2024 10:38:00 +0800 Subject: [PATCH 038/100] fix issue --- source/util/src/tscalablebf.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 1d6ef29987..ffcfdfeaf1 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -87,6 +87,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t le pSBf->status = SBF_INVALID; if (code == TSDB_CODE_OUT_OF_BUFFER) { code = TSDB_CODE_SUCCESS; + return code; } QUERY_CHECK_CODE(code, lino, _error); } @@ -126,6 +127,8 @@ int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int3 pSBf->status = SBF_INVALID; if (code == TSDB_CODE_OUT_OF_BUFFER) { code = TSDB_CODE_SUCCESS; + (*winRes) = TSDB_CODE_FAILED; + goto _end; } QUERY_CHECK_CODE(code, lino, _end); } From 820661f192d61f5188bca14539cb33890351a439 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Aug 2024 10:43:28 +0800 Subject: [PATCH 039/100] enh: remove some asserts --- include/util/tcoding.h | 3 -- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 36 ++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbUtil2.c | 2 +- source/dnode/vnode/src/vnd/vnodeAsync.c | 15 ++------- source/dnode/vnode/src/vnd/vnodeCommit.c | 3 +- source/dnode/vnode/src/vnd/vnodeHash.c | 1 - source/dnode/vnode/src/vnd/vnodeOpen.c | 4 ++- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 1040adf431..b4f62d349c 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -213,7 +213,6 @@ static FORCE_INLINE int32_t taosEncodeVariantU16(void **buf, uint16_t value) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (uint8_t)(value | ENCODE_LIMIT); value >>= 7; i++; - ASSERT(i < 3); } if (buf != NULL) { @@ -261,7 +260,6 @@ static FORCE_INLINE int32_t taosEncodeVariantU32(void **buf, uint32_t value) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; i++; - ASSERT(i < 5); } if (buf != NULL) { @@ -309,7 +307,6 @@ static FORCE_INLINE int32_t taosEncodeVariantU64(void **buf, uint64_t value) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (uint8_t)(value | ENCODE_LIMIT); value >>= 7; i++; - ASSERT(i < 10); } if (buf != NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index e3d7f9d45f..0bd00a100c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -61,7 +61,9 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con // // open each segment reader int64_t offset = config->file->size - sizeof(SSttFooter); - ASSERT(offset >= TSDB_FHDR_SIZE); + if (offset < TSDB_FHDR_SIZE) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey; @@ -115,7 +117,9 @@ int32_t tsdbSttFileReaderClose(SSttFileReader **reader) { int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray **statisBlkArray) { if (!reader->ctx->statisBlkLoaded) { if (reader->footer->statisBlkPtr->size > 0) { - ASSERT(reader->footer->statisBlkPtr->size % sizeof(SStatisBlk) == 0); + if (reader->footer->statisBlkPtr->size % sizeof(SStatisBlk) != 0) { + return TSDB_CODE_FILE_CORRUPTED; + } int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk); void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size); @@ -147,7 +151,9 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray * int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tombBlkArray) { if (!reader->ctx->tombBlkLoaded) { if (reader->footer->tombBlkPtr->size > 0) { - ASSERT(reader->footer->tombBlkPtr->size % sizeof(STombBlk) == 0); + if (reader->footer->tombBlkPtr->size % sizeof(STombBlk) != 0) { + return TSDB_CODE_FILE_CORRUPTED; + } int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk); void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size); @@ -179,7 +185,9 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBlkArray) { if (!reader->ctx->sttBlkLoaded) { if (reader->footer->sttBlkPtr->size > 0) { - ASSERT(reader->footer->sttBlkPtr->size % sizeof(SSttBlk) == 0); + if (reader->footer->sttBlkPtr->size % sizeof(SSttBlk) != 0) { + return TSDB_CODE_FILE_CORRUPTED; + } int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk); void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size); @@ -256,7 +264,9 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit); - ASSERT(hdr.delimiter == TSDB_FILE_DLMT); + if (hdr.delimiter != TSDB_FILE_DLMT) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } // set data container tBlockDataReset(bData); @@ -266,7 +276,9 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * // key part TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit); - ASSERT(br.offset == buffer0->size); + if (br.offset != buffer0->size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } bool loadExtra = false; for (int i = 0; i < ncid; i++) { @@ -376,7 +388,10 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk br.offset += tombBlk->size[i]; } - ASSERT(br.offset == tombBlk->dp->size); + if (br.offset != tombBlk->dp->size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } + _exit: if (code) { tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, @@ -444,7 +459,9 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta } } - ASSERT(br.offset == buffer0->size); + if (br.offset != buffer0->size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } _exit: if (code) { @@ -814,8 +831,6 @@ _exit: } static void tsdbSttFWriterDoClose(SSttFileWriter *writer) { - ASSERT(writer->fd == NULL); - for (int32_t i = 0; i < ARRAY_SIZE(writer->local); ++i) { tBufferDestroy(writer->local + i); } @@ -854,7 +869,6 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o tsdbCloseFile(&writer->fd); - ASSERT(writer->file->size > 0); STFileOp op = (STFileOp){ .optype = TSDB_FOP_CREATE, .fid = writer->config->fid, diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.c b/source/dnode/vnode/src/tsdb/tsdbUtil2.c index 7ada3085b1..e13e520cbf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil2.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.c @@ -171,7 +171,7 @@ static int32_t tStatisBlockUpdate(STbStatisBlock *block, SRowInfo *row) { TAOS_CHECK_RETURN(tBufferPutAt(&block->counts, (block->numOfRecords - 1) * sizeof(record.count), &record.count, sizeof(record.count))); } else { - ASSERT(0); + return TSDB_CODE_INVALID_PARA; } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 2ddd3c9d3e..1208b06337 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -165,9 +165,7 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { } ret = vHashDrop(async->taskTable, task); - if (ret != 0) { - ASSERT(0); - } + TAOS_UNUSED(ret); async->numTasks--; if (task->numWait == 0) { @@ -403,7 +401,6 @@ static int32_t vnodeAsyncDestroy(SVAsync **async) { } (void)taosThreadJoin((*async)->workers[i].thread, NULL); - ASSERT((*async)->workers[i].state == EVA_WORKER_STATE_STOP); (*async)->workers[i].state = EVA_WORKER_STATE_UINIT; } @@ -413,18 +410,11 @@ static int32_t vnodeAsyncDestroy(SVAsync **async) { channel->prev->next = channel->next; int32_t ret = vHashDrop((*async)->channelTable, channel); - if (ret) { - ASSERT(0); - } + TAOS_UNUSED(ret); (*async)->numChannels--; taosMemoryFree(channel); } - ASSERT((*async)->numLaunchWorkers == 0); - ASSERT((*async)->numIdleWorkers == 0); - ASSERT((*async)->numChannels == 0); - ASSERT((*async)->numTasks == 0); - (void)taosThreadMutexDestroy(&(*async)->mutex); (void)taosThreadCondDestroy(&(*async)->hasTask); @@ -438,7 +428,6 @@ static int32_t vnodeAsyncDestroy(SVAsync **async) { static int32_t vnodeAsyncLaunchWorker(SVAsync *async) { for (int32_t i = 0; i < async->numWorkers; i++) { - ASSERT(async->workers[i].state != EVA_WORKER_STATE_IDLE); if (async->workers[i].state == EVA_WORKER_STATE_ACTIVE) { continue; } else if (async->workers[i].state == EVA_WORKER_STATE_STOP) { diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 8fcbe49f9a..70b40e8d0b 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -302,7 +302,6 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); (void)taosThreadMutexLock(&pVnode->mutex); - ASSERT(pVnode->onCommit == NULL); pVnode->onCommit = pVnode->inUse; pVnode->inUse = NULL; (void)taosThreadMutexUnlock(&pVnode->mutex); @@ -339,7 +338,7 @@ static void vnodeReturnBufPool(SVnode *pVnode) { pVnode->recycleTail = pPool; } } else { - ASSERT(0); + vError("vgId:%d, buffer pool %p of id %d nRef:%d", TD_VID(pVnode), pPool, pPool->id, nRef); } (void)taosThreadMutexUnlock(&pVnode->mutex); diff --git a/source/dnode/vnode/src/vnd/vnodeHash.c b/source/dnode/vnode/src/vnd/vnodeHash.c index 00fc2dfc00..96ad759a90 100644 --- a/source/dnode/vnode/src/vnd/vnodeHash.c +++ b/source/dnode/vnode/src/vnd/vnodeHash.c @@ -77,7 +77,6 @@ int32_t vHashDestroy(SVHashTable** ht) { } if (*ht) { - ASSERT((*ht)->numEntries == 0); taosMemoryFree((*ht)->buckets); taosMemoryFree(*ht); (*ht) = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ed008d4f88..989faa3a0f 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -558,7 +558,9 @@ void vnodeClose(SVnode *pVnode) { // start the sync timer after the queue is ready int32_t vnodeStart(SVnode *pVnode) { - ASSERT(pVnode); + if (pVnode == NULL) { + return TSDB_CODE_INVALID_PARA; + } return vnodeSyncStart(pVnode); } From 7073f204ac4594d7271efc6f8e82594bf2bbc8f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Aug 2024 11:00:02 +0800 Subject: [PATCH 040/100] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 160ff2e13c..2288e8bbce 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -903,6 +903,7 @@ int32_t tLDataIterNextRow(SLDataIter *pIter, const char *idStr, bool* hasNext) { pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); _exit: + tsdbError("failed to exec stt-file nextIter, lino:%d, code:%s, %s", lino, tstrerror(code), idStr); *hasNext = (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL) && (pBlockData != NULL); return code; } From bbdd1f655b62698733e99f0f1188d9761c68b70f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Aug 2024 11:04:44 +0800 Subject: [PATCH 041/100] fix(stream): send kill checkpoint trans to mnode when trying to close vnode. --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/stream/src/streamMeta.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a70a04f23d..5fc550da32 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1204,11 +1204,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) } if (req.mndTrigger) { - qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ", pTask->id.idStr, + tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ", pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId); } else { const char* pPrevStatus = streamTaskGetStatusStr(streamTaskGetPrevStatus(pTask)); - qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 + tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d after transfer-state, prev status:%s", pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a9976760b6..b6be1d04ca 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1260,7 +1260,9 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) pMeta->stage = stage; // mark the sign to send msg before close all tasks - if ((!isLeader) && (pMeta->role == NODE_ROLE_LEADER)) { + // 1. for leader vnode, always send msg before closing + // 2. for follower vnode, if it's is changed from leader, also sending msg before closing. + if (pMeta->role == NODE_ROLE_LEADER) { pMeta->sendMsgBeforeClosing = true; } From bdabe66aaba6a3b8be99cfea7e4b6492cc684404 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 20 Aug 2024 03:48:42 +0000 Subject: [PATCH 042/100] fix/TD-31520-use-global-counter --- include/libs/monitor/monitor.h | 2 ++ source/dnode/vnode/src/inc/vnodeInt.h | 15 ------------ source/dnode/vnode/src/vnd/vnodeOpen.c | 18 -------------- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 ++++-- source/libs/monitor/src/monMain.c | 34 ++++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 35 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 6007d52bb4..832926bf98 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -30,6 +30,8 @@ extern "C" { #define MON_VER_LEN 12 #define MON_LOG_LEN 1024 +#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" + typedef struct { int64_t ts; ELogLevel level; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index e81b858d0e..6f562cef4a 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -101,20 +101,6 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count" - -#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" -#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" -#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" -#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" -#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" -#define VNODE_METRIC_TAG_NAME_USERNAME "username" -#define VNODE_METRIC_TAG_NAME_RESULT "result" - -#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" -// #define VNODE_METRIC_TAG_VALUE_INSERT "insert" -// #define VNODE_METRIC_TAG_VALUE_DELETE "delete" - // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); struct SQueryNode { @@ -452,7 +438,6 @@ typedef struct SVMonitorObj { char strClusterId[TSDB_CLUSTER_ID_LEN]; char strDnodeId[TSDB_NODE_ID_LEN]; char strVgId[TSDB_VGROUP_ID_LEN]; - taos_counter_t* insertCounter; } SVMonitorObj; typedef struct { diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ed008d4f88..264c76be3a 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -494,24 +494,6 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%" PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId); snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%" PRId32, pVnode->config.vgId); - if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) { - taos_counter_t *counter = NULL; - int32_t label_count = 7; - const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID, - VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP, - VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME, - VNODE_METRIC_TAG_NAME_RESULT}; - counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels); - vInfo("vgId:%d, new metric:%p", TD_VID(pVnode), counter); - if (taos_collector_registry_register_metric(counter) == 1) { - (void)taos_counter_destroy(counter); - counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); - vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter); - } - pVnode->monitor.insertCounter = counter; - vInfo("vgId:%d, succeed to set metric:%p", TD_VID(pVnode), counter); - } - return pVnode; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index dc84b73c10..4ad6ae4744 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,6 +15,7 @@ #include "audit.h" #include "cos.h" +#include "monitor.h" #include "tencode.h" #include "tglobal.h" #include "tmsg.h" @@ -23,6 +24,8 @@ #include "vnode.h" #include "vnodeInt.h" +extern taos_counter_t *tsInsertCounter; + static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -1901,7 +1904,8 @@ _exit: (void)atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); (void)atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); - if (tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0) { + if (tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0 && + tsInsertCounter != NULL) { const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, pVnode->monitor.strDnodeId, @@ -1909,7 +1913,7 @@ _exit: pVnode->monitor.strVgId, pOriginalMsg->info.conn.user, "Success"}; - (void)taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); + (void)taos_counter_add(tsInsertCounter, pSubmitRsp->affectedRows, sample_labels); } if (code == 0) { diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 9669eac508..86704ed242 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -21,11 +21,25 @@ #include "thttp.h" #include "ttime.h" +#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count" + +#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" +#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" +#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" +#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" +#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" +#define VNODE_METRIC_TAG_NAME_USERNAME "username" +#define VNODE_METRIC_TAG_NAME_RESULT "result" + +// #define VNODE_METRIC_TAG_VALUE_INSERT "insert" +// #define VNODE_METRIC_TAG_VALUE_DELETE "delete" + SMonitor tsMonitor = {0}; char *tsMonUri = "/report"; char *tsMonFwUri = "/general-metric"; char *tsMonSlowLogUri = "/slow-sql-detail-batch"; char *tsMonFwBasicUri = "/taosd-cluster-basic"; +taos_counter_t *tsInsertCounter = NULL; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { (void)taosThreadMutexLock(&tsMonitor.lock); @@ -114,6 +128,26 @@ int32_t monInit(const SMonCfg *pCfg) { monInitMonitorFW(); + if (tsEnableMonitor && tsInsertCounter == NULL) { + taos_counter_t *counter = NULL; + int32_t label_count = 7; + const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID, + VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP, + VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME, + VNODE_METRIC_TAG_NAME_RESULT}; + counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels); + uDebug("new metric:%p", counter); + if (taos_collector_registry_register_metric(counter) == 1) { + (void)taos_counter_destroy(counter); + uError("failed to register metric:%p", counter); + } else { + tsInsertCounter = counter; + uInfo("succeed to set inserted row metric:%p", tsInsertCounter); + } + } else { + uError("failed to set insert counter, already set"); + } + return 0; } From 6470e5f6e3b3a7318ca1fce15aed9685496949f9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 20 Aug 2024 12:24:01 +0800 Subject: [PATCH 043/100] refact: refact function syncUtilNodeInfo2RaftId --- source/dnode/mnode/impl/src/mndSync.c | 2 +- source/libs/sync/src/syncMain.c | 52 +++++++++++---------------- source/libs/sync/src/syncUtil.c | 12 +++---- 3 files changed, 27 insertions(+), 39 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index cf7769b932..3c5724dde3 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -486,7 +486,7 @@ int32_t mndInitSync(SMnode *pMnode) { int32_t code = 0; (void)tsem_init(&pMgmt->syncSem, 0, 0); - pMgmt->sync = syncOpen(&syncInfo, true); + pMgmt->sync = syncOpen(&syncInfo, 1); // always check if (pMgmt->sync <= 0) { if (terrno != 0) code = terrno; mError("failed to open sync since %s", tstrerror(code)); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8b8a9e1279..ffd180ee01 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1113,37 +1113,6 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { goto _error; } - // init internal - pSyncNode->myNodeInfo = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex]; - if (!syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d, failed to determine my raft member id", pSyncNode->vgId); - goto _error; - } - - pSyncNode->arbTerm = -1; - (void)taosThreadMutexInit(&pSyncNode->arbTokenMutex, NULL); - syncUtilGenerateArbToken(pSyncNode->myNodeInfo.nodeId, pSyncInfo->vgId, pSyncNode->arbToken); - sInfo("vgId:%d, arb token:%s", pSyncNode->vgId, pSyncNode->arbToken); - - // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; - int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { - if (i != pSyncNode->raftCfg.cfg.myIndex) { - pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; - syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); - j++; - } - } - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - if (!syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d, failed to determine raft member id, peer:%d", pSyncNode->vgId, i); - goto _error; - } - } - // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; pSyncNode->totalReplicaNum = pSyncNode->raftCfg.cfg.totalReplicaNum; @@ -1155,6 +1124,27 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { } } + // init internal + pSyncNode->myNodeInfo = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex]; + pSyncNode->myRaftId = pSyncNode->replicasId[pSyncNode->raftCfg.cfg.myIndex]; + + // init peersNum, peers, peersId + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; + int32_t j = 0; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { + if (i != pSyncNode->raftCfg.cfg.myIndex) { + pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; + pSyncNode->peersId[j] = pSyncNode->replicasId[i]; + syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); + j++; + } + } + + pSyncNode->arbTerm = -1; + (void)taosThreadMutexInit(&pSyncNode->arbTokenMutex, NULL); + syncUtilGenerateArbToken(pSyncNode->myNodeInfo.nodeId, pSyncInfo->vgId, pSyncNode->arbToken); + sInfo("vgId:%d, generate arb token:%s", pSyncNode->vgId, pSyncNode->arbToken); + // init raft algorithm pSyncNode->pFsm = pSyncInfo->pFsm; pSyncInfo->pFsm = NULL; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index ca879f70d9..69abbcdea7 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -43,13 +43,11 @@ void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = 0xFFFFFFFF; - sDebug("vgId:%d, resolve sync addr from fqdn, dnode:%d cluster:%" PRId64 " fqdn:%s port:%u", vgId, pInfo->nodeId, - pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort); + sDebug("vgId:%d, resolve sync addr from fqdn, ep:%s:%u", vgId, pInfo->nodeFqdn, pInfo->nodePort); for (int32_t i = 0; i < tsResolveFQDNRetryTime; i++) { int32_t code = taosGetIpv4FromFqdn(pInfo->nodeFqdn, &ipv4); if (code) { - sError("vgId:%d, failed to resolve sync addr, dnode:%d fqdn:%s, wait one second", vgId, pInfo->nodeId, - pInfo->nodeFqdn); + sError("vgId:%d, failed to resolve sync addr, dnode:%d fqdn:%s, retry", vgId, pInfo->nodeId, pInfo->nodeFqdn); taosSsleep(1); } else { break; @@ -57,7 +55,7 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* } if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { - sError("vgId:%d, failed to resolve sync addr, fqdn:%s", vgId, pInfo->nodeFqdn); + sError("vgId:%d, failed to resolve sync addr, dnode:%d fqdn:%s", vgId, pInfo->nodeId, pInfo->nodeFqdn); terrno = TSDB_CODE_TSC_INVALID_FQDN; return false; } @@ -67,8 +65,8 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId->addr = SYNC_ADDR(pInfo); raftId->vgId = vgId; - sInfo("vgId:%d, sync addr:%" PRIu64 " is resolved, dnode:%d cluster:%" PRId64 " fqdn:%s port:%u ip:%s ipv4:%u", vgId, - raftId->addr, pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, pInfo->nodePort, ipbuf, ipv4); + sInfo("vgId:%d, sync addr:%" PRIu64 " is resolved, ep:%s:%u ip:%s ipv4:%u dnode:%d cluster:%" PRId64, vgId, + raftId->addr, pInfo->nodeFqdn, pInfo->nodePort, ipbuf, ipv4, pInfo->nodeId, pInfo->clusterId); return true; } From 54dbf92517ccd514a9e0a3c29fc4b52e65d65f4e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 20 Aug 2024 13:28:53 +0800 Subject: [PATCH 044/100] fix mem leak for max delay --- source/libs/executor/src/streamtimewindowoperator.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 5c12db1ab9..9147ff9b27 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -439,6 +439,11 @@ void destroyFlusedPos(void* pRes) { } } +void destroyFlusedppPos(void* ppRes) { + void *pRes = *(void **)ppRes; + destroyFlusedPos(pRes); +} + void clearGroupResInfo(SGroupResInfo* pGroupResInfo) { if (pGroupResInfo->freeItem) { int32_t size = taosArrayGetSize(pGroupResInfo->pRows); @@ -1920,6 +1925,7 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); + tSimpleHashSetFreeFp(pInfo->aggSup.pResultRowHashTable, destroyFlusedppPos); code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); QUERY_CHECK_CODE(code, lino, _error); @@ -5283,6 +5289,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); + tSimpleHashSetFreeFp(pInfo->aggSup.pResultRowHashTable, destroyFlusedppPos); if (pIntervalPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; From 7ac14bca97d841839bf299cdc994b512c149a110 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 20 Aug 2024 14:06:35 +0800 Subject: [PATCH 045/100] fix: memory leak of geos --- source/util/src/tgeosctx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index 99655ed7f7..82a360edd1 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -104,7 +104,8 @@ void taosGeosDestroy() { int32_t size = taosArrayGetSize(sGeosPool.poolArray); for (int32_t i = 0; i < size; ++i) { SGeosContext *pool = *(SGeosContext **)TARRAY_GET_ELEM(sGeosPool.poolArray, i); - for (int32_t j = 0; j < GEOS_POOL_CAPACITY; ++j) { + int32_t poolSize = i == size - 1 ? sGeosPool.size : GEOS_POOL_CAPACITY; + for (int32_t j = 0; j < poolSize; ++j) { destroyGeosCtx(pool + j); } taosMemoryFree(pool); From 7759e6aea41fd2cd57f27a1b5095d1e0b98c1c6e Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 20 Aug 2024 14:12:30 +0800 Subject: [PATCH 046/100] fix:[TD-31558] Handle error of tDecodeSSchemaWrapper. --- source/dnode/vnode/src/meta/metaQuery.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index bee4727260..3b10182c90 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -694,9 +694,13 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv SSchemaWrapper *pSchemaWrapper = &schema; tDecoderInit(&dc, pData, nData); - (void)tDecodeSSchemaWrapper(&dc, pSchemaWrapper); + code = tDecodeSSchemaWrapper(&dc, pSchemaWrapper); tDecoderClear(&dc); tdbFree(pData); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(pSchemaWrapper->pSchema); + goto _exit; + } // convert STSchema *pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); From b70ad8d3bb5cd2030051c8254c478754519444ed Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 20 Aug 2024 14:13:06 +0800 Subject: [PATCH 047/100] fix: return code check issue --- source/libs/executor/src/dynqueryctrloperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 02932cd278..772222f9ed 100644 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -320,7 +320,7 @@ static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initPara return code; } (*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES); - if (NULL == *ppRes) { + if (NULL == (*ppRes)->pChildren) { code = terrno; freeOperatorParam(pChild0, OP_GET_PARAM); freeOperatorParam(pChild1, OP_GET_PARAM); From e20c36c02202d5d16595ef4a7d870a83f1e2bf06 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Aug 2024 14:47:05 +0800 Subject: [PATCH 048/100] enh: clear some useless asserts --- source/dnode/vnode/src/tsdb/tsdbCache.c | 19 ++++--- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 14 ----- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 54 ++++++++++++------- source/dnode/vnode/src/tsdb/tsdbFS.c | 25 ++++----- source/dnode/vnode/src/tsdb/tsdbFS2.c | 5 +- source/dnode/vnode/src/tsdb/tsdbFile2.c | 8 +-- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 17 ++---- source/dnode/vnode/src/tsdb/tsdbMerge.c | 16 ------ .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 34 ++++++++---- source/dnode/vnode/src/tsdb/tsdbRetention.c | 2 - source/dnode/vnode/src/tsdb/tsdbUtil.c | 11 +--- source/dnode/vnode/src/tsdb/tsdbUtil2.c | 4 +- source/dnode/vnode/src/vnd/vnodeBufPool.c | 25 ++++++--- 13 files changed, 113 insertions(+), 121 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index fb72784229..6e29ac74ed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1413,7 +1413,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr SIdxKey *idxKey = taosArrayGet(remainCols, 0); if (idxKey->key.cid != PRIMARYKEY_TIMESTAMP_COL_ID) { // ignore 'ts' loaded from cache and load it from tsdb - SLastCol* pLastCol = taosArrayGet(pLastArray, 0); + SLastCol *pLastCol = taosArrayGet(pLastArray, 0); tsdbCacheUpdateLastColToNone(pLastCol, TSDB_LAST_CACHE_NO_CACHE); SLastKey *key = &(SLastKey){.lflag = ltype, .uid = uid, .cid = PRIMARYKEY_TIMESTAMP_COL_ID}; @@ -1745,12 +1745,12 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache goto _exit; } } - if (taosArrayPush(remainCols, &(SIdxKey){i, key}) ==NULL) { + if (taosArrayPush(remainCols, &(SIdxKey){i, key}) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } bool ignoreRocks = pLastCol ? (pLastCol->cacheStatus == TSDB_LAST_CACHE_NO_CACHE) : false; - if (taosArrayPush(ignoreFromRocks, &ignoreRocks) ==NULL) { + if (taosArrayPush(ignoreFromRocks, &ignoreRocks) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -1802,12 +1802,12 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache } _exit: - if (remainCols) { - taosArrayDestroy(remainCols); - } - if (ignoreFromRocks) { - taosArrayDestroy(ignoreFromRocks); - } + if (remainCols) { + taosArrayDestroy(remainCols); + } + if (ignoreFromRocks) { + taosArrayDestroy(ignoreFromRocks); + } TAOS_RETURN(code); } @@ -2653,7 +2653,6 @@ static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarli TAOS_RETURN(code); } default: - ASSERT(0); break; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 3c407b31cf..d920e289e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -208,8 +208,6 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { int32_t code = 0; int32_t lino = 0; - ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0); - if (committer->ctx->info->fset == NULL // || committer->sttTrigger > 1 // || TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0 // @@ -264,11 +262,6 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { int32_t code = 0; int32_t lino = 0; - ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0); - ASSERT(committer->dataIterMerger == NULL); - ASSERT(TARRAY2_SIZE(committer->tombIterArray) == 0); - ASSERT(committer->tombIterMerger == NULL); - STsdbIter *iter; STsdbIterConfig config = {0}; @@ -342,10 +335,6 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; - ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0); - ASSERT(committer->dataIterMerger == NULL); - ASSERT(committer->writer == NULL); - TAOS_CHECK_GOTO(tsdbCommitOpenReader(committer), &lino, _exit); TAOS_CHECK_GOTO(tsdbCommitOpenIter(committer), &lino, _exit); TAOS_CHECK_GOTO(tsdbCommitOpenWriter(committer), &lino, _exit); @@ -641,9 +630,6 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) { ASSERT(0); } - ASSERT(committer->writer == NULL); - ASSERT(committer->dataIterMerger == NULL); - ASSERT(committer->tombIterMerger == NULL); TARRAY2_DESTROY(committer->dataIterArray, NULL); TARRAY2_DESTROY(committer->tombIterArray, NULL); TARRAY2_DESTROY(committer->sttReaderArray, NULL); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 7e7ea59a5b..51f8f15537 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -283,7 +283,9 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB } } - ASSERT(br.offset == br.buffer->size); + if (br.offset != br.buffer->size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } _exit: if (code) { @@ -313,7 +315,10 @@ int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *re // decompress SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer); TAOS_CHECK_GOTO(tBlockDataDecompress(&br, bData, assist), &lino, _exit); - ASSERT(br.offset == buffer->size); + + if (br.offset != buffer->size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } _exit: if (code) { @@ -345,7 +350,9 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit); - ASSERT(hdr.delimiter == TSDB_FILE_DLMT); + if (hdr.delimiter != TSDB_FILE_DLMT) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } tBlockDataReset(bData); bData->suid = hdr.suid; @@ -354,7 +361,9 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe // Key part TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit); - ASSERT(br.offset == buffer0->size); + if (br.offset != buffer0->size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } int extraColIdx = -1; for (int i = 0; i < ncid; i++) { @@ -526,7 +535,9 @@ int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *rec TAOS_CHECK_GOTO(tGetColumnDataAgg(&br, sma), &lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(columnDataAggArray, sma), &lino, _exit); } - ASSERT(br.offset == record->smaSize); + if (br.offset != record->smaSize) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } } _exit: @@ -661,7 +672,8 @@ struct SDataFileWriter { }; static int32_t tsdbDataFileWriterCloseAbort(SDataFileWriter *writer) { - ASSERT(0); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, __LINE__, + "not implemented"); return 0; } @@ -980,7 +992,9 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData return 0; } - ASSERT(bData->uid); + if (!bData->uid) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t lino = 0; @@ -1234,7 +1248,6 @@ static int32_t tsdbDataFileWriteTableDataEnd(SDataFileWriter *writer) { if (writer->ctx->tbHasOldData) { TAOS_CHECK_GOTO(tsdbDataFileDoWriteTableOldData(writer, NULL /* as the largest key */), &lino, _exit); - ASSERT(writer->ctx->tbHasOldData == false); } TAOS_CHECK_GOTO(tsdbDataFileDoWriteBlockData(writer, writer->blockData), &lino, _exit); @@ -1251,9 +1264,6 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA int32_t code = 0; int32_t lino = 0; - ASSERT(writer->ctx->blockDataIdx == writer->ctx->blockData->nRow); - ASSERT(writer->blockData->nRow == 0); - SMetaInfo info; bool drop = false; TABLEID tbid1[1]; @@ -1451,7 +1461,9 @@ int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFD } static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) { - ASSERT(TARRAY2_SIZE(writer->tombBlkArray) > 0); + if (TARRAY2_SIZE(writer->tombBlkArray) <= 0) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t lino = 0; @@ -1523,7 +1535,9 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom TAOS_CHECK_GOTO(tsdbDataFileDoWriteTombBlock(writer), &lino, _exit); } } else { - ASSERT(0); + tsdbError("vgId:%d duplicate tomb record, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 ", version:%" PRId64, + TD_VID(writer->config->tsdb->pVnode), writer->config->cid, record->suid, record->uid, + record->version); } } @@ -1566,7 +1580,9 @@ _exit: int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize, int32_t encryptAlgorithm, char *encryptKey) { - ASSERT(TARRAY2_SIZE(brinBlkArray) > 0); + if (TARRAY2_SIZE(brinBlkArray) <= 0) { + return TSDB_CODE_INVALID_PARA; + } ptr->offset = *fileSize; ptr->size = TARRAY2_DATA_LEN(brinBlkArray); @@ -1852,7 +1868,9 @@ int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { int32_t code = 0; int32_t lino = 0; - ASSERT(bData->uid); + if (!bData->uid) { + return TSDB_CODE_INVALID_PARA; + } if (!writer->ctx->opened) { TAOS_CHECK_GOTO(tsdbDataFileWriterDoOpen(writer), &lino, _exit); @@ -1895,7 +1913,9 @@ _exit: } int32_t tsdbDataFileFlush(SDataFileWriter *writer) { - ASSERT(writer->ctx->opened); + if (!writer->ctx->opened) { + return TSDB_CODE_INVALID_PARA; + } if (writer->blockData->nRow == 0) return 0; if (writer->ctx->tbHasOldData) return 0; @@ -1910,8 +1930,6 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) { char fname[TSDB_FILENAME_LEN]; int32_t ftype = TSDB_FTYPE_TOMB; - ASSERT(writer->files[ftype].size == 0); - int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); int32_t lcn = writer->files[ftype].lcn; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index cc89b106da..a516114c29 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -83,7 +83,10 @@ static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, STsdbFS *pFS) { } } - ASSERT(n + sizeof(TSCKSUM) == nData); + if (n + sizeof(TSCKSUM) != nData) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } _exit: return code; @@ -450,8 +453,9 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe taosMemoryFree(pHeadF); } } else { - ASSERT(pHeadF->offset == pSetNew->pHeadF->offset); - ASSERT(pHeadF->size == pSetNew->pHeadF->size); + if (pHeadF->offset != pSetNew->pHeadF->offset || pHeadF->size != pSetNew->pHeadF->size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } } // data @@ -499,7 +503,6 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe // stt if (sameDisk) { if (pSetNew->nSttF > pSetOld->nSttF) { - ASSERT(pSetNew->nSttF == pSetOld->nSttF + 1); pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -509,7 +512,6 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe pSetOld->aSttF[pSetOld->nSttF]->nRef = 1; pSetOld->nSttF++; } else if (pSetNew->nSttF < pSetOld->nSttF) { - ASSERT(pSetNew->nSttF == 1); for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { SSttFile *pSttFile = pSetOld->aSttF[iStt]; nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); @@ -548,8 +550,10 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; pSetOld->aSttF[iStt]->nRef = 1; } else { - ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size); - ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset); + if (pSetOld->aSttF[iStt]->size != pSetOld->aSttF[iStt]->size || + pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } } } } @@ -777,8 +781,6 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback) { // empty one code = tsdbSaveFSToFile(&pTsdb->fs, current); TSDB_CHECK_CODE(code, lino, _exit); - - ASSERT(!rollback); } // scan and fix FS @@ -796,7 +798,6 @@ int32_t tsdbFSClose(STsdb *pTsdb) { int32_t code = 0; if (pTsdb->fs.pDelFile) { - ASSERT(pTsdb->fs.pDelFile->nRef == 1); taosMemoryFree(pTsdb->fs.pDelFile); } @@ -804,20 +805,16 @@ int32_t tsdbFSClose(STsdb *pTsdb) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); // head - ASSERT(pSet->pHeadF->nRef == 1); taosMemoryFree(pSet->pHeadF); // data - ASSERT(pSet->pDataF->nRef == 1); taosMemoryFree(pSet->pDataF); // sma - ASSERT(pSet->pSmaF->nRef == 1); taosMemoryFree(pSet->pSmaF); // stt for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - ASSERT(pSet->aSttF[iStt]->nRef == 1); taosMemoryFree(pSet->aSttF[iStt]); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index c3b612b227..b8669055d4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -208,7 +208,9 @@ static int32_t load_fs(STsdb *pTsdb, const char *fname, TFileSetArray *arr) { /* fmtv */ item1 = cJSON_GetObjectItem(json, "fmtv"); if (cJSON_IsNumber(item1)) { - ASSERT(item1->valuedouble == 1); + if (item1->valuedouble != 1) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } } else { TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } @@ -1193,7 +1195,6 @@ int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) { ASSERT(fset != NULL); (*fset)->numWaitTask--; - ASSERT((*fset)->numWaitTask >= 0); } else { (*fset)->taskRunning = true; break; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 3d6a259254..57db838752 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -242,18 +242,18 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { int32_t tsdbTFileObjRef(STFileObj *fobj) { int32_t nRef; - (void)(void)taosThreadMutexLock(&fobj->mutex); + (void)taosThreadMutexLock(&fobj->mutex); ASSERT(fobj->ref > 0 && fobj->state == TSDB_FSTATE_LIVE); nRef = ++fobj->ref; - (void)(void)taosThreadMutexUnlock(&fobj->mutex); + (void)taosThreadMutexUnlock(&fobj->mutex); tsdbTrace("ref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); return 0; } int32_t tsdbTFileObjUnref(STFileObj *fobj) { - (void)(void)taosThreadMutexLock(&fobj->mutex); + (void)taosThreadMutexLock(&fobj->mutex); int32_t nRef = --fobj->ref; - (void)(void)taosThreadMutexUnlock(&fobj->mutex); + (void)taosThreadMutexUnlock(&fobj->mutex); ASSERT(nRef >= 0); tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); if (nRef == 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d8ec014985..79565f887b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -170,7 +170,6 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid goto _err; } - ASSERT(pPool != NULL); // do delete SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData)); if (pDelData == NULL) { @@ -183,7 +182,6 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pDelData->pNext = NULL; taosWLockLatch(&pTbData->lock); if (pTbData->pHead == NULL) { - ASSERT(pTbData->pTail == NULL); pTbData->pHead = pTbData->pTail = pDelData; } else { pTbData->pTail->pNext = pDelData; @@ -263,8 +261,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, S bool tsdbTbDataIterNext(STbDataIter *pIter) { pIter->pRow = NULL; if (pIter->backward) { - ASSERT(pIter->pNode != pIter->pTbData->sl.pTail); - if (pIter->pNode == pIter->pTbData->sl.pHead) { return false; } @@ -274,8 +270,6 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return false; } } else { - ASSERT(pIter->pNode != pIter->pTbData->sl.pHead); - if (pIter->pNode == pIter->pTbData->sl.pTail) { return false; } @@ -366,7 +360,6 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; - ASSERT(pPool != NULL); pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = terrno; @@ -516,8 +509,6 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->pTSRow->len); } else if (pRow->type == TSDBROW_COL_FMT) { pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize); - } else { - ASSERT(0); } if (pNode == NULL) { code = terrno; @@ -582,10 +573,6 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); - ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aColData[0].flag == HAS_VALUE); - // copy and construct block data SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData)); if (pBlockData == NULL) { @@ -740,7 +727,9 @@ int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode) { int32_t code = 0; int32_t nRef = atomic_fetch_add_32(&pMemTable->nRef, 1); - ASSERT(nRef > 0); + if (nRef <= 0) { + tsdbError("vgId:%d, memtable ref count is invalid, ref:%d", TD_VID(pMemTable->pTsdb->pVnode), nRef); + } (void)vnodeBufPoolRegisterQuery(pMemTable->pPool, pQNode); diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 1bbda8a249..5587f29440 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -69,13 +69,6 @@ static int32_t tsdbMergerClose(SMerger *merger) { int32_t lino = 0; SVnode *pVnode = merger->tsdb->pVnode; - ASSERT(merger->writer == NULL); - ASSERT(merger->dataIterMerger == NULL); - ASSERT(merger->tombIterMerger == NULL); - ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0); - ASSERT(TARRAY2_SIZE(merger->tombIterArr) == 0); - ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0); - // clear the merge TARRAY2_DESTROY(merger->tombIterArr, NULL); TARRAY2_DESTROY(merger->dataIterArr, NULL); @@ -156,8 +149,6 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { } } - ASSERT(merger->ctx->level > 0); - if (merger->ctx->level <= TSDB_MAX_LEVEL) { TARRAY2_FOREACH_REVERSE(merger->ctx->fset->lvlArr, lvl) { if (TARRAY2_SIZE(lvl->fobjArr) == 0) { @@ -180,8 +171,6 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { numFile = numFile - TARRAY2_SIZE(lvl->fobjArr) * pow(merger->sttTrigger, lvl->level); } - ASSERT(numFile >= 0); - // get file system operations TARRAY2_FOREACH(merger->ctx->fset->lvlArr, lvl) { if (lvl->level >= merger->ctx->level) { @@ -323,11 +312,6 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) { int32_t code = 0; int32_t lino = 0; - ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0); - ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0); - ASSERT(merger->dataIterMerger == NULL); - ASSERT(merger->writer == NULL); - TARRAY2_CLEAR(merger->fopArr, NULL); merger->ctx->tbid->suid = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8e52074807..71dce80845 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -77,7 +77,9 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { } } - ASSERT(pFD->szFile % szPage == 0); + if (pFD->szFile % szPage != 0) { + TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit); + } pFD->szFile = pFD->szFile / szPage; _exit: @@ -202,7 +204,6 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor int32_t code = 0; int32_t lino; - // ASSERT(pgno <= pFD->szFile); if (!pFD->pFD) { code = tsdbOpenFileImpl(pFD); TSDB_CHECK_CODE(code, lino, _exit); @@ -317,8 +318,9 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6 int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); int64_t bOffset = fOffset % pFD->szPage; - // ASSERT(pgno && pgno <= pFD->szFile); - ASSERT(bOffset < szPgCont); + if (bOffset < szPgCont) { + TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit); + } while (n < size) { if (pFD->pgno != pgno) { @@ -417,7 +419,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t bOffset = fOffset % pFD->szPage; - ASSERT(bOffset < szPgCont); + if (bOffset >= szPgCont) { + TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit); + } // 1, find pgnoStart & pgnoEnd to fetch from s3, if all pgs are local, no need to fetch // 2, fetch pgnoStart ~ pgnoEnd from s3 @@ -690,7 +694,9 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } } - ASSERT(n == size); + if (n != size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } _exit: if (code) { @@ -731,7 +737,9 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) { TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } } - ASSERT(n == size); + if (n != size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } _exit: if (code) { @@ -760,7 +768,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m int32_t n; code = tGetMapData(pReader->aBuf[0], mDataBlk, &n); if (code) goto _exit; - ASSERT(n == size); + if (n != size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } _exit: if (code) { @@ -861,7 +871,9 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa } } - ASSERT(n == size); + if (n != size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } _exit: if (code) { @@ -901,7 +913,9 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { } } - ASSERT(n == size); + if (n != size) { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index bdd174a8f8..96b752104d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -403,8 +403,6 @@ static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3Kee nowSec = nowSec * 1000000l; } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { nowSec = nowSec * 1000000000l; - } else { - ASSERT(0); } nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision]; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 667d7ffb7c..9e3accb469 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -106,7 +106,6 @@ _exit: #endif void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *)) { - ASSERT(idx >= 0 && idx < pMapData->nItem); (void)tGetItemFn(pMapData->pData + pMapData->aOffset[idx], pItem); } @@ -628,8 +627,6 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); } } - } else { - ASSERT(0); } } @@ -697,8 +694,6 @@ int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) if (code) return code; } else if (pRow->type == TSDBROW_COL_FMT) { pIter->iColData = 0; - } else { - ASSERT(0); } return 0; @@ -730,8 +725,8 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { return NULL; } } else { - ASSERT(0); - return NULL; // suppress error report by compiler + tsdbError("invalid row type:%d", pIter->pRow->type); + return NULL; } } @@ -1745,8 +1740,6 @@ int32_t tBlockDataDecompressColData(const SDiskDataHdr *hdr, const SBlockCol *bl code = tBlockDataAddColData(blockData, blockCol->cid, blockCol->type, blockCol->cflag, &colData); TSDB_CHECK_CODE(code, lino, _exit); - // ASSERT(blockCol->flag != HAS_NONE); - SColDataCompressInfo info = { .cmprAlg = blockCol->alg, .columnFlag = blockCol->cflag, diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.c b/source/dnode/vnode/src/tsdb/tsdbUtil2.c index e13e520cbf..d87a05bcf4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil2.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.c @@ -276,7 +276,9 @@ int32_t tBrinBlockClear(SBrinBlock *brinBlock) { } int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record) { - ASSERT(record->firstKey.key.numOfPKs == record->lastKey.key.numOfPKs); + if (record->firstKey.key.numOfPKs != record->lastKey.key.numOfPKs) { + return TSDB_CODE_INVALID_PARA; + } if (brinBlock->numOfRecords == 0) { // the first row brinBlock->numOfPKs = record->firstKey.key.numOfPKs; diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index cbd6fbbe52..2071125175 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -103,17 +103,18 @@ int vnodeCloseBufPool(SVnode *pVnode) { } void vnodeBufPoolReset(SVBufPool *pPool) { - ASSERT(pPool->nQuery == 0); + if (pPool->nQuery != 0) { + vError("vgId:%d, buffer pool %p of id %d has %d queries, reset it may cause problem", TD_VID(pPool->pVnode), pPool, + pPool->id, pPool->nQuery); + } + for (SVBufPoolNode *pNode = pPool->pTail; pNode->prev; pNode = pPool->pTail) { - ASSERT(pNode->pnext == &pPool->pTail); pNode->prev->pnext = &pPool->pTail; pPool->pTail = pNode->prev; pPool->size = pPool->size - sizeof(*pNode) - pNode->size; taosMemoryFree(pNode); } - ASSERT(pPool->size == pPool->ptr - pPool->node.data); - pPool->size = 0; pPool->ptr = pPool->node.data; } @@ -123,7 +124,11 @@ void *vnodeBufPoolMallocAligned(SVBufPool *pPool, int size) { void *p = NULL; uint8_t *ptr = NULL; int paddingLen = 0; - ASSERT(pPool != NULL); + + if (pPool == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } if (pPool->lock) taosThreadSpinLock(pPool->lock); @@ -162,7 +167,11 @@ void *vnodeBufPoolMallocAligned(SVBufPool *pPool, int size) { void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p = NULL; - ASSERT(pPool != NULL); + + if (pPool == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } if (pPool->lock) taosThreadSpinLock(pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { @@ -207,7 +216,9 @@ void vnodeBufPoolFree(SVBufPool *pPool, void *p) { void vnodeBufPoolRef(SVBufPool *pPool) { int32_t nRef = atomic_fetch_add_32(&pPool->nRef, 1); - ASSERT(nRef > 0); + if (nRef <= 0) { + vError("vgId:%d, buffer pool %p of id %d is referenced by %d", TD_VID(pPool->pVnode), pPool, pPool->id, nRef); + } } void vnodeBufPoolAddToFreeList(SVBufPool *pPool) { From 557dbd883880fac40d815ec6b3f546b6596fdfbd Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 20 Aug 2024 14:53:28 +0800 Subject: [PATCH 049/100] fix: data sink memory leak --- source/libs/executor/src/dataDispatcher.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3964422411..f616cb05da 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -318,6 +318,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->sink.fGetCacheSize = getCacheSize; dispatcher->pManager = pManager; + pManager = NULL; dispatcher->pSchema = pDataSink->pInputDataBlockDesc; dispatcher->status = DS_BUF_EMPTY; dispatcher->queryEnd = false; @@ -336,6 +337,9 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD return TSDB_CODE_SUCCESS; _return: + + taosMemoryFree(pManager); + if (dispatcher) { dsDestroyDataSinker(dispatcher); } From 99bdcb3f580946808bb3923f765db62a99a63429 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 20 Aug 2024 15:02:09 +0800 Subject: [PATCH 050/100] enh(stream):remove assert of stream operator --- include/util/tutil.h | 7 ++ source/libs/executor/src/executor.c | 64 +++++++------ source/libs/executor/src/executorInt.c | 23 +++-- source/libs/executor/src/filloperator.c | 52 ---------- source/libs/executor/src/scanoperator.c | 56 +++++------ source/libs/executor/src/sortoperator.c | 7 +- .../executor/src/streamcountwindowoperator.c | 8 +- .../executor/src/streameventwindowoperator.c | 7 +- source/libs/executor/src/streamfilloperator.c | 14 ++- .../executor/src/streamtimewindowoperator.c | 53 ++++++----- source/libs/executor/src/timesliceoperator.c | 3 - source/libs/executor/src/timewindowoperator.c | 94 ++++++++++++------- source/libs/stream/src/streamSessionState.c | 3 +- source/libs/stream/src/streamUpdate.c | 8 +- source/libs/stream/src/tstreamFileState.c | 14 +-- 15 files changed, 208 insertions(+), 205 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index 1ee3bb0e83..6c7517f630 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -139,6 +139,13 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define QUERY_CHECK_CODE TSDB_CHECK_CODE +#define QUERY_CHECK_CONDITION(condition, CODE, LINO, LABEL, ERRNO) \ + if (!condition) { \ + (CODE) = (ERRNO); \ + (LINO) = __LINE__; \ + goto LABEL; \ + } + #define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \ if ((ptr) == NULL) { \ (CODE) = (ERRNO); \ diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b1c9207ab7..833371cfc5 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -158,7 +158,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SStreamScanInfo* pInfo = pOperator->info; qDebug("s-task:%s in this batch, %d blocks need to be processed", id, (int32_t)numOfBlocks); - ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); + QUERY_CHECK_CONDITION((pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); if (type == STREAM_INPUT__MERGED_SUBMIT) { for (int32_t i = 0; i < numOfBlocks; i++) { @@ -189,7 +190,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pInfo->blockType = STREAM_INPUT__CHECKPOINT; } else { - ASSERT(0); + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); } return TSDB_CODE_SUCCESS; @@ -543,7 +545,9 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table int32_t* tversion, int32_t idx, bool* tbGet) { *tbGet = false; - ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL); + if (tinfo == NULL || dbName == NULL || tableName == NULL) { + return TSDB_CODE_INVALID_PARA; + } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; if (taosArrayGetSize(pTaskInfo->schemaInfos) <= idx) { @@ -722,7 +726,8 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo blockIndex += 1; current += p->info.rows; - ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT); + QUERY_CHECK_CONDITION((p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); void* tmp = taosArrayPush(pResList, &p); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); @@ -987,15 +992,17 @@ void qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { *scanner = pOperator->info; break; } else { - ASSERT(pOperator->numOfDownstream == 1); pOperator = pOperator->pDownstream[0]; } } } int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange* pVerRange, STimeWindow* pWindow) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + QUERY_CHECK_CONDITION((pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo; @@ -1007,12 +1014,19 @@ int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRan ", window:%" PRId64 " - %" PRId64, GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey, pWindow->ekey); - return 0; +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange* pVerRange, STimeWindow* pWindow) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + QUERY_CHECK_CONDITION((pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo; @@ -1024,14 +1038,26 @@ int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRan "-%" PRId64, GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey, pWindow->ekey); - return 0; +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t qStreamRecoverFinish(qTaskInfo_t tinfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + QUERY_CHECK_CONDITION((pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; - return 0; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { @@ -1046,9 +1072,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; STimeWindowAggSupp* pSup = &pInfo->twAggSup; - ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - qInfo("save stream param for interval: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); pSup->calTriggerSaved = pSup->calTrigger; @@ -1063,9 +1086,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; STimeWindowAggSupp* pSup = &pInfo->twAggSup; - ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - qInfo("save stream param for session: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); pSup->calTriggerSaved = pSup->calTrigger; @@ -1078,9 +1098,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; STimeWindowAggSupp* pSup = &pInfo->twAggSup; - ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); pSup->calTriggerSaved = pSup->calTrigger; @@ -1093,9 +1110,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { SStreamEventAggOperatorInfo* pInfo = pOperator->info; STimeWindowAggSupp* pSup = &pInfo->twAggSup; - ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); pSup->calTriggerSaved = pSup->calTrigger; @@ -1108,9 +1122,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { SStreamCountAggOperatorInfo* pInfo = pOperator->info; STimeWindowAggSupp* pSup = &pInfo->twAggSup; - ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); pSup->calTriggerSaved = pSup->calTrigger; @@ -1126,7 +1137,6 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { if (pOperator->numOfDownstream != 1 || pOperator->pDownstream[0] == NULL) { if (pOperator->numOfDownstream > 1) { qError("unexpected stream, multiple downstream"); - ASSERT(0); return -1; } return 0; diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index cbb124b9e0..031ffbb50e 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -163,7 +163,11 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR return NULL; } - ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); + if (pResult->pageId != p1->pageId || pResult->offset != p1->offset) { + terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + pTaskInfo->code = terrno; + return NULL; + } } } else { // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the @@ -176,7 +180,11 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR return NULL; } - ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); + if (pResult->pageId != p1->pageId || pResult->offset != p1->offset) { + terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + pTaskInfo->code = terrno; + return NULL; + } } } @@ -324,6 +332,7 @@ _end: static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SqlFunctionCtx* pCtx = pExprSup->pCtx; for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) { @@ -363,7 +372,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int if (hasPk && (j == pkParamIdx)) { pInput->pPrimaryKey = pInput->pData[j]; } - ASSERT(pInput->pData[j] != NULL); + QUERY_CHECK_CONDITION((pInput->pData[j] != NULL), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { // todo avoid case: top(k, 12), 12 is the value parameter. // sum(11), 11 is also the value parameter. @@ -374,14 +383,16 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int pInput->blankFill = pBlock->info.blankFill; code = doCreateConstantValColumnInfo(pInput, pFuncParam, j, pBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + QUERY_CHECK_CODE(code, lino, _end); } } } } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 5ece57cad1..fe9d0d3cf0 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -55,7 +55,6 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; -static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static int32_t fillResetPrevForNewGroup(SFillInfo* pFillInfo); @@ -168,56 +167,6 @@ _end: return code; } -// todo refactor: decide the start key according to the query time range. -static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order) { - if (order == TSDB_ORDER_ASC) { - int64_t skey = pBlock->info.window.skey; - if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the - ASSERT(taosFillNotStarted(pInfo->pFillInfo)); - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); - } else if (pInfo->pFillInfo->start < skey) { - int64_t t = skey; - SInterval* pInterval = &pInfo->pFillInfo->interval; - - while (1) { - int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (prev <= pInfo->pFillInfo->start) { - t = prev; - break; - } - t = prev; - } - - // todo time window chosen problem: t or prev value? - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); - } - } else { - int64_t ekey = pBlock->info.window.ekey; - if (ekey > pInfo->pFillInfo->start) { - ASSERT(taosFillNotStarted(pInfo->pFillInfo)); - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); - } else if (ekey < pInfo->pFillInfo->start) { - int64_t t = ekey; - SInterval* pInterval = &pInfo->pFillInfo->interval; - int64_t prev = t; - while (1) { - int64_t next = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (next >= pInfo->pFillInfo->start) { - prev = t; - t = next; - break; - } - prev = t; - t = next; - } - - // todo time window chosen problem: t or next value? - if (t > pInfo->pFillInfo->start) t = prev; - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); - } - } -} - static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; @@ -602,7 +551,6 @@ static void reviseFillStartAndEndKey(SFillOperatorInfo* pInfo, int32_t order) { } pInfo->win.ekey = ekey; } else { - assert(order == TSDB_ORDER_DESC); skey = taosTimeTruncate(pInfo->win.skey, &pInfo->pFillInfo->interval); next = skey; while (next < pInfo->win.skey) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b3655e16f2..04aaa3c210 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1241,14 +1241,11 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { taosRUnLockLatch(&pTaskInfo->lock); QUERY_CHECK_CODE(code, lino, _end); - ASSERT(pInfo->base.dataReader == NULL); + QUERY_CHECK_CONDITION((pInfo->base.dataReader == NULL), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), &pInfo->pIgnoreTables); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - T_LONG_JMP(pTaskInfo->env, code); - } + QUERY_CHECK_CODE(code, lino, _end); if (pInfo->filesetDelimited) { pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader); @@ -1590,7 +1587,6 @@ static bool isCountWindow(SStreamScanInfo* pInfo) { static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex); uint64_t* groupCol = (uint64_t*)pColInfo->pData; - ASSERT(rowIndex < pBlock->info.rows); pInfo->groupId = groupCol[rowIndex]; } @@ -1659,9 +1655,8 @@ _end: bool comparePrimaryKey(SColumnInfoData* pCol, int32_t rowId, void* pVal) { // coverity scan - ASSERTS(pVal != NULL, "pVal should not be NULL"); - if (!pVal) { - qError("failed to compare primary key, since primary key is null"); + if (!pVal || !pCol) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_INVALID_PARA)); return false; } void* pData = colDataGetData(pCol, rowId); @@ -1738,7 +1733,6 @@ static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ goto _end; } - ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startData = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -1770,27 +1764,22 @@ static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ win.skey = TMIN(win.skey, startData[*pRowIndex]); continue; } - - ASSERT(!(win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) || - !(isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0))); break; } STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; // coverity scan - ASSERTS(pInfo->pUpdateInfo != NULL, "Failed to set data version, since pInfo->pUpdateInfo is NULL"); - if (pInfo->pUpdateInfo) { - qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, - pInfo->pUpdateInfo->maxDataVersion); - resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); + if (pInfo->pUpdateInfo == NULL){ + qError("Failed to set data version, since pInfo->pUpdateInfo is NULL"); + return; } + qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, + pInfo->pUpdateInfo->maxDataVersion); + resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; if (pRes) { (*pRes) = true; } - -_end: - qTrace("%s success", __func__); } static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval, @@ -2198,7 +2187,6 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS } uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; - ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; int64_t ver = pSrcBlock->info.version - 1; @@ -2296,7 +2284,6 @@ static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData; - ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; int64_t ver = pSrcBlock->info.version - 1; @@ -2357,7 +2344,6 @@ static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock pSrcPkCol = taosArrayGet(pSrcBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX); } - ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; int64_t ver = pSrcBlock->info.version - 1; @@ -2477,7 +2463,6 @@ static int32_t checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBl QUERY_CHECK_CODE(code, lino, _end); } SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); - ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; SColumnInfoData* pPkColDataInfo = NULL; if (hasPrimaryKeyCol(pInfo)) { @@ -2554,7 +2539,7 @@ static int32_t doBlockDataWindowFilter(SSDataBlock* pBlock, int32_t tsIndex, STi if (pWindow->skey != INT64_MIN) { qDebug("%s filter for additional history window, skey:%" PRId64, id, pWindow->skey); - ASSERT(pCol->pData != NULL); + QUERY_CHECK_NULL(pCol->pData, code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); for (int32_t i = 0; i < pBlock->info.rows; ++i) { int64_t* ts = (int64_t*)colDataGetData(pCol, i); p[i] = (*ts >= pWindow->skey); @@ -2603,7 +2588,8 @@ static int32_t doBlockDataPrimaryKeyFilter(SSDataBlock* pBlock, STqOffsetVal* of SColumnInfoData* pColPk = taosArrayGet(pBlock->pDataBlock, 1); qDebug("doBlockDataWindowFilter primary key, ts:%" PRId64 " %" PRId64, offset->ts, offset->primaryKey.val); - ASSERT(pColPk->info.type == offset->primaryKey.type); + QUERY_CHECK_CONDITION((pColPk->info.type == offset->primaryKey.type), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); __compar_fn_t func = getComparFunc(pColPk->info.type, 0); for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -3945,7 +3931,9 @@ void streamScanReloadState(SOperatorInfo* pOperator) { pInfo->stateStore.windowSBfDelete(pInfo->pUpdateInfo, 1); code = pInfo->stateStore.windowSBfAdd(pInfo->pUpdateInfo, 1); QUERY_CHECK_CODE(code, lino, _end); - ASSERT(pInfo->pUpdateInfo->minTS > pUpInfo->minTS); + + QUERY_CHECK_CONDITION((pInfo->pUpdateInfo->minTS > pUpInfo->minTS), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pUpInfo->maxDataVersion); SHashObj* curMap = pInfo->pUpdateInfo->pMap; void* pIte = taosHashIterate(curMap, NULL); @@ -4105,12 +4093,11 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* } if (pHandle->initTqReader) { - ASSERT(pHandle->tqReader == NULL); pInfo->tqReader = pAPI->tqReaderFn.tqReaderOpen(pHandle->vnode); - ASSERT(pInfo->tqReader); + QUERY_CHECK_NULL(pInfo->tqReader, code, lino, _error, terrno); } else { - ASSERT(pHandle->tqReader); pInfo->tqReader = pHandle->tqReader; + QUERY_CHECK_NULL(pInfo->tqReader, code, lino, _error, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } pInfo->pUpdateInfo = NULL; @@ -5887,7 +5874,10 @@ void destroyTableMergeScanOperatorInfo(void* param) { } int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { - ASSERT(pOptr != NULL); + if (pOptr == NULL) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_INVALID_PARA)); + return TSDB_CODE_INVALID_PARA; + } // TODO: merge these two info into one struct STableMergeScanExecInfo* execInfo = taosMemoryCalloc(1, sizeof(STableMergeScanExecInfo)); if (!execInfo) { @@ -6200,7 +6190,7 @@ int32_t fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, ch int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pSupp->dbNameSlotId != -1) { - ASSERT(strlen(dbName)); + QUERY_CHECK_CONDITION((strlen(dbName) > 0), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); QUERY_CHECK_NULL(colInfoData, code, lino, _end, terrno); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 36f9ac0954..cb15c3c836 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -735,7 +735,12 @@ int32_t doGroupSort(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { } // beginSortGroup would fetch all child blocks of pInfo->currGroupId; - ASSERT(pInfo->childOpStatus != CHILD_OP_SAME_GROUP); + if (pInfo->childOpStatus == CHILD_OP_SAME_GROUP) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + pOperator->pTaskInfo->code = code; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + T_LONG_JMP(pOperator->pTaskInfo->env, code); + } code = getGroupSortedBlockData(pInfo->pCurrSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pInfo, &pBlock); if (pBlock != NULL && (code == 0)) { diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 44a383772d..b4f8d6837a 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -88,7 +88,7 @@ int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t group winCode = TSDB_CODE_FAILED; } else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) { - ASSERT(pBuffInfo->pCur); + QUERY_CHECK_NULL(pBuffInfo->pCur, code, lino, _end, terrno); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, (void**)&pCurWin->winInfo.pStatePos, &size); @@ -345,7 +345,6 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (slidingRows + winRows > pAggSup->windowSliding) { buffInfo.winBuffOp = CREATE_NEW_WINDOW; winRows = pAggSup->windowSliding - slidingRows; - ASSERT(i >= 0); } } else { buffInfo.winBuffOp = MOVE_NEXT_WINDOW; @@ -690,7 +689,10 @@ static int32_t doStreamCountAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRe QUERY_CHECK_CODE(code, lino, _end); continue; } else { - ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } } if (pInfo->scalarSupp.pExprInfo != NULL) { diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index ff1ff579fc..38a813bf33 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -378,7 +378,6 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild, &winRows); QUERY_CHECK_CODE(code, lino, _end); - ASSERT(winRows >= 1); if (rebuild) { uint64_t uid = 0; code = appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, @@ -660,7 +659,10 @@ static int32_t doStreamEventAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRe QUERY_CHECK_CODE(code, lino, _end); continue; } else { - ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -779,7 +781,6 @@ void streamEventReloadState(SOperatorInfo* pOperator) { int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== event window operator reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; - ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY)); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 75b15dbea4..fac1cf48c7 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -331,7 +331,7 @@ void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillS pFillInfo->pLinearInfo->winIndex = 0; } break; default: - ASSERT(0); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); break; } } @@ -419,7 +419,6 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS pFillSup->next.pRowVal = pFillSup->cur.pRowVal; pFillInfo->preRowKey = INT64_MIN; } else { - ASSERT(hasNextWindow(pFillSup)); setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); pFillInfo->pos = FILL_POS_START; } @@ -447,7 +446,6 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS pFillInfo->pResRow = &pFillSup->prev; pFillInfo->pLinearInfo->hasNext = false; } else { - ASSERT(hasNextWindow(pFillSup)); setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); pFillInfo->pos = FILL_POS_START; pFillInfo->pLinearInfo->nextEnd = INT64_MIN; @@ -458,10 +456,9 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS } } break; default: - ASSERT(0); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); break; } - ASSERT(pFillInfo->pos != FILL_POS_INVALID); } static int32_t checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId, bool* pRes) { @@ -628,7 +625,9 @@ static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SRes SWinKey key = {.groupId = groupId, .ts = pRow->key}; int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); - ASSERT(code == TSDB_CODE_SUCCESS); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { @@ -795,7 +794,6 @@ static int32_t buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint if (winCode != TSDB_CODE_SUCCESS) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { - ASSERT(tbname); char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); @@ -1125,7 +1123,7 @@ static int32_t doStreamFillNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { return code; } break; default: - ASSERTS(false, "invalid SSDataBlock type"); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 5c12db1ab9..071c82c970 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -213,7 +213,6 @@ static void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins) { } bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) { - ASSERTS(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0, "maxts should greater than 0"); return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark; } @@ -415,7 +414,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL); QUERY_CHECK_CODE(code, lino, _end); } else { - ASSERT(tbname); + QUERY_CHECK_CONDITION((tbname), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName); @@ -916,7 +915,6 @@ void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBl } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - ASSERT(pBlock->info.rows > 0); break; } pGroupResInfo->index += 1; @@ -1366,7 +1364,7 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera int32_t winCode = TSDB_CODE_SUCCESS; code = pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize, &winCode); QUERY_CHECK_CODE(code, lino, _end); - ASSERT(winCode == TSDB_CODE_SUCCESS); + QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES); QUERY_CHECK_CODE(code, lino, _end); @@ -1694,7 +1692,10 @@ static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBloc } else if (IS_FINAL_INTERVAL_OP(pOperator) && pBlock->info.type == STREAM_MID_RETRIEVE) { continue; } else { - ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + if (pBlock->info.type != STREAM_INVALID) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -1883,7 +1884,6 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN .deleteMarkSaved = 0, .calTriggerSaved = 0, }; - ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 4096); @@ -2090,7 +2090,6 @@ int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pSup->pCtx[i].saveHandle.pBuf = NULL; } - ASSERT(numOfCols > 0); return TSDB_CODE_SUCCESS; } @@ -2391,7 +2390,6 @@ _end: static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset) { - ASSERT(pWinInfo->sessionWin.win.skey <= pWinInfo->sessionWin.win.ekey); *pResult = (SResultRow*)pWinInfo->pStatePos->pRowBuff; // set time window for current result (*pResult)->win = pWinInfo->sessionWin.win; @@ -3022,7 +3020,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - ASSERT(pBlock->info.rows > 0); break; } @@ -3262,7 +3259,7 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe code = pAggSup->stateStore.streamStateSessionAddIfNotExist( pAggSup->pState, &winfo.sessionWin, pAggSup->gap, (void**)&winfo.pStatePos, &pAggSup->resultRowSize, &winCode); QUERY_CHECK_CODE(code, lino, _end); - ASSERT(winCode == TSDB_CODE_SUCCESS); + QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); code = @@ -3276,7 +3273,6 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe // 3.pChildren int32_t size = 0; buf = taosDecodeFixedI32(buf, &size); - ASSERT(size <= taosArrayGetSize(pInfo->pChildren)); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i); code = doStreamSessionDecodeOpState(buf, 0, pChOp, false, &buf); @@ -3447,7 +3443,10 @@ static int32_t doStreamSessionAggNext(SOperatorInfo* pOperator, SSDataBlock** pp continue; } else { - ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -3621,7 +3620,6 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; - ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo); @@ -3667,7 +3665,6 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; - ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY)); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); @@ -3990,7 +3987,10 @@ static int32_t doStreamSessionSemiAggNext(SOperatorInfo* pOperator, SSDataBlock* doStreamSessionSaveCheckpoint(pOperator); continue; } else { - ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -4205,7 +4205,8 @@ int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey; code = getSessionWindowInfoByKey(pAggSup, pKey, &pCurWin->winInfo); QUERY_CHECK_CODE(code, lino, _end); - ASSERT(IS_VALID_SESSION_WIN(pCurWin->winInfo)); + QUERY_CHECK_CONDITION((IS_VALID_SESSION_WIN(pCurWin->winInfo)), code, lino, _end, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize)); @@ -4575,7 +4576,6 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera // 3.pChildren int32_t size = 0; buf = taosDecodeFixedI32(buf, &size); - ASSERT(size <= taosArrayGetSize(pInfo->pChildren)); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i); code = doStreamStateDecodeOpState(buf, 0, pChOp, false, &buf); @@ -4717,7 +4717,10 @@ static int32_t doStreamStateAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRe continue; } else { - ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -4829,7 +4832,6 @@ void streamStateReloadState(SOperatorInfo* pOperator) { int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; - ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY)); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); @@ -5135,7 +5137,10 @@ static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** p continue; } else { - ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } } if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) { @@ -5254,8 +5259,6 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* .minTs = INT64_MAX, .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; - ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); - pOperator->pTaskInfo = pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -5635,7 +5638,6 @@ static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock* } else { pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; } - ASSERT(taosArrayGetSize(pInfo->pUpdated) == 0); (*ppRes) = pInfo->pDelRes; return code; } @@ -5684,7 +5686,10 @@ static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock* pInfo->clearState = true; break; } else { - ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + if (pBlock->info.type != STREAM_INVALID) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } } if (pInfo->scalarSupp.pExprInfo != NULL) { diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 258f886805..f48393273f 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -114,7 +114,6 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); char* p = colDataGetData(pColInfoData, rowIndex); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - ASSERT(varDataTLen(p) <= pColInfoData->info.bytes); memcpy(pLinearInfo->start.val, p, varDataTLen(p)); } else { memcpy(pLinearInfo->start.val, p, pLinearInfo->bytes); @@ -127,7 +126,6 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo char* p = colDataGetData(pColInfoData, rowIndex); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - ASSERT(varDataTLen(p) <= pColInfoData->info.bytes); memcpy(pLinearInfo->end.val, p, varDataTLen(p)); } else { memcpy(pLinearInfo->end.val, p, pLinearInfo->bytes); @@ -143,7 +141,6 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo char* p = colDataGetData(pColInfoData, rowIndex); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - ASSERT(varDataTLen(p) <= pColInfoData->info.bytes); memcpy(pLinearInfo->end.val, p, varDataTLen(p)); } else { memcpy(pLinearInfo->end.val, p, pLinearInfo->bytes); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6a74c6a093..91583d03ed 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -136,7 +136,6 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn // } } - ASSERT(forwardRows >= 0); return forwardRows; } @@ -211,8 +210,6 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) { - ASSERT(startPos >= 0 && startPos < pDataBlockInfo->rows); - int32_t num = -1; int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); @@ -259,8 +256,6 @@ void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY pr SFunctParam* pParam = &pCtx[k].param[0]; SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId); - ASSERT(pColInfo->info.type == pParam->pCol->type && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { SGroupKeys* p = taosArrayGet(pPrevValues, index); @@ -356,9 +351,11 @@ static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, i return true; } -static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex, +static int32_t setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex, int32_t nextRowIndex, SArray* pDataBlock, const TSKEY* tsCols, - TSKEY blockEkey, STimeWindow* win) { + TSKEY blockEkey, STimeWindow* win, bool* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t order = pInfo->binfo.inputTsOrder; TSKEY actualEndKey = tsCols[endRowIndex]; @@ -367,21 +364,27 @@ static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SEx // not ended in current data block, do not invoke interpolation if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) { setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP); - return false; + (*pRes) = false; + return code; } // there is actual end point of current time window, no interpolation needs if (key == actualEndKey) { setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP); - return true; + (*pRes) = true; + return code; } - ASSERT(nextRowIndex >= 0); + if (nextRowIndex < 0) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR)); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } TSKEY nextKey = tsCols[nextRowIndex]; doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, RESULT_ROW_END_INTERP, pSup); - return true; + (*pRes) = true; + return code; } bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType) { @@ -437,13 +440,7 @@ int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBl * This time window does not cover any data, try next time window, * this case may happen when the time window is too small */ - if (primaryKeys == NULL) { - if (ascQuery) { - ASSERT(pDataBlockInfo->window.skey <= pNext->ekey); - } else { - ASSERT(pDataBlockInfo->window.ekey >= pNext->skey); - } - } else { + if (primaryKeys != NULL) { if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { @@ -469,7 +466,6 @@ int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBl } static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { - ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { return pResult->startInterp == true; } else { @@ -485,15 +481,21 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { } } -static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult, - STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) { +static int32_t doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult, + STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pInfo->timeWindowInterpo) { - return; + return code; + } + + if (pBlock == NULL) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + return code; } - ASSERT(pBlock != NULL); if (pBlock->pDataBlock == NULL) { - return; + return code; } SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); @@ -528,14 +530,22 @@ static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataB } TSKEY endKey = (pInfo->binfo.inputTsOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; - bool interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, nextRowIndex, pBlock->pDataBlock, tsCols, - endKey, win); + bool interp = false; + code = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, nextRowIndex, pBlock->pDataBlock, tsCols, + endKey, win, &interp); + QUERY_CHECK_CODE(code, lino, _end); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } } else { setNotInterpoWindowKey(pSup->pCtx, pSup->numOfExprs, RESULT_ROW_END_INTERP); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) { @@ -558,7 +568,6 @@ static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, S char* val = colDataGetData(pColInfo, i); if (IS_VAR_DATA_TYPE(pkey->type)) { memcpy(pkey->pData, val, varDataTLen(val)); - ASSERT(varDataTLen(val) <= pkey->bytes); } else { memcpy(pkey->pData, val, pkey->bytes); } @@ -594,11 +603,17 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num T_LONG_JMP(pTaskInfo->env, terrno); } - ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId); + if (!(pr->offset == p1->offset && pr->pageId == p1->pageId)) { + pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + T_LONG_JMP(pTaskInfo->env, terrno); + } if (pr->closed) { - ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && - isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); + if (!(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && + isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)) ) { + pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + T_LONG_JMP(pTaskInfo->env, terrno); + } SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow); taosMemoryFree(pNode); continue; @@ -611,7 +626,10 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } - ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + if(isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { + pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + T_LONG_JMP(pTaskInfo->env, terrno); + } SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0); if (!pTsKey) { @@ -869,7 +887,10 @@ int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo } tsCols = (int64_t*)pColDataInfo->pData; - ASSERT(tsCols[0] != 0); + if(tsCols[0] == 0) { + pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + T_LONG_JMP(pTaskInfo->env, terrno); + } // no data in primary ts if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) { @@ -1959,7 +1980,10 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { } else { if (pMiaInfo->groupId != pBlock->info.id.groupId) { // if there are unclosed time window, close it firstly. - ASSERT(pMiaInfo->curTs != INT64_MIN); + if (pMiaInfo->curTs == INT64_MIN) { + pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + T_LONG_JMP(pTaskInfo->env, terrno); + } finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo); resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow)); @@ -2216,7 +2240,9 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->binfo.inputTsOrder); - ASSERT(forwardRows > 0); + if(forwardRows <= 0) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } // prev time window not interpolation yet. if (iaInfo->timeWindowInterpo) { diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 50a587e353..5eb55467b1 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -282,7 +282,6 @@ _end: int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, int32_t* pWinCode) { SWinKey* pTmpkey = pKey; - ASSERT(keyLen == sizeof(SWinKey)); SSessionKey pWinKey = {.groupId = pTmpkey->groupId, .win.skey = pTmpkey->ts, .win.ekey = pTmpkey->ts}; return getSessionWinResultBuff(pFileState, &pWinKey, 0, pVal, pVLen, pWinCode); } @@ -455,7 +454,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream SSessionKey pTmpKey = *pWinKey; int32_t winCode = TSDB_CODE_SUCCESS; code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen, &winCode); - ASSERT(winCode == TSDB_CODE_FAILED); + QUERY_CHECK_CONDITION((winCode == TSDB_CODE_FAILED), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); QUERY_CHECK_CODE(code, lino, _end); goto _end; } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 6a2c85323a..bc2253d4bd 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -564,7 +564,7 @@ _end: int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - ASSERT(pInfo); + QUERY_CHECK_NULL(pInfo, code, lino, _error, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; @@ -572,6 +572,8 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { int32_t size = 0; if (tDecodeI32(&decoder, &size) < 0) return -1; pInfo->pTsBuckets = taosArrayInit(size, sizeof(TSKEY)); + QUERY_CHECK_NULL(pInfo->pTsBuckets, code, lino, _error, terrno); + TSKEY ts = INT64_MIN; for (int32_t i = 0; i < size; i++) { if (tDecodeI64(&decoder, &ts) < 0) return -1; @@ -623,7 +625,8 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { code = taosHashPut(pInfo->pMap, &uid, sizeof(uint64_t), pVal, valSize); QUERY_CHECK_CODE(code, lino, _error); } - ASSERT(mapSize == taosHashGetSize(pInfo->pMap)); + QUERY_CHECK_CONDITION((mapSize == taosHashGetSize(pInfo->pMap)), code, lino, _error, + TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); if (tDecodeU64(&decoder, &pInfo->maxDataVersion) < 0) return -1; if (tDecodeI32(&decoder, &pInfo->pkColLen) < 0) return -1; @@ -634,7 +637,6 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { if (pInfo->pkColLen != 0) { pInfo->comparePkRowFn = compareKeyTsAndPk; pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC); - ; } else { pInfo->comparePkRowFn = compareKeyTs; pInfo->comparePkCol = NULL; diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 3cdbad2dd5..cbaaec0964 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -125,7 +125,6 @@ static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) { (*pVal) = taosMemoryCalloc(1, *pLen); void* buff = *pVal; int32_t tmp = taosEncodeFixedI64(&buff, *pKey); - ASSERT(tmp == sizeof(TSKEY)); } SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, @@ -204,7 +203,7 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ int32_t len = 0; int32_t tmpRes = streamDefaultGet_rocksdb(pFileState->pFileStore, STREAM_STATE_INFO_NAME, &valBuf, &len); if (tmpRes == TSDB_CODE_SUCCESS) { - ASSERT(len == sizeof(TSKEY)); + QUERY_CHECK_CONDITION((len == sizeof(TSKEY)), code, lino, _error, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); streamFileStateDecode(&pFileState->flushMark, valBuf, len); qDebug("===stream===flushMark read:%" PRId64, pFileState->flushMark); } @@ -361,7 +360,7 @@ int32_t popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; if (pPos->beUsed == used) { if (used && !pPos->pRowBuff) { - ASSERT(pPos->needFree == true); + QUERY_CHECK_CONDITION((pPos->needFree == true), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); continue; } code = tdListAppend(pFlushList, &pPos); @@ -496,13 +495,13 @@ _end: code = tdListAppend(pFileState->usedBuffs, &pPos); QUERY_CHECK_CODE(code, lino, _error); + QUERY_CHECK_CONDITION((pPos->pRowBuff != NULL), code, lino, _error, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); _error: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; } - ASSERT(pPos->pRowBuff != NULL); return pPos; } @@ -636,7 +635,7 @@ int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** QUERY_CHECK_CODE(code, lino, _end); pPos->pRowBuff = getFreeBuff(pFileState); } - ASSERT(pPos->pRowBuff); + QUERY_CHECK_CONDITION((pPos->pRowBuff != NULL), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } code = recoverSessionRowBuff(pFileState, pPos); @@ -877,7 +876,10 @@ void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { taosMemoryFreeClear(pVal); break; } - ASSERT(vlen == pFileState->rowSize); + if (vlen != pFileState->rowSize) { + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + QUERY_CHECK_CODE(code, lino, _end); + } memcpy(pNewPos->pRowBuff, pVal, vlen); taosMemoryFreeClear(pVal); pNewPos->beFlushed = true; From b06d16162dbbefa7cbc04493173481bdee977ef8 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 20 Aug 2024 15:10:58 +0800 Subject: [PATCH 051/100] fix: tudf crash --- include/util/taoserror.h | 1 + source/libs/function/src/tudf.c | 5 +++++ source/util/src/terror.c | 1 + 3 files changed, 7 insertions(+) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 32e3ba1e43..b772edbf22 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -897,6 +897,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2909) #define TSDB_CODE_UDF_FUNC_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290A) #define TSDB_CODE_UDF_UV_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290B) +#define TSDB_CODE_UDF_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2920) // sml #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 9a751db801..d5ecf09cee 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -796,9 +796,14 @@ void *decodeUdfResponse(const void *buf, SUdfResponse *rsp) { buf = decodeUdfTeardownResponse(buf, &rsp->teardownRsp); break; default: + rsp->code = TSDB_CODE_UDF_INTERNAL_ERROR; fnError("decode udf response, invalid udf response type %d", rsp->type); break; } + if(buf == NULL) { + rsp->code = terrno; + fnError("decode udf response failed, code:0x%x", rsp->code); + } return (void *)buf; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b627a2aa80..b307c4ac4b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -742,6 +742,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output TAOS_DEFINE_ERROR(TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED, "udf program language not supported") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_FUNC_EXEC_FAILURE, "udf function execution failure") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_UV_EXEC_FAILURE, "udf uvlib function execution failure") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INTERNAL_ERROR, "udf internal error") //schemaless TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") From 9900643657b94237c35aeab683c6e74861e8f48b Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 20 Aug 2024 15:33:28 +0800 Subject: [PATCH 052/100] enh:[TD-31564] Remove ASSERT in client. --- source/client/src/clientImpl.c | 30 ++++++++++++++++++------------ source/client/src/clientMain.c | 27 ++++++++++++++------------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e12c761fcc..563d70ab08 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -166,7 +166,10 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass pInst = &p; } else { - ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL); + if (NULL == *pInst || NULL == (*pInst)->pAppHbMgr) { + tscError("*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL); + TSC_ERR_JRET(TSDB_CODE_TSC_INTERNAL_ERROR); + } // reset to 0 in case of conn with duplicated user key but its user has ever been dropped. atomic_store_8(&(*pInst)->pAppHbMgr->connHbFlag, 0); } @@ -2036,9 +2039,9 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // length | int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); - if (ASSERT(numOfCols == cols)) { + if (numOfCols != cols) { tscError("estimateJsonLen error: numOfCols:%d != cols:%d", numOfCols, cols); - return -1; + return TSDB_CODE_TSC_INTERNAL_ERROR; } int32_t len = getVersion1BlockMetaSize(p, numOfCols); @@ -2123,7 +2126,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int int32_t totalLen = 0; int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); - if (ASSERT(numOfCols == cols)) { + if (numOfCols != cols) { tscError("doConvertJson error: numOfCols:%d != cols:%d", numOfCols, cols); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2148,7 +2151,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int for (int32_t i = 0; i < numOfCols; ++i) { int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i]; int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i]; - if (ASSERT(colLen < dataLen)) { + if (colLen >= dataLen) { tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2236,7 +2239,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, bool convertUcs4) { - if (ASSERT(numOfCols > 0 && pFields != NULL && pResultInfo != NULL)) { + if (numOfCols <= 0 || pFields == NULL || pResultInfo == NULL) { tscError("setResultDataPtr paras error"); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2269,7 +2272,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t cols = *(int32_t*)p; p += sizeof(int32_t); - if (ASSERT(rows == numOfRows && cols == numOfCols)) { + if (rows != numOfRows || cols != numOfCols) { tscError("setResultDataPtr paras error:rows;%d numOfRows:%d cols:%d numOfCols:%d", rows, numOfRows, cols, numOfCols); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -2288,8 +2291,6 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); - - /*ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);*/ } int32_t* colLength = (int32_t*)p; @@ -2411,18 +2412,23 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR if (pRsp->compressed && compLen < rawLen) { int32_t len = tsDecompressString(pStart, compLen, 1, pResultInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0); - ASSERT(len == rawLen); if (len < 0) { tscError("tsDecompressString failed"); return terrno ? terrno : TSDB_CODE_FAILED; } - + if (len != rawLen) { + tscError("tsDecompressString failed, len:%d != rawLen:%d", len, rawLen); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } pResultInfo->pData = pResultInfo->decompBuf; pResultInfo->payloadLen = rawLen; } else { pResultInfo->pData = pStart; pResultInfo->payloadLen = htonl(pRsp->compLen); - ASSERT(pRsp->compLen == pRsp->payloadLen); + if (pRsp->compLen != pRsp->payloadLen) { + tscError("pRsp->compLen:%d != pRsp->payloadLen:%d", pRsp->compLen, pRsp->payloadLen); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index d007dae7f7..de56a4844a 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -477,7 +477,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } else if (TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return NULL; } else { - // assert to avoid un-initialization error tscError("invalid result passed to taos_fetch_row"); terrno = TSDB_CODE_TSC_INTERNAL_ERROR; return NULL; @@ -557,12 +556,14 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) { - if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) { + if (charLen > fields[i].bytes || charLen < 0) { tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); + break; } } else { - if (ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)) { + if (charLen > fields[i].bytes * TSDB_NCHAR_SIZE || charLen < 0) { tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); + break; } } @@ -1315,11 +1316,11 @@ void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - if (ASSERT(res != NULL && fp != NULL)) { + if (res == NULL || fp == NULL) { tscError("taos_fetch_rows_a invalid paras"); return; } - if (ASSERT(TD_RES_QUERY(res))) { + if (!TD_RES_QUERY(res)) { tscError("taos_fetch_rows_a res is NULL"); return; } @@ -1334,12 +1335,12 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - if (ASSERT(res != NULL && fp != NULL)) { - tscError("taos_fetch_rows_a invalid paras"); + if (res == NULL || fp == NULL) { + tscError("taos_fetch_raw_block_a invalid paras"); return; } - if (ASSERT(TD_RES_QUERY(res))) { - tscError("taos_fetch_rows_a res is NULL"); + if (!TD_RES_QUERY(res)) { + tscError("taos_fetch_raw_block_a res is NULL"); return; } SRequestObj *pRequest = res; @@ -1353,12 +1354,12 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } const void *taos_get_raw_block(TAOS_RES *res) { - if (ASSERT(res != NULL)) { - tscError("taos_fetch_rows_a invalid paras"); + if (res == NULL) { + tscError("taos_get_raw_block invalid paras"); return NULL; } - if (ASSERT(TD_RES_QUERY(res))) { - tscError("taos_fetch_rows_a res is NULL"); + if (!TD_RES_QUERY(res)) { + tscError("taos_get_raw_block res is NULL"); return NULL; } SRequestObj *pRequest = res; From 1f8e712a64d20c94d9dabbf5c577a32aacd8129e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 20 Aug 2024 16:47:22 +0800 Subject: [PATCH 053/100] fix compile error --- source/libs/executor/src/cachescanoperator.c | 6 +++++- source/libs/executor/src/countwindowoperator.c | 13 ++++++++----- source/libs/executor/src/scanoperator.c | 13 +++++++++---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 2751cf2851..6b5fadbaa1 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -298,7 +298,11 @@ int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { int32_t resultRows = pBufRes->info.rows; // the results may be null, if last values are all null - ASSERT(resultRows == 0 || resultRows == taosArrayGetSize(pInfo->pUidList)); + if (resultRows != 0 && resultRows != taosArrayGetSize(pInfo->pUidList)) { + pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(pTaskInfo->code)); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } pInfo->indexOfBufferedRes = 0; } diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c index a9858eeb96..f4eb0dd87e 100644 --- a/source/libs/executor/src/countwindowoperator.c +++ b/source/libs/executor/src/countwindowoperator.c @@ -66,14 +66,17 @@ static void clearWinStateBuff(SCountWindowResult* pBuff) { pBuff->winRows = 0; } static SCountWindowResult* getCountWinStateInfo(SCountWindowSupp* pCountSup) { SCountWindowResult* pBuffInfo = taosArrayGet(pCountSup->pWinStates, pCountSup->stateIndex); if (!pBuffInfo) { + terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return NULL; } - int32_t size = taosArrayGetSize(pCountSup->pWinStates); - // coverity scan - ASSERTS(size > 0, "WinStates is empty"); - if (size > 0) { - pCountSup->stateIndex = (pCountSup->stateIndex + 1) % size; + int32_t size = taosArrayGetSize(pCountSup->pWinStates); + if (size == 0) { + terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return NULL; } + pCountSup->stateIndex = (pCountSup->stateIndex + 1) % size; return pBuffInfo; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 04aaa3c210..ca3d08be78 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1720,6 +1720,8 @@ static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, } static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex, bool* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pBlock->info.rows == 0) { if (pRes) { (*pRes) = false; @@ -1769,10 +1771,8 @@ static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; // coverity scan - if (pInfo->pUpdateInfo == NULL){ - qError("Failed to set data version, since pInfo->pUpdateInfo is NULL"); - return; - } + QUERY_CHECK_NULL(pInfo->pUpdateInfo, code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, pInfo->pUpdateInfo->maxDataVersion); resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); @@ -1780,6 +1780,11 @@ static void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ if (pRes) { (*pRes) = true; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval, From 8d32fb4b89e6de920bc00c0bad335e550aa9b34b Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 20 Aug 2024 16:50:43 +0800 Subject: [PATCH 054/100] fix: close file --- source/client/src/clientMonitor.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index ddd2595be5..d33322b5fd 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -122,7 +122,7 @@ static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) { p->fileName = NULL; } } - return code; + return TSDB_CODE_SUCCESS; } static int32_t sendReport(void* pTransporter, SEpSet* epSet, char* pCont, MONITOR_TYPE type, void* param) { @@ -165,6 +165,7 @@ static int32_t sendReport(void* pTransporter, SEpSet* epSet, char* pCont, MONITO return asyncSendMsgToServer(pTransporter, epSet, &transporterId, pInfo); FAILED: + taosCloseFile(&(((MonitorSlowLogData*)param)->pFile)); monitorFreeSlowLogDataEx(param); return TAOS_GET_TERRNO(TSDB_CODE_TSC_INTERNAL_ERROR); } @@ -462,11 +463,13 @@ static int64_t getFileSize(char* path) { static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64_t offset, SLOW_LOG_QUEUE_TYPE type, char* fileName, void* pTransporter, SEpSet* epSet) { if (data == NULL) { + taosCloseFile(&pFile); taosMemoryFree(fileName); return TSDB_CODE_INVALID_PARA; } MonitorSlowLogData* pParam = taosMemoryMalloc(sizeof(MonitorSlowLogData)); if (pParam == NULL) { + taosCloseFile(&pFile); taosMemoryFree(data); taosMemoryFree(fileName); return terrno; @@ -485,6 +488,7 @@ static int32_t monitorReadSend(int64_t clusterId, TdFilePtr pFile, int64_t* offs SAppInstInfo* pInst = getAppInstByClusterId(clusterId); if (pInst == NULL) { tscError("failed to get app instance by clusterId:%" PRId64, clusterId); + taosCloseFile(&pFile); taosMemoryFree(fileName); return terrno; } @@ -681,7 +685,6 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) { } char* tmp = taosStrdup(filename); monitorSendSlowLogAtBeginning(clusterId, &tmp, pFile, 0); - (void)taosCloseFile(&pFile); taosMemoryFree(tmp); } @@ -730,9 +733,9 @@ static void* monitorThreadFunc(void* param) { break; } } + monitorFreeSlowLogData(slowLogData); + taosFreeQitem(slowLogData); } - monitorFreeSlowLogData(slowLogData); - taosFreeQitem(slowLogData); if (quitCnt == 0) { monitorSendAllSlowLog(); From 3cdef9f0bfd8968a42cf17847d3068f9d87618c2 Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 20 Aug 2024 17:51:28 +0800 Subject: [PATCH 055/100] fix:[TD-31567] Fix repeatFunction accessing out-of-bounds array index. --- source/libs/scalar/src/sclfunc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 37daff1d63..12f0137fc5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1768,7 +1768,7 @@ int32_t repeatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu continue; } int32_t count = 0; - GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, i)); + GET_TYPED_DATA(count, int32_t, GET_PARAM_TYPE(&pInput[1]), colDataGetData(pInput[1].columnData, 0)); if (count <= 0) { varDataSetLen(output, 0); SCL_ERR_JRET(colDataSetVal(pOutputData, i, outputBuf, false)); From 22aa4f15288edd8d621c04cd4e33eb659b5d0277 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 20 Aug 2024 19:38:47 +0800 Subject: [PATCH 056/100] fix: report error when the ttl value is too large --- source/libs/parser/src/parAstCreater.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2bf82ba98f..20081b7535 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1905,10 +1905,11 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType case TABLE_OPTION_TTL: { int64_t ttl = taosStr2Int64(((SToken*)pVal)->z, NULL, 10); if (ttl > INT32_MAX) { - ttl = INT32_MAX; + pCxt->errCode = TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } else { + // ttl can not be smaller than 0, because there is a limitation in sql.y (TTL NK_INTEGER) + ((STableOptions*)pOptions)->ttl = ttl; } - // ttl can not be smaller than 0, because there is a limitation in sql.y (TTL NK_INTEGER) - ((STableOptions*)pOptions)->ttl = ttl; break; } case TABLE_OPTION_SMA: @@ -2516,7 +2517,7 @@ SNode* addCreateUserStmtWhiteList(SAstCreateContext* pCxt, SNode* pCreateUserStm return pCreateUserStmt; } -SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo, +SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo, int8_t createDb, int8_t is_import) { CHECK_PARSER_STATUS(pCxt); char password[TSDB_USET_PASSWORD_LEN + 3] = {0}; From c867f93e0e02b139336bccdf530cb8fa70f2d55f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 20 Aug 2024 19:41:07 +0800 Subject: [PATCH 057/100] fix compile error --- source/libs/executor/src/timewindowoperator.c | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 91583d03ed..95d415f85c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -490,7 +490,7 @@ static int32_t doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDa } if (pBlock == NULL) { - code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + code = TSDB_CODE_INVALID_PARA; return code; } @@ -782,11 +782,14 @@ static bool hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, ret); } // window start key interpolation - doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup); + ret = doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup); + if (ret != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, ret); + } } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, 1); @@ -814,7 +817,10 @@ static bool hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->binfo.inputTsOrder); // window start(end) key interpolation - doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup); + code = doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } // TODO: add to open window? how to close the open windows after input blocks exhausted? #if 0 if ((ascScan && ekey <= pBlock->info.window.ekey) || @@ -2253,11 +2259,14 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, ret); } // window start key interpolation - doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup); + ret = doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup); + if (ret != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, ret); + } } updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, 1); @@ -2294,7 +2303,10 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* iaInfo->binfo.inputTsOrder); // window start(end) key interpolation - doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup); + code = doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, From 9f8f7591696558b4ba433e03c71ee33f477d3dff Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 20 Aug 2024 21:10:14 +0800 Subject: [PATCH 058/100] tetst:update windows test case in full ci --- tests/script/win-test-file | 14 +++-- tests/system-test/win-test-file | 108 +++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/tests/script/win-test-file b/tests/script/win-test-file index acc4c74d21..8c96722c9f 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -1,3 +1,5 @@ +./test.sh -f tsim/query/timeline.sim +./test.sh -f tsim/join/join.sim ./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim ./test.sh -f tsim/parser/where.sim ./test.sh -f tsim/parser/join_manyblocks.sim @@ -150,6 +152,7 @@ ./test.sh -f tsim/parser/join_multivnode.sim ./test.sh -f tsim/parser/join.sim ./test.sh -f tsim/parser/last_cache.sim +./test.sh -f tsim/parser/last_both.sim ./test.sh -f tsim/parser/last_groupby.sim ./test.sh -f tsim/parser/lastrow.sim ./test.sh -f tsim/parser/lastrow2.sim @@ -192,6 +195,7 @@ ./test.sh -f tsim/query/session.sim ./test.sh -f tsim/query/join_interval.sim ./test.sh -f tsim/query/join_pk.sim +./test.sh -f tsim/query/join_order.sim ./test.sh -f tsim/query/count_spread.sim ./test.sh -f tsim/query/unionall_as_table.sim ./test.sh -f tsim/query/multi_order_by.sim @@ -214,6 +218,9 @@ ./test.sh -f tsim/query/bug3398.sim ./test.sh -f tsim/query/explain_tsorder.sim ./test.sh -f tsim/query/apercentile.sim +./test.sh -f tsim/query/query_count0.sim +./test.sh -f tsim/query/query_count_sliding0.sim +./test.sh -f tsim/query/union_precision.sim ./test.sh -f tsim/qnode/basic1.sim ./test.sh -f tsim/snode/basic1.sim ./test.sh -f tsim/mnode/basic1.sim @@ -287,9 +294,6 @@ ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ./test.sh -f tsim/sma/rsmaCreateInsertQueryDelete.sim - -### refactor stream backend, open case after rsma refactored -#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim ./test.sh -f tsim/sync/vnodesnapshot-rsma-test.sim ./test.sh -f tsim/valgrind/checkError1.sim ./test.sh -f tsim/valgrind/checkError2.sim @@ -326,6 +330,7 @@ ./test.sh -f tsim/compress/commitlog.sim ./test.sh -f tsim/compress/compress2.sim ./test.sh -f tsim/compress/compress.sim +./test.sh -f tsim/compress/compress_col.sim ./test.sh -f tsim/compress/uncompress.sim ./test.sh -f tsim/compute/avg.sim ./test.sh -f tsim/compute/block_dist.sim @@ -401,8 +406,9 @@ ./test.sh -f tsim/tag/tbNameIn.sim ./test.sh -f tmp/monitor.sim ./test.sh -f tsim/tagindex/add_index.sim -./test.sh -f tsim/tagindex/indexOverflow.sim ./test.sh -f tsim/tagindex/sma_and_tag_index.sim +./test.sh -f tsim/tagindex/indexOverflow.sim ./test.sh -f tsim/view/view.sim ./test.sh -f tsim/query/cache_last.sim ./test.sh -f tsim/query/const.sim +./test.sh -f tsim/query/nestedJoinView.sim diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index 69688e7450..41eb28e071 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -1,5 +1,13 @@ +python3 ./test.py -f 2-query/pk_error.py +python3 ./test.py -f 2-query/pk_func.py +python3 ./test.py -f 2-query/pk_varchar.py +python3 ./test.py -f 2-query/pk_func_group.py +python3 ./test.py -f 2-query/partition_expr.py +python3 ./test.py -f 2-query/project_group.py python3 ./test.py -f 2-query/tbname_vgroup.py +python3 ./test.py -f 2-query/count_interval.py python3 ./test.py -f 2-query/compact-col.py +python3 ./test.py -f 2-query/tms_memleak.py python3 ./test.py -f 2-query/stbJoin.py python3 ./test.py -f 2-query/stbJoin.py -Q 2 python3 ./test.py -f 2-query/stbJoin.py -Q 3 @@ -8,6 +16,8 @@ python3 ./test.py -f 2-query/hint.py python3 ./test.py -f 2-query/hint.py -Q 2 python3 ./test.py -f 2-query/hint.py -Q 3 python3 ./test.py -f 2-query/hint.py -Q 4 +python3 ./test.py -f 2-query/para_tms.py +python3 ./test.py -f 2-query/para_tms2.py python3 ./test.py -f 2-query/nestedQuery.py python3 ./test.py -f 2-query/nestedQuery_str.py python3 ./test.py -f 2-query/nestedQuery_math.py @@ -52,6 +62,20 @@ python3 ./test.py -f 2-query/last_cache_scan.py python3 ./test.py -f 2-query/last_cache_scan.py -Q 2 python3 ./test.py -f 2-query/last_cache_scan.py -Q 3 python3 ./test.py -f 2-query/last_cache_scan.py -Q 4 +python3 ./test.py -f 2-query/tbname.py +python3 ./test.py -f 2-query/tbname.py -Q 2 +python3 ./test.py -f 2-query/tbname.py -Q 3 +python3 ./test.py -f 2-query/tbname.py -Q 4 +python3 ./test.py -f 2-query/tsma.py +python3 ./test.py -f 2-query/tsma.py -R +python3 ./test.py -f 2-query/tsma.py -Q 2 +python3 ./test.py -f 2-query/tsma.py -Q 3 +python3 ./test.py -f 2-query/tsma.py -Q 4 +python3 ./test.py -f 2-query/tsma2.py +python3 ./test.py -f 2-query/tsma2.py -R +python3 ./test.py -f 2-query/tsma2.py -Q 2 +python3 ./test.py -f 2-query/tsma2.py -Q 3 +python3 ./test.py -f 2-query/tsma2.py -Q 4 python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqDropStb.py python3 ./test.py -f 7-tmq/subscribeStb0.py @@ -67,11 +91,14 @@ python3 ./test.py -f 7-tmq/tmqClientConsLog.py python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py python3 ./test.py -f 7-tmq/tmqOffset.py +python3 ./test.py -f 7-tmq/tmq_primary_key.py python3 ./test.py -f 7-tmq/tmqDropConsumer.py python3 ./test.py -f 1-insert/insert_stb.py python3 ./test.py -f 1-insert/delete_stable.py +python3 ./test.py -f 1-insert/stt_blocks_check.py python3 ./test.py -f 2-query/out_of_order.py -Q 3 python3 ./test.py -f 2-query/out_of_order.py +python3 ./test.py -f 2-query/agg_null.py python3 ./test.py -f 2-query/insert_null_none.py python3 ./test.py -f 2-query/insert_null_none.py -R python3 ./test.py -f 2-query/insert_null_none.py -Q 2 @@ -108,6 +135,18 @@ python3 ./test.py -f 2-query/match.py python3 ./test.py -f 2-query/match.py -Q 2 python3 ./test.py -f 2-query/match.py -Q 3 python3 ./test.py -f 2-query/match.py -Q 4 +python3 ./test.py -f 2-query/td-28068.py +python3 ./test.py -f 2-query/td-28068.py -Q 2 +python3 ./test.py -f 2-query/td-28068.py -Q 3 +python3 ./test.py -f 2-query/td-28068.py -Q 4 +python3 ./test.py -f 2-query/agg_group_AlwaysReturnValue.py +python3 ./test.py -f 2-query/agg_group_AlwaysReturnValue.py -Q 2 +python3 ./test.py -f 2-query/agg_group_AlwaysReturnValue.py -Q 3 +python3 ./test.py -f 2-query/agg_group_AlwaysReturnValue.py -Q 4 +python3 ./test.py -f 2-query/agg_group_NotReturnValue.py +python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 2 +python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 3 +python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 4 python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 -i False @@ -158,7 +197,8 @@ python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -# python3 ./test.py -f 7-tmq/tmq_taosx.py +python3 ./test.py -f 7-tmq/tmq_taosx.py +python3 ./test.py -f 7-tmq/tmq_ts4563.py python3 ./test.py -f 7-tmq/tmq_replay.py python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py python3 ./test.py -f 7-tmq/tmq_offset.py @@ -168,15 +208,21 @@ python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True +python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1 +python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3 python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-false.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-stb-false.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-column-false.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 +python3 test.py -f 7-tmq/tmqVnodeSplit-db-false.py -N 3 -n 3 python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 python3 ./test.py -f 99-TDcase/TD-19201.py python3 ./test.py -f 99-TDcase/TD-21561.py @@ -184,6 +230,7 @@ python3 ./test.py -f 99-TDcase/TS-3404.py python3 ./test.py -f 99-TDcase/TS-3581.py python3 ./test.py -f 99-TDcase/TS-3311.py python3 ./test.py -f 99-TDcase/TS-3821.py +python3 ./test.py -f 99-TDcase/TS-5130.py python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py @@ -217,6 +264,10 @@ python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 3 python3 ./test.py -f 0-others/timeRangeWise.py -N 3 python3 ./test.py -f 0-others/delete_check.py python3 ./test.py -f 0-others/test_hot_refresh_configurations.py +python3 ./test.py -f 0-others/empty_identifier.py +python3 ./test.py -f 1-insert/composite_primary_key_create.py +python3 ./test.py -f 1-insert/composite_primary_key_insert.py +python3 ./test.py -f 1-insert/composite_primary_key_delete.py python3 ./test.py -f 1-insert/insert_double.py python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/alter_replica.py -N 3 @@ -270,7 +321,10 @@ python3 ./test.py -f 1-insert/test_ts4219.py python3 ./test.py -f 1-insert/ts-4272.py python3 ./test.py -f 1-insert/test_ts4295.py python3 ./test.py -f 1-insert/test_td27388.py +python3 ./test.py -f 1-insert/test_ts4479.py +python3 ./test.py -f 1-insert/test_td29793.py python3 ./test.py -f 1-insert/insert_timestamp.py +python3 ./test.py -f 1-insert/test_td29157.py python3 ./test.py -f 0-others/show.py python3 ./test.py -f 0-others/show_tag_index.py python3 ./test.py -f 0-others/information_schema.py @@ -308,6 +362,11 @@ python3 ./test.py -f 2-query/concat_ws2.py python3 ./test.py -f 2-query/concat_ws2.py -R python3 ./test.py -f 2-query/cos.py python3 ./test.py -f 2-query/cos.py -R +python3 ./test.py -f 2-query/group_partition.py +python3 ./test.py -f 2-query/group_partition.py -R +python3 ./test.py -f 2-query/group_partition.py -Q 2 +python3 ./test.py -f 2-query/group_partition.py -Q 3 +python3 ./test.py -f 2-query/group_partition.py -Q 4 python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/count_partition.py -R python3 ./test.py -f 2-query/count.py @@ -361,6 +420,40 @@ python3 ./test.py -f 2-query/last_row.py python3 ./test.py -f 2-query/last_row.py -R python3 ./test.py -f 2-query/last.py python3 ./test.py -f 2-query/last.py -R +python3 ./test.py -f 2-query/last_and_last_row.py +python3 ./test.py -f 2-query/last_and_last_row.py -R +python3 ./test.py -f 2-query/last_and_last_row.py -Q 2 +python3 ./test.py -f 2-query/last_and_last_row.py -Q 3 +python3 ./test.py -f 2-query/last_and_last_row.py -Q 4 +python3 ./test.py -f 2-query/last+last_row.py +python3 ./test.py -f 2-query/last+last_row.py -Q 2 +python3 ./test.py -f 2-query/last+last_row.py -Q 3 +python3 ./test.py -f 2-query/last+last_row.py -Q 4 +python3 ./test.py -f 2-query/primary_ts_base_1.py +python3 ./test.py -f 2-query/primary_ts_base_1.py -R +python3 ./test.py -f 2-query/primary_ts_base_1.py -Q 2 +python3 ./test.py -f 2-query/primary_ts_base_1.py -Q 3 +python3 ./test.py -f 2-query/primary_ts_base_1.py -Q 4 +python3 ./test.py -f 2-query/primary_ts_base_2.py +python3 ./test.py -f 2-query/primary_ts_base_2.py -R +python3 ./test.py -f 2-query/primary_ts_base_2.py -Q 2 +python3 ./test.py -f 2-query/primary_ts_base_2.py -Q 3 +python3 ./test.py -f 2-query/primary_ts_base_2.py -Q 4 +python3 ./test.py -f 2-query/primary_ts_base_3.py +python3 ./test.py -f 2-query/primary_ts_base_3.py -R +python3 ./test.py -f 2-query/primary_ts_base_3.py -Q 2 +python3 ./test.py -f 2-query/primary_ts_base_3.py -Q 3 +python3 ./test.py -f 2-query/primary_ts_base_3.py -Q 4 +python3 ./test.py -f 2-query/primary_ts_base_4.py +python3 ./test.py -f 2-query/primary_ts_base_4.py -R +python3 ./test.py -f 2-query/primary_ts_base_4.py -Q 2 +python3 ./test.py -f 2-query/primary_ts_base_4.py -Q 3 +python3 ./test.py -f 2-query/primary_ts_base_4.py -Q 4 +python3 ./test.py -f 2-query/primary_ts_base_5.py +python3 ./test.py -f 2-query/primary_ts_base_5.py -R +python3 ./test.py -f 2-query/primary_ts_base_5.py -Q 2 +python3 ./test.py -f 2-query/primary_ts_base_5.py -Q 3 +python3 ./test.py -f 2-query/primary_ts_base_5.py -Q 4 python3 ./test.py -f 2-query/leastsquares.py python3 ./test.py -f 2-query/leastsquares.py -R python3 ./test.py -f 2-query/length.py @@ -368,6 +461,8 @@ python3 ./test.py -f 2-query/length.py -R python3 ./test.py -f 2-query/limit.py python3 ./test.py -f 2-query/log.py python3 ./test.py -f 2-query/log.py -R +python3 ./test.py -f 2-query/logical_operators.py +python3 ./test.py -f 2-query/logical_operators.py -R python3 ./test.py -f 2-query/lower.py python3 ./test.py -f 2-query/lower.py -R python3 ./test.py -f 2-query/ltrim.py @@ -376,6 +471,8 @@ python3 ./test.py -f 2-query/mavg.py python3 ./test.py -f 2-query/mavg.py -R python3 ./test.py -f 2-query/max_partition.py python3 ./test.py -f 2-query/max_partition.py -R +python3 ./test.py -f 2-query/partition_limit_interval.py +python3 ./test.py -f 2-query/partition_limit_interval.py -R python3 ./test.py -f 2-query/max_min_last_interval.py python3 ./test.py -f 2-query/last_row_interval.py python3 ./test.py -f 2-query/max.py @@ -478,6 +575,8 @@ python3 ./test.py -f 2-query/json_tag.py python3 ./test.py -f 2-query/nestedQueryInterval.py python3 ./test.py -f 2-query/systable_func.py python3 ./test.py -f 2-query/test_ts4382.py +python3 ./test.py -f 2-query/test_ts4403.py +python3 ./test.py -f 2-query/test_td28163.py python3 ./test.py -f 2-query/stablity.py python3 ./test.py -f 2-query/stablity_1.py python3 ./test.py -f 2-query/elapsed.py @@ -486,6 +585,10 @@ python3 ./test.py -f 2-query/function_diff.py python3 ./test.py -f 2-query/tagFilter.py python3 ./test.py -f 2-query/projectionDesc.py python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 +python3 ./test.py -f 2-query/ts-4348-td-27939.py +python3 ./test.py -f 2-query/backslash_g.py +python3 ./test.py -f 2-query/test_ts4467.py +python3 ./test.py -f 2-query/geometry.py python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 @@ -612,6 +715,7 @@ python3 ./test.py -f 2-query/irate.py -Q 2 python3 ./test.py -f 2-query/function_null.py -Q 2 python3 ./test.py -f 2-query/count_partition.py -Q 2 python3 ./test.py -f 2-query/max_partition.py -Q 2 +python3 ./test.py -f 2-query/partition_limit_interval.py -Q 2 python3 ./test.py -f 2-query/max_min_last_interval.py -Q 2 python3 ./test.py -f 2-query/last_row_interval.py -Q 2 python3 ./test.py -f 2-query/last_row.py -Q 2 @@ -707,6 +811,7 @@ python3 ./test.py -f 2-query/irate.py -Q 3 python3 ./test.py -f 2-query/function_null.py -Q 3 python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 +python3 ./test.py -f 2-query/partition_limit_interval.py -Q 3 python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 python3 ./test.py -f 2-query/last_row_interval.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3 @@ -802,6 +907,7 @@ python3 ./test.py -f 2-query/irate.py -Q 4 python3 ./test.py -f 2-query/function_null.py -Q 4 python3 ./test.py -f 2-query/count_partition.py -Q 4 python3 ./test.py -f 2-query/max_partition.py -Q 4 +python3 ./test.py -f 2-query/partition_limit_interval.py -Q 4 python3 ./test.py -f 2-query/max_min_last_interval.py -Q 4 python3 ./test.py -f 2-query/last_row_interval.py -Q 4 python3 ./test.py -f 2-query/last_row.py -Q 4 From 5a0a03e46df07b06e2f76f300b30edf95d1b04df Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 21 Aug 2024 09:04:12 +0800 Subject: [PATCH 059/100] fix: memleak --- source/client/src/clientMonitor.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index d33322b5fd..d40dc16690 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -120,6 +120,8 @@ static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) { .data = NULL}; if (monitorPutData2MonitorQueue(tmp) == 0) { p->fileName = NULL; + } else { + taosCloseFile(&(p->pFile)); } } return TSDB_CODE_SUCCESS; @@ -714,7 +716,7 @@ static void* monitorThreadFunc(void* param) { MonitorSlowLogData* slowLogData = NULL; (void)taosReadQitem(monitorQueue, (void**)&slowLogData); if (slowLogData != NULL) { - if (slowLogData->type == SLOW_LOG_READ_BEGINNIG) { + if (slowLogData->type == SLOW_LOG_READ_BEGINNIG && quitCnt == 0) { if (slowLogData->pFile != NULL) { monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile, slowLogData->offset); @@ -857,6 +859,7 @@ int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data) { if (taosWriteQitem(monitorQueue, slowLogData) == 0) { (void)tsem2_post(&monitorSem); } else { + taosCloseFile(&(slowLogData->pFile)); monitorFreeSlowLogData(slowLogData); taosFreeQitem(slowLogData); } From b6084e64cec987892135ff6de141218fd4d2f302 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Aug 2024 10:00:35 +0800 Subject: [PATCH 060/100] refactor: remove assert in stream module. --- include/util/taoserror.h | 1 + source/libs/stream/src/streamCheckpoint.c | 23 +++++++++++++++------ source/libs/stream/src/streamData.c | 2 -- source/libs/stream/src/streamMsg.c | 13 ++++++++---- source/libs/stream/src/streamQueue.c | 4 +++- source/libs/stream/src/streamSched.c | 5 ++--- source/libs/stream/src/streamSnapshot.c | 9 ++++++-- source/libs/stream/src/streamStartHistory.c | 5 +---- source/libs/stream/src/streamStartTask.c | 8 ++++--- source/libs/stream/src/streamTaskSm.c | 6 +----- source/libs/stream/src/streamTimer.c | 5 ++++- source/libs/stream/src/streamUpdate.c | 11 ++++++++-- source/util/src/tcompression.c | 4 ++-- source/util/src/terror.c | 2 ++ 14 files changed, 63 insertions(+), 35 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 1911c48d26..58c8c55a53 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -958,6 +958,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_STREAM_INVALID_STATETRANS TAOS_DEF_ERROR_CODE(0, 0x4103) #define TSDB_CODE_STREAM_TASK_IVLD_STATUS TAOS_DEF_ERROR_CODE(0, 0x4104) #define TSDB_CODE_STREAM_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x4105) +#define TSDB_CODE_STREAM_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x4106) // TDLite #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 0ef7c2312a..17edc3c30b 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -37,10 +37,14 @@ int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int6 return code; } + if (srcTaskId <= 0) { + stDebug("s-task:0x%x invalid src task id:%d for creating checkpoint trigger block", pTask->id.idStr, srcTaskId); + return TSDB_CODE_INVALID_PARA; + } + pChkpoint->type = checkpointType; if (checkpointType == STREAM_INPUT__CHECKPOINT_TRIGGER && (pTask->info.taskLevel != TASK_LEVEL__SOURCE)) { pChkpoint->srcTaskId = srcTaskId; - ASSERT(srcTaskId != 0); } SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); @@ -189,10 +193,13 @@ int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) { pBlock->srcTaskId = pTask->id.taskId; pBlock->srcVgId = pTask->pMeta->vgId; + if (pTask->chkInfo.pActiveInfo->dispatchTrigger == true) { + stError("s-task:%s already dispatch checkpoint-trigger, not dispatch again", pTask->id.idStr); + return 0; + } int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); if (code == 0) { - ASSERT(pTask->chkInfo.pActiveInfo->dispatchTrigger == false); code = streamDispatchStreamBlock(pTask); } else { stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code)); @@ -593,8 +600,11 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV } } - ASSERT(pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer && + bool valid = (pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer && pInfo->processedVer <= pReq->checkpointVer); + if (!valid) { + stFatal(""); + } // update only it is in checkpoint status, or during restore procedure. if (pStatus.state == TASK_STATUS__CK || (!restored)) { @@ -1102,7 +1112,10 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { streamMutexLock(&pInfo->lock); // outputQ should be empty here - ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0); + if (streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) > 0) { + stFatal("s-task:%s items are still in outputQ, failed to init trigger dispatch info", pTask->id.idStr); + return; + } pInfo->dispatchTrigger = true; if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { @@ -1375,9 +1388,7 @@ int32_t streamTaskSendNegotiateChkptIdMsg(SStreamTask* pTask) { pTask->pBackend = NULL; } - ASSERT(pTask->pBackend == NULL); pTask->status.requireConsensusChkptId = true; - stDebug("s-task:%s set the require consensus-checkpointId flag", id); return 0; } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 0602bf9334..d85435d21c 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -33,7 +33,6 @@ int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t return code; } - ASSERT((pReq->blockNum == taosArrayGetSize(pReq->data)) && (pReq->blockNum == taosArrayGetSize(pReq->dataLen))); for (int32_t i = 0; i < blockNum; i++) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); @@ -52,7 +51,6 @@ int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t } int32_t len = tsDecompressString(pInput, compLen, 1, p, fullLen, ONE_STAGE_COMP, NULL, 0); - ASSERT(len == fullLen); pInput = p; } diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c index 6fe2d818b3..586f9ac692 100644 --- a/source/libs/stream/src/streamMsg.c +++ b/source/libs/stream/src/streamMsg.c @@ -229,8 +229,11 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p if (tEncodeI32(pEncoder, pReq->upstreamRelTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1; if (tEncodeI64(pEncoder, pReq->totalLen) < 0) return -1; - ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum); - ASSERT(taosArrayGetSize(pReq->dataLen) == pReq->blockNum); + + if (taosArrayGetSize(pReq->data) != pReq->blockNum || taosArrayGetSize(pReq->dataLen) == pReq->blockNum) { + return TSDB_CODE_INVALID_MSG; + } + for (int32_t i = 0; i < pReq->blockNum; i++) { int32_t* pLen = taosArrayGet(pReq->dataLen, i); void* data = taosArrayGetP(pReq->data, i); @@ -261,7 +264,6 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->totalLen) < 0) return -1; - ASSERT(pReq->blockNum > 0); pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*)); pReq->dataLen = taosArrayInit(pReq->blockNum, sizeof(int32_t)); for (int32_t i = 0; i < pReq->blockNum; i++) { @@ -270,7 +272,10 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { void* data; if (tDecodeI32(pDecoder, &len1) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &data, &len2) < 0) return -1; - ASSERT(len1 == len2); + + if (len1 != len2) { + return TSDB_CODE_INVALID_MSG; + } void* p = taosArrayPush(pReq->dataLen, &len1); if (p == NULL) { diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 752101afbd..3703ed07aa 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -452,7 +452,9 @@ static void fillTokenBucket(STokenBucket* pBucket, const char* id) { int64_t now = taosGetTimestampMs(); int64_t deltaToken = now - pBucket->tokenFillTimestamp; - ASSERT(pBucket->numOfToken >= 0); + if (pBucket->numOfToken < 0) { + return; + } int32_t incNum = (deltaToken / 1000.0) * pBucket->numRate; if (incNum > 0) { diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index e8c7be5204..585cf63cfc 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -22,9 +22,8 @@ static void streamTaskSchedHelper(void* param, void* tmrId); int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); - ASSERT(ref == 2 && pTask->schedInfo.pDelayTimer == NULL); - - stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.delaySchedParam); + stDebug("s-task:%s setup scheduler trigger, ref:%d delay:%" PRId64 " ms", pTask->id.idStr, ref, + pTask->info.delaySchedParam); pTask->schedInfo.pDelayTimer = taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.delaySchedParam, pTask, streamTimer); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 9122af0e12..0390422623 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -441,7 +441,9 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta SBackendSnapFile2 snapFile = {0}; code = streamBackendSnapInitFile(path, pSnap, &snapFile); - ASSERT(code == 0); + if (code) { + goto _err; + } void* p = taosArrayPush(pDbSnapSet, &snapFile); if (p == NULL) { @@ -767,7 +769,10 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa if (!taosIsDir(path)) { code = taosMulMkDir(path); stInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); - ASSERT(code == 0); + if (code) { + stError("s-task:0x%x failed to mkdir:%s", (int32_t) snapInfo.taskId, path); + return code; + } } pDbSnapFile->path = path; diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 0dbb6ed454..e68c088b9a 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -53,10 +53,9 @@ static int32_t streamTaskSetReady(SStreamTask* pTask) { pTask->id.idStr, pTask->info.taskLevel, numOfUps, p.name); } - ASSERT(pTask->status.downstreamReady == 0); pTask->status.downstreamReady = 1; - pTask->execInfo.readyTs = taosGetTimestampMs(); + int64_t el = (pTask->execInfo.readyTs - pTask->execInfo.checkTs); stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", pTask->id.idStr, numOfDowns, el, p.name); @@ -212,8 +211,6 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { int64_t now = taosGetTimestampMs(); int32_t code = 0; - ASSERT(hTaskId != 0); - // check stream task status in the first place. SStreamTaskState pStatus = streamTaskGetStatus(pTask); if (pStatus.state != TASK_STATUS__READY && pStatus.state != TASK_STATUS__HALT && diff --git a/source/libs/stream/src/streamStartTask.c b/source/libs/stream/src/streamStartTask.c index 99f4e84951..92aad2ece1 100644 --- a/source/libs/stream/src/streamStartTask.c +++ b/source/libs/stream/src/streamStartTask.c @@ -50,8 +50,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { SArray* pTaskList = NULL; code = prepareBeforeStartTasks(pMeta, &pTaskList, now); if (code != TSDB_CODE_SUCCESS) { - ASSERT(pTaskList == NULL); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; // ignore the error and return directly } // broadcast the check downstream tasks msg only for tasks with related fill-history tasks. @@ -364,7 +363,10 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } - ASSERT(pTask->status.downstreamReady == 0); + if(pTask->status.downstreamReady != 0) { + stFatal("s-task:0x%x downstream should be not ready, but it ready here, internal error happens", taskId); + return TSDB_CODE_STREAM_INTERNAL_ERROR; + } // avoid initialization and destroy running concurrently. streamMutexLock(&pTask->lock); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 8fd26dda27..a8ed0f7639 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -168,9 +168,7 @@ static STaskStateTrans* streamTaskFindTransform(ETaskStatus state, const EStream } if (isInvalidStateTransfer(state, event)) { - return NULL; - } else { - ASSERT(0); + stError("invalid state transfer %d, handle event:%d", state, GET_EVT_NAME(event)); } return NULL; @@ -182,8 +180,6 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, pEventName, el, pSM->prev.state.name, pSM->current.name); - ASSERT(taosArrayGetSize(pSM->pWaitingEventList) == 1); - SFutureHandleEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); if (pEvtInfo == NULL) { return terrno; diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 8f2fc83b72..b6275c0eb1 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -57,6 +57,9 @@ int32_t streamCleanBeforeQuitTmr(SStreamTmrInfo* pInfo, SStreamTask* pTask) { atomic_store_8(&pInfo->isActive, 0); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - ASSERT(ref >= 0); + if (ref < 0) { + stFatal("invalid task timer ref value:%d, %s", ref, pTask->id.idStr); + } + return ref; } \ No newline at end of file diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 6a2c85323a..1ac14defa9 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -564,7 +564,10 @@ _end: int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - ASSERT(pInfo); + if (pInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } + SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; @@ -623,7 +626,11 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { code = taosHashPut(pInfo->pMap, &uid, sizeof(uint64_t), pVal, valSize); QUERY_CHECK_CODE(code, lino, _error); } - ASSERT(mapSize == taosHashGetSize(pInfo->pMap)); + + if (mapSize != taosHashGetSize(pInfo->pMap)) { + return TSDB_CODE_INVALID_MSG; + } + if (tDecodeU64(&decoder, &pInfo->maxDataVersion) < 0) return -1; if (tDecodeI32(&decoder, &pInfo->pkColLen) < 0) return -1; diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 8402d2a658..bec9a5797d 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -252,7 +252,7 @@ int32_t l2ComressInitImpl_xz(char *lossyColumns, float fPrecision, double dPreci } int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char *const output, int32_t outputSize, const char type, int8_t lvl) { - size_t len = FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); + size_t len = 0;//FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); if (len > inputSize) { output[0] = 0; memcpy(output + 1, input, inputSize); @@ -264,7 +264,7 @@ int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char int32_t l2DecompressImpl_xz(const char *const input, const int32_t compressedSize, char *const output, int32_t outputSize, const char type) { if (input[0] == 1) { - return FL2_decompress(output, outputSize, input + 1, compressedSize - 1); + return 0;//FL2_decompress(output, outputSize, input + 1, compressedSize - 1); } else if (input[0] == 0) { memcpy(output, input + 1, compressedSize - 1); return compressedSize - 1; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 396abf21a7..79f25bc8be 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -802,6 +802,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exi TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_EXEC_CANCELLED, "Stream task exec cancelled") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INVALID_STATETRANS, "Invalid task state to handle event") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_IVLD_STATUS, "Invalid task status to proceed") +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INTERNAL_ERROR, "Stream internal error") +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_NOT_LEADER, "Stream task not on leader vnode") // TDLite TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags") From b10aeeea87b68a13d875d96e2ece2aaf5ba2319d Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 21 Aug 2024 10:00:53 +0800 Subject: [PATCH 061/100] fix: update ttl test case --- tests/system-test/2-query/ttl_comment.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py index ba24579611..b332cd3afb 100644 --- a/tests/system-test/2-query/ttl_comment.py +++ b/tests/system-test/2-query/ttl_comment.py @@ -36,12 +36,12 @@ class TDTestCase: tdSql.error(f"create table {dbname}.ttl_table1(ts timestamp, i int) ttl 1.1") tdSql.error(f"create table {dbname}.ttl_table2(ts timestamp, i int) ttl 1e1") tdSql.error(f"create table {dbname}.ttl_table3(ts timestamp, i int) ttl -1") + tdSql.error(f"create table {dbname}.ttl_table4(ts timestamp, i int) ttl 2147483648") print("============== STEP 1 ===== test normal table") tdSql.execute(f"create table {dbname}.normal_table1(ts timestamp, i int)") tdSql.execute(f"create table {dbname}.normal_table2(ts timestamp, i int) comment '' ttl 3") - tdSql.execute(f"create table {dbname}.normal_table3(ts timestamp, i int) ttl 2100000000020 comment 'hello'") tdSql.query("select * from information_schema.ins_tables where table_name like 'normal_table1'") tdSql.checkData(0, 0, 'normal_table1') @@ -54,12 +54,6 @@ class TDTestCase: tdSql.checkData(0, 7, 3) tdSql.checkData(0, 8, '') - - tdSql.query("select * from information_schema.ins_tables where table_name like 'normal_table3'") - tdSql.checkData(0, 0, 'normal_table3') - tdSql.checkData(0, 7, 2147483647) - tdSql.checkData(0, 8, 'hello') - tdSql.execute(f"alter table {dbname}.normal_table1 comment 'nihao'") tdSql.query("select * from information_schema.ins_tables where table_name like 'normal_table1'") tdSql.checkData(0, 0, 'normal_table1') @@ -75,19 +69,14 @@ class TDTestCase: tdSql.checkData(0, 0, 'normal_table2') tdSql.checkData(0, 8, 'fly') - tdSql.execute(f"alter table {dbname}.normal_table3 comment 'fly'") - tdSql.query("select * from information_schema.ins_tables where table_name like 'normal_table3'") - tdSql.checkData(0, 0, 'normal_table3') - tdSql.checkData(0, 8, 'fly') - tdSql.execute(f"alter table {dbname}.normal_table1 ttl 1") tdSql.query("select * from information_schema.ins_tables where table_name like 'normal_table1'") tdSql.checkData(0, 0, 'normal_table1') tdSql.checkData(0, 7, 1) - tdSql.execute(f"alter table {dbname}.normal_table3 ttl 0") - tdSql.query("select * from information_schema.ins_tables where table_name like 'normal_table3'") - tdSql.checkData(0, 0, 'normal_table3') + tdSql.execute(f"alter table {dbname}.normal_table2 ttl 0") + tdSql.query("select * from information_schema.ins_tables where table_name like 'normal_table2'") + tdSql.checkData(0, 0, 'normal_table2') tdSql.checkData(0, 7, 0) From 1cd0e35443a438699e4013d2deee65a210b8379b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 21 Aug 2024 10:00:21 +0800 Subject: [PATCH 062/100] fix(stream):ignore some bf error --- source/util/src/tscalablebf.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index ffcfdfeaf1..407223e937 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -71,8 +71,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t le int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - code = TSDB_CODE_OUT_OF_BUFFER; - QUERY_CHECK_CODE(code, lino, _error); + return code; } int32_t size = taosArrayGetSize(pSBf->bfArray); SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); @@ -105,8 +104,8 @@ int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int3 int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pSBf->status == SBF_INVALID) { - code = TSDB_CODE_OUT_OF_BUFFER; - QUERY_CHECK_CODE(code, lino, _end); + (*winRes) = TSDB_CODE_FAILED; + return code; } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); From 87e56d3d67f394394759efd066e7925451267d4b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Aug 2024 10:23:22 +0800 Subject: [PATCH 063/100] Revert "fix: memory leak of geos" --- include/util/tgeosctx.h | 8 +- source/client/src/clientMain.c | 2 - source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 2 - source/libs/executor/src/sysscanoperator.c | 2 +- source/libs/geometry/src/geosWrapper.c | 28 +----- source/libs/parser/src/parInsertSql.c | 4 +- source/libs/scalar/src/sclvector.c | 8 +- source/util/src/tgeosctx.c | 111 +++++---------------- source/util/src/tsched.c | 2 + source/util/src/tworker.c | 2 + tools/shell/src/shellEngine.c | 5 +- 11 files changed, 45 insertions(+), 129 deletions(-) diff --git a/include/util/tgeosctx.h b/include/util/tgeosctx.h index a4355db29a..267ba9e049 100644 --- a/include/util/tgeosctx.h +++ b/include/util/tgeosctx.h @@ -32,16 +32,14 @@ typedef struct SGeosContext { GEOSWKBReader *WKBReader; GEOSWKBWriter *WKBWriter; - pcre2_code *WKTRegex; + pcre2_code *WKTRegex; pcre2_match_data *WKTMatchData; char errMsg[512]; } SGeosContext; -SGeosContext *acquireThreadLocalGeosCtx(); -SGeosContext *getThreadLocalGeosCtx(); -const char *getGeosErrMsg(int32_t code); -void taosGeosDestroy(); +SGeosContext* getThreadLocalGeosCtx(); +void destroyThreadLocalGeosCtx(); #ifdef __cplusplus } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index c0eaf27077..d007dae7f7 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -23,7 +23,6 @@ #include "query.h" #include "scheduler.h" #include "tdatablock.h" -#include "tgeosctx.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" @@ -87,7 +86,6 @@ void taos_cleanup(void) { tscDebug("rpc cleanup"); taosConvDestroy(); - taosGeosDestroy(); tmqMgmtClose(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index ae6efa7af4..fdce9fd4c9 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -19,7 +19,6 @@ #include "index.h" #include "qworker.h" #include "tcompression.h" -#include "tgeosctx.h" #include "tglobal.h" #include "tgrant.h" #include "tstream.h" @@ -122,7 +121,6 @@ void dmCleanupDnode(SDnode *pDnode) { streamMetaCleanup(); indexCleanup(); taosConvDestroy(); - taosGeosDestroy(); // compress destroy tsCompressExit(); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 082d4e7789..d8a2331980 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -979,7 +979,7 @@ static int32_t sysTableGetGeomText(char* iGeom, int32_t nGeom, char** output, in if (TSDB_CODE_SUCCESS != (code = initCtxAsText()) || TSDB_CODE_SUCCESS != (code = doAsText(iGeom, nGeom, &outputWKT))) { - qError("geo text for systable failed:%s", getGeosErrMsg(code)); + qError("geo text for systable failed:%s", getThreadLocalGeosCtx()->errMsg); *output = NULL; *nOutput = 0; return code; diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 7372521276..dde34edc91 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -23,8 +23,7 @@ typedef char (*_geosPreparedRelationFunc_t)(GEOSContextHandle_t handle, const GE void geosFreeBuffer(void *buffer) { if (buffer) { - SGeosContext *pCtx = acquireThreadLocalGeosCtx(); - if (pCtx) GEOSFree_r(pCtx->handle, buffer); + GEOSFree_r(getThreadLocalGeosCtx()->handle, buffer); } } @@ -37,8 +36,6 @@ int32_t initCtxMakePoint() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); if (geosCtx->handle == NULL) { @@ -64,8 +61,6 @@ int32_t doMakePoint(double x, double y, unsigned char **outputGeom, size_t *size int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - GEOSGeometry *geom = NULL; unsigned char *wkb = NULL; @@ -171,8 +166,6 @@ int32_t initCtxGeomFromText() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); if (geosCtx->handle == NULL) { @@ -209,8 +202,6 @@ int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - GEOSGeometry *geom = NULL; unsigned char *wkb = NULL; @@ -246,8 +237,6 @@ int32_t initCtxAsText() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); if (geosCtx->handle == NULL) { @@ -284,8 +273,6 @@ int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT) int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - GEOSGeometry *geom = NULL; char *wkt = NULL; @@ -317,8 +304,6 @@ int32_t initCtxRelationFunc() { int32_t code = TSDB_CODE_FAILED; SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - if (geosCtx->handle == NULL) { geosCtx->handle = GEOS_init_r(); if (geosCtx->handle == NULL) { @@ -344,8 +329,6 @@ int32_t doGeosRelation(const GEOSGeometry *geom1, const GEOSPreparedGeometry *pr _geosPreparedRelationFunc_t swappedPreparedRelationFn) { SGeosContext *geosCtx = getThreadLocalGeosCtx(); - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - if (!preparedGeom1) { if (!swapped) { ASSERT(relationFn); @@ -406,6 +389,8 @@ int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry // need to call destroyGeometry(outputGeom, outputPreparedGeom) later int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom) { + SGeosContext *geosCtx = getThreadLocalGeosCtx(); + ASSERT(outputGeom); // it is not allowed if outputGeom is NULL *outputGeom = NULL; @@ -417,10 +402,6 @@ int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, return TSDB_CODE_SUCCESS; } - SGeosContext *geosCtx = getThreadLocalGeosCtx(); - - if (!geosCtx) return TSDB_CODE_OUT_OF_MEMORY; - *outputGeom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, varDataVal(input), varDataLen(input)); if (*outputGeom == NULL) { return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; @@ -437,8 +418,7 @@ int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, } void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom) { - SGeosContext *geosCtx = acquireThreadLocalGeosCtx(); - if (!geosCtx) return; + SGeosContext *geosCtx = getThreadLocalGeosCtx(); if (preparedGeom && *preparedGeom) { GEOSPreparedGeom_destroy_r(geosCtx->handle, *preparedGeom); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index aa6116287e..cb94cd42f7 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -655,7 +655,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, code = parseGeometry(pToken, &output, &size); if (code != TSDB_CODE_SUCCESS) { - code = buildSyntaxErrMsg(pMsgBuf, getGeosErrMsg(code), pToken->z); + code = buildSyntaxErrMsg(pMsgBuf, getThreadLocalGeosCtx()->errMsg, pToken->z); } else if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { // Too long values will raise the invalid sql error message code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); @@ -1646,7 +1646,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, code = parseGeometry(pToken, &output, &size); if (code != TSDB_CODE_SUCCESS) { - code = buildSyntaxErrMsg(&pCxt->msg, getGeosErrMsg(code), pToken->z); + code = buildSyntaxErrMsg(&pCxt->msg, getThreadLocalGeosCtx()->errMsg, pToken->z); } // Too long values will raise the invalid sql error message else if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index bc8a2ae233..5556108a52 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -446,12 +446,12 @@ static FORCE_INLINE int32_t varToGeometry(char *buf, SScalarParam *pOut, int32_t unsigned char *t = NULL; char *output = NULL; - if ((code = initCtxGeomFromText()) != 0) { - sclError("failed to init geometry ctx, %s", getGeosErrMsg(code)); + if (initCtxGeomFromText()) { + sclError("failed to init geometry ctx, %s", getThreadLocalGeosCtx()->errMsg); SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } - if ((code = doGeomFromText(buf, &t, &len)) != 0) { - sclError("failed to convert text to geometry, %s", getGeosErrMsg(code)); + if (doGeomFromText(buf, &t, &len)) { + sclInfo("failed to convert text to geometry, %s", getThreadLocalGeosCtx()->errMsg); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); } diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c index 82a360edd1..a05734c911 100644 --- a/source/util/src/tgeosctx.c +++ b/source/util/src/tgeosctx.c @@ -14,102 +14,39 @@ */ #include "tgeosctx.h" -#include "tarray.h" #include "tdef.h" -#include "tlockfree.h" -#include "tlog.h" -#define GEOS_POOL_CAPACITY 64 -typedef struct { - SArray *poolArray; // totalSize: (GEOS_POOL_CAPACITY * (taosArrayGetSize(poolArray) - 1)) + size - SGeosContext *pool; // current SGeosContext pool - int32_t size; // size of current SGeosContext pool, size <= GEOS_POOL_CAPACITY - SRWLatch lock; -} SGeosContextPool; +static threadlocal SGeosContext tlGeosCtx = {0}; -static SGeosContextPool sGeosPool = {0}; -static threadlocal SGeosContext *tlGeosCtx = NULL; +SGeosContext* getThreadLocalGeosCtx() { return &tlGeosCtx; } -SGeosContext *acquireThreadLocalGeosCtx() { return tlGeosCtx; } - -SGeosContext *getThreadLocalGeosCtx() { - if (tlGeosCtx) return tlGeosCtx; - - taosWLockLatch(&sGeosPool.lock); - if (!sGeosPool.pool || sGeosPool.size >= GEOS_POOL_CAPACITY) { - if (!(sGeosPool.pool = (SGeosContext *)taosMemoryCalloc(GEOS_POOL_CAPACITY, sizeof(SGeosContext)))) { - taosWUnLockLatch(&sGeosPool.lock); - return NULL; - } - if (!sGeosPool.poolArray) { - if (!(sGeosPool.poolArray = taosArrayInit(16, POINTER_BYTES))) { - taosMemoryFreeClear(sGeosPool.pool); - taosWUnLockLatch(&sGeosPool.lock); - return NULL; - } - } - if (!taosArrayPush(sGeosPool.poolArray, &sGeosPool.pool)) { - taosMemoryFreeClear(sGeosPool.pool); - taosWUnLockLatch(&sGeosPool.lock); - return NULL; - } - sGeosPool.size = 0; +void destroyThreadLocalGeosCtx() { + if (tlGeosCtx.WKTReader) { + GEOSWKTReader_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKTReader); + tlGeosCtx.WKTReader = NULL; } - tlGeosCtx = sGeosPool.pool + sGeosPool.size; - ++sGeosPool.size; - taosWUnLockLatch(&sGeosPool.lock); - return tlGeosCtx; -} + if (tlGeosCtx.WKTWriter) { + GEOSWKTWriter_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKTWriter); + tlGeosCtx.WKTWriter = NULL; + } -const char *getGeosErrMsg(int32_t code) { return tlGeosCtx ? tlGeosCtx->errMsg : (code != 0 ? tstrerror(code) : ""); } + if (tlGeosCtx.WKBReader) { + GEOSWKBReader_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKBReader); + tlGeosCtx.WKBReader = NULL; + } -static void destroyGeosCtx(SGeosContext *pCtx) { - if (pCtx) { - if (pCtx->WKTReader) { - GEOSWKTReader_destroy_r(pCtx->handle, pCtx->WKTReader); - pCtx->WKTReader = NULL; - } + if (tlGeosCtx.WKBWriter) { + GEOSWKBWriter_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKBWriter); + tlGeosCtx.WKBWriter = NULL; + } - if (pCtx->WKTWriter) { - GEOSWKTWriter_destroy_r(pCtx->handle, pCtx->WKTWriter); - pCtx->WKTWriter = NULL; - } + if (tlGeosCtx.WKTRegex) { + destroyRegexes(tlGeosCtx.WKTRegex, tlGeosCtx.WKTMatchData); + } - if (pCtx->WKBReader) { - GEOSWKBReader_destroy_r(pCtx->handle, pCtx->WKBReader); - pCtx->WKBReader = NULL; - } - - if (pCtx->WKBWriter) { - GEOSWKBWriter_destroy_r(pCtx->handle, pCtx->WKBWriter); - pCtx->WKBWriter = NULL; - } - - if (pCtx->WKTRegex) { - destroyRegexes(pCtx->WKTRegex, pCtx->WKTMatchData); - pCtx->WKTRegex = NULL; - pCtx->WKTMatchData = NULL; - } - - if (pCtx->handle) { - GEOS_finish_r(pCtx->handle); - pCtx->handle = NULL; - } + if (tlGeosCtx.handle) { + GEOS_finish_r(tlGeosCtx.handle); + tlGeosCtx.handle = NULL; } } - -void taosGeosDestroy() { - uInfo("geos is cleaned up"); - int32_t size = taosArrayGetSize(sGeosPool.poolArray); - for (int32_t i = 0; i < size; ++i) { - SGeosContext *pool = *(SGeosContext **)TARRAY_GET_ELEM(sGeosPool.poolArray, i); - int32_t poolSize = i == size - 1 ? sGeosPool.size : GEOS_POOL_CAPACITY; - for (int32_t j = 0; j < poolSize; ++j) { - destroyGeosCtx(pool + j); - } - taosMemoryFree(pool); - } - taosArrayDestroy(sGeosPool.poolArray); - sGeosPool.poolArray = NULL; -} \ No newline at end of file diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 55a927f340..6779e8dee5 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -178,6 +178,8 @@ void *taosProcessSchedQueue(void *scheduler) { (*(msg.tfp))(msg.ahandle, msg.thandle); } + destroyThreadLocalGeosCtx(); + return NULL; } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index ebec134c91..b2064d6787 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -105,6 +105,7 @@ static void *tQWorkerThreadFp(SQueueWorker *worker) { taosUpdateItemSize(qinfo.queue, 1); } + destroyThreadLocalGeosCtx(); DestoryThreadLocalRegComp(); return NULL; @@ -664,6 +665,7 @@ static void *tQueryAutoQWorkerThreadFp(SQueryAutoQWorker *worker) { } } + destroyThreadLocalGeosCtx(); DestoryThreadLocalRegComp(); return NULL; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 2c8330c433..0ccbd683dc 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -611,14 +611,14 @@ void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width) code = initCtxAsText(); if (code != TSDB_CODE_SUCCESS) { - shellPrintString(getGeosErrMsg(code), width); + shellPrintString(getThreadLocalGeosCtx()->errMsg, width); return; } char *outputWKT = NULL; code = doAsText(val, length, &outputWKT); if (code != TSDB_CODE_SUCCESS) { - shellPrintString(getGeosErrMsg(code), width); // should NOT happen + shellPrintString(getThreadLocalGeosCtx()->errMsg, width); // should NOT happen return; } @@ -1282,6 +1282,7 @@ void *shellThreadLoop(void *arg) { taosResetTerminalMode(); } while (shellRunCommand(command, true) == 0); + destroyThreadLocalGeosCtx(); taosMemoryFreeClear(command); shellWriteHistory(); shellExit(); From 7e8e9ac7f1f536c451bedf64882a6b62e61c2b68 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Aug 2024 10:41:23 +0800 Subject: [PATCH 064/100] fix: wrong change --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 71dce80845..eaedbca464 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -318,7 +318,7 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6 int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); int64_t bOffset = fOffset % pFD->szPage; - if (bOffset < szPgCont) { + if (bOffset >= szPgCont) { TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit); } From 110394ed699c593c86244abf22083ef95dad520d Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 21 Aug 2024 11:07:42 +0800 Subject: [PATCH 065/100] fix: return value --- source/client/src/clientMonitor.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index d40dc16690..10e6695ee5 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -121,7 +121,9 @@ static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) { if (monitorPutData2MonitorQueue(tmp) == 0) { p->fileName = NULL; } else { - taosCloseFile(&(p->pFile)); + if(taosCloseFile(&(p->pFile)) != 0) { + tscError("failed to close file:%p", p->pFile); + } } } return TSDB_CODE_SUCCESS; @@ -166,8 +168,10 @@ static int32_t sendReport(void* pTransporter, SEpSet* epSet, char* pCont, MONITO int64_t transporterId = 0; return asyncSendMsgToServer(pTransporter, epSet, &transporterId, pInfo); - FAILED: - taosCloseFile(&(((MonitorSlowLogData*)param)->pFile)); +FAILED: + if (taosCloseFile(&(((MonitorSlowLogData*)param)->pFile)) != 0) { + tscError("failed to close file:%p", ((MonitorSlowLogData*)param)->pFile); + } monitorFreeSlowLogDataEx(param); return TAOS_GET_TERRNO(TSDB_CODE_TSC_INTERNAL_ERROR); } @@ -465,13 +469,17 @@ static int64_t getFileSize(char* path) { static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64_t offset, SLOW_LOG_QUEUE_TYPE type, char* fileName, void* pTransporter, SEpSet* epSet) { if (data == NULL) { - taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + tscError("failed to close file:%p", pFile); + } taosMemoryFree(fileName); return TSDB_CODE_INVALID_PARA; } MonitorSlowLogData* pParam = taosMemoryMalloc(sizeof(MonitorSlowLogData)); if (pParam == NULL) { - taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + tscError("failed to close file:%p", pFile); + } taosMemoryFree(data); taosMemoryFree(fileName); return terrno; @@ -490,7 +498,9 @@ static int32_t monitorReadSend(int64_t clusterId, TdFilePtr pFile, int64_t* offs SAppInstInfo* pInst = getAppInstByClusterId(clusterId); if (pInst == NULL) { tscError("failed to get app instance by clusterId:%" PRId64, clusterId); - taosCloseFile(&pFile); + if (taosCloseFile(&pFile) != 0) { + tscError("failed to close file:%p", pFile); + } taosMemoryFree(fileName); return terrno; } @@ -859,7 +869,9 @@ int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data) { if (taosWriteQitem(monitorQueue, slowLogData) == 0) { (void)tsem2_post(&monitorSem); } else { - taosCloseFile(&(slowLogData->pFile)); + if (taosCloseFile(&(slowLogData->pFile)) != 0) { + tscError("failed to close file:%p", slowLogData->pFile); + } monitorFreeSlowLogData(slowLogData); taosFreeQitem(slowLogData); } From 693cf34cf7248dba6ad2dbfc1da53b37c3912bbc Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 21 Aug 2024 11:30:51 +0800 Subject: [PATCH 066/100] fix(tfs): multi-level storage --- source/libs/tfs/src/tfs.c | 2 +- tests/system-test/1-insert/mutil_stage.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index f81bbbdeb7..4954e81837 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -554,7 +554,7 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { } STfsDisk *pDisk = NULL; - if ((code = tfsGetDiskByName(pTfs, dirName, NULL)) != 0) { + if ((code = tfsGetDiskByName(pTfs, dirName, &pDisk)) != 0) { fError("failed to mount %s to FS since %s", pCfg->dir, tstrerror(code)); TAOS_RETURN(code); } diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index ebaa1e7dc3..720b49bb54 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -93,6 +93,11 @@ class TDTestCase: @property def __err_cfg(self): cfg_list = [] + err_case0 = [ + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L2} {NON_PRIMARY_DIR}" + ] err_case1 = [ f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L1} {PRIMARY_DIR}", @@ -131,6 +136,7 @@ class TDTestCase: f"dataDir {self.taos_data_dir}{os.sep}data33 -1 {NON_PRIMARY_DIR}" ] + cfg_list.append(err_case0) cfg_list.append(err_case1) cfg_list.append(err_case2) cfg_list.append(err_case3) From 0fd3e3d1a3b27a43603787322ef810c7572f1e33 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 21 Aug 2024 11:53:54 +0800 Subject: [PATCH 067/100] chore: restore test case --- tests/system-test/1-insert/mutil_stage.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 720b49bb54..ebaa1e7dc3 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -93,11 +93,6 @@ class TDTestCase: @property def __err_cfg(self): cfg_list = [] - err_case0 = [ - f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L2} {NON_PRIMARY_DIR}" - ] err_case1 = [ f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L1} {PRIMARY_DIR}", @@ -136,7 +131,6 @@ class TDTestCase: f"dataDir {self.taos_data_dir}{os.sep}data33 -1 {NON_PRIMARY_DIR}" ] - cfg_list.append(err_case0) cfg_list.append(err_case1) cfg_list.append(err_case2) cfg_list.append(err_case3) From 81c4f26c59ac74e65671e6b3920bbaa4e77cbba0 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 21 Aug 2024 04:23:42 +0000 Subject: [PATCH 068/100] fix/TD-31560 --- source/dnode/mnode/sdb/src/sdbFile.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6fb395e63f..9073032738 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -52,6 +52,8 @@ static void sdbResetData(SSdb *pSdb) { SHashObj *hash = pSdb->hashObjs[i]; if (hash == NULL) continue; + sdbWriteLock(pSdb, i); + SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { SSdbRow *pRow = *ppRow; @@ -60,15 +62,13 @@ static void sdbResetData(SSdb *pSdb) { sdbFreeRow(pSdb, pRow, true); ppRow = taosHashIterate(hash, ppRow); } - } - - for (ESdbType i = 0; i < SDB_MAX; ++i) { - SHashObj *hash = pSdb->hashObjs[i]; - if (hash == NULL) continue; taosHashClear(pSdb->hashObjs[i]); pSdb->tableVer[i] = 0; pSdb->maxId[i] = 0; + + sdbUnLock(pSdb, i); + mInfo("sdb:%s is reset", sdbTableName(i)); } From a3bebe896650b238ae0bc00295476771ed1f6d55 Mon Sep 17 00:00:00 2001 From: sima Date: Wed, 21 Aug 2024 10:01:32 +0800 Subject: [PATCH 069/100] fix:[TD-31571] fix CONCAT function to return NULL when all input parameters are NULL Previously, the CONCAT function did not correctly handle cases where all input parameters were NULL, resulting in an unexpected output. This commit ensures that CONCAT returns NULL when all input arguments are NULL, aligning with expected SQL behavior. --- source/libs/scalar/src/sclfunc.c | 59 +++++++++++++------------------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 12f0137fc5..477a768ccf 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -860,31 +860,27 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t numOfRows = 0; bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false; for (int32_t i = 0; i < inputNum; ++i) { - if (pInput[i].numOfRows > numOfRows) { - numOfRows = pInput[i].numOfRows; - } + numOfRows = TMAX(pInput[i].numOfRows, numOfRows); } + int32_t outputLen = VARSTR_HEADER_SIZE; for (int32_t i = 0; i < inputNum; ++i) { + if (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) { + colDataSetNNULL(pOutputData, 0, numOfRows); + pOutput->numOfRows = numOfRows; + goto _return; + } pInputData[i] = pInput[i].columnData; int32_t factor = 1; if (hasNchar && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { factor = TSDB_NCHAR_SIZE; } - - int32_t numOfNulls = getNumOfNullEntries(pInputData[i], pInput[i].numOfRows); - if (pInput[i].numOfRows == 1) { - inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * factor * (numOfRows - numOfNulls); - } else { - inputLen += (pInputData[i]->varmeta.length - (numOfRows - numOfNulls) * VARSTR_HEADER_SIZE) * factor; - } + outputLen += pInputData[i]->info.bytes * factor; } - int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; outputBuf = taosMemoryCalloc(outputLen, 1); if (NULL == outputBuf) { SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - char *output = outputBuf; for (int32_t k = 0; k < numOfRows; ++k) { bool hasNull = false; @@ -901,6 +897,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } VarDataLenT dataLen = 0; + char *output = outputBuf; for (int32_t i = 0; i < inputNum; ++i) { int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; input[i] = colDataGetData(pInputData[i], rowIdx); @@ -908,8 +905,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); } varDataSetLen(output, dataLen); - SCL_ERR_JRET(colDataSetVal(pOutputData, k, output, false)); - output += varDataTLen(output); + SCL_ERR_JRET(colDataSetVal(pOutputData, k, outputBuf, false)); } pOutput->numOfRows = numOfRows; @@ -926,51 +922,44 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t code = TSDB_CODE_SUCCESS; SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; - char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); char *outputBuf = NULL; if (NULL == pInputData) { SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - if (NULL == input) { - SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } int32_t inputLen = 0; int32_t numOfRows = 0; bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false; - for (int32_t i = 1; i < inputNum; ++i) { - if (pInput[i].numOfRows > numOfRows) { - numOfRows = pInput[i].numOfRows; - } - } for (int32_t i = 0; i < inputNum; ++i) { + numOfRows = TMAX(pInput[i].numOfRows, numOfRows); + } + int32_t outputLen = VARSTR_HEADER_SIZE; + for (int32_t i = 0; i < inputNum; ++i) { + if (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) { + colDataSetNNULL(pOutputData, 0, numOfRows); + pOutput->numOfRows = numOfRows; + goto _return; + } pInputData[i] = pInput[i].columnData; int32_t factor = 1; if (hasNchar && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { factor = TSDB_NCHAR_SIZE; } - int32_t numOfNulls = getNumOfNullEntries(pInputData[i], pInput[i].numOfRows); if (i == 0) { // calculate required separator space - inputLen += - (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * (numOfRows - numOfNulls) * (inputNum - 2) * factor; - } else if (pInput[i].numOfRows == 1) { - inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * (numOfRows - numOfNulls) * factor; + outputLen += pInputData[i]->info.bytes * factor * (inputNum - 2); } else { - inputLen += (pInputData[i]->varmeta.length - (numOfRows - numOfNulls) * VARSTR_HEADER_SIZE) * factor; + outputLen += pInputData[i]->info.bytes * factor; } } - int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; outputBuf = taosMemoryCalloc(outputLen, 1); if (NULL == outputBuf) { SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - char *output = outputBuf; - for (int32_t k = 0; k < numOfRows; ++k) { if (colDataIsNull_s(pInputData[0], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0]))) { colDataSetNULL(pOutputData, k); @@ -979,6 +968,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p VarDataLenT dataLen = 0; bool hasNull = false; + char *output = outputBuf; for (int32_t i = 1; i < inputNum; ++i) { if (colDataIsNull_s(pInputData[i], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) { hasNull = true; @@ -986,9 +976,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; - input[i] = colDataGetData(pInputData[i], rowIdx); - SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); + SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); if (i < inputNum - 1) { // insert the separator @@ -1003,14 +992,12 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } else { varDataSetLen(output, dataLen); SCL_ERR_JRET(colDataSetVal(pOutputData, k, output, false)); - output += varDataTLen(output); } } pOutput->numOfRows = numOfRows; _return: - taosMemoryFree(input); taosMemoryFree(outputBuf); taosMemoryFree(pInputData); From 62f227ed66b6a03b57cf811695dc975cfab75d91 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Aug 2024 14:13:49 +0800 Subject: [PATCH 070/100] fix(tsdb): check code before load print log. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 2288e8bbce..5fe3e9a787 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -903,7 +903,10 @@ int32_t tLDataIterNextRow(SLDataIter *pIter, const char *idStr, bool* hasNext) { pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); _exit: - tsdbError("failed to exec stt-file nextIter, lino:%d, code:%s, %s", lino, tstrerror(code), idStr); + if (code) { + tsdbError("failed to exec stt-file nextIter, lino:%d, code:%s, %s", lino, tstrerror(code), idStr); + } + *hasNext = (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL) && (pBlockData != NULL); return code; } From 2f446afe2872f97a0c88f9608887f9e78f3f768f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Aug 2024 14:15:09 +0800 Subject: [PATCH 071/100] fix: fix syntax error. --- source/libs/stream/src/streamTaskSm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index a8ed0f7639..4524b14435 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -168,7 +168,7 @@ static STaskStateTrans* streamTaskFindTransform(ETaskStatus state, const EStream } if (isInvalidStateTransfer(state, event)) { - stError("invalid state transfer %d, handle event:%d", state, GET_EVT_NAME(event)); + stError("invalid state transfer %d, handle event:%s", state, GET_EVT_NAME(event)); } return NULL; From 726f41697dd1e207e8324cab98a8059b2833029e Mon Sep 17 00:00:00 2001 From: sima Date: Tue, 20 Aug 2024 11:35:54 +0800 Subject: [PATCH 072/100] enh:[TD-31548] Remove ASSERT in multi files. --- source/dnode/vnode/src/meta/metaOpen.c | 4 +++ source/dnode/vnode/src/meta/metaQuery.c | 15 ++++++++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 12 +++++--- source/libs/executor/src/scanoperator.c | 1 + source/libs/executor/src/sysscanoperator.c | 18 +++++++++++ source/libs/index/inc/indexTfile.h | 2 +- source/libs/index/src/indexCache.c | 23 ++++++++++++-- source/libs/index/src/indexComm.c | 18 +++++++++++ source/libs/index/src/indexFilter.c | 15 ++++++++++ source/libs/index/src/indexTfile.c | 35 ++++++++++++++++++---- source/libs/scalar/src/filter.c | 20 ++++++++++++- source/util/src/tcompare.c | 14 ++++++--- source/util/src/thashutil.c | 2 ++ 13 files changed, 161 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 13163aef05..2ba35a3987 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -332,6 +332,10 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL } else if (!pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { // all not NULL, compr tag vals __compar_fn_t func = getComparFunc(pTagIdxKey1->type, 0); + if (func == NULL) { + metaError("meta/open: %s", terrstr()); + return TSDB_CODE_FAILED; + } c = func(pTagIdxKey1->data, pTagIdxKey2->data); if (c) return c; } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 3b10182c90..45f9baf6fd 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1103,7 +1103,12 @@ int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) { SBtimeIdxKey *p = entryKey; if (count > TRY_ERROR_LIMIT) break; + terrno = TSDB_CODE_SUCCESS; int32_t cmp = (*param->filterFunc)((void *)&p->btime, (void *)&pBtimeKey->btime, param->type); + if (terrno != TSDB_CODE_SUCCESS) { + ret = terrno; + break; + } if (cmp == 0) { if (taosArrayPush(pUids, &p->uid) == NULL) { ret = terrno; @@ -1167,7 +1172,12 @@ int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) { if (count > TRY_ERROR_LIMIT) break; char *pTableKey = (char *)pEntryKey; + terrno = TSDB_CODE_SUCCESS; cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type); + if (terrno != TSDB_CODE_SUCCESS) { + ret = terrno; + goto END; + } if (cmp == 0) { tb_uid_t tuid = *(tb_uid_t *)pEntryVal; if (taosArrayPush(pUids, &tuid) == NULL) { @@ -1361,7 +1371,12 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { } } + terrno = TSDB_CODE_SUCCESS; int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); + if (terrno != TSDB_CODE_SUCCESS) { + TAOS_CHECK_GOTO(terrno, NULL, END); + break; + } if (cmp == 0) { // match tb_uid_t tuid = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 904b29bf43..5849ef9fcc 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -19,7 +19,6 @@ #define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ do { \ int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \ - ASSERT(newVal >= 0); \ if (newVal < 0) { \ vWarn("vgId:%d, %s, abnormal val:%" PRIi64 ", old val:%" PRIi64, TD_VID(pVnode), tags, newVal, (oVal)); \ } \ @@ -37,7 +36,10 @@ int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol int8_t tblType = reader->me.type; if (useCompress(tblType)) { SColCmprWrapper *p = &(reader->me.colCmpr); - ASSERT(numOfCol == p->nCols); + if (numOfCol != p->nCols) { + vError("fillTableColCmpr table type:%d, col num:%d, col cmpr num:%d mismatch", tblType, numOfCol, p->nCols); + return TSDB_CODE_APP_ERROR; + } for (int i = 0; i < p->nCols; i++) { SColCmpr *pCmpr = &p->pColCmpr[i]; pExt[i].colId = pCmpr->id; @@ -104,7 +106,8 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { } else if (mer1.me.type == TSDB_NORMAL_TABLE) { schema = mer1.me.ntbEntry.schemaRow; } else { - ASSERT(0); + vError("vnodeGetTableMeta get invalid table type:%d", mer1.me.type); + return TSDB_CODE_APP_ERROR; } metaRsp.numOfTags = schemaTag.nCols; @@ -262,7 +265,8 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { } } } else { - ASSERT(0); + vError("vnodeGetTableCfg get invalid table type:%d", mer1.me.type); + return TSDB_CODE_APP_ERROR; } cfgRsp.numOfTags = schemaTag.nCols; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 82b0a729b0..1a6bb8a5cc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2592,6 +2592,7 @@ static int32_t doBlockDataPrimaryKeyFilter(SSDataBlock* pBlock, STqOffsetVal* of TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); __compar_fn_t func = getComparFunc(pColPk->info.type, 0); + QUERY_CHECK_NULL(func, code, lino, _end, terrno); for (int32_t i = 0; i < pBlock->info.rows; ++i) { int64_t* ts = (int64_t*)colDataGetData(pColTs, i); void* data = colDataGetData(pColPk, i); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index d8a2331980..fc1159cc9d 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -343,27 +343,45 @@ int optSysDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { static int optSysFilterFuncImpl__LowerThan(void* a, void* b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return optSysDoCompare(func, OP_TYPE_LOWER_THAN, a, b); } static int optSysFilterFuncImpl__LowerEqual(void* a, void* b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return optSysDoCompare(func, OP_TYPE_LOWER_EQUAL, a, b); } static int optSysFilterFuncImpl__GreaterThan(void* a, void* b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return optSysDoCompare(func, OP_TYPE_GREATER_THAN, a, b); } static int optSysFilterFuncImpl__GreaterEqual(void* a, void* b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return optSysDoCompare(func, OP_TYPE_GREATER_EQUAL, a, b); } static int optSysFilterFuncImpl__Equal(void* a, void* b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return optSysDoCompare(func, OP_TYPE_EQUAL, a, b); } static int optSysFilterFuncImpl__NoEqual(void* a, void* b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return optSysDoCompare(func, OP_TYPE_NOT_EQUAL, a, b); } diff --git a/source/libs/index/inc/indexTfile.h b/source/libs/index/inc/indexTfile.h index 51d3cc0e7b..115d5b3894 100644 --- a/source/libs/index/inc/indexTfile.h +++ b/source/libs/index/inc/indexTfile.h @@ -115,7 +115,7 @@ int32_t tfileWriterOpen(char* path, uint64_t suid, int64_t version, const char* void tfileWriterClose(TFileWriter* tw); int32_t tfileWriterCreate(IFileCtx* ctx, TFileHeader* header, TFileWriter** pWriter); void tfileWriterDestroy(TFileWriter* tw); -int tfileWriterPut(TFileWriter* tw, void* data, bool order); +int32_t tfileWriterPut(TFileWriter* tw, void* data, bool order); int tfileWriterFinish(TFileWriter* tw); // diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index adf555edd2..734c4067c6 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -123,6 +123,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* if (cache == NULL) { return 0; } + int32_t code = TSDB_CODE_SUCCESS; MemTable* mem = cache; IndexCache* pCache = mem->pCache; @@ -146,7 +147,12 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + terrno = TSDB_CODE_SUCCESS; TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType); + if (terrno != TSDB_CODE_SUCCESS) { + code = terrno; + goto _return; + } if (cond == MATCH) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) @@ -161,9 +167,11 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* break; } } + +_return: taosMemoryFree(pCt); (void)tSkipListDestroyIter(iter); - return TSDB_CODE_SUCCESS; + return code; } static int32_t cacheSearchLessThan(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { return cacheSearchCompareFunc(cache, term, tr, s, LT); @@ -265,6 +273,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR } _cache_range_compare cmpFn = idxGetCompare(type); + int32_t code = TSDB_CODE_SUCCESS; MemTable* mem = cache; IndexCache* pCache = mem->pCache; @@ -308,9 +317,18 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR continue; } else { char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1); + if (NULL == p) { + code = terrno; + goto _return; + } memcpy(p, c->colVal, strlen(c->colVal)); + terrno = TSDB_CODE_SUCCESS; cond = cmpFn(p + skip, term->colVal, dType); taosMemoryFree(p); + if (terrno != TSDB_CODE_SUCCESS) { + code = terrno; + goto _return; + } } } if (cond == MATCH) { @@ -327,11 +345,12 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR } } +_return: taosMemoryFree(pCt); taosMemoryFree(exBuf); (void)tSkipListDestroyIter(iter); - return TSDB_CODE_SUCCESS; + return code; } static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { // impl later diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 6337b2c556..e62f506b1a 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -84,27 +84,45 @@ __compar_fn_t idxGetCompar(int8_t type) { } static FORCE_INLINE TExeCond tCompareLessThan(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); + if (func == NULL) { + return BREAK; + } return tCompare(func, QUERY_LESS_THAN, a, b, type); } static FORCE_INLINE TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); + if (func == NULL) { + return BREAK; + } return tCompare(func, QUERY_LESS_EQUAL, a, b, type); } static FORCE_INLINE TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); + if (func == NULL) { + return BREAK; + } return tCompare(func, QUERY_GREATER_THAN, a, b, type); } static FORCE_INLINE TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); + if (func == NULL) { + return BREAK; + } return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } static FORCE_INLINE TExeCond tCompareContains(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); + if (func == NULL) { + return BREAK; + } return tCompare(func, QUERY_TERM, a, b, type); } static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); + if (func == NULL) { + return BREAK; + } return tCompare(func, QUERY_TERM, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index c362932da3..2858d09c30 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -432,22 +432,37 @@ typedef int (*FilterFunc)(void *a, void *b, int16_t dtype); static FORCE_INLINE int sifGreaterThan(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return tDoCompare(func, QUERY_GREATER_THAN, a, b); } static FORCE_INLINE int sifGreaterEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return tDoCompare(func, QUERY_GREATER_EQUAL, a, b); } static FORCE_INLINE int sifLessEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return tDoCompare(func, QUERY_LESS_EQUAL, a, b); } static FORCE_INLINE int sifLessThan(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } return (int)tDoCompare(func, QUERY_LESS_THAN, a, b); } static FORCE_INLINE int sifEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); + if (func == NULL) { + return -1; + } //__compar_fn_t func = idxGetCompar(dtype); return (int)tDoCompare(func, QUERY_TERM, a, b); } diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 99c7b6bc76..839bf7d42f 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -315,7 +315,7 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { } static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, RangeType type) { - int ret = 0; + int32_t code = TSDB_CODE_SUCCESS; char* p = tem->colVal; int skip = 0; _cache_range_compare cmpFn = idxGetCompare(type); @@ -335,7 +335,13 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, FstSlice* s = &rt->data; char* ch = (char*)fstSliceData(s, NULL); + terrno = TSDB_CODE_SUCCESS; TExeCond cond = cmpFn(ch, p, tem->colType); + if (TSDB_CODE_SUCCESS != terrno) { + swsResultDestroy(rt); + code = terrno; + goto _return; + } if (MATCH == cond) { (void)tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); } else if (CONTINUE == cond) { @@ -345,10 +351,11 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, } swsResultDestroy(rt); } +_return: stmStDestroy(st); stmBuilderDestroy(sb); taosArrayDestroy(offsets); - return TSDB_CODE_SUCCESS; + return code; } static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { return tfSearchCompareFunc(reader, tem, tr, LT); @@ -427,8 +434,8 @@ static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) } static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr, RangeType ctype) { - int ret = 0; - int skip = 0; + int32_t code = TSDB_CODE_SUCCESS; + int skip = 0; char* p = NULL; if (ctype == CONTAINS) { @@ -469,9 +476,20 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt continue; } char* tBuf = taosMemoryCalloc(1, sz + 1); + if (NULL == tBuf) { + swsResultDestroy(rt); + code = terrno; + goto _return; + } memcpy(tBuf, ch, sz); + terrno = TSDB_CODE_SUCCESS; cond = cmpFn(tBuf + skip, tem->colVal, IDX_TYPE_GET_TYPE(tem->colType)); taosMemoryFree(tBuf); + if (TSDB_CODE_SUCCESS != terrno) { + swsResultDestroy(rt); + code = terrno; + goto _return; + } } if (MATCH == cond) { (void)tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); @@ -482,12 +500,14 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt } swsResultDestroy(rt); } + +_return: stmStDestroy(st); stmBuilderDestroy(sb); taosArrayDestroy(offsets); taosMemoryFree(p); - return TSDB_CODE_SUCCESS; + return code; } int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr) { int ret = 0; @@ -558,7 +578,7 @@ int32_t tfileWriterCreate(IFileCtx* ctx, TFileHeader* header, TFileWriter** pWri return code; } -int tfileWriterPut(TFileWriter* tw, void* data, bool order) { +int32_t tfileWriterPut(TFileWriter* tw, void* data, bool order) { // sort by coltype and write to tindex if (order == false) { __compar_fn_t fn; @@ -571,6 +591,9 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { } else { fn = getComparFunc(colType, 0); } + if (fn == NULL) { + return terrno; + } (void)taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 988aeba46f..c50e807675 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -521,6 +521,10 @@ int32_t filterInitRangeCtx(int32_t type, int32_t options, SFilterRangeCtx **ctx) (*ctx)->type = type; (*ctx)->options = options; (*ctx)->pCompareFunc = getComparFunc(type, 0); + if ((*ctx)->pCompareFunc == NULL) { + taosMemoryFree(*ctx); + return terrno; + } return TSDB_CODE_SUCCESS; } @@ -556,6 +560,9 @@ int32_t filterReuseRangeCtx(SFilterRangeCtx *ctx, int32_t type, int32_t options) ctx->options = options; ctx->pCompareFunc = getComparFunc(type, 0); + if (ctx->pCompareFunc == NULL) { + return terrno; + } return TSDB_CODE_SUCCESS; } @@ -1454,6 +1461,9 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); + if (func == NULL) { + FLT_ERR_RET(terrno); + } if (func(&ra->s, &ra->e) == 0) { void *data = taosMemoryMalloc(sizeof(int64_t)); if (data == NULL) { @@ -1565,6 +1575,9 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); + if (func == NULL) { + FLT_ERR_RET(terrno); + } if (func(&r->ra.s, &r->ra.e) == 0) { void *data = taosMemoryMalloc(sizeof(int64_t)); if (data == NULL) { @@ -2461,7 +2474,12 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t } if (colIdxi > 1) { - taosSort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); + __compar_fn_t cmpFn = getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0); + if (cmpFn == NULL) { + filterFreeGroupCtx(gRes[gResIdx]); + FLT_ERR_JRET(terrno); + } + taosSort(colIdx, colIdxi, sizeof(uint32_t), cmpFn); } for (uint32_t l = 0; l < colIdxi; ++l) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 670a70a309..0dee177faf 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -267,7 +267,7 @@ int32_t compareJsonVal(const void *pLeft, const void *pRight) { } else if (leftType == TSDB_DATA_TYPE_NULL) { return 0; } else { - ASSERTS(0, "data type unexpected"); + uError("data type unexpected leftType:%d rightType:%d", leftType, rightType); return 0; } } @@ -1497,7 +1497,9 @@ int32_t taosArrayCompareString(const void *a, const void *b) { int32_t comparestrPatternMatch(const void *pLeft, const void *pRight) { SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER; - ASSERT(varDataTLen(pRight) <= TSDB_MAX_FIELD_LEN); + if (varDataTLen(pRight) > TSDB_MAX_FIELD_LEN) { + return 1; + } size_t pLen = varDataLen(pRight); size_t sz = varDataLen(pLeft); @@ -1546,7 +1548,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_TIMESTAMP: return setChkInBytes8; default: - ASSERTS(0, "data type unexpected"); + uError("getComparFunc data type unexpected type:%d, optr:%d", type, optr); + terrno = TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + return NULL; } } @@ -1570,7 +1574,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_TIMESTAMP: return setChkNotInBytes8; default: - ASSERTS(0, "data type unexpected"); + uError("getComparFunc data type unexpected type:%d, optr:%d", type, optr); + terrno = TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + return NULL; } } diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 9d65977ff1..b466e1b351 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -226,10 +226,12 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { } int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + // getComparFunc(TSDB_DATA_TYPE_FLOAT, -1) will always get function compareFloatVal, which will never be NULL. return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b); } int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + // getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1) will always get function compareDoubleVal, which will never be NULL. return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); } From 044f7d66587ade3f966fcda100aedd3a55c9f9fe Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 21 Aug 2024 14:41:14 +0800 Subject: [PATCH 073/100] fix: remove assert --- source/libs/catalog/inc/catalogInt.h | 40 +++++++-- source/libs/catalog/src/catalog.c | 2 - source/libs/catalog/src/ctgAsync.c | 52 ++++++------ source/libs/catalog/src/ctgRemote.c | 7 +- source/libs/catalog/src/ctgUtil.c | 2 +- source/libs/executor/src/dataDeleter.c | 10 ++- source/libs/executor/src/dataDispatcher.c | 6 +- source/libs/executor/src/dataInserter.c | 6 +- .../libs/executor/src/dynqueryctrloperator.c | 4 +- source/libs/executor/src/hashjoin.c | 6 +- source/libs/executor/src/hashjoinoperator.c | 3 - source/libs/executor/src/mergejoin.c | 46 +++++----- source/libs/executor/src/mergejoinoperator.c | 14 ++-- source/libs/qworker/inc/qwInt.h | 84 ++++++++++++------- source/libs/scheduler/inc/schInt.h | 40 +++++++-- source/os/src/osFile.c | 3 - source/os/src/osMemory.c | 8 +- source/os/src/osSocket.c | 11 +-- source/os/src/osString.c | 49 ----------- source/os/src/osSysinfo.c | 3 +- source/os/src/osSystem.c | 3 - source/os/src/osThread.c | 3 +- 22 files changed, 214 insertions(+), 188 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f3b1852ce1..5580b0cadc 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -852,34 +852,58 @@ typedef struct SCtgCacheItemInfo { #define CTG_LOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before read lock"); \ + break; \ + } \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value after read lock"); \ + break; \ + } \ } else { \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before write lock"); \ + break; \ + } \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value after write lock"); \ + break; \ + } \ } \ } while (0) #define CTG_UNLOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value before read unlock"); \ + break; \ + } \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after read unlock"); \ + break; \ + } \ } else { \ - ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value before write unlock"); \ + break; \ + } \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after write unlock"); \ + break; \ + } \ } \ } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d4c79a6c8d..334dce9c1a 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -773,8 +773,6 @@ int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaNa } CTG_ERR_JRET(code); - - ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1); *pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0); taosArrayDestroy(tsmaRsp.pTsmas); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 4caa66445a..031d61554c 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -2448,30 +2448,33 @@ int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* STableTSMAInfoRsp* pOut = pMsgCtx->out; pRes->code = 0; - if (pOut->pTsmas->size > 0) { - ASSERT(pOut->pTsmas->size == 1); - pRes->pRes = pOut; - pMsgCtx->out = NULL; - TSWAP(pTask->res, pCtx->pResList); + if (1 != pOut->pTsmas->size) { + ctgError("invalid tsma num:%d", (int32_t)pOut->pTsmas->size); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } - STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0); - if (NULL == pTsma) { - ctgError("fail to get the 0th STableTSMAInfo, totalNum:%d", (int32_t)taosArrayGetSize(pOut->pTsmas)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - int32_t exists = false; - CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); - if (!exists) { - TSWAP(pMsgCtx->lastOut, pMsgCtx->out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); - } + pRes->pRes = pOut; + pMsgCtx->out = NULL; + TSWAP(pTask->res, pCtx->pResList); + + STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0); + if (NULL == pTsma) { + ctgError("fail to get the 0th STableTSMAInfo, totalNum:%d", (int32_t)taosArrayGetSize(pOut->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } + int32_t exists = false; + CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); + if (!exists) { + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); + } + break; } default: - ASSERT(0); + ctgError("invalid reqType:%d while getting tsma rsp", reqType); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } _return: @@ -2635,14 +2638,14 @@ int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf break; } default: - ASSERT(0); + ctgError("invalid fetchType:%d while getting tb tsma rsp", pFetch->fetchType); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } break; } case TDMT_VND_TABLE_META: { // handle source tb meta - ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META); STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; pFetch->fetchType = FETCH_TB_TSMA; pFetch->tsmaSourceTbName = *pTbName; @@ -2663,7 +2666,8 @@ int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf break; } default: - ASSERT(0); + ctgError("invalid reqType:%d while getting tsma rsp", reqType); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } _return: @@ -3628,7 +3632,8 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { break; } default: - ASSERT(0); + ctgError("invalid fetchType:%d in getting tb tsma task", pFetch->fetchType); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); break; } } @@ -3644,14 +3649,12 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; // currently, only support fetching one tsma - ASSERT(pCtx->pNames->size == 1); STablesReq* pReq = taosArrayGet(pCtx->pNames, 0); if (NULL == pReq) { ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - ASSERT(pReq->pTables->size == 1); SName* pTsmaName = taosArrayGet(pReq->pTables, 0); if (NULL == pReq) { ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pReq->pTables)); @@ -3686,7 +3689,6 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { } STableTSMAInfoRsp* pRsp = (STableTSMAInfoRsp*)pRes->pRes; - ASSERT(pRsp->pTsmas->size == 1); const STSMACache* pTsma = taosArrayGetP(pRsp->pTsmas, 0); if (NULL == pTsma) { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index a312dce164..96a700d2d6 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -42,7 +42,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = taosArrayGetSize(batchRsp.pRsps); } - if (ASSERTS(taskNum == msgNum || 0 == msgNum, "taskNum %d mis-match msgNum %d", taskNum, msgNum)) { + if (taskNum != msgNum && 0 != msgNum) { + ctgError("taskNum %d mis-match msgNum %d", taskNum, msgNum); msgNum = 0; } @@ -77,7 +78,9 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu if (msgNum > 0) { pRsp = taosArrayGet(batchRsp.pRsps, i); - if (ASSERTS(pRsp->msgIdx == *msgIdx, "rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx)) { + if (pRsp->msgIdx != *msgIdx) { + ctgError("rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx); + pRsp = &rsp; pRsp->msgIdx = *msgIdx; pRsp->reqType = -1; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index fa4df70f59..5d8107ca44 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -2631,7 +2631,7 @@ bool hasOutOfDateTSMACache(SArray* pTsmas) { for (int32_t i = 0; i < pTsmas->size; ++i) { STSMACache* pTsmaInfo = taosArrayGetP(pTsmas, i); if (NULL == pTsmaInfo) { - ASSERT(0); + continue; } if (isCtgTSMACacheOutOfDate(pTsmaInfo)) { return true; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 9f0ea0a87f..60bfb58ef5 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -87,7 +87,10 @@ static int32_t toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* p if (pRes->affectedRows) { pRes->skey = *(int64_t*)pColSKey->pData; pRes->ekey = *(int64_t*)pColEKey->pData; - ASSERT(pRes->skey <= pRes->ekey); + if (pRes->skey > pRes->ekey) { + qError("data delter skey:%" PRId64 " is bigger than ekey:%" PRId64, pRes->skey, pRes->ekey); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } } else { pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; @@ -205,7 +208,10 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; if (NULL == pDeleter->nextOutput.pData) { - ASSERT(pDeleter->queryEnd); + if (!pDeleter->queryEnd) { + qError("empty res while query not end in data deleter"); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } pOutput->useconds = pDeleter->useconds; pOutput->precision = pDeleter->pSchema->precision; pOutput->bufStatus = DS_BUF_EMPTY; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3964422411..d96b78a342 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -242,7 +242,11 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (NULL == pDispatcher->nextOutput.pData) { - ASSERT(pDispatcher->queryEnd); + if (!pDispatcher->queryEnd) { + qError("empty res while query not end in data dispatcher"); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->pSchema->precision; pOutput->bufStatus = DS_BUF_EMPTY; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 6f226ecb21..44342b0ac9 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -234,7 +234,11 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - ASSERT(pColInfoData->info.type == pCol->type); + if (pColInfoData->info.type != pCol->type) { + qError("column:%d type:%d in block dismatch with schema col:%d type:%d", colIdx, pColInfoData->info.type, k, pCol->type); + terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + goto _end; + } if (colDataIsNull_s(pColInfoData, j)) { SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); if (NULL == taosArrayPush(pVals, &cv)) { diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 02932cd278..ea2f81a4b8 100644 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -823,12 +823,14 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) { pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache); qDebug("more than 1 ref build table num %" PRId64, (int64_t)tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache)); +/* // debug only iter = 0; uint32_t* num = NULL; while (NULL != (num = tSimpleHashIterate(pStbJoin->ctx.prev.leftCache, num, &iter))) { - ASSERT(*num > 1); + A S S E R T(*num > 1); } +*/ } static void buildStbJoinTableList(SOperatorInfo* pOperator) { diff --git a/source/libs/executor/src/hashjoin.c b/source/libs/executor/src/hashjoin.c index d4a84afea2..f63b4093db 100755 --- a/source/libs/executor/src/hashjoin.c +++ b/source/libs/executor/src/hashjoin.c @@ -60,7 +60,7 @@ int32_t hInnerJoinDo(struct SOperatorInfo* pOperator) { /* size_t keySize = 0; int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize); - ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen)); + A S S E R T(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen)); int64_t rows = getSingleKeyRowsNum(pGroup->rows); pJoin->execInfo.expectRows += rows; qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows); @@ -145,7 +145,7 @@ int32_t hLeftJoinHandleSeqProbeRows(struct SOperatorInfo* pOperator, SHJoinOpera /* size_t keySize = 0; int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize); - ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen)); + A S S E R T(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen)); int64_t rows = getSingleKeyRowsNum(pGroup->rows); pJoin->execInfo.expectRows += rows; qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows); @@ -248,7 +248,7 @@ int32_t hLeftJoinHandleProbeRows(struct SOperatorInfo* pOperator, SHJoinOperator /* size_t keySize = 0; int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize); - ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen)); + A S S E R T(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen)); int64_t rows = getSingleKeyRowsNum(pGroup->rows); pJoin->execInfo.expectRows += rows; qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows); diff --git a/source/libs/executor/src/hashjoinoperator.c b/source/libs/executor/src/hashjoinoperator.c index 807d6b9785..15819cd94a 100644 --- a/source/libs/executor/src/hashjoinoperator.c +++ b/source/libs/executor/src/hashjoinoperator.c @@ -38,8 +38,6 @@ bool hJoinBlkReachThreshold(SHJoinOperatorInfo* pInfo, int64_t blkRows) { } int32_t hJoinHandleMidRemains(SHJoinOperatorInfo* pJoin, SHJoinCtx* pCtx) { - ASSERT(0 < pJoin->midBlk->info.rows); - TSWAP(pJoin->midBlk, pJoin->finBlk); pCtx->midRemains = false; @@ -1153,7 +1151,6 @@ int32_t hJoinInitResBlocks(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinN if (NULL == pJoin->finBlk) { QRY_ERR_RET(terrno); } - ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0); int32_t code = blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode)); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index d3abaaab6d..302dd31788 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -321,7 +321,7 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { int32_t buildEndIdx = buildGrp->endIdx; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; - ASSERT(buildGrp->endIdx >= buildGrp->readIdx); + //A S S E R T(buildGrp->endIdx >= buildGrp->readIdx); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; @@ -1383,9 +1383,9 @@ static int32_t mSemiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p continue; } - ASSERT(1 == pCtx->midBlk->info.rows); + //A S S E R T(1 == pCtx->midBlk->info.rows); MJ_ERR_RET(mJoinCopyMergeMidBlk(pCtx, &pCtx->midBlk, &pCtx->finBlk)); - ASSERT(false == pCtx->midRemains); + //A S S E R T(false == pCtx->midRemains); break; } while (true); @@ -1449,10 +1449,10 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) { } build->pHashCurGrp = *(SArray**)pGrp; - ASSERT(1 == taosArrayGetSize(build->pHashCurGrp)); + //A S S E R T(1 == taosArrayGetSize(build->pHashCurGrp)); build->grpRowIdx = 0; MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, NULL)); - ASSERT(build->grpRowIdx < 0); + //A S S E R T(build->grpRowIdx < 0); } pCtx->grpRemains = probeGrp->readIdx <= probeGrp->endIdx; @@ -1497,7 +1497,7 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { int32_t buildEndIdx = buildGrp->endIdx; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; - ASSERT(buildGrp->endIdx >= buildGrp->readIdx); + //A S S E R T(buildGrp->endIdx >= buildGrp->readIdx); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; @@ -1513,9 +1513,9 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { continue; } } else { - ASSERT(1 == pCtx->midBlk->info.rows); + //A S S E R T(1 == pCtx->midBlk->info.rows); MJ_ERR_RET(mJoinCopyMergeMidBlk(pCtx, &pCtx->midBlk, &pCtx->finBlk)); - ASSERT(false == pCtx->midRemains); + //A S S E R T(false == pCtx->midRemains); if (build->grpIdx == buildGrpNum) { continue; @@ -1555,8 +1555,8 @@ static int32_t mSemiJoinMergeFullCart(SMJoinMergeCtx* pCtx) { int32_t probeRows = GRP_REMAIN_ROWS(probeGrp); int32_t probeEndIdx = probeGrp->endIdx; - ASSERT(1 == taosArrayGetSize(build->eqGrps)); - ASSERT(buildGrp->beginIdx == buildGrp->endIdx); + //A S S E R T(1 == taosArrayGetSize(build->eqGrps)); + //A S S E R T(buildGrp->beginIdx == buildGrp->endIdx); if (probeRows <= rowsLeft) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); @@ -1826,7 +1826,7 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { int32_t buildEndIdx = buildGrp->endIdx; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; - ASSERT(buildGrp->endIdx >= buildGrp->readIdx); + //A S S E R T(buildGrp->endIdx >= buildGrp->readIdx); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; @@ -2381,7 +2381,7 @@ int32_t mAsofForwardTrimCacheBlk(SMJoinWindowCtx* pCtx) { if (pGrp->blk == pCtx->cache.outBlk && pCtx->pJoin->build->blkRowIdx > 0) { MJ_ERR_RET(blockDataTrimFirstRows(pGrp->blk, pCtx->pJoin->build->blkRowIdx)); pCtx->pJoin->build->blkRowIdx = 0; - ASSERT(pCtx->pJoin->build->blk == pGrp->blk); + //A S S E R T(pCtx->pJoin->build->blk == pGrp->blk); MJOIN_SAVE_TB_BLK(&pCtx->cache, pCtx->pJoin->build); } @@ -2419,16 +2419,16 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) { MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } - ASSERT(pGrp->blk == pCache->outBlk); + //A S S E R T(pGrp->blk == pCache->outBlk); //pGrp->endIdx = pGrp->blk->info.rows - pGrp->beginIdx; } - //ASSERT((pGrp->endIdx - pGrp->beginIdx + 1) == pCtx->cache.rowNum); + //A S S E R T((pGrp->endIdx - pGrp->beginIdx + 1) == pCtx->cache.rowNum); } - ASSERT(taosArrayGetSize(pCache->grps) == 1); - ASSERT(pGrp->blk->info.rows - pGrp->beginIdx == pCtx->cache.rowNum); + //A S S E R T(taosArrayGetSize(pCache->grps) == 1); + //A S S E R T(pGrp->blk->info.rows - pGrp->beginIdx == pCtx->cache.rowNum); } do { @@ -2473,7 +2473,7 @@ int32_t mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) { return TSDB_CODE_SUCCESS; } - ASSERT(pCtx->jLimit > (pGrp->blk->info.rows - pGrp->beginIdx)); + //A S S E R T(pCtx->jLimit > (pGrp->blk->info.rows - pGrp->beginIdx)); pGrp->endIdx = pGrp->blk->info.rows - 1; int64_t remainRows = pCtx->jLimit - (pGrp->endIdx - pGrp->beginIdx + 1); @@ -2550,8 +2550,8 @@ int32_t mAsofForwardSkipAllEqRows(SMJoinWindowCtx* pCtx, int64_t timestamp) { MJOIN_RESTORE_TB_BLK(cache, pTable); } while (!MJOIN_BUILD_TB_ROWS_DONE(pTable)); - ASSERT(pCtx->cache.rowNum == 0); - ASSERT(taosArrayGetSize(pCtx->cache.grps) == 0); + //A S S E R T(pCtx->cache.rowNum == 0); + //A S S E R T(taosArrayGetSize(pCtx->cache.grps) == 0); if (pTable->dsFetchDone) { return TSDB_CODE_SUCCESS; @@ -2648,7 +2648,7 @@ static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo if ((probeGot || MJOIN_DS_NEED_INIT(pOperator, pJoin->build)) && pCtx->cache.rowNum < pCtx->jLimit) { pJoin->build->newBlk = false; MJOIN_SAVE_TB_BLK(&pCtx->cache, pCtx->pJoin->build); - ASSERT(taosArrayGetSize(pCtx->cache.grps) <= 1); + //A S S E R T(taosArrayGetSize(pCtx->cache.grps) <= 1); buildGot = mJoinRetrieveBlk(pJoin, &pJoin->build->blkRowIdx, &pJoin->build->blk, pJoin->build); } @@ -3375,7 +3375,7 @@ int32_t mWinJoinMoveAscWinEnd(SMJoinWindowCtx* pCtx) { continue; } - ASSERT(pGrp->endIdx > startIdx); + //A S S E R T(pGrp->endIdx > startIdx); pGrp->endIdx--; break; @@ -3419,7 +3419,7 @@ int32_t mWinJoinMoveDescWinEnd(SMJoinWindowCtx* pCtx) { continue; } - ASSERT(pGrp->endIdx > startIdx); + //A S S E R T(pGrp->endIdx > startIdx); pGrp->endIdx--; break; @@ -3676,7 +3676,7 @@ int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJ MJ_ERR_RET(terrno); } - ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0); + //A S S E R T(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0); MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode))); diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 52b0da7c92..fced682312 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -490,8 +490,6 @@ int32_t mJoinCopyMergeMidBlk(SMJoinMergeCtx* pCtx, SSDataBlock** ppMid, SSDataBl int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx) { - ASSERT(0 < pCtx->midBlk->info.rows); - TSWAP(pCtx->midBlk, pCtx->finBlk); pCtx->midRemains = false; @@ -567,7 +565,6 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app int32_t currRows = append ? pRes->info.rows : 0; int32_t firstRows = GRP_REMAIN_ROWS(pFirst); int32_t secondRows = GRP_REMAIN_ROWS(pSecond); - ASSERT(secondRows > 0); for (int32_t c = 0; c < probe->finNum; ++c) { SMJoinColMap* pFirstCol = probe->finCols + c; @@ -581,9 +578,15 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app if (colDataIsNull_s(pInCol, pFirst->readIdx + r)) { colDataSetNItemsNull(pOutCol, currRows + r * secondRows, secondRows); } else { - ASSERT(pRes->info.capacity >= (pRes->info.rows + firstRows * secondRows)); + if (pRes->info.capacity < (pRes->info.rows + firstRows * secondRows)) { + qError("capacity:%d not enough, rows:%" PRId64 ", firstRows:%d, secondRows:%d", pRes->info.capacity, pRes->info.rows, firstRows, secondRows); + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } uint32_t startOffset = (IS_VAR_DATA_TYPE(pOutCol->info.type)) ? pOutCol->varmeta.length : ((currRows + r * secondRows) * pOutCol->info.bytes); - ASSERT((startOffset + 1 * pOutCol->info.bytes) <= pRes->info.capacity * pOutCol->info.bytes); + if ((startOffset + 1 * pOutCol->info.bytes) > pRes->info.capacity * pOutCol->info.bytes) { + qError("col buff not enough, startOffset:%d, bytes:%d, capacity:%d", startOffset, pOutCol->info.bytes, pRes->info.capacity); + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } MJ_ERR_RET(colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true)); } } @@ -1097,7 +1100,6 @@ SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTa } SMJoinTableCtx* pProbe = pJoin->probe; - ASSERT(pProbe->lastInGid); while (true) { if (pTable->remainInBlk) { diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 93bbc44a25..68355b628e 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -338,38 +338,62 @@ typedef struct SQWorkerMgmt { #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define QW_LOCK(type, _lock) \ - do { \ - if (QW_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ - QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ - } else { \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ - QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ - } \ +#define QW_LOCK(type, _lock) \ + do { \ + if (QW_READ == (type)) { \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before read lock"); \ + break; \ + } \ + QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value after read lock"); \ + break; \ + } \ + } else { \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before write lock"); \ + break; \ + } \ + QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value after write lock"); \ + break; \ + } \ + } \ } while (0) -#define QW_UNLOCK(type, _lock) \ - do { \ - if (QW_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ - QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ - } else { \ - ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ - QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ - } \ +#define QW_UNLOCK(type, _lock) \ + do { \ + if (QW_READ == (type)) { \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value before read unlock"); \ + break; \ + } \ + QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after read unlock"); \ + break; \ + } \ + } else { \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value before write unlock"); \ + break; \ + } \ + QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after write unlock"); \ + break; \ + } \ + } \ } while (0) extern SQWorkerMgmt gQwMgmt; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 5ea79c6ae9..6a64ff67e4 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -501,34 +501,58 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before read lock"); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before read lock"); \ + break; \ + } \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32(_lock) > 0, "invalid lock value after read lock"); \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value after read lock"); \ + break; \ + } \ } else { \ - ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before write lock"); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value before write lock"); \ + break; \ + } \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value after write lock"); \ + break; \ + } \ } \ } while (0) #define SCH_UNLOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ + if (atomic_load_32((_lock)) <= 0) { \ + qError("invalid lock value before read unlock"); \ + break; \ + } \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after read unlock"); \ + break; \ + } \ } else { \ - ASSERTS(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ + if (atomic_load_32((_lock)) != TD_RWLATCH_WRITE_FLAG_COPY) { \ + qError("invalid lock value before write unlock"); \ + break; \ + } \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ + if (atomic_load_32((_lock)) < 0) { \ + qError("invalid lock value after write unlock"); \ + break; \ + } \ } \ } while (0) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index a5df4f63f3..16ebe05520 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -454,7 +454,6 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - ASSERT(pFile->hFile != NULL); // Please check if you have closed the file. if (pFile->hFile == NULL) { #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -867,7 +866,6 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #endif int32_t code = 0; - ASSERT(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS int64_t ret = _lseeki64(pFile->fd, offset, whence); @@ -1479,7 +1477,6 @@ int32_t taosEOFFile(TdFilePtr pFile) { terrno = TSDB_CODE_INVALID_PARA; return -1; } - ASSERT(pFile->fp != NULL); if (pFile->fp == NULL) { terrno = TSDB_CODE_INVALID_PARA; return -1; diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 7a5a547354..91eb7763bc 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -326,10 +326,8 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { if (ptr == NULL) return taosMemoryMalloc(size); TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); if (tpTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { - +return NULL; - + + return NULL; } TdMemoryInfo tdMemoryInfo; @@ -366,7 +364,6 @@ char *taosStrdup(const char *ptr) { if (ptr == NULL) return NULL; TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { return NULL; } @@ -413,7 +410,6 @@ int64_t taosMemorySize(void *ptr) { #ifdef USE_TD_MEMORY TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { return NULL; } @@ -441,7 +437,7 @@ void taosMemoryTrim(int32_t size) { void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #ifdef USE_TD_MEMORY - ASSERT(0); + return NULL; #else #if defined(LINUX) #ifdef BUILD_WITH_RAND_ERR diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 081ed46c9a..2d160b277b 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -312,8 +312,7 @@ int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return -1; } #ifdef WINDOWS - ASSERT(0); - return 0; + return -1; #else return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); #endif @@ -681,8 +680,7 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { int taosGetLocalIp(const char *eth, char *ip) { #if defined(WINDOWS) // DO NOTHAING - ASSERT(0); - return 0; + return -1; #else int fd; struct ifreq ifr; @@ -708,8 +706,7 @@ int taosGetLocalIp(const char *eth, char *ip) { int taosValidIp(uint32_t ip) { #if defined(WINDOWS) // DO NOTHAING - ASSERT(0); - return 0; + return -1; #else int ret = -1; int fd; @@ -1111,7 +1108,7 @@ int32_t taosIgnSIGPIPE() { int32_t taosSetMaskSIGPIPE() { #ifdef WINDOWS - // ASSERT(0); + return -1; #else sigset_t signal_mask; (void)sigemptyset(&signal_mask); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index b0a3615ee5..1c99355e34 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -425,10 +425,6 @@ int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { int64_t tmp = strtoll(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; -#endif -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); #endif return tmp; } @@ -437,10 +433,6 @@ uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { uint64_t tmp = strtoull(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; -#endif -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); #endif return tmp; } @@ -449,10 +441,6 @@ int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; -#endif -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); #endif return tmp; } @@ -461,10 +449,6 @@ uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { uint32_t tmp = strtol(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; -#endif -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); #endif return tmp; } @@ -473,12 +457,6 @@ int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; -#endif -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); - ASSERT(tmp >= SHRT_MIN); - ASSERT(tmp <= SHRT_MAX); #endif return (int16_t)tmp; } @@ -487,23 +465,12 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; -#endif -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); - ASSERT(tmp <= USHRT_MAX); #endif return (uint16_t)tmp; } int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); - ASSERT(tmp >= SCHAR_MIN); - ASSERT(tmp <= SCHAR_MAX); -#endif return tmp; } @@ -511,33 +478,17 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); #if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; -#endif -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); - ASSERT(tmp <= UCHAR_MAX); #endif return tmp; } double taosStr2Double(const char *str, char **pEnd) { double tmp = strtod(str, pEnd); -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); - ASSERT(tmp != HUGE_VAL); -#endif return tmp; } float taosStr2Float(const char *str, char **pEnd) { float tmp = strtof(str, pEnd); -#ifdef TD_CHECK_STR_TO_INT_ERROR - ASSERT(errno != ERANGE); - ASSERT(errno != EINVAL); - ASSERT(tmp != HUGE_VALF); - ASSERT(tmp != NAN); -#endif return tmp; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 92e5967416..9a4f0f8a98 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -287,7 +287,7 @@ void taosGetSystemInfo() { int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS - // ASSERT(0); + return 0; #elif defined(_TD_DARWIN_64) #ifdef CUS_PROMPT const char *filepath = "/usr/local/"CUS_PROMPT"/email"; @@ -1040,7 +1040,6 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { char *taosGetCmdlineByPID(int pid) { #ifdef WINDOWS - ASSERT(0); return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 843ee20d5b..a8a9ff681b 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -91,7 +91,6 @@ typedef struct FILE TdCmd; #ifdef BUILD_NO_CALL void* taosLoadDll(const char* filename) { #if defined(WINDOWS) - ASSERT(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -110,7 +109,6 @@ void* taosLoadDll(const char* filename) { void* taosLoadSym(void* handle, char* name) { #if defined(WINDOWS) - ASSERT(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -131,7 +129,6 @@ void* taosLoadSym(void* handle, char* name) { void taosCloseDll(void* handle) { #if defined(WINDOWS) - ASSERT(0); return; #elif defined(_TD_DARWIN_64) return; diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 3e37d12759..5a24e7775f 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -784,8 +784,7 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) { #ifdef TD_USE_SPINLOCK_AS_MUTEX - ASSERT(pshared == 0); - if (pshared != 0) return -1; + if (pshared != 0) return TSDB_CODE_INVALID_PARA; return pthread_mutex_init((pthread_mutex_t *)lock, NULL); #else int32_t code = pthread_spin_init((pthread_spinlock_t *)lock, pshared); From 1a458109ca8d7260d3076f602680e54431382145 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Aug 2024 14:54:41 +0800 Subject: [PATCH 074/100] fix: fix syntax error. --- source/libs/stream/src/streamCheckpoint.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 17edc3c30b..c616709147 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -38,7 +38,7 @@ int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int6 } if (srcTaskId <= 0) { - stDebug("s-task:0x%x invalid src task id:%d for creating checkpoint trigger block", pTask->id.idStr, srcTaskId); + stDebug("s-task:%s invalid src task id:%d for creating checkpoint trigger block", pTask->id.idStr, srcTaskId); return TSDB_CODE_INVALID_PARA; } From ecf65dd7f610c13eab029e4bb21d9733cdbf9d89 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Aug 2024 15:16:04 +0800 Subject: [PATCH 075/100] fix(stream): fix syntax error. --- source/libs/stream/src/streamCheckpoint.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index c616709147..fc9e449841 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -602,8 +602,12 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV bool valid = (pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer && pInfo->processedVer <= pReq->checkpointVer); + if (!valid) { - stFatal(""); + stFatal("invalid checkpoint id check, current checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " processedVer:%" PRId64 " req checkpointId:%" PRId64 " checkpointVer:%" PRId64, + pInfo->checkpointId, pInfo->checkpointVer, pInfo->processedVer, pReq->checkpointId, pReq->checkpointVer); + return TSDB_CODE_STREAM_INTERNAL_ERROR; } // update only it is in checkpoint status, or during restore procedure. From 0f82052392828dd18b3491bbb2711efb4b3b9757 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Aug 2024 15:28:25 +0800 Subject: [PATCH 076/100] enh: clear tdb asserts --- include/util/taoserror.h | 3 +- source/libs/tdb/src/db/tdbBtree.c | 48 ++++++++++++++----------------- source/libs/tdb/src/db/tdbPage.c | 10 +++++-- source/libs/tdb/src/db/tdbPager.c | 14 --------- source/libs/tdb/src/db/tdbTxn.c | 1 - source/libs/tdb/src/inc/tdbInt.h | 7 +++-- source/libs/tdb/src/inc/tdbUtil.h | 6 ---- source/util/src/terror.c | 1 + 8 files changed, 37 insertions(+), 53 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b772edbf22..3b6ed7143a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -91,7 +91,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_RPC_NETWORK_BUSY TAOS_DEF_ERROR_CODE(0, 0x0024) #define TSDB_CODE_HTTP_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0025) #define TSDB_CODE_RPC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0026) -#define TSDB_CODE_RPC_ASYNC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0027) +#define TSDB_CODE_RPC_ASYNC_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0027) @@ -152,6 +152,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_FAILED_TO_CONNECT_S3 TAOS_DEF_ERROR_CODE(0, 0x0135) #define TSDB_CODE_MSG_PREPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0136) // internal #define TSDB_CODE_OUT_OF_BUFFER TAOS_DEF_ERROR_CODE(0, 0x0137) +#define TSDB_CODE_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0138) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 59b7f4af2e..f878766f77 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -161,7 +161,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg if (pgno == 0) { tdbError("tdb/btree-open: pgno cannot be zero."); tdbOsFree(pBt); - ASSERT(0); + return TSDB_CODE_INTERNAL_ERROR; } pBt->root = pgno; /* @@ -418,10 +418,6 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 int mlen; int cret; - if (ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL)) { - // -1 is less than - } - mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; cret = memcmp(pKey1, pKey2, mlen); if (cret == 0) { @@ -571,14 +567,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx nOlds = 3; } for (int i = 0; i < nOlds; i++) { - if (ASSERT(sIdx + i <= nCells)) { + if (!(sIdx + i <= nCells)) { return TSDB_CODE_FAILED; } SPgno pgno; if (sIdx + i == nCells) { - if (ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent))) { - return TSDB_CODE_FAILED; + if (TDB_BTREE_PAGE_IS_LEAF(pParent)) { + return TSDB_CODE_INTERNAL_ERROR; } pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { @@ -685,8 +681,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx // page is full, use a new page nNews++; - ASSERT(infoNews[nNews].size + cellBytes <= TDB_PAGE_USABLE_SIZE(pPage)); - if (childNotLeaf) { // for non-child page, this cell is used as the right-most child, // the divider cell to parent as well @@ -732,7 +726,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx szRCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); } - if (ASSERT(infoNews[iNew - 1].cnt > 0)) { + if (!(infoNews[iNew - 1].cnt > 0)) { return TSDB_CODE_FAILED; } @@ -822,10 +816,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx pCell = tdbPageGetCell(pPage, oIdx); szCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); - if (ASSERT(nNewCells <= infoNews[iNew].cnt)) { + if (!(nNewCells <= infoNews[iNew].cnt)) { return TSDB_CODE_FAILED; } - if (ASSERT(iNew < nNews)) { + if (!(iNew < nNews)) { return TSDB_CODE_FAILED; } @@ -866,10 +860,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } } else { - if (ASSERT(childNotLeaf)) { + if (!(childNotLeaf)) { return TSDB_CODE_FAILED; } - if (ASSERT(iNew < nNews - 1)) { + if (!(iNew < nNews - 1)) { return TSDB_CODE_FAILED; } @@ -877,7 +871,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // insert to parent as divider cell - if (ASSERT(iNew < nNews - 1)) { + if (!(iNew < nNews - 1)) { return TSDB_CODE_FAILED; } ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); @@ -894,7 +888,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } if (childNotLeaf) { - if (ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt)) { + if (!(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt)) { return TSDB_CODE_FAILED; } ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; @@ -1091,7 +1085,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= kLen; // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { - if (ASSERT(pVal != NULL && vLen != 0)) { + if (!(pVal != NULL && vLen != 0)) { tdbFree(pBuf); return TSDB_CODE_FAILED; } @@ -1259,14 +1253,14 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo int nPayload; int ret; - if (ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen)) { - return TSDB_CODE_FAILED; + if (!(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen)) { + return TSDB_CODE_INVALID_PARA; } - if (ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen)) { - return TSDB_CODE_FAILED; + if (!(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen)) { + return TSDB_CODE_INVALID_PARA; } - if (ASSERT(pKey != NULL && kLen > 0)) { - return TSDB_CODE_FAILED; + if (!(pKey != NULL && kLen > 0)) { + return TSDB_CODE_INVALID_PARA; } nPayload = 0; @@ -1645,7 +1639,6 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * SArray *ofps = pPage->pPager->ofps; if (ofps) { if (taosArrayPush(ofps, &ofp) == NULL) { - ASSERT(0); return terrno; } } @@ -2438,7 +2431,10 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { lidx = 0; ridx = nCells - 1; - ASSERT(nCells > 0); + if (nCells <= 0) { + tdbError("tdb/btc-move-to: empty page."); + return TSDB_CODE_FAILED; + } // compare first cell pBtc->idx = lidx; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 26c1c108d2..eab8f6ef19 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -522,7 +522,9 @@ static int tdbPageDefragment(SPage *pPage) { SCell *pCell = TDB_PAGE_CELL_AT(pPage, aCellIdx[iCell].iCell); int32_t szCell = pPage->xCellSize(pPage, pCell, 0, NULL, NULL); - ASSERT(pNextCell - szCell >= pCell); + if (pNextCell - szCell < pCell) { + return TSDB_CODE_INTERNAL_ERROR; + } pNextCell -= szCell; if (pNextCell > pCell) { @@ -535,7 +537,11 @@ static int tdbPageDefragment(SPage *pPage) { TDB_PAGE_FCELL_SET(pPage, 0); tdbOsFree(aCellIdx); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { + tdbError("tdb/page-defragment: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, + pPage->pFreeEnd); + return TSDB_CODE_INTERNAL_ERROR; + } return 0; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index a650847e1e..b0f32136a3 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -16,19 +16,7 @@ #include "crypt.h" #include "tdbInt.h" #include "tglobal.h" -/* -#pragma pack(push, 1) -typedef struct { - u8 hdrString[16]; - u16 pageSize; - SPgno freePage; - u32 nFreePages; - u8 reserved[102]; -} SFileHdr; -#pragma pack(pop) -TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); -*/ struct hashset_st { size_t nbits; size_t mask; @@ -450,7 +438,6 @@ static char *tdbEncryptPage(SPager *pPager, char *pPageData, int32_t pageSize, c if (encryptAlgorithm == DND_CA_SM4) { // tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); - // ASSERT(strlen(encryptKey) > 0); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]); @@ -915,7 +902,6 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage if (encryptAlgorithm == DND_CA_SM4) { // tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); - // ASSERT(strlen(encryptKey) > 0); // uint8_t flags = pPage->pData[0]; // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags); diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index 24a70f62b2..71560e3e85 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -40,7 +40,6 @@ int tdbTxnCloseImpl(TXN *pTxn) { if (pTxn->jfd) { TAOS_UNUSED(tdbOsClose(pTxn->jfd)); - ASSERT(pTxn->jfd == NULL); } tdbOsFree(pTxn); diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 605fe6a1a4..7e97be962b 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -319,7 +319,6 @@ static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { } else if (ret == EBUSY) { return P_LOCK_BUSY; } else { - ASSERT(0); return P_LOCK_FAIL; } } @@ -354,7 +353,10 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { int iOvfl; int lidx; - ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); + if (idx < 0 || idx >= TDB_PAGE_TOTAL_CELLS(pPage)) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -367,7 +369,6 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { } lidx = idx - iOvfl; - ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); return pCell; diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 4382513f73..22468e6579 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -22,12 +22,6 @@ extern "C" { #endif -#if __STDC_VERSION__ >= 201112LL -#define TDB_STATIC_ASSERT(op, info) static_assert(op, info) -#else -#define TDB_STATIC_ASSERT(op, info) -#endif - #define TDB_ROUND8(x) (((x) + 7) & ~7) int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b307c4ac4b..1d2e2357d1 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -111,6 +111,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_IP_NOT_IN_WHITE_LIST, "Not allowed to connec TAOS_DEFINE_ERROR(TSDB_CODE_FAILED_TO_CONNECT_S3, "Failed to connect to s3 server") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_PREPROCESSED, "Message has been processed in preprocess") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_BUFFER, "Out of buffer") +TAOS_DEFINE_ERROR(TSDB_CODE_INTERNAL_ERROR, "Internal error") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") From bbd097f0fd81f712dd8fac3b18b9c84dd8761da7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Aug 2024 15:54:07 +0800 Subject: [PATCH 077/100] fix(stream): fix typo --- source/libs/stream/src/streamMsg.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c index 586f9ac692..8105614d76 100644 --- a/source/libs/stream/src/streamMsg.c +++ b/source/libs/stream/src/streamMsg.c @@ -16,6 +16,7 @@ #include "streamMsg.h" #include "os.h" #include "tstream.h" +#include "streamInt.h" int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) { if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1; @@ -230,7 +231,8 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1; if (tEncodeI64(pEncoder, pReq->totalLen) < 0) return -1; - if (taosArrayGetSize(pReq->data) != pReq->blockNum || taosArrayGetSize(pReq->dataLen) == pReq->blockNum) { + if (taosArrayGetSize(pReq->data) != pReq->blockNum || taosArrayGetSize(pReq->dataLen) != pReq->blockNum) { + stError("invalid dispatch req msg"); return TSDB_CODE_INVALID_MSG; } From 48055beb1f0ac9761f88ed23fa8c999e76ce1614 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 21 Aug 2024 07:54:39 +0000 Subject: [PATCH 078/100] fix/TD-31520-move-to-vnode-init-stage --- include/libs/monitor/monitor.h | 1 + source/dnode/vnode/src/vnd/vnodeModule.c | 3 +++ source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- source/libs/monitor/src/monMain.c | 8 +++++--- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 832926bf98..96ac53ef4e 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -218,6 +218,7 @@ typedef struct { } SDmNotifyHandle; int32_t monInit(const SMonCfg *pCfg); +void monInitVnode(); void monCleanup(); void monRecordLog(int64_t ts, ELogLevel level, const char *content); int32_t monGetLogs(SMonLogs *logs); diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 709bfa19bc..2a2d01ada8 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -14,6 +14,7 @@ */ #include "cos.h" +#include "monitor.h" #include "vnd.h" static volatile int32_t VINIT = 0; @@ -26,6 +27,8 @@ int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) { TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads)); TAOS_CHECK_RETURN(walInit(stopDnodeFp)); + monInitVnode(); + return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4ad6ae4744..44cba41785 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1904,8 +1904,8 @@ _exit: (void)atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); (void)atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); - if (tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0 && - tsInsertCounter != NULL) { + if (tsEnableMonitor && tsMonitorFqdn[0] != 0 && tsMonitorPort != 0 && pSubmitRsp->affectedRows > 0 && + strlen(pOriginalMsg->info.conn.user) > 0 && tsInsertCounter != NULL) { const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, pVnode->monitor.strDnodeId, diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 86704ed242..4abc8958df 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -128,7 +128,11 @@ int32_t monInit(const SMonCfg *pCfg) { monInitMonitorFW(); - if (tsEnableMonitor && tsInsertCounter == NULL) { + return 0; +} + +void monInitVnode() { + if (tsEnableMonitor && tsMonitorFqdn[0] != 0 && tsMonitorPort != 0 && tsInsertCounter == NULL) { taos_counter_t *counter = NULL; int32_t label_count = 7; const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID, @@ -147,8 +151,6 @@ int32_t monInit(const SMonCfg *pCfg) { } else { uError("failed to set insert counter, already set"); } - - return 0; } void monCleanup() { From 92286cd8c7f91d77c1a986d8478d2b981e77158d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Aug 2024 16:11:26 +0800 Subject: [PATCH 079/100] refactor: do some internal refactor. --- source/libs/stream/src/streamCheckpoint.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index fc9e449841..1a0b1e8665 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -37,14 +37,13 @@ int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int6 return code; } - if (srcTaskId <= 0) { - stDebug("s-task:%s invalid src task id:%d for creating checkpoint trigger block", pTask->id.idStr, srcTaskId); - return TSDB_CODE_INVALID_PARA; - } - pChkpoint->type = checkpointType; if (checkpointType == STREAM_INPUT__CHECKPOINT_TRIGGER && (pTask->info.taskLevel != TASK_LEVEL__SOURCE)) { pChkpoint->srcTaskId = srcTaskId; + if (srcTaskId <= 0) { + stDebug("s-task:%s invalid src task id:%d for creating checkpoint trigger block", pTask->id.idStr, srcTaskId); + return TSDB_CODE_INVALID_PARA; + } } SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); From 95fc56b8f6d15f5bd92aa938a31ef200f1d1c0d4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 21 Aug 2024 16:38:54 +0800 Subject: [PATCH 080/100] fix(s3/cfg): fix s3MigrateEnabled & s3MigrateIntervalSec loading --- source/common/src/tglobal.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cf0a4725c1..4d486518ee 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1599,6 +1599,12 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minDiskFreeSize"); tsMinDiskFreeSize = pItem->i64; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3MigrateIntervalSec"); + tsS3MigrateIntervalSec = pItem->i32; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3MigrateEnabled"); + tsS3MigrateEnabled = (bool)pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "s3PageCacheSize"); tsS3PageCacheSize = pItem->i32; From 59e2849756ba95ca2a857e8424a4055eb5c037f2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 21 Aug 2024 18:37:17 +0800 Subject: [PATCH 081/100] fix(tsdb/cache): check array init's retval --- source/dnode/vnode/src/tsdb/tsdbCache.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index fb72784229..cd72768379 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1413,7 +1413,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr SIdxKey *idxKey = taosArrayGet(remainCols, 0); if (idxKey->key.cid != PRIMARYKEY_TIMESTAMP_COL_ID) { // ignore 'ts' loaded from cache and load it from tsdb - SLastCol* pLastCol = taosArrayGet(pLastArray, 0); + SLastCol *pLastCol = taosArrayGet(pLastArray, 0); tsdbCacheUpdateLastColToNone(pLastCol, TSDB_LAST_CACHE_NO_CACHE); SLastKey *key = &(SLastKey){.lflag = ltype, .uid = uid, .cid = PRIMARYKEY_TIMESTAMP_COL_ID}; @@ -1745,12 +1745,12 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache goto _exit; } } - if (taosArrayPush(remainCols, &(SIdxKey){i, key}) ==NULL) { + if (taosArrayPush(remainCols, &(SIdxKey){i, key}) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } bool ignoreRocks = pLastCol ? (pLastCol->cacheStatus == TSDB_LAST_CACHE_NO_CACHE) : false; - if (taosArrayPush(ignoreFromRocks, &ignoreRocks) ==NULL) { + if (taosArrayPush(ignoreFromRocks, &ignoreRocks) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -1802,12 +1802,12 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache } _exit: - if (remainCols) { - taosArrayDestroy(remainCols); - } - if (ignoreFromRocks) { - taosArrayDestroy(ignoreFromRocks); - } + if (remainCols) { + taosArrayDestroy(remainCols); + } + if (ignoreFromRocks) { + taosArrayDestroy(ignoreFromRocks); + } TAOS_RETURN(code); } @@ -2368,6 +2368,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (!state->pIndexList) { state->pIndexList = taosArrayInit(1, sizeof(SBrinBlk)); + if (!state->pIndexList) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _err); + } } else { taosArrayClear(state->pIndexList); } @@ -2653,7 +2656,6 @@ static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarli TAOS_RETURN(code); } default: - ASSERT(0); break; } From 16d314a6ae8d51f8bde2c36e804d0faf8f83e029 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Wed, 21 Aug 2024 16:28:44 +0800 Subject: [PATCH 082/100] fix:[TD-31588] Handle error when initFunc failed . --- source/libs/executor/src/projectoperator.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 4d2bdc62f8..dd0acc42ed 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -681,6 +681,7 @@ int32_t doApplyIndefinitFunction(SOperatorInfo* pOperator, SSDataBlock** pResBlo } int32_t initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { + int32_t code = TSDB_CODE_SUCCESS; for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || @@ -688,7 +689,10 @@ int32_t initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { continue; } - (void)pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo); + code = pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo); + if (code) { + return code; + } } return 0; @@ -1033,8 +1037,8 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc // do nothing } else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx); - (void) pfCtx->fpSet.init(pfCtx, pResInfo); - + code = pfCtx->fpSet.init(pfCtx, pResInfo); + TSDB_CHECK_CODE(code, lino, _exit); pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); if (pfCtx->pOutput == NULL) { code = terrno; From 4dc81efb8b4524f4df50313b6c301ac0e3d05335 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Aug 2024 20:00:34 +0800 Subject: [PATCH 083/100] refactor: update logs. --- source/libs/executor/inc/executil.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 46fc618f76..b68ea5a781 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -24,10 +24,10 @@ #include "tpagedbuf.h" #include "tsimplehash.h" -#define T_LONG_JMP(_obj, _c) \ - do { \ - ASSERT(1); \ - longjmp((_obj), (_c)); \ +#define T_LONG_JMP(_obj, _c) \ + do { \ + qError("error happens at %s, line:%d, code:%s", __func__, __LINE__, tstrerror((_c))); \ + longjmp((_obj), (_c)); \ } while (0) #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ From a823dc3a31a482173eeefd119ff65f4a01888b7c Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Wed, 21 Aug 2024 21:10:44 +0800 Subject: [PATCH 084/100] Update shellEngine.c --- tools/shell/src/shellEngine.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 0ccbd683dc..6fd5f7402f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -164,11 +164,13 @@ int32_t shellRunCommand(char *command, bool recordHistory) { } // add help or help; - if (strncasecmp(command, "help;", 5) == 0) { - showHelp(); - return 0; + if (strncasecmp(command, "help", 4) == 0) { + if(command[4] == ';' || command[4] == ' ' || command[4] == 0) { + showHelp(); + return 0; + } } - + if (recordHistory) shellRecordCommandToHistory(command); char quote = 0, *cmd = command; From 78ca5511c2a030016209f006dc5d3ba41a375f31 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Wed, 21 Aug 2024 21:12:52 +0800 Subject: [PATCH 085/100] Update shellAuto.c --- tools/shell/src/shellAuto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 2c92d19fcf..9b1c949807 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -585,7 +585,7 @@ void showHelp() { balance vgroup ;\n\ balance vgroup leader on \n\ compact database ; \n\ - crate view as select ...\n\ + create view as select ...\n\ redistribute vgroup dnode ;\n\ split vgroup ;\n\ show compacts;\n\ From 08031cf1808de8dff68426bfeb546911d00715cc Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Thu, 22 Aug 2024 09:45:57 +0800 Subject: [PATCH 086/100] fix:[TD-31600] Refactor error handling logic in taos_connect_internal. --- source/client/src/clientImpl.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 563d70ab08..eccc63f427 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -176,15 +176,19 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass _return: - code = taosThreadMutexUnlock(&appInfo.mutex); if (TSDB_CODE_SUCCESS != code) { - tscError("failed to unlock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code))); + (void)taosThreadMutexUnlock(&appInfo.mutex); + taosMemoryFreeClear(key); return code; + } else { + code = taosThreadMutexUnlock(&appInfo.mutex); + taosMemoryFreeClear(key); + if (TSDB_CODE_SUCCESS != code) { + tscError("failed to unlock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code))); + return code; + } + return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType, pObj); } - - taosMemoryFreeClear(key); - - return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType, pObj); } //SAppInstInfo* getAppInstInfo(const char* clusterKey) { From 02c74741672af3d6580d7ae0bd95182ba9a6d4c2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Aug 2024 09:47:44 +0800 Subject: [PATCH 087/100] enh: remove some useless asserts --- source/dnode/vnode/src/meta/metaCache.c | 1 - source/dnode/vnode/src/tsdb/tsdbCommit2.c | 4 +- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 3 - source/dnode/vnode/src/tsdb/tsdbFSet2.c | 13 ++- source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 7 +- source/dnode/vnode/src/tsdb/tsdbFSetRW.c | 2 - source/dnode/vnode/src/tsdb/tsdbFile2.c | 29 +++++- source/dnode/vnode/src/tsdb/tsdbIter.c | 28 ++---- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 - source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 23 +---- source/dnode/vnode/src/tsdb/tsdbUtil.c | 96 ++++--------------- source/dnode/vnode/src/vnd/vnodeSvr.c | 12 ++- 14 files changed, 72 insertions(+), 156 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 98f5615f5e..7577b9ec0c 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -503,7 +503,6 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv } static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) { - // ASSERT(keyLen == sizeof(int64_t) * 2); memcpy(&pBuf[2], key, keyLen); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index d920e289e2..df572ff852 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -626,8 +626,8 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) { if (eno == 0) { TAOS_CHECK_GOTO(tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT), &lino, _exit); } else { - // TODO - ASSERT(0); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(eno)); } TARRAY2_DESTROY(committer->dataIterArray, NULL); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index 6469160536..886a6f31a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -113,7 +113,7 @@ _exit: } static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { - ASSERT(0); + tsdbError("vgId:%d %s failed since not implemented", TD_VID(writer->config->tsdb->pVnode), __func__); return 0; } @@ -122,8 +122,6 @@ static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) { int32_t code = 0; int32_t lino = 0; - ASSERT(writer->ctx->offset <= writer->file.size); - ASSERT(writer->config->fid == writer->file.fid); STFileOp op = (STFileOp){ .optype = TSDB_FOP_CREATE, diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index b8669055d4..6dec456be7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -894,7 +894,6 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { // commit code = commit_edit(fs); - ASSERT(code == 0); TSDB_CHECK_CODE(code, lino, _exit); // schedule merge @@ -1052,7 +1051,6 @@ static SHashObj *tsdbFSetRangeArrayToHash(TFileSetRangeArray *pRanges) { STFileSetRange *u = TARRAY2_GET(pRanges, i); int32_t fid = u->fid; int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); - ASSERT(code == 0); tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); } return pHash; @@ -1192,7 +1190,6 @@ int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) { (void)taosThreadCondWait(&(*fset)->beginTask, &tsdb->mutex); (void)tsdbFSGetFSet(tsdb->pFS, fid, fset); - ASSERT(fset != NULL); (*fset)->numWaitTask--; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 78e1a6a13d..05ce1b23f5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -106,7 +106,9 @@ static void tsdbSttLvlRemove(SSttLvl **lvl) { static int32_t tsdbSttLvlApplyEdit(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl *lvl2) { int32_t code = 0; - ASSERT(lvl1->level == lvl2->level); + if (lvl1->level != lvl2->level) { + return TSDB_CODE_INVALID_PARA; + } int32_t i1 = 0, i2 = 0; while (i1 < TARRAY2_SIZE(lvl1->fobjArr) || i2 < TARRAY2_SIZE(lvl2->fobjArr)) { @@ -331,29 +333,24 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { code = TARRAY2_SORT_INSERT(lvl->fobjArr, fobj, tsdbTFileObjCmpr); if (code) return code; } else { - ASSERT(fset->farr[fobj->f->type] == NULL); fset->farr[fobj->f->type] = fobj; } } else if (op->optype == TSDB_FOP_REMOVE) { // delete a file if (op->of.type == TSDB_FTYPE_STT) { SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level); - ASSERT(lvl); STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}; STFileObj *tfobjp = &tfobj; int32_t idx = TARRAY2_SEARCH_IDX(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); - ASSERT(idx >= 0); TARRAY2_REMOVE(lvl->fobjArr, idx, tsdbSttLvlClearFObj); } else { - ASSERT(tsdbIsSameTFile(&op->of, fset->farr[op->of.type]->f)); (void)tsdbTFileObjUnref(fset->farr[op->of.type]); fset->farr[op->of.type] = NULL; } } else { if (op->nf.type == TSDB_FTYPE_STT) { SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level); - ASSERT(lvl); STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}, *tfobjp = &tfobj; STFileObj **fobjPtr = TARRAY2_SEARCH(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); @@ -374,7 +371,9 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset2) { int32_t code = 0; - ASSERT(fset1->fid == fset2->fid); + if (fset1->fid != fset2->fid) { + return TSDB_CODE_INVALID_PARA; + } for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { if (!fset1->farr[ftype] && !fset2->farr[ftype]) continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index 1adca31347..09db2ef4ef 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -154,13 +154,11 @@ _exit: return code; } -int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm, - char* encryptKey) { +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm, + char *encryptKey) { int32_t code = 0; int32_t lino = 0; - ASSERT(writer->ctx->offset >= 0 && writer->ctx->offset <= writer->ctx->file.size); - if (writer->ctx->offset == writer->ctx->file.size) { code = tsdbFSetRAWWriteFileDataEnd(writer); TSDB_CHECK_CODE(code, lino, _exit); @@ -173,7 +171,6 @@ int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeade TSDB_CHECK_CODE(code, lino, _exit); writer->ctx->offset += bHdr->dataLength; - ASSERT(writer->ctx->offset == writer->dataWriter->ctx->offset); _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index 3ccf833ddd..4a35316f35 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -73,8 +73,6 @@ static int32_t tsdbFSetWriteTableDataEnd(SFSetWriter *writer) { int32_t numRow = ((writer->blockData[pidx].nRow + writer->blockData[cidx].nRow) >> 1); if (writer->blockData[pidx].nRow > 0 && numRow >= writer->config->minRow) { - ASSERT(writer->blockData[pidx].nRow == writer->config->maxRow); - SRowInfo row = { .suid = writer->ctx->tbid->suid, .uid = writer->ctx->tbid->uid, diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 57db838752..50272a689c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -243,7 +243,13 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { int32_t tsdbTFileObjRef(STFileObj *fobj) { int32_t nRef; (void)taosThreadMutexLock(&fobj->mutex); - ASSERT(fobj->ref > 0 && fobj->state == TSDB_FSTATE_LIVE); + + if (fobj->ref <= 0 || fobj->state != TSDB_FSTATE_LIVE) { + (void)taosThreadMutexUnlock(&fobj->mutex); + tsdbError("file %s, fobj:%p ref %d", fobj->fname, fobj, fobj->ref); + return TSDB_CODE_FAILED; + } + nRef = ++fobj->ref; (void)taosThreadMutexUnlock(&fobj->mutex); tsdbTrace("ref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); @@ -254,7 +260,12 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) { (void)taosThreadMutexLock(&fobj->mutex); int32_t nRef = --fobj->ref; (void)taosThreadMutexUnlock(&fobj->mutex); - ASSERT(nRef >= 0); + + if (nRef < 0) { + tsdbError("file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); + return TSDB_CODE_FAILED; + } + tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); if (nRef == 0) { if (fobj->state == TSDB_FSTATE_DEAD) { @@ -319,7 +330,11 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { int32_t tsdbTFileObjRemove(STFileObj *fobj) { (void)taosThreadMutexLock(&fobj->mutex); - ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0); + if (fobj->state != TSDB_FSTATE_LIVE || fobj->ref <= 0) { + (void)taosThreadMutexUnlock(&fobj->mutex); + tsdbError("file %s, fobj:%p ref %d", fobj->fname, fobj, fobj->ref); + return TSDB_CODE_FAILED; + } fobj->state = TSDB_FSTATE_DEAD; int32_t nRef = --fobj->ref; (void)taosThreadMutexUnlock(&fobj->mutex); @@ -333,7 +348,13 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) { int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) { (void)taosThreadMutexLock(&fobj->mutex); - ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0); + + if (fobj->state != TSDB_FSTATE_LIVE || fobj->ref <= 0) { + (void)taosThreadMutexUnlock(&fobj->mutex); + tsdbError("file %s, fobj:%p ref %d", fobj->fname, fobj, fobj->ref); + return TSDB_CODE_FAILED; + } + fobj->state = TSDB_FSTATE_DEAD; int32_t nRef = --fobj->ref; (void)taosThreadMutexUnlock(&fobj->mutex); diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.c b/source/dnode/vnode/src/tsdb/tsdbIter.c index 0de9ba9822..5f9e838ccc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbIter.c @@ -547,8 +547,7 @@ int32_t tsdbIterOpen(const STsdbIterConfig *config, STsdbIter **iter) { code = tsdbMemTombIterOpen(iter[0]); break; default: - code = TSDB_CODE_INVALID_PARA; - ASSERTS(false, "Not implemented"); + return TSDB_CODE_INVALID_PARA; } if (code) { @@ -588,7 +587,7 @@ int32_t tsdbIterClose(STsdbIter **iter) { case TSDB_ITER_TYPE_MEMT_TOMB: break; default: - ASSERT(false); + return TSDB_CODE_INVALID_PARA; } taosMemoryFree(iter[0]); iter[0] = NULL; @@ -610,7 +609,7 @@ int32_t tsdbIterNext(STsdbIter *iter) { case TSDB_ITER_TYPE_MEMT_TOMB: return tsdbMemTombIterNext(iter, NULL); default: - ASSERT(false); + return TSDB_CODE_INVALID_PARA; } return 0; } @@ -630,7 +629,7 @@ static int32_t tsdbIterSkipTableData(STsdbIter *iter, const TABLEID *tbid) { case TSDB_ITER_TYPE_MEMT_TOMB: return tsdbMemTombIterNext(iter, tbid); default: - ASSERT(false); + return TSDB_CODE_INVALID_PARA; } return 0; } @@ -691,7 +690,10 @@ int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger TARRAY2_FOREACH(iterArray, iter) { if (iter->noMoreData) continue; node = tRBTreePut(merger[0]->iterTree, iter->node); - ASSERT(node); + if (node == NULL) { + taosMemoryFree(merger[0]); + return TSDB_CODE_INVALID_PARA; + } } return tsdbIterMergerNext(merger[0]); @@ -718,10 +720,8 @@ int32_t tsdbIterMergerNext(SIterMerger *merger) { merger->iter = NULL; } else if ((node = tRBTreeMin(merger->iterTree))) { c = merger->iterTree->cmprFn(merger->iter->node, node); - ASSERT(c); if (c > 0) { node = tRBTreePut(merger->iterTree, merger->iter->node); - ASSERT(node); merger->iter = NULL; } } @@ -734,15 +734,9 @@ int32_t tsdbIterMergerNext(SIterMerger *merger) { return 0; } -SRowInfo *tsdbIterMergerGetData(SIterMerger *merger) { - ASSERT(!merger->isTomb); - return merger->iter ? merger->iter->row : NULL; -} +SRowInfo *tsdbIterMergerGetData(SIterMerger *merger) { return merger->iter ? merger->iter->row : NULL; } -STombRecord *tsdbIterMergerGetTombRecord(SIterMerger *merger) { - ASSERT(merger->isTomb); - return merger->iter ? merger->iter->record : NULL; -} +STombRecord *tsdbIterMergerGetTombRecord(SIterMerger *merger) { return merger->iter ? merger->iter->record : NULL; } int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid) { int32_t code; @@ -757,10 +751,8 @@ int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid) { merger->iter = NULL; } else if ((node = tRBTreeMin(merger->iterTree))) { c = merger->iterTree->cmprFn(merger->iter->node, node); - ASSERT(c); if (c > 0) { node = tRBTreePut(merger->iterTree, merger->iter->node); - ASSERT(node); merger->iter = NULL; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 9f5f72e0aa..57405c0e48 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -194,7 +194,7 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in break; } default: - ASSERT(0); + return TSDB_CODE_INVALID_PARA; } } else { switch (type) { @@ -223,7 +223,7 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in pKey->pks[0].val = UINT8_MAX; break; default: - ASSERT(0); + return TSDB_CODE_INVALID_PARA; } } } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index eaedbca464..8632603fd6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -151,7 +151,6 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e offset -= chunkoffset; } - ASSERT(offset >= 0); int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); if (n < 0) { @@ -217,7 +216,6 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor offset -= chunkoffset; } - ASSERT(offset >= 0); // seek int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 2b63d5b6e6..680b1250f9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -67,9 +67,6 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; - ASSERT(reader->dataReader == NULL); - ASSERT(TARRAY2_SIZE(reader->sttReaderArr) == 0); - // data SDataFileReaderConfig config = { .tsdb = reader->tsdb, @@ -125,11 +122,6 @@ static int32_t tsdbSnapReadFileSetOpenIter(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; - ASSERT(reader->dataIterMerger == NULL); - ASSERT(reader->tombIterMerger == NULL); - ASSERT(TARRAY2_SIZE(reader->dataIterArr) == 0); - ASSERT(TARRAY2_SIZE(reader->tombIterArr) == 0); - STsdbIter* iter; STsdbIterConfig config = { .filterByVersion = true, @@ -210,8 +202,6 @@ static int32_t tsdbSnapReadRangeBegin(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; - ASSERT(reader->ctx->fsr == NULL); - if (reader->ctx->fsrArrIdx < TARRAY2_SIZE(reader->fsrArr)) { reader->ctx->fsr = TARRAY2_GET(reader->fsrArr, reader->ctx->fsrArrIdx++); reader->ctx->isDataDone = false; @@ -335,7 +325,6 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat } if (reader->blockData->nRow > 0) { - ASSERT(reader->blockData->suid || reader->blockData->uid); code = tsdbSnapCmprData(reader, data); TSDB_CHECK_CODE(code, lino, _exit); } @@ -616,7 +605,6 @@ static int32_t tsdbSnapWriteTimeSeriesRow(STsdbSnapWriter* writer, SRowInfo* row } if (row->suid == INT64_MAX) { - ASSERT(writer->ctx->hasData == false); goto _exit; } @@ -634,9 +622,6 @@ static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - ASSERT(writer->ctx->dataReader == NULL); - ASSERT(TARRAY2_SIZE(writer->ctx->sttReaderArr) == 0); - if (writer->ctx->fset) { // open data reader SDataFileReaderConfig dataFileReaderConfig = { @@ -825,8 +810,6 @@ static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) { int32_t code = 0; int32_t lino = 0; - ASSERT(writer->ctx->fsetWriteBegin == false); - STFileSet* fset = &(STFileSet){.fid = fid}; writer->ctx->fid = fid; @@ -885,7 +868,6 @@ static int32_t tsdbSnapWriteTombRecord(STsdbSnapWriter* writer, const STombRecor } if (record->suid == INT64_MAX) { - ASSERT(writer->ctx->hasTomb == false); goto _exit; } @@ -996,7 +978,6 @@ static int32_t tsdbSnapWriteDecmprTombBlock(SSnapDataHdr* hdr, STombBlock* tombB TAOS_UNUSED(tTombBlockClear(tombBlock)); int64_t size = hdr->size; - ASSERT(size % TOMB_RECORD_ELEM_NUM == 0); size = size / TOMB_RECORD_ELEM_NUM; tombBlock->numOfRecords = size / sizeof(int64_t); @@ -1043,8 +1024,6 @@ static int32_t tsdbSnapWriteTombData(STsdbSnapWriter* writer, SSnapDataHdr* hdr) TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(writer->ctx->hasData == false); - for (int32_t i = 0; i < TOMB_BLOCK_SIZE(tombBlock); ++i) { code = tTombBlockGet(tombBlock, i, &record); TSDB_CHECK_CODE(code, lino, _exit); @@ -1177,7 +1156,7 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* writer, SSnapDataHdr* hdr) { code = tsdbSnapWriteTombData(writer, hdr); TSDB_CHECK_CODE(code, lino, _exit); } else { - ASSERT(0); + TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_PARA, lino, _exit); } _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 9e3accb469..f70b0aad26 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -583,7 +583,8 @@ int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t nowSec) { } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { nowSec = nowSec * 1000000000l; } else { - ASSERT(0); + tsdbError("invalid time precision:%d", pKeepCfg->precision); + return 0; } nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision]; @@ -897,76 +898,6 @@ int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } -/* -// delete skyline ====================================================== -static int32_t tsdbMergeSkyline2(SArray *aSkyline1, SArray *aSkyline2, SArray *aSkyline) { - int32_t code = 0; - int32_t i1 = 0; - int32_t n1 = taosArrayGetSize(aSkyline1); - int32_t i2 = 0; - int32_t n2 = taosArrayGetSize(aSkyline2); - TSDBKEY *pSkyline1; - TSDBKEY *pSkyline2; - TSDBKEY item; - int64_t version1 = 0; - int64_t version2 = 0; - - ASSERT(n1 > 0 && n2 > 0); - - taosArrayClear(aSkyline); - - while (i1 < n1 && i2 < n2) { - pSkyline1 = (TSDBKEY *)taosArrayGet(aSkyline1, i1); - pSkyline2 = (TSDBKEY *)taosArrayGet(aSkyline2, i2); - - if (pSkyline1->ts < pSkyline2->ts) { - version1 = pSkyline1->version; - i1++; - } else if (pSkyline1->ts > pSkyline2->ts) { - version2 = pSkyline2->version; - i2++; - } else { - version1 = pSkyline1->version; - version2 = pSkyline2->version; - i1++; - i2++; - } - - item.ts = TMIN(pSkyline1->ts, pSkyline2->ts); - item.version = TMAX(version1, version2); - if (taosArrayPush(aSkyline, &item) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - - while (i1 < n1) { - pSkyline1 = (TSDBKEY *)taosArrayGet(aSkyline1, i1); - item.ts = pSkyline1->ts; - item.version = pSkyline1->version; - if (taosArrayPush(aSkyline, &item) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - i1++; - } - - while (i2 < n2) { - pSkyline2 = (TSDBKEY *)taosArrayGet(aSkyline2, i2); - item.ts = pSkyline2->ts; - item.version = pSkyline2->version; - if (taosArrayPush(aSkyline, &item) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - i2++; - } - -_exit: - return code; -} -*/ - // delete skyline ====================================================== static void tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyline) { int32_t i1 = 0; @@ -978,8 +909,6 @@ static void tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyl int64_t version1 = 0; int64_t version2 = 0; - ASSERT(n1 > 0 && n2 > 0); - taosArrayClear(pSkyline); TSDBKEY **pItem = TARRAY_GET_ELEM(pSkyline, 0); @@ -1212,7 +1141,9 @@ _exit: int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid) { int32_t code = 0; - ASSERT(pId->suid || pId->uid); + if (!pId->suid && !pId->uid) { + return TSDB_CODE_INVALID_PARA; + } pBlockData->suid = pId->suid; pBlockData->uid = pId->uid; @@ -1272,8 +1203,6 @@ void tBlockDataReset(SBlockData *pBlockData) { } void tBlockDataClear(SBlockData *pBlockData) { - ASSERT(pBlockData->suid || pBlockData->uid); - pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { tColDataClear(tBlockDataGetColDataByIdx(pBlockData, iColData)); @@ -1281,7 +1210,9 @@ void tBlockDataClear(SBlockData *pBlockData) { } int32_t tBlockDataAddColData(SBlockData *pBlockData, int16_t cid, int8_t type, int8_t cflag, SColData **ppColData) { - ASSERT(pBlockData->nColData == 0 || pBlockData->aColData[pBlockData->nColData - 1].cid < cid); + if (pBlockData->nColData != 0 && pBlockData->aColData[pBlockData->nColData - 1].cid >= cid) { + return TSDB_CODE_INVALID_PARA; + } SColData *newColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SColData) * (pBlockData->nColData + 1)); if (newColData == NULL) { @@ -1345,7 +1276,9 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS // uid if (pBlockData->uid == 0) { - ASSERT(uid); + if (!uid) { + return TSDB_CODE_INVALID_PARA; + } code = tRealloc((uint8_t **)&pBlockData->aUid, sizeof(int64_t) * (pBlockData->nRow + 1)); if (code) goto _exit; pBlockData->aUid[pBlockData->nRow] = uid; @@ -1379,7 +1312,9 @@ int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS // version int64_t lversion = pBlockData->aVersion[pBlockData->nRow - 1]; int64_t rversion = TSDBROW_VERSION(pRow); - ASSERT(lversion != rversion); + if (lversion == rversion) { + return TSDB_CODE_INVALID_PARA; + } if (rversion > lversion) { pBlockData->aVersion[pBlockData->nRow - 1] = rversion; } @@ -1393,7 +1328,8 @@ int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS code = tBlockDataUpsertBlockRow(pBlockData, pRow->pBlockData, pRow->iRow, (rversion > lversion) ? 1 : -1); if (code) goto _exit; } else { - ASSERT(0); + code = TSDB_CODE_INVALID_PARA; + goto _exit; } _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index dc84b73c10..e34862247f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -934,20 +934,23 @@ _exit: } static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { + int ret = 0; SVDropTtlTableReq ttlReq = {0}; if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + ret = TSDB_CODE_INVALID_MSG; goto end; } - ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids)); + if (ttlReq.nUids != taosArrayGetSize(ttlReq.pTbUids)) { + ret = TSDB_CODE_INVALID_MSG; + goto end; + } if (ttlReq.nUids != 0) { vInfo("vgId:%d, drop ttl table req will be processed, time:%d, ntbUids:%d", pVnode->config.vgId, ttlReq.timestampSec, ttlReq.nUids); } - int ret = 0; if (ttlReq.nUids > 0) { metaDropTables(pVnode->pMeta, ttlReq.pTbUids); (void)tqUpdateTbUidList(pVnode->pTq, ttlReq.pTbUids, false); @@ -1787,8 +1790,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in } if (info.suid) { - code = metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); - ASSERT(code == 0); + (void)metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); } if (pSubmitTbData->sver != info.skmVer) { From 378dadf7c45012dcd625edde65894ec23a4fd37c Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 22 Aug 2024 09:54:12 +0800 Subject: [PATCH 088/100] fix: set auto del --- source/os/src/osFile.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 16ebe05520..c274c7fbab 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1633,7 +1633,14 @@ int taosCloseCFile(FILE *f) { return fclose(f); } int taosSetAutoDelFile(char *path) { #ifdef WINDOWS - return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY); + bool succ = SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY); + if (succ) { + return 0; + } else { + DWORD error = GetLastError(); + terrno = TAOS_SYSTEM_ERROR(error); + return terrno; + } #else if (-1 == unlink(path)) { terrno = TAOS_SYSTEM_ERROR(errno); From da72fca693a643e54cb90a97b2180cb06c3bea12 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 22 Aug 2024 10:03:09 +0800 Subject: [PATCH 089/100] fix: local policy memory leak --- source/client/src/clientImpl.c | 1 + source/libs/executor/src/exchangeoperator.c | 1 + 2 files changed, 2 insertions(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 563d70ab08..11b053df26 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1792,6 +1792,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg)); if (NULL == arg) { pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(tEpSet); rpcFreeCont(pMsg->pCont); destroySendMsgInfo(pMsg->info.ahandle); return; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 60faba3e3a..d90f59047d 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -521,6 +521,7 @@ void doDestroyExchangeOperatorInfo(void* param) { int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; + taosMemoryFreeClear(pMsg->pEpSet); SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); if (pExchangeInfo == NULL) { qWarn("failed to acquire exchange operator, since it may have been released, %p", pExchangeInfo); From d3ea7ad6d15f383f5bb408cdb404519fab68f95e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 22 Aug 2024 10:08:26 +0800 Subject: [PATCH 090/100] change QID log --- docs/examples/c/with_reqid_demo.c | 10 +- source/client/src/clientEnv.c | 152 ++++---- source/client/src/clientImpl.c | 100 ++--- source/client/src/clientMain.c | 97 ++--- source/client/src/clientRawBlockWrite.c | 141 +++---- source/client/src/clientTmq.c | 235 ++++++------ source/dnode/mgmt/node_util/inc/dmUtil.h | 14 +- source/dnode/mnode/impl/inc/mndInt.h | 14 +- source/dnode/vnode/src/inc/vnd.h | 12 +- source/dnode/vnode/src/tq/tq.c | 14 +- source/dnode/vnode/src/tq/tqRead.c | 145 ++++---- source/dnode/vnode/src/tq/tqUtil.c | 32 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 73 ++-- source/libs/catalog/src/ctgAsync.c | 411 +++++++++++---------- source/libs/catalog/src/ctgRemote.c | 76 ++-- source/libs/stream/src/streamCheckStatus.c | 98 ++--- source/libs/stream/src/streamDispatch.c | 51 +-- source/libs/stream/src/streamExec.c | 81 ++-- source/libs/stream/src/streamTask.c | 2 +- source/libs/transport/inc/transLog.h | 12 +- 20 files changed, 903 insertions(+), 867 deletions(-) diff --git a/docs/examples/c/with_reqid_demo.c b/docs/examples/c/with_reqid_demo.c index 1a1a53acc6..21de28ffe8 100644 --- a/docs/examples/c/with_reqid_demo.c +++ b/docs/examples/c/with_reqid_demo.c @@ -33,7 +33,8 @@ static int DemoWithReqId() { // connect TAOS *taos = taos_connect(host, user, password, NULL, port); if (taos == NULL) { - fprintf(stderr, "Failed to connect to %s:%hu, ErrCode: 0x%x, ErrMessage: %s.\n", host, port, taos_errno(NULL), taos_errstr(NULL)); + fprintf(stderr, "Failed to connect to %s:%hu, ErrCode: 0x%x, ErrMessage: %s.\n", host, port, taos_errno(NULL), + taos_errstr(NULL)); taos_cleanup(); return -1; } @@ -44,7 +45,8 @@ static int DemoWithReqId() { TAOS_RES *result = taos_query_with_reqid(taos, sql, reqid); code = taos_errno(result); if (code != 0) { - fprintf(stderr, "Failed to execute sql with reqId: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, taos_errstr(result)); + fprintf(stderr, "Failed to execute sql with QID: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, + taos_errstr(result)); taos_close(taos); taos_cleanup(); return -1; @@ -73,6 +75,4 @@ static int DemoWithReqId() { // ANCHOR_END: with_reqid } -int main(int argc, char *argv[]) { - return DemoWithReqId(); -} +int main(int argc, char *argv[]) { return DemoWithReqId(); } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d63701c529..491a49778c 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -26,6 +26,7 @@ #include "qworker.h" #include "scheduler.h" #include "tcache.h" +#include "tcompare.h" #include "tglobal.h" #include "thttp.h" #include "tmsg.h" @@ -35,7 +36,6 @@ #include "tsched.h" #include "ttime.h" #include "tversion.h" -#include "tcompare.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -44,16 +44,16 @@ #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 -#define ENV_JSON_FALSE_CHECK(c) \ - do { \ - if (!c) { \ - tscError("faild to add item to JSON object");\ - code = TSDB_CODE_TSC_FAIL_GENERATE_JSON; \ - goto _end; \ - } \ +#define ENV_JSON_FALSE_CHECK(c) \ + do { \ + if (!c) { \ + tscError("faild to add item to JSON object"); \ + code = TSDB_CODE_TSC_FAIL_GENERATE_JSON; \ + goto _end; \ + } \ } while (0) -#define ENV_ERR_RET(c,info) \ +#define ENV_ERR_RET(c, info) \ do { \ int32_t _code = c; \ if (_code != TSDB_CODE_SUCCESS) { \ @@ -94,109 +94,110 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1); int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 - ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, + ", current:%d, app current:%d, total:%d, QID:0x%" PRIx64, pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } return code; } -static void concatStrings(SArray *list, char* buf, int size){ - int len = 0; - for(int i = 0; i < taosArrayGetSize(list); i++){ - char* db = taosArrayGet(list, i); +static void concatStrings(SArray *list, char *buf, int size) { + int len = 0; + for (int i = 0; i < taosArrayGetSize(list); i++) { + char *db = taosArrayGet(list, i); if (NULL == db) { tscError("get dbname failed, buf:%s", buf); break; } - char* dot = strchr(db, '.'); + char *dot = strchr(db, '.'); if (dot != NULL) { db = dot + 1; } - if (i != 0){ + if (i != 0) { (void)strcat(buf, ","); len += 1; } int ret = snprintf(buf + len, size - len, "%s", db); - if (ret < 0) { + if (ret < 0) { tscError("snprintf failed, buf:%s, ret:%d", buf, ret); break; } len += ret; - if (len >= size){ + if (len >= size) { tscInfo("dbList is truncated, buf:%s, len:%d", buf, len); break; } } } -static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration){ - cJSON* json = cJSON_CreateObject(); +static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration) { + cJSON *json = cJSON_CreateObject(); int32_t code = TSDB_CODE_SUCCESS; if (json == NULL) { tscError("[monitor] cJSON_CreateObject failed"); return TSDB_CODE_OUT_OF_MEMORY; } char clusterId[32] = {0}; - if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0){ + if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0) { tscError("failed to generate clusterId:%" PRId64, pTscObj->pAppInfo->clusterId); code = TSDB_CODE_FAILED; goto _end; } char startTs[32] = {0}; - if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start/1000) < 0){ - tscError("failed to generate startTs:%" PRId64, pRequest->metric.start/1000); + if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start / 1000) < 0) { + tscError("failed to generate startTs:%" PRId64, pRequest->metric.start / 1000); code = TSDB_CODE_FAILED; goto _end; } char requestId[32] = {0}; - if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0){ + if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0) { tscError("failed to generate requestId:%" PRIu64, pRequest->requestId); code = TSDB_CODE_FAILED; goto _end; } - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId))); - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs))); - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId))); - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration/1000))); - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code))); - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code)))); - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType))); - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows))); - if(pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){ + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration / 1000))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code)))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject( + json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows))); + if (pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen) { char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen]; pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0'; - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr))); pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = tmp; - }else{ - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr))); + } else { + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr))); } - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_name", cJSON_CreateString(appInfo.appName))); - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn))); + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn))); char pid[32] = {0}; - if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0){ + if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0) { tscError("failed to generate pid:%d", appInfo.pid); code = TSDB_CODE_FAILED; goto _end; } - ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid))); - if(pRequest->dbList != NULL){ + ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid))); + if (pRequest->dbList != NULL) { char dbList[1024] = {0}; concatStrings(pRequest->dbList, dbList, sizeof(dbList) - 1); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(dbList))); - }else if(pRequest->pDb != NULL){ + } else if (pRequest->pDb != NULL) { ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(pRequest->pDb))); - }else{ + } else { ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(""))); } - char* value = cJSON_PrintUnformatted(json); + char *value = cJSON_PrintUnformatted(json); MonitorSlowLogData data = {0}; data.clusterId = pTscObj->pAppInfo->clusterId; data.type = SLOW_LOG_WRITE; @@ -212,7 +213,7 @@ _end: return code; } -static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) { +static bool checkSlowLogExceptDb(SRequestObj *pRequest, char *exceptDb) { if (pRequest->pDb != NULL) { return strcmp(pRequest->pDb, exceptDb) != 0; } @@ -227,7 +228,7 @@ static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) { if (dot != NULL) { db = dot + 1; } - if(strcmp(db, exceptDb) == 0){ + if (strcmp(db, exceptDb) == 0) { return false; } } @@ -248,15 +249,14 @@ static void deregisterRequest(SRequestObj *pRequest) { int32_t reqType = SLOW_LOG_TYPE_OTHERS; int64_t duration = taosGetTimestampUs() - pRequest->metric.start; - tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", QID:0x%" PRIx64 " elapsed:%.2f ms, " "current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) { - if ((pRequest->pQuery && pRequest->pQuery->pRoot && - QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type && - (0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) || + if ((pRequest->pQuery && pRequest->pQuery->pRoot && QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type && + (0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) || QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) { tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", @@ -279,7 +279,7 @@ static void deregisterRequest(SRequestObj *pRequest) { } } - if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){ + if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) { if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) { sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { @@ -289,14 +289,15 @@ static void deregisterRequest(SRequestObj *pRequest) { } } - if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL || duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) && + if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL || + duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) && checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) { (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) { taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s", taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); - if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){ + if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) { slowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration); if (TSDB_CODE_SUCCESS != generateWriteSlowLog(pTscObj, pRequest, reqType, duration)) { tscError("failed to generate write slow log"); @@ -389,7 +390,7 @@ void destroyAllRequests(SHashObj *pRequests) { SRequestObj *pRequest = acquireRequest(*rid); if (pRequest) { destroyRequest(pRequest); - (void)releaseRequest(*rid); // ignore error + (void)releaseRequest(*rid); // ignore error } pIter = taosHashIterate(pRequests, pIter); @@ -404,7 +405,7 @@ void stopAllRequests(SHashObj *pRequests) { SRequestObj *pRequest = acquireRequest(*rid); if (pRequest) { taos_stop_query(pRequest); - (void)releaseRequest(*rid); // ignore error + (void)releaseRequest(*rid); // ignore error } pIter = taosHashIterate(pRequests, pIter); @@ -412,7 +413,7 @@ void stopAllRequests(SHashObj *pRequests) { } void destroyAppInst(void *info) { - SAppInstInfo* pAppInfo = *(SAppInstInfo**)info; + SAppInstInfo *pAppInfo = *(SAppInstInfo **)info; tscDebug("destroy app inst mgr %p", pAppInfo); int32_t code = taosThreadMutexLock(&appInfo.mutex); @@ -597,28 +598,28 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } /// return the most previous req ref id -int64_t removeFromMostPrevReq(SRequestObj* pRequest) { - int64_t mostPrevReqRefId = pRequest->self; - SRequestObj* pTmp = pRequest; +int64_t removeFromMostPrevReq(SRequestObj *pRequest) { + int64_t mostPrevReqRefId = pRequest->self; + SRequestObj *pTmp = pRequest; while (pTmp->relation.prevRefId) { pTmp = acquireRequest(pTmp->relation.prevRefId); if (pTmp) { mostPrevReqRefId = pTmp->self; - (void)releaseRequest(mostPrevReqRefId); // ignore error + (void)releaseRequest(mostPrevReqRefId); // ignore error } else { break; } } - (void)removeRequest(mostPrevReqRefId); // ignore error + (void)removeRequest(mostPrevReqRefId); // ignore error return mostPrevReqRefId; } void destroyNextReq(int64_t nextRefId) { if (nextRefId) { - SRequestObj* pObj = acquireRequest(nextRefId); + SRequestObj *pObj = acquireRequest(nextRefId); if (pObj) { - (void)releaseRequest(nextRefId); // ignore error - (void)releaseRequest(nextRefId); // ignore error + (void)releaseRequest(nextRefId); // ignore error + (void)releaseRequest(nextRefId); // ignore error } } } @@ -638,7 +639,7 @@ void destroySubRequests(SRequestObj *pRequest) { pTmp = acquireRequest(tmpRefId); if (pTmp) { pReqList[++reqIdx] = pTmp; - (void)releaseRequest(tmpRefId); // ignore error + (void)releaseRequest(tmpRefId); // ignore error } else { tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); break; @@ -646,7 +647,7 @@ void destroySubRequests(SRequestObj *pRequest) { } for (int32_t i = reqIdx; i >= 0; i--) { - (void)removeRequest(pReqList[i]->self); // ignore error + (void)removeRequest(pReqList[i]->self); // ignore error } tmpRefId = pRequest->relation.nextRefId; @@ -654,8 +655,8 @@ void destroySubRequests(SRequestObj *pRequest) { pTmp = acquireRequest(tmpRefId); if (pTmp) { tmpRefId = pTmp->relation.nextRefId; - (void)removeRequest(pTmp->self); // ignore error - (void)releaseRequest(pTmp->self); // ignore error + (void)removeRequest(pTmp->self); // ignore error + (void)releaseRequest(pTmp->self); // ignore error } else { tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); break; @@ -758,7 +759,7 @@ void stopAllQueries(SRequestObj *pRequest) { for (int32_t i = reqIdx; i >= 0; i--) { taosStopQueryImpl(pReqList[i]); - (void)releaseRequest(pReqList[i]->self); // ignore error + (void)releaseRequest(pReqList[i]->self); // ignore error } taosStopQueryImpl(pRequest); @@ -769,7 +770,7 @@ void stopAllQueries(SRequestObj *pRequest) { if (pTmp) { tmpRefId = pTmp->relation.nextRefId; taosStopQueryImpl(pTmp); - (void)releaseRequest(pTmp->self); // ignore error + (void)releaseRequest(pTmp->self); // ignore error } else { tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); break; @@ -929,7 +930,8 @@ void taos_init_imp(void) { appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - appInfo.pInstMapByClusterId = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + appInfo.pInstMapByClusterId = + taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (NULL == appInfo.pInstMap || NULL == appInfo.pInstMapByClusterId) { tscError("failed to allocate memory when init appInfo"); tscInitRes = TSDB_CODE_OUT_OF_MEMORY; @@ -993,8 +995,8 @@ int taos_init() { return tscInitRes; } -const char* getCfgName(TSDB_OPTION option) { - const char* name = NULL; +const char *getCfgName(TSDB_OPTION option) { + const char *name = NULL; switch (option) { case TSDB_OPTION_SHELL_ACTIVITY_TIMER: @@ -1025,12 +1027,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { char newstr[PATH_MAX]; int len = strlen(str); if (len > 1 && str[0] != '"' && str[0] != '\'') { - if (len + 2 >= PATH_MAX) { + if (len + 2 >= PATH_MAX) { tscError("Too long path %s", str); return -1; } newstr[0] = '"'; - (void)memcpy(newstr+1, str, len); + (void)memcpy(newstr + 1, str, len); newstr[len + 1] = '"'; newstr[len + 2] = '\0'; str = newstr; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 563d70ab08..7135e8f070 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -15,8 +15,8 @@ #include "cJSON.h" #include "clientInt.h" -#include "clientMonitor.h" #include "clientLog.h" +#include "clientMonitor.h" #include "command.h" #include "scheduler.h" #include "tdatablock.h" @@ -28,7 +28,7 @@ #include "tref.h" #include "tsched.h" #include "tversion.h" -static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); +static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo); static bool stringLengthCheck(const char* str, size_t maxsize) { @@ -68,15 +68,13 @@ bool chkRequestKilled(void* param) { return killed; } -void cleanupAppInfo() { - taosHashCleanup(appInfo.pInstMap); -} +void cleanupAppInfo() { taosHashCleanup(appInfo.pInstMap); } static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj); int32_t taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, - uint16_t port, int connType, STscObj** pObj) { + uint16_t port, int connType, STscObj** pObj) { TSC_ERR_RET(taos_init()); if (!validateUserName(user)) { TSC_ERR_RET(TSDB_CODE_TSC_INVALID_USER_LENGTH); @@ -126,7 +124,7 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass } SAppInstInfo** pInst = NULL; - int32_t code = taosThreadMutexLock(&appInfo.mutex); + int32_t code = taosThreadMutexLock(&appInfo.mutex); if (TSDB_CODE_SUCCESS != code) { tscError("failed to lock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code))); TSC_ERR_RET(code); @@ -187,14 +185,14 @@ _return: return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType, pObj); } -//SAppInstInfo* getAppInstInfo(const char* clusterKey) { -// SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey)); -// if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) { -// return *ppAppInstInfo; -// } else { -// return NULL; -// } -//} +// SAppInstInfo* getAppInstInfo(const char* clusterKey) { +// SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey)); +// if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) { +// return *ppAppInstInfo; +// } else { +// return NULL; +// } +// } void freeQueryParam(SSyncQueryParam* param) { if (param == NULL) return; @@ -230,7 +228,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self)); if (err) { - tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", + tscError("%" PRId64 " failed to add to request container, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; @@ -241,7 +239,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { if (TSDB_CODE_SUCCESS != nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { - tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", + tscError("%" PRId64 " failed to create node allocator, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; @@ -249,7 +247,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, } } - tscDebugL("0x%" PRIx64 " SQL: %s, reqId:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId); + tscDebugL("0x%" PRIx64 " SQL: %s, QID:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId); return TSDB_CODE_SUCCESS; } @@ -363,10 +361,10 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; - tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); } @@ -432,7 +430,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) { return TSDB_CODE_SUCCESS; } -int32_t qnodeRequired(SRequestObj* pRequest, bool *required) { +int32_t qnodeRequired(SRequestObj* pRequest, bool* required) { if (QUERY_POLICY_VNODE == tsQueryPolicy || QUERY_POLICY_CLIENT == tsQueryPolicy) { *required = false; return TSDB_CODE_SUCCESS; @@ -554,7 +552,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr if (NULL == nodeList) { return terrno; } - char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; + char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; int32_t dbNum = taosArrayGetSize(pDbVgList); for (int32_t i = 0; i < dbNum; ++i) { @@ -568,7 +566,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } for (int32_t j = 0; j < vgNum; ++j) { - SVgroupInfo* pInfo = taosArrayGet(pVg, j); + SVgroupInfo* pInfo = taosArrayGet(pVg, j); if (NULL == pInfo) { taosArrayDestroy(nodeList); return TSDB_CODE_OUT_OF_RANGE; @@ -972,8 +970,8 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { break; } default: - tscError("0x%" PRIx64 ", invalid exec result for request type %d, reqId:0x%" PRIx64, pRequest->self, - pRequest->type, pRequest->requestId); + tscError("0x%" PRIx64 ", invalid exec result for request type %d, QID:0x%" PRIx64, pRequest->self, pRequest->type, + pRequest->requestId); code = TSDB_CODE_APP_ERROR; } @@ -1017,12 +1015,12 @@ void returnToUser(SRequestObj* pRequest) { (void)releaseRequest(pRequest->relation.userRefId); return; } else { - tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self, pRequest->relation.userRefId, pRequest->requestId); } } -static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock**pBlock) { +static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock** pBlock) { int64_t lastTs = 0; TAOS_FIELD* pResFields = taos_fetch_fields(pRes); int32_t numOfFields = taos_num_fields(pRes); @@ -1032,7 +1030,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock* return code; } - for(int32_t i = 0; i < numOfFields; ++i) { + for (int32_t i = 0; i < numOfFields; ++i) { SColumnInfoData colInfoData = createColumnInfoData(pResFields[i].type, pResFields[i].bytes, i + 1); code = blockDataAppendColInfo(*pBlock, &colInfoData); if (TSDB_CODE_SUCCESS != code) { @@ -1054,7 +1052,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock* lastTs = ts; } - for(int32_t j = 0; j < numOfFields; ++j) { + for (int32_t j = 0; j < numOfFields; ++j) { SColumnInfoData* pColInfoData = taosArrayGet((*pBlock)->pDataBlock, j); code = colDataSetVal(pColInfoData, i, pRow[j], false); if (TSDB_CODE_SUCCESS != code) { @@ -1069,7 +1067,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock* (*pBlock)->info.window.ekey = lastTs; (*pBlock)->info.rows = numOfRows; - tscDebug("lastKey:%"PRId64" numOfRows:%d from all vgroups", lastTs, numOfRows); + tscDebug("lastKey:%" PRId64 " numOfRows:%d from all vgroups", lastTs, numOfRows); return TSDB_CODE_SUCCESS; } @@ -1082,7 +1080,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) { SSDataBlock* pBlock = NULL; if (TSDB_CODE_SUCCESS != createResultBlock(res, rowNum, &pBlock)) { - tscError("0x%" PRIx64 ", create result block failed, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); + tscError("0x%" PRIx64 ", create result block failed, QID:0x%" PRIx64, pRequest->self, pRequest->requestId); return; } @@ -1091,7 +1089,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) { continuePostSubQuery(pNextReq, pBlock); (void)releaseRequest(pRequest->relation.nextRefId); } else { - tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self, pRequest->relation.nextRefId, pRequest->requestId); } @@ -1110,7 +1108,7 @@ void handlePostSubQuery(SSqlCallbackWrapper* pWrapper) { continuePostSubQuery(pNextReq, NULL); (void)releaseRequest(pRequest->relation.nextRefId); } else { - tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self, pRequest->relation.nextRefId, pRequest->requestId); } } @@ -1141,11 +1139,11 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { } taosMemoryFree(pResult); - tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { - tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, QID:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId); (void)removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)); restartAsyncQuery(pRequest, code); @@ -1215,7 +1213,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue } break; case QUERY_EXEC_MODE_SCHEDULE: { - SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); + SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); if (NULL == pMnodeList) { code = terrno; break; @@ -1533,7 +1531,7 @@ int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* p } int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, - SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj) { + SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj) { *pTscObj = NULL; int32_t code = createTscObj(user, auth, db, connType, pAppInfo, pTscObj); if (TSDB_CODE_SUCCESS != code) { @@ -1560,7 +1558,8 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta } int64_t transporterId = 0; - code = asyncSendMsgToServer((*pTscObj)->pAppInfo->pTransporter, &(*pTscObj)->pAppInfo->mgmtEp.epSet, &transporterId, body); + code = asyncSendMsgToServer((*pTscObj)->pAppInfo->pTransporter, &(*pTscObj)->pAppInfo->mgmtEp.epSet, &transporterId, + body); if (TSDB_CODE_SUCCESS != code) { destroyTscObj(*pTscObj); tscError("failed to send connect msg to server, code:%s", tstrerror(code)); @@ -1577,7 +1576,7 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta *pTscObj = NULL; return terrno; } else { - tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, (*pTscObj)->id, + tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, QID:0x%" PRIx64, (*pTscObj)->id, (*pTscObj)->connId, (*pTscObj)->pAppInfo->pTransporter, pRequest->requestId); destroyRequest(pRequest); } @@ -1708,7 +1707,7 @@ int32_t doProcessMsgFromServer(void* param) { char tbuf[40] = {0}; TRACE_TO_STR(trace, tbuf); - tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, gtid: %s", pMsg->info.handle, + tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, QID: %s", pMsg->info.handle, TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); if (pSendInfo->requestObjRefId != 0) { @@ -1820,7 +1819,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons } STscObj* pObj = NULL; - int32_t code = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY, &pObj); + int32_t code = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY, &pObj); if (TSDB_CODE_SUCCESS == code) { int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t)); if (NULL == rid) { @@ -1890,7 +1889,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) } SReqResultInfo* pResInfo = &pRequest->body.resInfo; - SSchedulerReq req = { .syncReq = true, .pFetchRes = (void**)&pResInfo->pData }; + SSchedulerReq req = {.syncReq = true, .pFetchRes = (void**)&pResInfo->pData}; pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req); if (pRequest->code != TSDB_CODE_SUCCESS) { @@ -1905,7 +1904,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) return NULL; } - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64, pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); STscObj* pTscObj = pRequest->pTscObj; @@ -2033,7 +2032,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { } static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { - char* p = (char*)pResultInfo->pData; + char* p = (char*)pResultInfo->pData; int32_t blockVersion = *(int32_t*)p; // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column @@ -2298,7 +2297,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 char* pStart = p; for (int32_t i = 0; i < numOfCols; ++i) { - if(blockVersion == BLOCK_VERSION_1){ + if (blockVersion == BLOCK_VERSION_1) { colLength[i] = htonl(colLength[i]); } if (colLength[i] >= dataLen) { @@ -2733,7 +2732,8 @@ void syncQueryFn(void* param, void* res, int32_t code) { (void)tsem_post(&pParam->sem); } -void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) { +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, + int8_t source) { if (sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; if (fp) { @@ -2854,7 +2854,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo; - tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, + tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); pResultInfo->pData = pResult; @@ -2877,10 +2877,10 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; - tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code), + tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); @@ -2932,7 +2932,7 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param int32_t code = schedulerFetchRows(pRequest->body.queryJob, &req); if (TSDB_CODE_SUCCESS != code) { tscError("0x%" PRIx64 " failed to schedule fetch rows", pRequest->requestId); - pRequest->body.fetchFp(param, pRequest, code); + pRequest->body.fetchFp(param, pRequest, code); } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index de56a4844a..55401d7eb2 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -16,19 +16,19 @@ #include "catalog.h" #include "clientInt.h" #include "clientLog.h" -#include "clientStmt.h" #include "clientMonitor.h" +#include "clientStmt.h" #include "functionMgt.h" #include "os.h" #include "query.h" #include "scheduler.h" +#include "tcompare.h" #include "tdatablock.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" #include "version.h" -#include "tcompare.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -120,7 +120,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha } STscObj *pObj = NULL; - int32_t code = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY, &pObj); + int32_t code = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY, &pObj); if (TSDB_CODE_SUCCESS == code) { int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); if (NULL == rid) { @@ -183,15 +183,15 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) return 0; } -typedef struct SFetchWhiteListInfo{ - int64_t connId; +typedef struct SFetchWhiteListInfo { + int64_t connId; __taos_async_whitelist_fn_t userCbFn; - void* userParam; + void *userParam; } SFetchWhiteListInfo; -int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { - SFetchWhiteListInfo* pInfo = (SFetchWhiteListInfo*)param; - TAOS* taos = &pInfo->connId; +int32_t fetchWhiteListCallbackFn(void *param, SDataBuf *pMsg, int32_t code) { + SFetchWhiteListInfo *pInfo = (SFetchWhiteListInfo *)param; + TAOS *taos = &pInfo->connId; if (code != TSDB_CODE_SUCCESS) { pInfo->userCbFn(pInfo->userParam, code, taos, 0, NULL); taosMemoryFree(pMsg->pData); @@ -209,7 +209,7 @@ int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { return terrno; } - uint64_t* pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t)); + uint64_t *pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t)); if (pWhiteLists == NULL) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -238,7 +238,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa return; } - int64_t connId = *(int64_t*)taos; + int64_t connId = *(int64_t *)taos; STscObj *pTsc = acquireTscObj(connId); if (NULL == pTsc) { @@ -255,7 +255,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa return; } - void* pReq = taosMemoryMalloc(msgLen); + void *pReq = taosMemoryMalloc(msgLen); if (pReq == NULL) { fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); releaseTscObj(connId); @@ -269,7 +269,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa return; } - SFetchWhiteListInfo* pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo)); + SFetchWhiteListInfo *pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo)); if (pParam == NULL) { fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); taosMemoryFree(pReq); @@ -280,9 +280,9 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa pParam->connId = connId; pParam->userCbFn = fp; pParam->userParam = param; - SMsgSendInfo* pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pSendInfo == NULL) { - fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); + fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); taosMemoryFree(pParam); taosMemoryFree(pReq); releaseTscObj(connId); @@ -297,7 +297,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa pSendInfo->msgType = TDMT_MND_GET_USER_WHITELIST; int64_t transportId = 0; - SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp); + SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp); if (TSDB_CODE_SUCCESS != asyncSendMsgToServer(pTsc->pAppInfo->pTransporter, &epSet, &transportId, pSendInfo)) { tscWarn("failed to async send msg to server"); } @@ -443,7 +443,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } - if(pRequest->inCallback) { + if (pRequest->inCallback) { tscError("can not call taos_fetch_row before query callback ends."); terrno = TSDB_CODE_TSC_INVALID_OPERATION; return NULL; @@ -454,7 +454,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SMqRspObj *msg = ((SMqRspObj *)res); SReqResultInfo *pResultInfo = NULL; if (msg->common.resIter == -1) { - if(tmqGetNextResInfo(res, true, &pResultInfo) != 0){ + if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) { return NULL; } } else { @@ -466,7 +466,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { pResultInfo->current += 1; return pResultInfo->row; } else { - if (tmqGetNextResInfo(res, true, &pResultInfo) != 0){ + if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) { return NULL; } @@ -540,22 +540,23 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) len += sprintf(str + len, "%lf", dv); } break; - case TSDB_DATA_TYPE_VARBINARY:{ - void* data = NULL; + case TSDB_DATA_TYPE_VARBINARY: { + void *data = NULL; uint32_t size = 0; - int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); - if(taosAscii2Hex(row[i], charLen, &data, &size) < 0){ + int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); + if (taosAscii2Hex(row[i], charLen, &data, &size) < 0) { break; } (void)memcpy(str + len, data, size); len += size; taosMemoryFree(data); - }break; + } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); - if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) { + if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY || + fields[i].type == TSDB_DATA_TYPE_GEOMETRY) { if (charLen > fields[i].bytes || charLen < 0) { tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); break; @@ -664,7 +665,8 @@ const char *taos_get_client_info() { return version; } // return int32_t int taos_affected_rows(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -675,7 +677,8 @@ int taos_affected_rows(TAOS_RES *res) { // return int64_t int64_t taos_affected_rows64(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -725,7 +728,8 @@ int taos_select_db(TAOS *taos, const char *db) { } void taos_stop_query(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + TD_RES_TMQ_BATCH_META(res)) { return; } @@ -784,7 +788,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { return pRequest->code; } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SReqResultInfo *pResultInfo = NULL; - int32_t code = tmqGetNextResInfo(res, true, &pResultInfo); + int32_t code = tmqGetNextResInfo(res, true, &pResultInfo); if (code != 0) return code; pResultInfo->current = pResultInfo->numOfRows; @@ -807,7 +811,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SReqResultInfo *pResultInfo = NULL; - int32_t code = tmqGetNextResInfo(res, false, &pResultInfo); + int32_t code = tmqGetNextResInfo(res, false, &pResultInfo); if (code != 0) { (*numOfRows) = 0; return 0; @@ -937,7 +941,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t SRequestObj *pRequest = pWrapper->pRequest; SQuery *pQuery = pRequest->pQuery; - qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); + qDebug("0x%" PRIx64 " start to semantic analysis, QID:0x%" PRIx64, pRequest->self, pRequest->requestId); int64_t analyseStart = taosGetTimestampUs(); pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart; @@ -953,7 +957,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t (void)memcpy(&pRequest->parseMeta, pResultMeta, sizeof(*pResultMeta)); (void)memset(pResultMeta, 0, sizeof(*pResultMeta)); } - + handleQueryAnslyseRes(pWrapper, pResultMeta, code); } @@ -999,7 +1003,7 @@ void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResult } pNewRequest->pQuery = NULL; - code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pNewRequest->pQuery); + code = nodesMakeNode(QUERY_NODE_QUERY, (SNode **)&pNewRequest->pQuery); if (pNewRequest->pQuery) { pNewRequest->pQuery->pRoot = pRoot; pRoot = NULL; @@ -1056,15 +1060,15 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta pRequest->pQuery = NULL; if (NEED_CLIENT_HANDLE_ERROR(code)) { - tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); restartAsyncQuery(pRequest, code); return; } // return to app directly - tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:0x%" PRIx64, pRequest->self, - tstrerror(code), pRequest->requestId); + tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, QID:0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); pRequest->code = code; returnToUser(pRequest); } @@ -1113,7 +1117,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c SQuery *pQuery = pRequest->pQuery; pRequest->metric.ctgCostUs += taosGetTimestampUs() - pRequest->metric.ctgStart; - qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId, + qDebug("0x%" PRIx64 " start to continue parse, QID:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId, tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { @@ -1126,7 +1130,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c } if (TSDB_CODE_SUCCESS != code) { - tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, + tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code, tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); pRequest->pWrapper = NULL; @@ -1143,7 +1147,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) } if (TSDB_CODE_SUCCESS != code) { - tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, + tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code, tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); pRequest->pWrapper = NULL; @@ -1261,7 +1265,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } if (TSDB_CODE_SUCCESS != code) { - tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), + tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); pRequest->pWrapper = NULL; @@ -1269,9 +1273,9 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { pRequest->pQuery = NULL; if (NEED_CLIENT_HANDLE_ERROR(code)) { - tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); - (void)refreshMeta(pRequest->pTscObj, pRequest); //ignore return code,try again + (void)refreshMeta(pRequest->pTscObj, pRequest); // ignore return code,try again pRequest->prevCode = code; doAsyncQuery(pRequest, true); return; @@ -1285,7 +1289,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest); - SRequestObj* pUserReq = pRequest; + SRequestObj *pUserReq = pRequest; (void)acquireRequest(pRequest->self); while (pUserReq) { if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) { @@ -1631,7 +1635,6 @@ TAOS_STMT *taos_stmt_init_with_options(TAOS *taos, TAOS_STMT_OPTIONS *options) { return pStmt; } - int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { if (stmt == NULL || sql == NULL) { tscError("NULL parameter for %s", __FUNCTION__); @@ -1874,7 +1877,7 @@ int taos_stmt_close(TAOS_STMT *stmt) { return stmtClose(stmt); } -int taos_set_conn_mode(TAOS* taos, int mode, int value) { +int taos_set_conn_mode(TAOS *taos, int mode, int value) { if (taos == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1897,6 +1900,4 @@ int taos_set_conn_mode(TAOS* taos, int mode, int value) { return 0; } -char* getBuildInfo(){ - return buildinfo; -} +char *getBuildInfo() { return buildinfo; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index f05120b366..e94a2d163c 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -23,32 +23,31 @@ #include "tglobal.h" #include "tmsgtype.h" -#define RAW_NULL_CHECK(c) \ - do { \ - if (c == NULL) { \ - code = TSDB_CODE_OUT_OF_MEMORY; \ - goto end; \ - } \ +#define RAW_NULL_CHECK(c) \ + do { \ + if (c == NULL) { \ + code = TSDB_CODE_OUT_OF_MEMORY; \ + goto end; \ + } \ } while (0) -#define RAW_FALSE_CHECK(c) \ - do { \ - if (!c) { \ - code = TSDB_CODE_INVALID_PARA; \ - goto end; \ - } \ +#define RAW_FALSE_CHECK(c) \ + do { \ + if (!c) { \ + code = TSDB_CODE_INVALID_PARA; \ + goto end; \ + } \ } while (0) -#define RAW_RETURN_CHECK(c) \ - do { \ - code = c; \ - if (code != 0) { \ - goto end; \ - } \ +#define RAW_RETURN_CHECK(c) \ + do { \ + code = c; \ + if (code != 0) { \ + goto end; \ + } \ } while (0) - -#define LOG_ID_TAG "connId:0x%" PRIx64 ",reqId:0x%" PRIx64 +#define LOG_ID_TAG "connId:0x%" PRIx64 ",QID:0x%" PRIx64 #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId #define TMQ_META_VERSION "1.0" @@ -57,10 +56,10 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } -static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, - int8_t t, SColCmprWrapper* pColCmprRow, cJSON** pJson) { +static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t, + SColCmprWrapper* pColCmprRow, cJSON** pJson) { int32_t code = TSDB_CODE_SUCCESS; - int8_t buildDefaultCompress = 0; + int8_t buildDefaultCompress = 0; if (pColCmprRow->nCols <= 0) { buildDefaultCompress = 1; } @@ -82,7 +81,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche cJSON* columns = cJSON_CreateArray(); RAW_NULL_CHECK(columns); for (int i = 0; i < schemaRow->nCols; i++) { - cJSON* column = cJSON_CreateObject(); + cJSON* column = cJSON_CreateObject(); RAW_NULL_CHECK(column); SSchema* s = schemaRow->pSchema + i; cJSON* cname = cJSON_CreateString(s->name); @@ -142,7 +141,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche cJSON* tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { - cJSON* tag = cJSON_CreateObject(); + cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); SSchema* s = schemaTag->pSchema + i; cJSON* tname = cJSON_CreateString(s->name); @@ -176,7 +175,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { if (encode != 0) { const char* encodeStr = columnEncodeStr(encode); RAW_NULL_CHECK(encodeStr); - cJSON* encodeJson = cJSON_CreateString(encodeStr); + cJSON* encodeJson = cJSON_CreateString(encodeStr); RAW_NULL_CHECK(encodeJson); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "encode", encodeJson)); return code; @@ -185,7 +184,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { if (compress != 0) { const char* compressStr = columnCompressStr(compress); RAW_NULL_CHECK(compressStr); - cJSON* compressJson = cJSON_CreateString(compressStr); + cJSON* compressJson = cJSON_CreateString(compressStr); RAW_NULL_CHECK(compressJson); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "compress", compressJson)); return code; @@ -194,7 +193,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { if (level != 0) { const char* levelStr = columnLevelStr(level); RAW_NULL_CHECK(levelStr); - cJSON* levelJson = cJSON_CreateString(levelStr); + cJSON* levelJson = cJSON_CreateString(levelStr); RAW_NULL_CHECK(levelJson); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "level", levelJson)); return code; @@ -235,7 +234,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** case TSDB_ALTER_TABLE_ADD_COLUMN: { TAOS_FIELD* field = taosArrayGet(req.pFields, 0); RAW_NULL_CHECK(field); - cJSON* colName = cJSON_CreateString(field->name); + cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); @@ -259,7 +258,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: { SFieldWithOptions* field = taosArrayGet(req.pFields, 0); RAW_NULL_CHECK(field); - cJSON* colName = cJSON_CreateString(field->name); + cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); @@ -285,7 +284,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** case TSDB_ALTER_TABLE_DROP_COLUMN: { TAOS_FIELD* field = taosArrayGet(req.pFields, 0); RAW_NULL_CHECK(field); - cJSON* colName = cJSON_CreateString(field->name); + cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); break; @@ -294,7 +293,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { TAOS_FIELD* field = taosArrayGet(req.pFields, 0); RAW_NULL_CHECK(field); - cJSON* colName = cJSON_CreateString(field->name); + cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); cJSON* colType = cJSON_CreateNumber(field->type); @@ -320,7 +319,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** RAW_NULL_CHECK(oldField); TAOS_FIELD* newField = taosArrayGet(req.pFields, 1); RAW_NULL_CHECK(newField); - cJSON* colName = cJSON_CreateString(oldField->name); + cJSON* colName = cJSON_CreateString(oldField->name); RAW_NULL_CHECK(colName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); cJSON* colNewName = cJSON_CreateString(newField->name); @@ -331,7 +330,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { TAOS_FIELD* field = taosArrayGet(req.pFields, 0); RAW_NULL_CHECK(field); - cJSON* colName = cJSON_CreateString(field->name); + cJSON* colName = cJSON_CreateString(field->name); RAW_NULL_CHECK(colName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); RAW_RETURN_CHECK(setCompressOption(json, field->bytes)); @@ -405,7 +404,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(tagNumJson); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tagNum", tagNumJson)); - cJSON* tags = cJSON_CreateArray(); + cJSON* tags = cJSON_CreateArray(); RAW_NULL_CHECK(tags); SArray* pTagVals = NULL; RAW_RETURN_CHECK(tTagToValArray(pTag, &pTagVals)); @@ -416,13 +415,13 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { uError("p->nTag == 0"); goto end; } - char* pJson = NULL; + char* pJson = NULL; parseTagDatatoJson(pTag, &pJson); - cJSON* tag = cJSON_CreateObject(); + cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); STagVal* pTagVal = taosArrayGet(pTagVals, 0); RAW_NULL_CHECK(pTagVal); - char* ptname = taosArrayGet(tagName, 0); + char* ptname = taosArrayGet(tagName, 0); RAW_NULL_CHECK(ptname); cJSON* tname = cJSON_CreateString(ptname); RAW_NULL_CHECK(tname); @@ -443,7 +442,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(pTagVal); cJSON* tag = cJSON_CreateObject(); RAW_NULL_CHECK(tag); - char* ptname = taosArrayGet(tagName, i); + char* ptname = taosArrayGet(tagName, i); RAW_NULL_CHECK(ptname); cJSON* tname = cJSON_CreateString(ptname); RAW_NULL_CHECK(tname); @@ -463,7 +462,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_NULL_CHECK(buf); if (!buf) goto end; - if(dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) { + if (dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) { taosMemoryFree(buf); goto end; } @@ -489,8 +488,8 @@ end: static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSON** pJson) { int32_t code = 0; - char* string = NULL; - cJSON* json = cJSON_CreateObject(); + char* string = NULL; + cJSON* json = cJSON_CreateObject(); RAW_NULL_CHECK(json); cJSON* type = cJSON_CreateString("create"); RAW_NULL_CHECK(type); @@ -534,8 +533,8 @@ static void processCreateTable(SMqMetaRsp* metaRsp, cJSON** pJson) { if (pCreateReq->type == TSDB_CHILD_TABLE) { buildCreateCTableJson(req.pReqs, req.nReqs, pJson); } else if (pCreateReq->type == TSDB_NORMAL_TABLE) { - buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, - TSDB_NORMAL_TABLE, &pCreateReq->colCmpr, pJson); + buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE, + &pCreateReq->colCmpr, pJson); } } @@ -548,7 +547,7 @@ end: static void processAutoCreateTable(STaosxRsp* rsp, char** string) { SDecoder* decoder = NULL; SVCreateTbReq* pCreateReq = NULL; - int32_t code = 0; + int32_t code = 0; uDebug("auto create table data:%p", rsp); if (rsp->createTableNum <= 0) { uError("processAutoCreateTable rsp->createTableNum <= 0"); @@ -563,7 +562,7 @@ static void processAutoCreateTable(STaosxRsp* rsp, char** string) { // loop to create table for (int32_t iReq = 0; iReq < rsp->createTableNum; iReq++) { // decode - void** data = taosArrayGet(rsp->createTableReq, iReq); + void** data = taosArrayGet(rsp->createTableReq, iReq); RAW_NULL_CHECK(data); int32_t* len = taosArrayGet(rsp->createTableLen, iReq); RAW_NULL_CHECK(len); @@ -735,7 +734,8 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1); } RAW_NULL_CHECK(buf); - if(dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) != TSDB_CODE_SUCCESS) { + if (dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) != + TSDB_CODE_SUCCESS) { taosMemoryFree(buf); goto end; } @@ -823,7 +823,7 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) { // getTbName(req.tableFName); char sql[256] = {0}; (void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, - req.tsColName, req.skey, req.tsColName, req.ekey); + req.tsColName, req.skey, req.tsColName, req.ekey); json = cJSON_CreateObject(); RAW_NULL_CHECK(json); @@ -865,7 +865,7 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(tableNameList); for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq* pDropTbReq = req.pReqs + iReq; - cJSON* tableName = cJSON_CreateString(pDropTbReq->name); + cJSON* tableName = cJSON_CreateString(pDropTbReq->name); RAW_NULL_CHECK(tableName); RAW_FALSE_CHECK(cJSON_AddItemToArray(tableNameList, tableName)); } @@ -910,7 +910,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { } // build create stable pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SFieldWithOptions)); - RAW_NULL_CHECK (pReq.pColumns); + RAW_NULL_CHECK(pReq.pColumns); for (int32_t i = 0; i < req.schemaRow.nCols; i++) { SSchema* pSchema = req.schemaRow.pSchema + i; SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes}; @@ -957,7 +957,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); RAW_NULL_CHECK(pCmdMsg.pMsg); - if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0){ + if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) { code = TSDB_CODE_INVALID_PARA; goto end; } @@ -968,7 +968,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - (void)launchQueryImpl(pRequest, &pQuery, true, NULL); //ignore, because return value is pRequest + (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest if (pRequest->code == TSDB_CODE_SUCCESS) { SCatalog* pCatalog = NULL; @@ -1021,7 +1021,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { .requestObjRefId = pRequest->self, .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; SName pName = {0}; - (void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); // ignore the return value, always return pName + (void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, + &pName); // ignore the return value, always return pName STableMeta* pTableMeta = NULL; code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -1059,19 +1060,18 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); RAW_NULL_CHECK(pCmdMsg.pMsg); - if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0){ + if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) { code = TSDB_CODE_INVALID_PARA; goto end; } - SQuery pQuery = {0}; pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.stableQuery = true; - (void)launchQueryImpl(pRequest, &pQuery, true, NULL); //ignore, because return value is pRequest + (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest if (pRequest->code == TSDB_CODE_SUCCESS) { // ignore the error code @@ -1391,9 +1391,9 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { } (void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, - req.tsColName, req.skey, req.tsColName, req.ekey); + req.tsColName, req.skey, req.tsColName, req.ekey); - TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX); + TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX); RAW_NULL_CHECK(res); SRequestObj* pRequest = (SRequestObj*)res; code = pRequest->code; @@ -1500,7 +1500,6 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { if (TSDB_CODE_SUCCESS != code) goto end; RAW_RETURN_CHECK(rewriteToVnodeModifyOpStmt(pQuery, pArray)); - (void)launchQueryImpl(pRequest, pQuery, true, NULL); pVgData = NULL; @@ -1573,7 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat RAW_RETURN_CHECK(smlInitHandle(&pQuery)); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); RAW_NULL_CHECK(pVgHash); - RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData))); + RAW_RETURN_CHECK( + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData))); RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); @@ -1632,7 +1632,8 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha RAW_RETURN_CHECK(smlInitHandle(&pQuery)); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); RAW_NULL_CHECK(pVgHash); - RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData))); + RAW_RETURN_CHECK( + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData))); RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); @@ -1736,7 +1737,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); RAW_NULL_CHECK(pSW); - TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); + TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); RAW_NULL_CHECK(fields); for (int i = 0; i < pSW->nCols; i++) { fields[i].type = pSW->pSchema[i].type; @@ -1845,7 +1846,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) // find schema data info for (int j = 0; j < rspObj.rsp.createTableNum; j++) { - void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); + void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); RAW_NULL_CHECK(dataTmp); int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); RAW_NULL_CHECK(dataTmp); @@ -1897,7 +1898,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); RAW_NULL_CHECK(pSW); - TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); + TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { SET_ERROR_MSG("calloc fields failed"); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1980,10 +1981,10 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) { RAW_NULL_CHECK(len); void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i); RAW_NULL_CHECK(tmpBuf); - SDecoder metaCoder = {0}; + SDecoder metaCoder = {0}; SMqMetaRsp metaRsp = {0}; tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), *len - sizeof(SMqRspHead)); - if(tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0 ) { + if (tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0) { goto end; } cJSON* pItem = NULL; @@ -2013,18 +2014,18 @@ char* tmq_get_json_meta(TAOS_RES* res) { if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res; - char* string = NULL; + char* string = NULL; processAutoCreateTable(&pMetaDataRspObj->rsp, &string); return string; } else if (TD_RES_TMQ_BATCH_META(res)) { SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res; - char* string = NULL; + char* string = NULL; processBatchMetaToJson(&pBatchMetaRspObj->rsp, &string); return string; } SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - cJSON* pJson = NULL; + cJSON* pJson = NULL; processSimpleMeta(&pMetaRspObj->metaRsp, &pJson); char* string = cJSON_PrintUnformatted(pJson); cJSON_Delete(pJson); @@ -2103,7 +2104,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { uDebug("tmq get raw type meta:%p", raw); } else if (TD_RES_TMQ(res)) { SMqRspObj* rspObj = ((SMqRspObj*)res); - int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw); + int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw); if (code != 0) { uError("tmq get raw type error:%d", terrno); return code; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c9f166e565..872fd305f5 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -37,7 +37,7 @@ struct SMqMgmt { static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once volatile int32_t tmqInitRes = 0; // initialize rsp code static struct SMqMgmt tmqMgmt = {0}; -static int8_t pollFlag = 0; +static int8_t pollFlag = 0; typedef struct { int32_t code; @@ -121,7 +121,7 @@ struct tmq_t { typedef struct SAskEpInfo { int32_t code; - tsem2_t sem; + tsem2_t sem; } SAskEpInfo; enum { @@ -191,7 +191,7 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - tsem2_t rspSem; + tsem2_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; @@ -219,12 +219,12 @@ typedef struct SMqVgCommon { } SMqVgCommon; typedef struct SMqSeekParam { - tsem2_t sem; + tsem2_t sem; int32_t code; } SMqSeekParam; typedef struct SMqCommittedParam { - tsem2_t sem; + tsem2_t sem; int32_t code; SMqVgOffset vgOffset; } SMqCommittedParam; @@ -242,18 +242,18 @@ typedef struct { int32_t waitingRspNum; int32_t code; tmq_commit_cb* callbackFn; - void* userParam; + void* userParam; } SMqCommitCbParamSet; typedef struct { SMqCommitCbParamSet* params; - char topicName[TSDB_TOPIC_FNAME_LEN]; - int32_t vgId; - int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + int32_t vgId; + int64_t consumerId; } SMqCommitCbParam; typedef struct SSyncCommitInfo { - tsem2_t sem; + tsem2_t sem; int32_t code; } SSyncCommitInfo; @@ -334,7 +334,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "session.timeout.ms") == 0) { int64_t tmp = taosStr2int64(value); - if (tmp < 6000 || tmp > 1800000){ + if (tmp < 6000 || tmp > 1800000) { return TMQ_CONF_INVALID; } conf->sessionTimeoutMs = tmp; @@ -343,7 +343,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "heartbeat.interval.ms") == 0) { int64_t tmp = taosStr2int64(value); - if (tmp < 1000 || tmp >= conf->sessionTimeoutMs){ + if (tmp < 1000 || tmp >= conf->sessionTimeoutMs) { return TMQ_CONF_INVALID; } conf->heartBeatIntervalMs = tmp; @@ -352,7 +352,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcasecmp(key, "max.poll.interval.ms") == 0) { int64_t tmp = taosStr2int64(value); - if (tmp < 1000 || tmp > INT32_MAX){ + if (tmp < 1000 || tmp > INT32_MAX) { return TMQ_CONF_INVALID; } conf->maxPollIntervalMs = tmp; @@ -515,7 +515,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); - if(tEncodeMqVgOffset(&encoder, &pOffset) < 0) { + if (tEncodeMqVgOffset(&encoder, &pOffset) < 0) { tEncoderClear(&encoder); taosMemoryFree(buf); return TSDB_CODE_INVALID_PARA; @@ -562,7 +562,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse return code; } -static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic **topic) { +static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic** topic) { int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); for (int32_t i = 0; i < numOfTopics; ++i) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); @@ -577,8 +577,8 @@ static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic return TSDB_CODE_TMQ_INVALID_TOPIC; } -static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, - int32_t rspNum, SMqCommitCbParamSet** ppParamSet) { +static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum, + SMqCommitCbParamSet** ppParamSet) { SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -595,7 +595,7 @@ static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, voi static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg) { SMqClientTopic* pTopic = NULL; - int32_t code = getTopicByName(tmq, pTopicName, &pTopic); + int32_t code = getTopicByName(tmq, pTopicName, &pTopic); if (code != 0) { tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); return code; @@ -723,7 +723,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us taosRUnLockLatch(&tmq->lock); goto end; } - int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); + int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { @@ -769,7 +769,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us if (pParamSet->waitingRspNum != 1) { // count down since waiting rsp num init as 1 code = commitRspCountDown(pParamSet, tmq->consumerId, "", 0); - if (code != 0){ + if (code != 0) { tscError("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code)); pParamSet = NULL; goto end; @@ -824,14 +824,14 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { } int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { - if (code != 0){ + if (code != 0) { goto _return; } if (pMsg == NULL || param == NULL) { code = TSDB_CODE_INVALID_PARA; goto _return; } - + SMqHbRsp rsp = {0}; code = tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp); if (code != 0) { @@ -858,7 +858,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { taosWUnLockLatch(&tmq->lock); (void)taosReleaseRef(tmqMgmt.rsetId, refId); } - + tDestroySMqHbRsp(&rsp); _return: @@ -881,32 +881,32 @@ void tmqSendHbReq(void* param, void* tmrId) { req.epoch = tmq->epoch; req.pollFlag = atomic_load_8(&pollFlag); req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); - if (req.topics == NULL){ + if (req.topics == NULL) { return; } taosRLockLatch(&tmq->lock); for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); if (pTopic == NULL) { continue; } int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); TopicOffsetRows* data = taosArrayReserve(req.topics, 1); - if (data == NULL){ + if (data == NULL) { continue; } (void)strcpy(data->topicName, pTopic->topicName); data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); - if (data->offsetRows == NULL){ + if (data->offsetRows == NULL) { continue; } for (int j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg == NULL){ + if (pVg == NULL) { continue; } - OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); - if (offRows == NULL){ + OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); + if (offRows == NULL) { continue; } offRows->vgId = pVg->vgId; @@ -964,7 +964,7 @@ void tmqSendHbReq(void* param, void* tmrId) { (void)atomic_val_compare_exchange_8(&pollFlag, 1, 0); OVER: tDestroySMqHbReq(&req); - if(tmrId != NULL){ + if (tmrId != NULL) { (void)taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); } (void)taosReleaseRef(tmqMgmt.rsetId, refId); @@ -1006,14 +1006,15 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { continue; } tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); - (void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer); + (void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, + &pTmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn; asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); (void)taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, - &pTmq->commitTimer); + &pTmq->commitTimer); } else { tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); } @@ -1100,16 +1101,16 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { taosRLockLatch(&tmq->lock); for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); - if(topic == NULL) { + if (topic == NULL) { tscError("topic is null"); continue; } char* tmp = strchr(topic->topicName, '.'); - if(tmp == NULL) { + if (tmp == NULL) { tscError("topic name is invalid:%s", topic->topicName); continue; } - if(tmq_list_append(*topics, tmp+ 1) != 0) { + if (tmq_list_append(*topics, tmp + 1) != 0) { tscError("failed to append topic:%s", tmp + 1); continue; } @@ -1227,27 +1228,31 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { } code = taosOpenQueue(&pTmq->mqueue); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + pTmq->groupId); SET_ERROR_MSG_TMQ("open queue failed") goto _failed; } code = taosOpenQueue(&pTmq->delayedTask); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + pTmq->groupId); SET_ERROR_MSG_TMQ("open delayed task queue failed") goto _failed; } code = taosAllocateQall(&pTmq->qall); if (code) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + pTmq->groupId); SET_ERROR_MSG_TMQ("allocate qall failed") goto _failed; } if (conf->groupId[0] == 0) { - tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), + pTmq->groupId); SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") goto _failed; } @@ -1287,8 +1292,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { // init semaphore if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) { - tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, tstrerror(TAOS_SYSTEM_ERROR(errno)), - pTmq->groupId); + tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, + tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId); SET_ERROR_MSG_TMQ("init t_sem failed") goto _failed; } @@ -1371,7 +1376,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SName name = {0}; code = tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic)); if (code) { - tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, code); + tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, + code); goto FAIL; } char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); @@ -1382,7 +1388,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = tNameExtractFullName(&name, topicFName); if (code) { - tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, code); + tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, + code); taosMemoryFree(topicFName); goto FAIL; } @@ -1459,7 +1466,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); - tmq->commitTimer =taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); + tmq->commitTimer = + taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); if (tmq->epTimer == NULL || tmq->commitTimer == NULL) { code = TSDB_CODE_TSC_INTERNAL_ERROR; goto FAIL; @@ -1516,12 +1524,12 @@ static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId) { } int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { - tmq_t* tmq = NULL; + tmq_t* tmq = NULL; SMqPollRspWrapper* pRspWrapper = NULL; - int8_t rspType = 0; - int32_t vgId = 0; - uint64_t requestId = 0; - SMqPollCbParam* pParam = (SMqPollCbParam*)param; + int8_t rspType = 0; + int32_t vgId = 0; + uint64_t requestId = 0; + SMqPollCbParam* pParam = (SMqPollCbParam*)param; if (pMsg == NULL) { return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -1530,7 +1538,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFreeClear(pMsg->pEpSet); return TSDB_CODE_TSC_INTERNAL_ERROR; } - int64_t refId = pParam->refId; + int64_t refId = pParam->refId; vgId = pParam->vgId; requestId = pParam->requestId; tmq = taosAcquireRef(tmqMgmt.rsetId, refId); @@ -1562,7 +1570,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (msgEpoch < clientEpoch) { // do not write into queue since updating epoch reset tscWarn("consumer:0x%" PRIx64 - " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, + " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, QID:0x%" PRIx64, tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; goto END; @@ -1589,7 +1597,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); - tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, QID:0x%" PRIx64, tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { SDecoder decoder = {0}; @@ -1621,23 +1629,24 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } tDecoderClear(&decoder); (void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead)); - tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId,requestId); + tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, QID:0x%" PRIx64, tmq->consumerId, vgId, + requestId); } else { // invalid rspType tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); } END: - if (pRspWrapper){ + if (pRspWrapper) { pRspWrapper->code = code; pRspWrapper->vgId = vgId; (void)strcpy(pRspWrapper->topicName, pParam->topicName); code = taosWriteQitem(tmq->mqueue, pRspWrapper); - if(code != 0){ + if (code != 0) { tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); } } int32_t total = taosQueueItemSize(tmq->mqueue); - tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, QID:0x%" PRIx64, tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId); if (tmq) (void)tsem2_post(&tmq->rspSem); @@ -1676,7 +1685,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic } for (int32_t j = 0; j < vgNumGet; j++) { SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - if (pVgEp == NULL){ + if (pVgEp == NULL) { continue; } (void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId); @@ -1712,7 +1721,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic clientVg.offsetInfo.committedOffset = offsetNew; clientVg.offsetInfo.beginOffset = offsetNew; } - if (taosArrayPush(pTopic->vgs, &clientVg) == NULL){ + if (taosArrayPush(pTopic->vgs, &clientVg) == NULL) { tscError("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId, pTopic->topicName); freeClientVg(&clientVg); @@ -1773,7 +1782,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) .commitOffset = pVgCur->offsetInfo.committedOffset, .numOfRows = pVgCur->numOfRows, .vgStatus = pVgCur->vgStatus}; - if(taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0){ + if (taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0) { tscError("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId); } } @@ -1787,7 +1796,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) continue; } initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq); - if(taosArrayPush(newTopics, &topic) == NULL){ + if (taosArrayPush(newTopics, &topic) == NULL) { tscError("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName); freeClientTopic(&topic); } @@ -1919,7 +1928,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable pDataRsp->withSchema = true; pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*)); - if (pDataRsp->blockSchema == NULL){ + if (pDataRsp->blockSchema == NULL) { tscError("failed to allocate memory for blockSchema"); return; } @@ -1938,7 +1947,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if (schema) { - if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL){ + if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL) { tscError("failed to push schema into blockSchema"); continue; } @@ -1947,7 +1956,8 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg } } -int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj** ppRspObj) { +int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, + SMqRspObj** ppRspObj) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); if (pRspObj == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1959,7 +1969,8 @@ int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, in return 0; } -int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqTaosxRspObj** ppRspObj) { +int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, + SMqTaosxRspObj** ppRspObj) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); if (pRspObj == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2030,7 +2041,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, reqId:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, QID:0x%" PRIx64, pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId); if (code != 0) { return code; @@ -2056,10 +2067,10 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { for (int i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (pTopic == NULL){ + if (pTopic == NULL) { continue; } - int32_t numOfVg = taosArrayGetSize(pTopic->vgs); + int32_t numOfVg = taosArrayGetSize(pTopic->vgs); if (pTopic->noPrivilege) { tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName); continue; @@ -2069,7 +2080,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { if (pVg == NULL) { continue; } - int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; + int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); @@ -2203,7 +2214,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); if (pDataRsp->blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 - ", total:%" PRId64 ", reqId:0x%" PRIx64, + ", total:%" PRId64 ", QID:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); pVg->emptyBlockReceiveTs = taosGetTimestampMs(); tmqFreeRspWrapper(pRspWrapper); @@ -2220,13 +2231,13 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { pVg->blockSleepForReplay = pRsp->rsp.sleepTime; if (pVg->blockSleepForReplay > 0) { if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) { - tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%"PRId64, tmq->consumerId, - pVg->vgId, pVg->blockSleepForReplay); + tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%" PRId64, + tmq->consumerId, pVg->vgId, pVg->blockSleepForReplay); } } } tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, + ", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); taosFreeQitem(pRspWrapper); @@ -2302,7 +2313,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever, tmq->consumerId, true); SMqBatchMetaRspObj* pRsp = NULL; - (void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp) ; + (void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); return pRsp; @@ -2337,7 +2348,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tmq->consumerId, pDataRsp->blockNum != 0); if (pDataRsp->blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", QID:0x%" PRIx64, tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); pVg->emptyBlockReceiveTs = taosGetTimestampMs(); tmqFreeRspWrapper(pRspWrapper); @@ -2349,9 +2360,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } // build rsp - int64_t numOfRows = 0; - SMqTaosxRspObj* pRsp = NULL; - if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) !=0 ) { + int64_t numOfRows = 0; + SMqTaosxRspObj* pRsp = NULL; + if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) != 0) { tscError("consumer:0x%" PRIx64 " build taosx rsp failed, vgId:%d", tmq->consumerId, pVg->vgId); } tmq->totalRows += numOfRows; @@ -2359,7 +2370,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, + ", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); @@ -2471,7 +2482,7 @@ static void displayConsumeStatistics(tmq_t* pTmq) { tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); } -static int32_t innerClose(tmq_t* tmq){ +static int32_t innerClose(tmq_t* tmq) { if (tmq->status != TMQ_CONSUMER_STATUS__READY) { tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId); return 0; @@ -2485,7 +2496,7 @@ static int32_t innerClose(tmq_t* tmq){ tmqSendHbReq((void*)(tmq->refId), NULL); tmq_list_t* lst = tmq_list_new(); - if (lst == NULL){ + if (lst == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = tmq_subscribe(tmq, lst); @@ -2499,7 +2510,7 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { int32_t code = 0; if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { code = innerClose(tmq); - if(code == 0){ + if (code == 0) { atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); } } @@ -2514,12 +2525,12 @@ int32_t tmq_consumer_close(tmq_t* tmq) { int32_t code = 0; if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { code = innerClose(tmq); - if(code == 0){ + if (code == 0) { atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); } } - if (code == 0){ + if (code == 0) { (void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId); } return code; @@ -2562,13 +2573,13 @@ const char* tmq_get_topic_name(TAOS_RES* res) { return NULL; } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - char *tmp = strchr(((SMqRspObjCommon*)res)->topic, '.'); + char* tmp = strchr(((SMqRspObjCommon*)res)->topic, '.'); if (tmp == NULL) { return NULL; } return tmp + 1; } else if (TD_RES_TMQ_META(res)) { - char *tmp = strchr(((SMqMetaRspObj*)res)->topic, '.'); + char* tmp = strchr(((SMqMetaRspObj*)res)->topic, '.'); if (tmp == NULL) { return NULL; } @@ -2584,13 +2595,13 @@ const char* tmq_get_db_name(TAOS_RES* res) { } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { - char *tmp = strchr(((SMqRspObjCommon*)res)->db, '.'); + char* tmp = strchr(((SMqRspObjCommon*)res)->db, '.'); if (tmp == NULL) { return NULL; } return tmp + 1; } else if (TD_RES_TMQ_META(res)) { - char *tmp = strchr(((SMqMetaRspObj*)res)->db, '.'); + char* tmp = strchr(((SMqMetaRspObj*)res)->db, '.'); if (tmp == NULL) { return NULL; } @@ -2690,7 +2701,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { int32_t code = 0; SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); - if(pInfo == NULL) { + if (pInfo == NULL) { tscError("failed to allocate memory for sync commit"); return TSDB_CODE_OUT_OF_MEMORY; } @@ -2836,7 +2847,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { if (param == NULL) { goto FAIL; } - + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); if (tmq == NULL) { @@ -2857,7 +2868,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); if (pParam->sync) { SMqAskEpRsp rsp = {0}; - if(tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL){ + if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL) { (void)doUpdateLocalEp(tmq, head->epoch, &rsp); } tDeleteSMqAskEpRsp(&rsp); @@ -2871,10 +2882,10 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; pWrapper->epoch = head->epoch; (void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL){ + if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL) { tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); taosFreeQitem(pWrapper); - }else{ + } else { (void)taosWriteQitem(tmq->mqueue, pWrapper); } } @@ -2972,7 +2983,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); + tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); } @@ -3015,7 +3026,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes if (common->withSchema) { doFreeReqResultInfo(&pRspObj->resInfo); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter); - if (pSW){ + if (pSW) { TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols)); } } @@ -3032,9 +3043,9 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes pRspObj->resInfo.precision = precision; pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows; - int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, pRspObj->resInfo.numOfRows, - convertUcs4); - if (code != 0){ + int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, + pRspObj->resInfo.numOfRows, convertUcs4); + if (code != 0) { return code; } *pResInfo = &pRspObj->resInfo; @@ -3062,18 +3073,18 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); code = tDecodeMqDataRsp(&decoder, &rsp); tDecoderClear(&decoder); - if (code != 0){ + if (code != 0) { goto END; } - SMqRspHead* pHead = pMsg->pData; + SMqRspHead* pHead = pMsg->pData; tmq_topic_assignment assignment = {.begin = pHead->walsver, .end = pHead->walever + 1, .currentOffset = rsp.common.rspOffset.version, .vgId = pParam->vgId}; (void)taosThreadMutexLock(&pCommon->mutex); - if(taosArrayPush(pCommon->pList, &assignment) == NULL){ + if (taosArrayPush(pCommon->pList, &assignment) == NULL) { tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId, pParam->vgId, pCommon->pTopicName); code = TSDB_CODE_TSC_INTERNAL_ERROR; @@ -3184,7 +3195,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep taosMemoryFree(sendInfo); return TSDB_CODE_OUT_OF_MEMORY; } - if (tsem2_init(&pParam->sem, 0, 0) != 0){ + if (tsem2_init(&pParam->sem, 0, 0) != 0) { taosMemoryFree(buf); taosMemoryFree(sendInfo); taosMemoryFree(pParam); @@ -3348,7 +3359,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a taosWLockLatch(&tmq->lock); SMqClientTopic* pTopic = NULL; - int32_t code = getTopicByName(tmq, tname, &pTopic); + int32_t code = getTopicByName(tmq, tname, &pTopic); if (code != 0) { tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); goto end; @@ -3358,10 +3369,10 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a *numOfAssignment = taosArrayGetSize(pTopic->vgs); for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - if (pClientVg == NULL){ + if (pClientVg == NULL) { continue; } - int32_t type = pClientVg->offsetInfo.beginOffset.type; + int32_t type = pClientVg->offsetInfo.beginOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; @@ -3381,7 +3392,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - if (pClientVg == NULL){ + if (pClientVg == NULL) { continue; } if (pClientVg->offsetInfo.beginOffset.type != TMQ_OFFSET__LOG) { @@ -3410,7 +3421,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - if (tsem2_init(&pCommon->rsp, 0, 0) != 0){ + if (tsem2_init(&pCommon->rsp, 0, 0) != 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } @@ -3420,7 +3431,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a for (int32_t i = 0; i < (*numOfAssignment); ++i) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - if (pClientVg == NULL){ + if (pClientVg == NULL) { continue; } SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); @@ -3479,8 +3490,8 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); - tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, - tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); + tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, QID:0x%" PRIx64, tmq->consumerId, + pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); if (code != 0) { goto end; @@ -3504,7 +3515,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a for (int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - if (pClientVg == NULL){ + if (pClientVg == NULL) { continue; } if (pClientVg->vgId != p->vgId) { @@ -3631,7 +3642,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ taosMemoryFree(sendInfo); return TSDB_CODE_OUT_OF_MEMORY; } - if (tsem2_init(&pParam->sem, 0, 0) != 0){ + if (tsem2_init(&pParam->sem, 0, 0) != 0) { taosMemoryFree(msg); taosMemoryFree(sendInfo); taosMemoryFree(pParam); diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 5be41f830d..83bc02ed1c 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -83,12 +83,12 @@ extern "C" { }\ } -#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", gtid:%s", __VA_ARGS__, buf);}} -#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", gtid:%s", __VA_ARGS__, buf);}} -#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", gtid:%s", __VA_ARGS__, buf);}} -#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", gtid:%s", __VA_ARGS__, buf);}} -#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", gtid:%s", __VA_ARGS__, buf);}} -#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", gtid:%s", __VA_ARGS__, buf);}} +#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", QID:%s", __VA_ARGS__, buf);}} +#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", QID:%s", __VA_ARGS__, buf);}} +#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", QID:%s", __VA_ARGS__, buf);}} +#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", QID:%s", __VA_ARGS__, buf);}} +#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", QID:%s", __VA_ARGS__, buf);}} +#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", QID:%s", __VA_ARGS__, buf);}} // clang-format on @@ -208,7 +208,7 @@ void dmGetMonitorSystemInfo(SMonSysInfo *pInfo); int32_t dmReadFile(const char *path, const char *name, bool *pDeployed); int32_t dmWriteFile(const char *path, const char *name, bool deployed); int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile); -//int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile); +// int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile); // dmodule.c int32_t dmInitDndInfo(SDnodeData *pData); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 072568eb0f..8681373198 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -41,12 +41,12 @@ extern "C" { #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} -#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", gtid:%s", __VA_ARGS__, buf);}} -#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", gtid:%s", __VA_ARGS__, buf);}} -#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", gtid:%s", __VA_ARGS__, buf);}} -#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);}} -#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);}} -#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);}} +#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", QID:%s", __VA_ARGS__, buf);}} +#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", QID:%s", __VA_ARGS__, buf);}} +#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", QID:%s", __VA_ARGS__, buf);}} +#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", QID:%s", __VA_ARGS__, buf);}} +#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", QID:%s", __VA_ARGS__, buf);}} +#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", QID:%s", __VA_ARGS__, buf);}} // clang-format on #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) @@ -54,7 +54,7 @@ extern "C" { #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) typedef int32_t (*MndMsgFp)(SRpcMsg *pMsg); -typedef int32_t (*MndMsgFpExt)(SRpcMsg *pMsg, SQueueInfo* pInfo); +typedef int32_t (*MndMsgFpExt)(SRpcMsg *pMsg, SQueueInfo *pInfo); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); typedef int32_t (*ShowRetrieveFp)(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 7bdc8e1c33..3c1f529ec4 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -32,12 +32,12 @@ extern "C" { #define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", DEBUG_DEBUG, vDebugFlag, __VA_ARGS__); }} while(0) #define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", DEBUG_TRACE, vDebugFlag, __VA_ARGS__); }} while(0) -#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) -#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) -#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) -#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) -#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) -#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) +#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0) +#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", QID:%s", __VA_ARGS__, buf);}} while(0) +#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", QID:%s", __VA_ARGS__, buf);}} while(0) +#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", QID:%s", __VA_ARGS__, buf);}} while(0) +#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", QID:%s", __VA_ARGS__, buf);}} while(0) +#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", QID:%s", __VA_ARGS__, buf);}} while(0) // clang-format on diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5fc550da32..4b132f682a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -173,7 +173,7 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { dataRsp.common.blockNum = 0; char buf[TSDB_OFFSET_LEN] = {0}; (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset); - tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf, + tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, QID:0x%" PRIx64, req.consumerId, vgId, buf, req.reqId); code = tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); @@ -193,7 +193,7 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* (void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset); (void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset); - tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, + tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, QID:0x%" PRIx64, vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId); return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); @@ -421,7 +421,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { char buf[TSDB_OFFSET_LEN] = {0}; (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset); - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, QID:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); @@ -1204,13 +1204,13 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) } if (req.mndTrigger) { - tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ", pTask->id.idStr, - vgId, pTask->info.taskLevel, req.checkpointId, req.transId); + tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ", + pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId); } else { const char* pPrevStatus = streamTaskGetStatusStr(streamTaskGetPrevStatus(pTask)); tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 - ", transId:%d after transfer-state, prev status:%s", - pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus); + ", transId:%d after transfer-state, prev status:%s", + pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus); } code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index c779a17301..6b07971909 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -74,7 +74,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { pCreateReq = req.pReqs + iReq; if (pCreateReq->type == TSDB_CHILD_TABLE && pCreateReq->ctb.suid == tbSuid) { reqNew.nReqs++; - if (taosArrayPush(reqNew.pArray, pCreateReq) == NULL){ + if (taosArrayPush(reqNew.pArray, pCreateReq) == NULL) { taosArrayDestroy(reqNew.pArray); tDeleteSVCreateTbBatchReq(&req); goto end; @@ -155,7 +155,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { pDropReq = req.pReqs + iReq; if (pDropReq->suid == tbSuid) { reqNew.nReqs++; - if (taosArrayPush(reqNew.pArray, pDropReq) == NULL){ + if (taosArrayPush(reqNew.pArray, pDropReq) == NULL) { taosArrayDestroy(reqNew.pArray); goto end; } @@ -214,12 +214,12 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t while (offset <= appliedVer) { if (walFetchHead(pHandle->pWalReader, offset) < 0) { tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 - ", no more log to return, reqId:0x%" PRIx64 " 0x%" PRIx64, + ", no more log to return, QID:0x%" PRIx64 " 0x%" PRIx64, pHandle->consumerId, pHandle->epoch, vgId, offset, reqId, id); goto END; } - tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64 " 0x%" PRIx64, + tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, QID:0x%" PRIx64 " 0x%" PRIx64, vgId, pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id); if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) { @@ -261,10 +261,10 @@ END: bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; } -void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid){ - bool ret = false; - SSchemaWrapper *schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1); - if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){ +void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { + bool ret = false; + SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1); + if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } tDeleteSchemaWrapper(schema); @@ -486,7 +486,7 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { (pReader->nextBlk + 1), numOfBlocks, idstr); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if (pSubmitTbData == NULL){ + if (pSubmitTbData == NULL) { return false; } if (pReader->tbIdHash == NULL) { @@ -639,9 +639,9 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) { tqTrace("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); - int32_t code = 0; - int32_t line = 0; - STSchema* pTSchema = NULL; + int32_t code = 0; + int32_t line = 0; + STSchema* pTSchema = NULL; SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); TSDB_CHECK_NULL(pSubmitTbData, code, line, END, terrno); SSDataBlock* pBlock = pReader->pResBlock; @@ -712,10 +712,11 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* continue; } - SColData* pCol = taosArrayGet(pCols, sourceIdx); + SColData* pCol = taosArrayGet(pCols, sourceIdx); TSDB_CHECK_NULL(pCol, code, line, END, terrno); - SColVal colVal = {0}; - tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual, sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId); + SColVal colVal = {0}; + tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual, + sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId); if (pCol->cid < pColData->info.colId) { sourceIdx++; } else if (pCol->cid == pColData->info.colId) { @@ -738,7 +739,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* TSDB_CHECK_NULL(pTSchema, code, line, END, terrno); for (int32_t i = 0; i < numOfRows; i++) { - SRow* pRow = taosArrayGetP(pRows, i); + SRow* pRow = taosArrayGetP(pRows, i); TSDB_CHECK_NULL(pRow, code, line, END, terrno); int32_t sourceIdx = 0; for (int32_t j = 0; j < colActual; j++) { @@ -765,44 +766,43 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* } } } - } END: - if (code != 0){ + if (code != 0) { tqError("tqRetrieveDataBlock failed, line:%d, code:%d", line, code); } taosMemoryFreeClear(pTSchema); return code; } -#define PROCESS_VAL \ - if (curRow == 0) {\ - assigned[j] = !COL_VAL_IS_NONE(&colVal);\ - buildNew = true;\ - } else {\ - bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal);\ - if (currentRowAssigned != assigned[j]) {\ - assigned[j] = currentRowAssigned;\ - buildNew = true;\ - }\ +#define PROCESS_VAL \ + if (curRow == 0) { \ + assigned[j] = !COL_VAL_IS_NONE(&colVal); \ + buildNew = true; \ + } else { \ + bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal); \ + if (currentRowAssigned != assigned[j]) { \ + assigned[j] = currentRowAssigned; \ + buildNew = true; \ + } \ } -#define SET_DATA \ - if (colVal.cid < pColData->info.colId) {\ - sourceIdx++;\ - } else if (colVal.cid == pColData->info.colId) {\ - TQ_ERR_GO_TO_END(doSetVal(pColData, curRow - lastRow, &colVal));\ - sourceIdx++;\ - targetIdx++;\ +#define SET_DATA \ + if (colVal.cid < pColData->info.colId) { \ + sourceIdx++; \ + } else if (colVal.cid == pColData->info.colId) { \ + TQ_ERR_GO_TO_END(doSetVal(pColData, curRow - lastRow, &colVal)); \ + sourceIdx++; \ + targetIdx++; \ } -static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, - SArray* schemas, SSchemaWrapper* pSchemaWrapper, char* assigned, - int32_t numOfRows, int32_t curRow, int32_t* lastRow){ - int32_t code = 0; - SSchemaWrapper* pSW = NULL; - SSDataBlock* block = NULL; +static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas, + SSchemaWrapper* pSchemaWrapper, char* assigned, int32_t numOfRows, int32_t curRow, + int32_t* lastRow) { + int32_t code = 0; + SSchemaWrapper* pSW = NULL; + SSDataBlock* block = NULL; if (taosArrayGetSize(blocks) > 0) { SSDataBlock* pLastBlock = taosArrayGetLast(blocks); TQ_NULL_GO_TO_END(pLastBlock); @@ -834,33 +834,34 @@ END: taosMemoryFree(block); return code; } -static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas){ - int32_t code = 0; - int32_t curRow = 0; - int32_t lastRow = 0; +static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) { + int32_t code = 0; + int32_t curRow = 0; + int32_t lastRow = 0; SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); + char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); TQ_NULL_GO_TO_END(assigned); - SArray* pCols = pSubmitTbData->aCol; - SColData* pCol = taosArrayGet(pCols, 0); + SArray* pCols = pSubmitTbData->aCol; + SColData* pCol = taosArrayGet(pCols, 0); TQ_NULL_GO_TO_END(pCol); - int32_t numOfRows = pCol->nVal; - int32_t numOfCols = taosArrayGetSize(pCols); + int32_t numOfRows = pCol->nVal; + int32_t numOfCols = taosArrayGetSize(pCols); for (int32_t i = 0; i < numOfRows; i++) { bool buildNew = false; for (int32_t j = 0; j < numOfCols; j++) { pCol = taosArrayGet(pCols, j); TQ_NULL_GO_TO_END(pCol); - SColVal colVal = {0}; + SColVal colVal = {0}; tColDataGetValue(pCol, i, &colVal); PROCESS_VAL } if (buildNew) { - TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, curRow, &lastRow)); + TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, + curRow, &lastRow)); } SSDataBlock* pBlock = taosArrayGetLast(blocks); @@ -877,7 +878,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData TQ_NULL_GO_TO_END(pCol); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); TQ_NULL_GO_TO_END(pColData); - SColVal colVal = {0}; + SColVal colVal = {0}; tColDataGetValue(pCol, i, &colVal); SET_DATA } @@ -885,30 +886,30 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData curRow++; } SSDataBlock* pLastBlock = taosArrayGetLast(blocks); - pLastBlock->info.rows = curRow - lastRow; + pLastBlock->info.rows = curRow - lastRow; END: taosMemoryFree(assigned); return code; } -int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas){ - int32_t code = 0; - STSchema* pTSchema = NULL; +int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) { + int32_t code = 0; + STSchema* pTSchema = NULL; SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); + char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); TQ_NULL_GO_TO_END(assigned); - int32_t curRow = 0; - int32_t lastRow = 0; - SArray* pRows = pSubmitTbData->aRowP; - int32_t numOfRows = taosArrayGetSize(pRows); - pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); + int32_t curRow = 0; + int32_t lastRow = 0; + SArray* pRows = pSubmitTbData->aRowP; + int32_t numOfRows = taosArrayGetSize(pRows); + pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); for (int32_t i = 0; i < numOfRows; i++) { bool buildNew = false; - SRow* pRow = taosArrayGetP(pRows, i); + SRow* pRow = taosArrayGetP(pRows, i); TQ_NULL_GO_TO_END(pRow); for (int32_t j = 0; j < pTSchema->numOfCols; j++) { @@ -918,7 +919,8 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra } if (buildNew) { - TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, curRow, &lastRow)); + TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, + curRow, &lastRow)); } SSDataBlock* pBlock = taosArrayGetLast(blocks); @@ -932,7 +934,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra int32_t colActual = blockDataGetNumOfCols(pBlock); while (targetIdx < colActual) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); - SColVal colVal = {0}; + SColVal colVal = {0}; TQ_ERR_GO_TO_END(tRowGet(pRow, pTSchema, sourceIdx, &colVal)); SET_DATA } @@ -940,7 +942,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra curRow++; } SSDataBlock* pLastBlock = taosArrayGetLast(blocks); - pLastBlock->info.rows = curRow - lastRow; + pLastBlock->info.rows = curRow - lastRow; END: taosMemoryFreeClear(pTSchema); @@ -950,10 +952,10 @@ END: int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk); - SSDataBlock* block = NULL; + SSDataBlock* block = NULL; SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if(pSubmitTbData == NULL){ + if (pSubmitTbData == NULL) { return terrno; } pReader->nextBlk++; @@ -1034,7 +1036,7 @@ bool tqCurrentBlockConsumed(const STqReader* pReader) { return pReader->msg.msgS void tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); - if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0){ + if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0) { tqError("failed to remove table uid:%" PRId64 " from hash", *pKey); } } @@ -1064,7 +1066,8 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t sz = taosArrayGetSize(tbUidList); for (int32_t i = 0; i < sz; i++) { int64_t* tbUid = (int64_t*)taosArrayGet(tbUidList, i); - if (tbUid && taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, tbUid, sizeof(int64_t), NULL, 0) != 0){ + if (tbUid && + taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, tbUid, sizeof(int64_t), NULL, 0) != 0) { tqError("failed to add table uid:%" PRId64 " to hash", *tbUid); continue; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index b3d2300996..07f053e088 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -92,7 +92,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand char formatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal); tqDebug("tmq poll: consumer:0x%" PRIx64 - ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64, + ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. QID:0x%" PRIx64, consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); return 0; } else { @@ -117,7 +117,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1); code = tqInitDataRsp(&dataRsp.common, *pOffsetVal); - if (code != 0){ + if (code != 0) { return code; } tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, @@ -145,7 +145,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, terrno = 0; SMqDataRsp dataRsp = {0}; - int code = tqInitDataRsp(&dataRsp.common, *pOffset); + int code = tqInitDataRsp(&dataRsp.common, *pOffset); if (code != 0) { goto end; } @@ -176,7 +176,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, end : { char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, QID:0x%" PRIx64 " code:%d", consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code); tDeleteMqDataRsp(&dataRsp); @@ -206,10 +206,10 @@ static void tDeleteCommon(void* parm) {} static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { - int32_t vgId = TD_VID(pTq->pVnode); - STaosxRsp taosxRsp = {0}; - SMqBatchMetaRsp btMetaRsp = {0}; - int32_t code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + STaosxRsp taosxRsp = {0}; + SMqBatchMetaRsp btMetaRsp = {0}; + int32_t code = 0; TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp.common, *offset)); if (offset->type != TMQ_OFFSET__LOG) { @@ -323,9 +323,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); continue; } - int32_t tLen = sizeof(SMqRspHead) + len; - void* tBuf = taosMemoryCalloc(1, tLen); - if (tBuf == NULL){ + int32_t tLen = sizeof(SMqRspHead) + len; + void* tBuf = taosMemoryCalloc(1, tLen); + if (tBuf == NULL) { code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); goto END; } @@ -339,11 +339,11 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); continue; } - if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL){ + if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL) { code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); goto END; } - if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL){ + if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL) { code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); goto END; } @@ -601,7 +601,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void* SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX); TSDB_CHECK_NULL(pUidCol, code, line, END, terrno) - int64_t* pUid = taosArrayGet(pRes->uidList, i); + int64_t* pUid = taosArrayGet(pRes->uidList, i); code = colDataSetVal(pUidCol, i, (const char*)pUid, false); TSDB_CHECK_CODE(code, line, END); void* tmp = taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -632,7 +632,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void* } END: - if (code != 0){ + if (code != 0) { tqError("failed to extract delete data block, line:%d code:%d", line, code); } tDecoderClear(pCoder); @@ -662,7 +662,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); - if (pId == NULL){ + if (pId == NULL) { continue; } if (pId->streamId != streamId) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 422ca16e50..020bc0aaae 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -207,7 +207,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList); // send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode - (void) streamTaskSendCheckpointsourceRsp(pTask); + (void)streamTaskSendCheckpointsourceRsp(pTask); streamTaskResetStatus(pTask); streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr); @@ -303,7 +303,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM streamMetaWUnLock(pMeta); taosArrayDestroy(req.pNodeList); - return rsp.code; // always return true + return rsp.code; // always return true } int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { @@ -324,7 +324,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId); SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask && (code == 0)) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) { @@ -384,7 +384,7 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { pRsp->downstreamTaskId, pRsp->downstreamNodeId); SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); if (pTask && (code == 0)) { code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pMeta, pTask); @@ -482,8 +482,8 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe } tDecoderClear(&decoder); - tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", - rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); + tqDebug("tq task:0x%x (vgId:%d) recv check rsp(QID:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, + rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); if (!isLeader) { tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, @@ -681,7 +681,8 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); if (code) { - tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, + (int32_t)hTaskId.taskId); } } @@ -701,7 +702,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen } streamMetaWUnLock(pMeta); - return 0; // always return success + return 0; // always return success } int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) { @@ -793,23 +794,23 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t vgId = pMeta->vgId; if (type == STREAM_EXEC_T_START_ONE_TASK) { - (void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); + (void)streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); return 0; } else if (type == STREAM_EXEC_T_START_ALL_TASKS) { - (void) streamMetaStartAllTasks(pMeta); + (void)streamMetaStartAllTasks(pMeta); return 0; } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { - (void) restartStreamTasks(pMeta, isLeader); + (void)restartStreamTasks(pMeta, isLeader); return 0; } else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) { - (void) streamMetaStopAllTasks(pMeta); + (void)streamMetaStopAllTasks(pMeta); return 0; } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) { int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); return code; } else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask != NULL && (code == 0)) { char* pStatus = NULL; @@ -831,13 +832,13 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead } SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, p, pTask->chkInfo.nextProcessVer); - (void) streamExecTask(pTask); + (void)streamExecTask(pTask); } else { int8_t status = streamTaskSetSchedStatusInactive(pTask); tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, @@ -900,7 +901,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask == NULL || (code != 0)) { tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, pReq->taskId); @@ -924,9 +925,9 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { streamTaskSetStatusReady(pTask); } else if (pState.state == TASK_STATUS__UNINIT) { -// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); -// ASSERT(pTask->status.downstreamReady == 0); -// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId); + // tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); + // ASSERT(pTask->status.downstreamReady == 0); + // tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); } else { tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); @@ -942,7 +943,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont; SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); if (pTask == NULL || (code != 0)) { tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", @@ -958,7 +959,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) pTask->id.idStr, (int32_t)pReq->downstreamTaskId); code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, - TSDB_CODE_STREAM_TASK_IVLD_STATUS); + TSDB_CODE_STREAM_TASK_IVLD_STATUS); streamMetaReleaseTask(pMeta, pTask); return code; } @@ -996,7 +997,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) pTask->id.idStr, recv, total); } code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, - TSDB_CODE_ACTION_IN_PROGRESS); + TSDB_CODE_ACTION_IN_PROGRESS); } } else { // upstream not recv the checkpoint-source/trigger till now ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT); @@ -1005,7 +1006,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) "upstream sending checkpoint-source/trigger", pTask->id.idStr); code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, - TSDB_CODE_ACTION_IN_PROGRESS); + TSDB_CODE_ACTION_IN_PROGRESS); } streamMetaReleaseTask(pMeta, pTask); @@ -1016,7 +1017,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); if (pTask == NULL || (code != 0)) { tqError( "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", @@ -1038,7 +1039,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask == NULL || (code != 0)) { tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, pReq->taskId); @@ -1122,7 +1123,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask == NULL || (code != 0)) { tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; @@ -1169,13 +1170,15 @@ int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { ret int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } -int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } +int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { + return doProcessDummyRspMsg(pMeta, pMsg); +} int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask); if (pTask == NULL || (code != 0)) { tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", pRsp->downstreamNodeId, pRsp->downstreamTaskId); @@ -1211,8 +1214,9 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask == NULL || (code != 0)) { - tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", - pMeta->vgId, req.taskId); + tqError( + "vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", + pMeta->vgId, req.taskId); (void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); return code; } @@ -1221,7 +1225,8 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { if (req.startTs < pTask->execInfo.created) { tqWarn("s-task:%s vgId:%d create time:%" PRId64 " recv expired consensus checkpointId:%" PRId64 " from task createTs:%" PRId64 " < task createTs:%" PRId64 ", discard", - pTask->id.idStr, pMeta->vgId, pTask->execInfo.created, req.checkpointId, req.startTs, pTask->execInfo.created); + pTask->id.idStr, pMeta->vgId, pTask->execInfo.created, req.checkpointId, req.startTs, + pTask->execInfo.created); streamMetaAddFailedTaskSelf(pTask, now); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; @@ -1234,15 +1239,15 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId); if (pTask->chkInfo.consensusTransId >= req.transId) { - tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", - pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId); + tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", pTask->id.idStr, vgId, + pTask->chkInfo.consensusTransId, req.transId); streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; } if (pTask->chkInfo.checkpointId != req.checkpointId) { - tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64" transId:%d", pTask->id.idStr, vgId, + tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64 " transId:%d", pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId, req.transId); pTask->chkInfo.checkpointId = req.checkpointId; tqSetRestoreVersionInfo(pTask); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 031d61554c..4954ff6675 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -20,12 +20,11 @@ #include "tref.h" #include "trpc.h" - void ctgIsTaskDone(SCtgJob* pJob, CTG_TASK_TYPE type, bool* done) { SCtgTask* pTask = NULL; *done = true; - + CTG_LOCK(CTG_READ, &pJob->taskLock); int32_t taskNum = taosArrayGetSize(pJob->pTasks); @@ -46,8 +45,8 @@ void ctgIsTaskDone(SCtgJob* pJob, CTG_TASK_TYPE type, bool* done) { int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param; - SName* name = pParam->pName; - SCtgTask task = {0}; + SName* name = pParam->pName; + SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_META; task.taskId = taskIdx; @@ -95,11 +94,12 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, (int32_t)sizeof(SMetaRes)); + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, + (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + if (NULL == taosArrayPush(pJob->pTasks, &task)) { ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -245,11 +245,12 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, (int32_t)sizeof(SMetaRes)); + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, + (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + if (NULL == taosArrayPush(pJob->pTasks, &task)) { ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -513,7 +514,8 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, (int32_t)sizeof(SMetaRes)); + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, + (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -539,21 +541,22 @@ int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { if (NULL == pTaskCtx) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + task.taskCtx = pTaskCtx; pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, (int32_t)sizeof(SMetaRes)); + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, + (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + if (NULL == taosArrayPush(pJob->pTasks, &task)) { ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + return TSDB_CODE_SUCCESS; } @@ -571,7 +574,8 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, (int32_t)sizeof(SMetaRes)); + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, + (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -580,11 +584,10 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + return TSDB_CODE_SUCCESS; } - int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { int32_t viewNum = taosArrayGetSize(pReq->pView); for (int32_t i = 0; i < viewNum; ++i) { @@ -593,20 +596,19 @@ int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { qError("taosArrayGet the %dth view in req failed", i); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - + int32_t viewNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < viewNum; ++m) { SName* name = taosArrayGet(p->pTables, m); CTG_ERR_RET(ctgDropViewMetaEnqueue(pCtg, p->dbFName, 0, name->tname, 0, true)); } } - + return TSDB_CODE_SUCCESS; } - int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (NULL == pDb || NULL == pTb) { @@ -650,7 +652,7 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); - + int32_t tbNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < tbNum; ++m) { SName* name = taosArrayGet(p->pTables, m); @@ -671,7 +673,7 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con } CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); - + int32_t tbNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < tbNum; ++m) { SName* name = taosArrayGet(p->pTables, m); @@ -700,7 +702,7 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - char dbFName[TSDB_DB_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(name, dbFName); CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); @@ -713,7 +715,7 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - char dbFName[TSDB_DB_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(name, dbFName); CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); @@ -776,7 +778,7 @@ _return: taosHashCleanup(pDb); taosHashCleanup(pTb); - + return code; } @@ -826,7 +828,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const *job = taosMemoryCalloc(1, sizeof(SCtgJob)); if (NULL == *job) { - ctgError("failed to calloc, size:%d, reqId:0x%" PRIx64, (int32_t)sizeof(SCtgJob), pConn->requestId); + ctgError("failed to calloc, size:%d, QID:0x%" PRIx64, (int32_t)sizeof(SCtgJob), pConn->requestId); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -953,7 +955,6 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL)); } - for (int32_t i = 0; i < indexNum; ++i) { char* indexName = taosArrayGet(pReq->pIndex, i); if (NULL == indexName) { @@ -1017,7 +1018,7 @@ _return: ctgFreeJob(*job); *job = NULL; - + CTG_RET(code); } @@ -1025,7 +1026,7 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { if (pTask->subTask) { return TSDB_CODE_SUCCESS; } - + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableMeta) { pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); @@ -1038,7 +1039,7 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { if (NULL == taosArrayPush(pJob->jobRes.pTableMeta, &res)) { CTG_ERR_RET(terrno); } - + return TSDB_CODE_SUCCESS; } @@ -1169,7 +1170,7 @@ int32_t ctgDumpTbTagRes(SCtgTask* pTask) { if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableTag, NULL, pRes)) { taosArrayDestroy(pRes); } - + if (NULL == pJob->jobRes.pTableTag) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -1183,7 +1184,6 @@ int32_t ctgDumpTbTagRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } - int32_t ctgDumpIndexRes(SCtgTask* pTask) { if (pTask->subTask) { return TSDB_CODE_SUCCESS; @@ -1350,7 +1350,6 @@ int32_t ctgDumpSvrVer(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } - int32_t ctgDumpViewsRes(SCtgTask* pTask) { if (pTask->subTask) { return TSDB_CODE_SUCCESS; @@ -1363,7 +1362,6 @@ int32_t ctgDumpViewsRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } - int32_t ctgCallSubCb(SCtgTask* pTask) { int32_t code = 0; @@ -1377,7 +1375,7 @@ int32_t ctgCallSubCb(SCtgTask* pTask) { qError("taosArrayGetP the %dth parent failed", i); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + pParent->subRes.code = pTask->code; if (TSDB_CODE_SUCCESS == pTask->code) { code = (*gCtgAsyncFps[pTask->type].cloneFp)(pTask, &pParent->subRes.res); @@ -1525,7 +1523,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf } ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // update cache not fatal error + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // update cache not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1644,23 +1642,23 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; bool taskDone = false; - + if (NULL == pMsgCtx) { ctgError("fail to get task msgCtx, taskType:%d", pTask->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); if (NULL == pFetch) { ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - - int32_t flag = pFetch->flag; - int32_t* vgId = &pFetch->vgId; - SName* pName = NULL; + + int32_t flag = pFetch->flag; + int32_t* vgId = &pFetch->vgId; + SName* pName = NULL; CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, pFetch, &pName)); - + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { @@ -1710,7 +1708,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu } ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1728,7 +1726,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu if (CTG_IS_META_NULL(pOut->metaType)) { ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1778,7 +1776,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - (void)ctgUpdateTbMetaToCache(pCtg, pOut, false); // cache update not fatal error + (void)ctgUpdateTbMetaToCache(pCtg, pOut, false); // cache update not fatal error if (CTG_IS_META_BOTH(pOut->metaType)) { TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); @@ -1806,10 +1804,11 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); if (NULL == pRes) { - ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, + (int32_t)taosArrayGetSize(ctx->pResList)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + pRes->code = 0; pRes->pRes = pOut->tbMeta; pOut->tbMeta = NULL; @@ -1827,7 +1826,8 @@ _return: if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); if (NULL == pRes) { - ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, + (int32_t)taosArrayGetSize(ctx->pResList)); } else { pRes->code = code; pRes->pRes = NULL; @@ -1904,7 +1904,8 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, pOut->dbVgroup, ctx->pName, (SVgroupInfo*)pTask->res)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, pOut->dbVgroup, ctx->pName, + (SVgroupInfo*)pTask->res)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false)); pOut->dbVgroup = NULL; @@ -1938,7 +1939,7 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); if (NULL == pFetch) { ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -1952,11 +1953,13 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); if (NULL == pReq) { - ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(ctx->pNames)); + ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, + (int32_t)taosArrayGetSize(ctx->pNames)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - - CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); + + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, tReq, pOut->dbVgroup, ctx, + pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); pOut->dbVgroup = NULL; @@ -1978,13 +1981,15 @@ _return: if (code) { STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); if (NULL == pReq) { - ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(ctx->pNames)); + ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, + (int32_t)taosArrayGetSize(ctx->pNames)); } else { - int32_t num = taosArrayGetSize(pReq->pTables); + int32_t num = taosArrayGetSize(pReq->pTables); for (int32_t i = 0; i < num; ++i) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); if (NULL == pRes) { - ctgError("fail to get the %dth res in ctx->pResList, resNum:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(ctx->pResList)); + ctgError("fail to get the %dth res in ctx->pResList, resNum:%d", pFetch->resIdx + i, + (int32_t)taosArrayGetSize(ctx->pResList)); } else { pRes->code = code; pRes->pRes = NULL; @@ -2027,7 +2032,7 @@ _return: if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { code = TSDB_CODE_SUCCESS; } - + newCode = ctgHandleTaskEnd(pTask, code); if (newCode && TSDB_CODE_SUCCESS == code) { code = newCode; @@ -2055,7 +2060,6 @@ _return: CTG_RET(code); } - int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; @@ -2090,11 +2094,11 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* taosMemoryFree(pJson); taosArrayDestroy(pTagVals); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } + } } else { CTG_ERR_JRET(tTagToValArray((const STag*)pRsp->pTags, &pTagVals)); } - + pTask->res = pTagVals; _return: @@ -2107,13 +2111,12 @@ _return: CTG_RET(code); } - int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgDbCfgCtx* ctx = pTask->taskCtx; - int32_t newCode = TSDB_CODE_SUCCESS; - + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); SDbCfgInfo* pCfg = NULL; @@ -2219,7 +2222,7 @@ int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - (void)ctgUpdateUserEnqueue(pCtg, pOut, true); // cache update not fatal error + (void)ctgUpdateUserEnqueue(pCtg, pOut, true); // cache update not fatal error taosMemoryFreeClear(pTask->msgCtx.out); CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); @@ -2275,9 +2278,9 @@ int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t flag = pFetch->flag; - int32_t* vgId = &pFetch->vgId; - SName* pName = NULL; + int32_t flag = pFetch->flag; + int32_t* vgId = &pFetch->vgId; + SName* pName = NULL; CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, pFetch, &pName)); @@ -2297,16 +2300,17 @@ int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* } ctgDebug("start to update view meta to cache, view:%s, querySQL:%s", pRsp->name, pRsp->querySql); - (void)ctgUpdateViewMetaToCache(pCtg, pRsp, false); // cache update not fatal error + (void)ctgUpdateViewMetaToCache(pCtg, pRsp, false); // cache update not fatal error taosMemoryFreeClear(pMsgCtx->out); pRsp = NULL; - + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); if (NULL == pRes) { - ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, + (int32_t)taosArrayGetSize(ctx->pResList)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + pRes->code = 0; pRes->pRes = pViewMeta; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -2319,7 +2323,8 @@ _return: if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); if (NULL == pRes) { - ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, + (int32_t)taosArrayGetSize(ctx->pResList)); } else { pRes->code = code; pRes->pRes = NULL; @@ -2347,7 +2352,6 @@ _return: CTG_RET(code); } - static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; @@ -2358,17 +2362,18 @@ static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, SVgroupInfo* pVgInfo = NULL; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); if (NULL == pFetch) { - ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, + (int32_t)taosArrayGetSize(pCtx->pFetches)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); if (NULL == pTbReq) { ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); if (NULL == pTbName) { ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -2392,42 +2397,41 @@ static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, .vgId = pVgInfo->vgId}; CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req)); pFetch->subFetchNum++; - + pVgInfo = taosHashIterate(pVgHash, pVgInfo); } } - + _return: CTG_RET(code); } - int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - int32_t newCode = TSDB_CODE_SUCCESS; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + int32_t newCode = TSDB_CODE_SUCCESS; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); if (NULL == pMsgCtx) { ctgError("fail to get the %dth SCtgMsgCtx", tReq->msgIdx); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); if (NULL == pTbReq) { ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - - SName* pName = taosArrayGet(pTbReq->pTables, 0); + + SName* pName = taosArrayGet(pTbReq->pTables, 0); if (NULL == pName) { ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pTbReq->pTables)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SRequestConnInfo* pConn = &pTask->pJob->conn; + SRequestConnInfo* pConn = &pTask->pJob->conn; CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { @@ -2436,9 +2440,9 @@ int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* if (!CTG_IS_META_NULL(pOut->metaType)) { CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP)); } - + break; - } + } case TDMT_MND_GET_TSMA: { SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); if (NULL == pRes) { @@ -2462,16 +2466,16 @@ int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* ctgError("fail to get the 0th STableTSMAInfo, totalNum:%d", (int32_t)taosArrayGetSize(pOut->pTsmas)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - - int32_t exists = false; + + int32_t exists = false; CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); if (!exists) { TSWAP(pMsgCtx->lastOut, pMsgCtx->out); CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); } - + break; - } + } default: ctgError("invalid reqType:%d while getting tsma rsp", reqType); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -2487,53 +2491,53 @@ _return: tstrerror(code)); } } - + newCode = ctgHandleTaskEnd(pTask, code); if (newCode && TSDB_CODE_SUCCESS == code) { code = newCode; } - + CTG_RET(code); } - int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - bool taskDone = false; - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SArray* pTsmas = NULL; - SHashObj* pVgHash = NULL; - SCtgDBCache* pDbCache = NULL; - STableTSMAInfo* pTsma = NULL; - SRequestConnInfo* pConn = &pTask->pJob->conn; + bool taskDone = false; + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + SArray* pTsmas = NULL; + SHashObj* pVgHash = NULL; + SCtgDBCache* pDbCache = NULL; + STableTSMAInfo* pTsma = NULL; + SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); if (NULL == pMsgCtx) { ctgError("fail to get the %dth SCtgMsgCtx", tReq->msgIdx); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); if (NULL == pFetch) { - ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, + (int32_t)taosArrayGetSize(pCtx->pFetches)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); if (NULL == pRes) { ctgError("fail to get the %dth SMetaRes, totalNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(pCtx->pResList)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); if (NULL == pTbReq) { ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); if (NULL == pTbName) { ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -2551,8 +2555,8 @@ int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) { // fetch progress - (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore cache error - + (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore cache error + if (!pDbCache) { // do not know which vnodes to fetch, fetch vnode list first SBuildUseDBInput input = {0}; @@ -2572,13 +2576,13 @@ int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf taskDone = true; } } - + break; - } + } case TDMT_VND_GET_STREAM_PROGRESS: { SStreamProgressRsp rsp = {0}; CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - + // update progress into res STableTSMAInfoRsp* pTsmasRsp = pRes->pRes; SArray* pTsmas = pTsmasRsp->pTsmas; @@ -2589,15 +2593,15 @@ int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf ctgError("fail to get the %dth STableTSMAInfo, totalNum:%d", tsmaIdx, (int32_t)taosArrayGetSize(pTsmas)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + if (pTsmaInfo->rspTs == 0) { pTsmaInfo->fillHistoryFinished = true; } - + pTsmaInfo->rspTs = taosGetTimestampMs(); pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration); pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished; - + qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name, pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay); @@ -2616,7 +2620,7 @@ int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf } break; - } + } case TDMT_MND_USE_DB: { SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; @@ -2629,42 +2633,42 @@ int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq)); break; - } + } case FETCH_TSMA_STREAM_PROGRESS: { STableTSMAInfoRsp* pTsmas = pRes->pRes; TSWAP(pOut->dbVgroup->vgHash, pVgHash); CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas)); break; - } + } default: ctgError("invalid fetchType:%d while getting tb tsma rsp", pFetch->fetchType); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } break; - } + } case TDMT_VND_TABLE_META: { // handle source tb meta STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; pFetch->fetchType = FETCH_TB_TSMA; pFetch->tsmaSourceTbName = *pTbName; - + if (CTG_IS_META_NULL(pOut->metaType)) { ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname); - (void)ctgRemoveTbMetaFromCache(pCtg, pTbName, false); // ignore cache error + (void)ctgRemoveTbMetaFromCache(pCtg, pTbName, false); // ignore cache error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - + if (META_TYPE_BOTH_TABLE == pOut->metaType) { // rewrite tsma fetch table with it's super table name (void)sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName); } - + CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA)); break; - } + } default: ctgError("invalid reqType:%d while getting tsma rsp", reqType); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -2683,7 +2687,7 @@ _return: taosHashCleanup(pVgHash); } if (code) { - int32_t newCode = TSDB_CODE_SUCCESS; + int32_t newCode = TSDB_CODE_SUCCESS; SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); if (NULL == pRes) { ctgError("fail to get the %dth SMetaRes, totalNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(pCtx->pResList)); @@ -2701,7 +2705,7 @@ _return: if (newCode && TSDB_CODE_SUCCESS == code) { code = newCode; } - + bool allSubFetchFinished = false; if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) { allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum; @@ -2711,7 +2715,7 @@ _return: taskDone = true; } } - + if (pTask->res && taskDone) { int32_t newCode = ctgHandleTaskEnd(pTask, code); if (newCode && TSDB_CODE_SUCCESS == code) { @@ -2722,7 +2726,6 @@ _return: CTG_RET(code); } - int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq* tReq, int32_t flag, SName* pName, int32_t* vgId) { SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; @@ -2818,7 +2821,7 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { ctgError("fail to get the %dth STablesReq, num:%d", i, dbNum); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - + ctgDebug("start to check tb metas in db %s, tbNum %ld", pReq->dbFName, taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); @@ -2837,16 +2840,16 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); if (NULL == pFetch) { ctgError("fail to get the %dth fetch in pCtx->pFetchs, fetchNum:%d", i, pCtx->fetchNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + CTG_ERR_RET(ctgGetFetchName(pCtx->pNames, pFetch, &pName)); - + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); if (NULL == pMsgCtx) { ctgError("fail to get the %dth pMsgCtx", i); @@ -2889,7 +2892,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask* pTask) { pMsgCtx->reqType = TDMT_MND_USE_DB; CTG_ERR_JRET(ctgBuildUseDbOutput((SUseDbOutput**)&pMsgCtx->out, dbCache->vgCache.vgInfo)); } - + CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgCache.vgInfo->vgHash, (SArray**)&pTask->res)); ctgReleaseVgInfoToCache(pCtg, dbCache); @@ -2940,7 +2943,8 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask* pTask) { if (NULL == pTask->res) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, dbCache->vgCache.vgInfo, pCtx->pName, + (SVgroupInfo*)pTask->res)); ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; @@ -2991,8 +2995,8 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { SCtgTaskReq tReq; tReq.pTask = pTask; tReq.msgIdx = -1; - CTG_ERR_JRET( - ctgGetVgInfosFromHashValue(pCtg, &pConn->mgmtEps, &tReq, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &pConn->mgmtEps, &tReq, dbCache->vgCache.vgInfo, pCtx, + pReq->dbFName, pReq->pTables, false)); ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; @@ -3003,7 +3007,7 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { baseResIdx += taosArrayGetSize(pReq->pTables); int32_t inc = baseResIdx - taosArrayGetSize(pCtx->pResList); - for(int32_t j = 0; j < inc; ++j) { + for (int32_t j = 0; j < inc; ++j) { if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -3026,12 +3030,12 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { } for (int32_t i = 0; i < pCtx->fetchNum; ++i) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); if (NULL == pFetch) { ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); if (NULL == pFetch) { ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); @@ -3106,7 +3110,7 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(pCtx->pName, dbFName); - + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); if (NULL == pMsgCtx) { ctgError("fail to get the %dth pMsgCtx", -1); @@ -3158,7 +3162,6 @@ _return: CTG_RET(code); } - int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { int32_t code = 0; SCatalog* pCtg = pTask->pJob->pCtg; @@ -3168,7 +3171,7 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(pCtx->pName, dbFName); - + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); if (NULL == pMsgCtx) { ctgError("fail to get the %dth pMsgCtx", -1); @@ -3205,7 +3208,6 @@ _return: CTG_RET(code); } - int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; @@ -3222,7 +3224,7 @@ int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) { CTG_CACHE_NHIT_INC(CTG_CI_QNODE, 1); CTG_ERR_RET(ctgGetQnodeListFromMnode(pCtg, pConn, NULL, pTask)); - + return TSDB_CODE_SUCCESS; } @@ -3242,7 +3244,7 @@ int32_t ctgLaunchGetDnodeTask(SCtgTask* pTask) { CTG_CACHE_NHIT_INC(CTG_CI_DNODE, 1); CTG_ERR_RET(ctgGetDnodeListFromMnode(pCtg, pConn, NULL, pTask)); - + return TSDB_CODE_SUCCESS; } @@ -3261,7 +3263,7 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { pMsgCtx->pBatchs = pJob->pBatchs; } - SDbCfgInfo cfgInfo; + SDbCfgInfo cfgInfo; CTG_ERR_RET(ctgReadDBCfgFromCache(pCtg, pCtx->dbFName, &cfgInfo)); if (cfgInfo.cfgVersion < 0) { @@ -3401,13 +3403,13 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { CTG_ERR_RET(pCtx->subTaskCode); } } - + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, &pCtx->user, tbNotExists, &inCache, &rsp)); if (inCache) { pTask->res = rsp.pRawRes; - ctgTaskDebug("Final res got, pass:[%d,%d], pCond:[%p,%p]", - rsp.pRawRes->pass[0], rsp.pRawRes->pass[1], rsp.pRawRes->pCond[0], rsp.pRawRes->pCond[1]); + ctgTaskDebug("Final res got, pass:[%d,%d], pCond:[%p,%p]", rsp.pRawRes->pass[0], rsp.pRawRes->pass[1], + rsp.pRawRes->pCond[0], rsp.pRawRes->pCond[1]); CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; @@ -3423,7 +3425,7 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { } else { CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); } - + return TSDB_CODE_SUCCESS; } @@ -3456,16 +3458,16 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { bool tbMetaDone = false; SName* pName = NULL; -/* - ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone); - if (tbMetaDone) { - CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx)); - TSWAP(pTask->res, pCtx->pResList); + /* + ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone); + if (tbMetaDone) { + CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx)); + TSWAP(pTask->res, pCtx->pResList); - CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); - return TSDB_CODE_SUCCESS; - } -*/ + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + */ int32_t dbNum = taosArrayGetSize(pCtx->pNames); int32_t fetchIdx = 0; @@ -3497,14 +3499,14 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { } for (int32_t i = 0; i < pCtx->fetchNum; ++i) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); if (NULL == pFetch) { ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } CTG_ERR_RET(ctgGetFetchName(pCtx->pNames, pFetch, &pName)); - + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); if (NULL == pMsgCtx) { ctgError("fail to get the %dth SCtgMsgCtx", i); @@ -3532,13 +3534,14 @@ int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) { SCtgTbTSMACtx* pTaskCtx = pTask->taskCtx; SCtgDBCache* pDbCache = NULL; - STablesReq* pTbReq = taosArrayGet(pTaskCtx->pNames, pFetch->dbIdx); + STablesReq* pTbReq = taosArrayGet(pTaskCtx->pNames, pFetch->dbIdx); if (NULL == pTbReq) { - ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pTaskCtx->pNames)); + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, + (int32_t)taosArrayGetSize(pTaskCtx->pNames)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore error + (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore error if (pDbCache) { ctgReleaseVgInfoToCache(pCtg, pDbCache); } else { @@ -3548,7 +3551,7 @@ int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) { CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pReq)); } - + _return: return code; @@ -3570,11 +3573,11 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { ctgError("fail to get the %dth STablesReq, dbNum:%d", idx, dbNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + CTG_ERR_RET(ctgGetTbTSMAFromCache(pCtg, pCtx, idx, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); } - + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetches); if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); @@ -3587,7 +3590,7 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, i); if (NULL == pFetch) { @@ -3595,19 +3598,19 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); if (NULL == pReq) { ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SName* pName = taosArrayGet(pReq->pTables, pFetch->tbIdx); + SName* pName = taosArrayGet(pReq->pTables, pFetch->tbIdx); if (NULL == pName) { ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); if (NULL == pMsgCtx) { ctgError("fail to get the %dth pMsgCtx", i); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -3647,14 +3650,14 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SArray* pRes = NULL; SCtgJob* pJob = pTask->pJob; - + // currently, only support fetching one tsma STablesReq* pReq = taosArrayGet(pCtx->pNames, 0); if (NULL == pReq) { ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + SName* pTsmaName = taosArrayGet(pReq->pTables, 0); if (NULL == pReq) { ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pReq->pTables)); @@ -3669,17 +3672,17 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { ctgError("fail to get the 0th SCtgMsgCtx, taskType:%d", pTask->type); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + if (!pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } - + SCtgTaskReq tReq = {.pTask = pTask, .msgIdx = 0}; if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { ctgError("taosArrayPush SMetaRes failed, code:%x", terrno); CTG_ERR_RET(terrno); } - + CTG_ERR_RET(ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, NULL, &tReq, TDMT_MND_GET_TSMA)); } else { SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); @@ -3689,13 +3692,13 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { } STableTSMAInfoRsp* pRsp = (STableTSMAInfoRsp*)pRes->pRes; - + const STSMACache* pTsma = taosArrayGetP(pRsp->pTsmas, 0); if (NULL == pTsma) { ctgError("fail to get the 0th STSMACache, totalNum:%d", (int32_t)taosArrayGetSize(pRsp->pTsmas)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + TSWAP(pTask->res, pCtx->pResList); // get tsma target stable meta if not existed in cache int32_t exists = false; @@ -3708,7 +3711,7 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { } else { CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); } - + return TSDB_CODE_SUCCESS; } @@ -3758,8 +3761,9 @@ int32_t ctgGetTbCfgCb(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { CTG_ERR_JRET(terrno); } - - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); + + CTG_ERR_JRET( + ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } CTG_RET(ctgLaunchGetTbCfgTask(pTask)); @@ -3775,17 +3779,18 @@ int32_t ctgGetTbTagCb(SCtgTask* pTask) { CTG_ERR_JRET(pTask->subRes.code); SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; - SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; + SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; if (NULL == pCtx->pVgInfo) { pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); if (NULL == pCtx->pVgInfo) { CTG_ERR_JRET(terrno); } - - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); + + CTG_ERR_JRET( + ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } - + CTG_RET(ctgLaunchGetTbTagTask(pTask)); _return: @@ -3793,14 +3798,12 @@ _return: CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); } - - int32_t ctgGetUserCb(SCtgTask* pTask) { int32_t code = 0; SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx; pCtx->subTaskCode = pTask->subRes.code; - + CTG_RET(ctgLaunchGetUserTask(pTask)); _return: @@ -3808,7 +3811,6 @@ _return: CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); } - int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) { SCtgDbVgCtx* ctx = pTask->taskCtx; @@ -3818,7 +3820,7 @@ int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) { } int32_t ctgCompTbMetaTasks(SCtgTask* pTask, void* param, bool* equal) { - SCtgTbMetaCtx* ctx = pTask->taskCtx; + SCtgTbMetaCtx* ctx = pTask->taskCtx; SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param; *equal = tNameTbNameEqual(ctx->pName, (SName*)pParam->pName); @@ -3890,7 +3892,7 @@ int32_t ctgSearchExistingTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, in qError("fail to get the %dth task, taskNum:%d", i, taskNum); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + if (type != pTask->type) { continue; } @@ -3923,7 +3925,7 @@ int32_t ctgSetSubTaskCb(SCtgTask* pSub, SCtgTask* pTask) { qError("fail to get the -1th SCtgMsgCtx"); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); if (NULL == pSubMsgCtx) { qError("fail to get the -1th sub SCtgMsgCtx"); @@ -3966,7 +3968,6 @@ SCtgTask* ctgGetTask(SCtgJob* pJob, int32_t taskId) { return NULL; } - int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param) { SCtgJob* pJob = (*ppTask)->pJob; int32_t subTaskId = -1; @@ -3989,7 +3990,7 @@ int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp f qError("fail to get the %dth sub SCtgTask, taskNum:%d", subTaskId, (int32_t)taosArrayGetSize(pJob->pTasks)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + if (newTask) { pSub->subTask = true; } @@ -4027,7 +4028,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { qError("fail to get the %dth task, taskNum:%d", i, taskNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 96a700d2d6..77501b990f 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -57,13 +57,13 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } for (int32_t i = 0; i < taskNum; ++i) { - int32_t* taskId = taosArrayGet(cbParam->taskId, i); + int32_t* taskId = taosArrayGet(cbParam->taskId, i); if (NULL == taskId) { ctgError("taosArrayGet %d taskId failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->taskId)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); + int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); if (NULL == msgIdx) { ctgError("taosArrayGet %d msgIdx failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->msgIdx)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -80,7 +80,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu if (pRsp->msgIdx != *msgIdx) { ctgError("rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx); - + pRsp = &rsp; pRsp->msgIdx = *msgIdx; pRsp->reqType = -1; @@ -117,7 +117,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); - (void)(*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal + (void)(*gCtgAsyncFps[pTask->type].handleRspFp)( + &tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal } CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -420,12 +421,12 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { if (TDMT_VND_BATCH_META == cbParam->reqType || TDMT_MND_BATCH_META == cbParam->reqType) { CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode)); } else { - int32_t* taskId = taosArrayGet(cbParam->taskId, 0); + int32_t* taskId = taosArrayGet(cbParam->taskId, 0); if (NULL == taskId) { ctgError("taosArrayGet %d taskId failed, total:%d", 0, (int32_t)taosArrayGetSize(cbParam->taskId)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); if (NULL == pTask) { ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks)); @@ -448,7 +449,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { ctgError("get task %d SCtgMsgCtx failed, taskType:%d", -1, pTask->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - + pMsgCtx->pBatchs = pBatchs; #endif @@ -537,7 +538,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob, CTG_ERR_JRET(code); } - ctgDebug("ctg req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType)); + ctgDebug("ctg req msg sent, QID:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: @@ -561,9 +562,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq->msgIdx, pTask->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - - SHashObj* pBatchs = pMsgCtx->pBatchs; - SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); + + SHashObj* pBatchs = pMsgCtx->pBatchs; + SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); if (NULL == pBatch) { newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); @@ -602,7 +603,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); - } else if (CTG_TASK_GET_TB_TSMA == pTask->type){ + } else if (CTG_TASK_GET_TB_TSMA == pTask->type) { SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); @@ -619,10 +620,11 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); if (NULL == pFetch) { - ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, + (int32_t)taosArrayGetSize(pCtx->pFetches)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); if (NULL == pTbReq) { ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -678,7 +680,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); - } else if (CTG_TASK_GET_TB_TSMA == pTask->type){ + } else if (CTG_TASK_GET_TB_TSMA == pTask->type) { SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); @@ -692,22 +694,23 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT pName = ctx->pName; } } else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) { - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); - if (NULL == pFetch) { - ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - if (NULL == pTbReq) { - ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); - if (NULL == pName) { - ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, + (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); @@ -1632,9 +1635,9 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* } int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out, - SCtgTaskReq* tReq, int32_t reqType) { - char* msg = NULL; - int32_t msgLen = 0; + SCtgTaskReq* tReq, int32_t reqType) { + char* msg = NULL; + int32_t msgLen = 0; SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -1723,7 +1726,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen)); #else - char dbFName[TSDB_DB_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; (void)tNameGetFullDbName(pTbName, dbFName); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1734,7 +1737,8 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen)); + CTG_RET( + ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen)); #endif } diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index b7fb48ddc6..28321d0099 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -36,7 +36,7 @@ static void setCheckDownstreamReqInfo(SStreamTaskCheckReq* pReq, int64_t reqId, static void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault, int32_t* numOfNotRsp, SArray* pTimeoutList, SArray* pNotReadyList, const char* id); static int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId); -static void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatusInfo** pStatusInfo); +static void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatusInfo** pStatusInfo); int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage) { @@ -107,11 +107,12 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pDispatch->taskId, pDispatch->nodeId, idstr); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64 - " window:%" PRId64 "-%" PRId64 " reqId:0x%" PRIx64, + " window:%" PRId64 "-%" PRId64 " QID:0x%" PRIx64, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); - (void) streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet); + (void)streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, + &pTask->outputInfo.fixedDispatcher.epSet); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamTaskStartMonitorCheckRsp(pTask); @@ -132,9 +133,9 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pVgInfo->taskId, pVgInfo->vgId, idstr); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 - " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64, + " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, QID:0x%" PRIx64, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId); - (void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + (void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { // for sink task, set it ready directly. stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId); @@ -164,20 +165,20 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS pRsp->status = TASK_DOWNSTREAM_NOT_LEADER; } else { SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, taskId, &pTask); + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, taskId, &pTask); if (pTask != NULL) { pRsp->status = streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); SStreamTaskState pState = streamTaskGetStatus(pTask); - stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64 + stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(QID:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); streamMetaReleaseTask(pMeta, pTask); } else { pRsp->status = TASK_DOWNSTREAM_NOT_READY; - stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 + stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(QID:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp check_status %d", pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); } @@ -261,7 +262,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, (uint8_t*)abuf, len); - (void) tEncodeStreamTaskCheckRsp(&encoder, pRsp); + (void)tEncodeStreamTaskCheckRsp(&encoder, pRsp); tEncoderClear(&encoder); SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; @@ -300,7 +301,8 @@ void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { if (pInfo->checkRspTmr == NULL) { pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer); } else { - streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor"); + streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, + "check-status-monitor"); } streamMutexUnlock(&pInfo->checkInfoLock); @@ -319,7 +321,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { pInfo->pList = NULL; if (pInfo->checkRspTmr != NULL) { - (void) taosTmrStop(pInfo->checkRspTmr); + (void)taosTmrStop(pInfo->checkRspTmr); pInfo->checkRspTmr = NULL; } @@ -329,11 +331,11 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void processDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - (void) streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); + (void)streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); int64_t checkTs = pTask->execInfo.checkTs; int64_t readyTs = pTask->execInfo.readyTs; - (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); + (void)streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); if (pTask->status.taskStatus == TASK_STATUS__HALT) { if (!HAS_RELATED_FILLHISTORY_TASK(pTask) || (pTask->info.fillHistory != 0)) { @@ -355,7 +357,7 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { // todo: let's retry if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr); - (void) streamLaunchFillHistoryTask(pTask); + (void)streamLaunchFillHistoryTask(pTask); } } @@ -379,7 +381,7 @@ void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { if (!existed) { SDownstreamTaskEpset t = {.nodeId = nodeId}; - void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); + void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); if (p == NULL) { // todo let's retry } @@ -430,7 +432,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { if (reqId != p->reqId) { - stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 + stError("s-task:%s QID:0x%" PRIx64 " expected:0x%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded", id, reqId, p->reqId, taskId); streamMutexUnlock(&pInfo->checkInfoLock); @@ -452,7 +454,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t } streamMutexUnlock(&pInfo->checkInfoLock); - stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId, + stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, QID:%" PRIx64 " discarded", id, taskId, reqId); return TSDB_CODE_FAILED; } @@ -539,10 +541,10 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher; setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId); - stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id, + stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) QID:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); - (void) streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); + (void)streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); } else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgs = taosArrayGetSize(vgInfo); @@ -557,9 +559,9 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { setCheckDownstreamReqInfo(&req, p->reqId, pVgInfo->taskId, pVgInfo->vgId); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 - " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64, + " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d QID:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId); - (void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + (void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); break; } } @@ -580,15 +582,15 @@ void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, i stDebug("s-task:%s recv status:NEW_STAGE/NOT_LEADER from downstream, task:0x%x, quit from check downstream", id, p->taskId); (*numOfFault) += 1; - } else { // TASK_DOWNSTREAM_NOT_READY - if (p->rspTs == 0) { // not response yet + } else { // TASK_DOWNSTREAM_NOT_READY + if (p->rspTs == 0) { // not response yet if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. - (void) taosArrayPush(pTimeoutList, &p->taskId); + (void)taosArrayPush(pTimeoutList, &p->taskId); } else { // el < CHECK_NOT_RSP_DURATION (*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp } } else { - (void) taosArrayPush(pNotReadyList, &p->taskId); + (void)taosArrayPush(pNotReadyList, &p->taskId); } } } @@ -613,14 +615,13 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { continue; } - int32_t taskId = *px; + int32_t taskId = *px; SDownstreamStatusInfo* p = NULL; findCheckRspStatus(pInfo, taskId, &p); if (p != NULL) { - if (p->status != -1 || p->rspTs != 0) { - stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%"PRId64, pTask->id.idStr, i, p->status, - p->rspTs); + stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%" PRId64, pTask->id.idStr, i, + p->status, p->rspTs); continue; } @@ -692,24 +693,24 @@ int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64 // this function is executed in timer thread void rspMonitorFn(void* param, void* tmrId) { - SStreamTask* pTask = param; - SStreamMeta* pMeta = pTask->pMeta; - STaskCheckInfo* pInfo = &pTask->taskCheckInfo; - int32_t vgId = pTask->pMeta->vgId; - int64_t now = taosGetTimestampMs(); - int64_t timeoutDuration = now - pInfo->timeoutStartTs; - const char* id = pTask->id.idStr; - int32_t numOfReady = 0; - int32_t numOfFault = 0; - int32_t numOfNotRsp = 0; - int32_t numOfNotReady = 0; - int32_t numOfTimeout = 0; - int32_t total = taosArrayGetSize(pInfo->pList); + SStreamTask* pTask = param; + SStreamMeta* pMeta = pTask->pMeta; + STaskCheckInfo* pInfo = &pTask->taskCheckInfo; + int32_t vgId = pTask->pMeta->vgId; + int64_t now = taosGetTimestampMs(); + int64_t timeoutDuration = now - pInfo->timeoutStartTs; + const char* id = pTask->id.idStr; + int32_t numOfReady = 0; + int32_t numOfFault = 0; + int32_t numOfNotRsp = 0; + int32_t numOfNotReady = 0; + int32_t numOfTimeout = 0; + int32_t total = taosArrayGetSize(pInfo->pList); stDebug("s-task:%s start to do check-downstream-rsp check in tmr", id); streamMutexLock(&pTask->lock); - SStreamTaskState state = streamTaskGetStatus(pTask); + SStreamTaskState state = streamTaskGetStatus(pTask); streamMutexUnlock(&pTask->lock); if (state.state == TASK_STATUS__STOP) { @@ -721,7 +722,7 @@ void rspMonitorFn(void* param, void* tmrId) { // not record the failed of the current task if try to close current vnode // otherwise, the put of message operation may incur invalid read of message queue. if (!pMeta->closeFlag) { - (void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); + (void)addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); } streamMetaReleaseTask(pMeta, pTask); @@ -740,8 +741,8 @@ void rspMonitorFn(void* param, void* tmrId) { streamMutexLock(&pInfo->checkInfoLock); if (pInfo->notReadyTasks == 0) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, state.name, - vgId, ref); + stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, state.name, vgId, + ref); streamTaskCompleteCheckRsp(pInfo, false, id); streamMutexUnlock(&pInfo->checkInfoLock); @@ -803,7 +804,7 @@ void rspMonitorFn(void* param, void* tmrId) { streamTaskCompleteCheckRsp(pInfo, false, id); streamMutexUnlock(&pInfo->checkInfoLock); - (void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); + (void)addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); streamMetaReleaseTask(pMeta, pTask); taosArrayDestroy(pNotReadyList); @@ -819,7 +820,8 @@ void rspMonitorFn(void* param, void* tmrId) { handleTimeoutDownstreamTasks(pTask, pTimeoutList); } - streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor"); + streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, + "check-status-monitor"); streamMutexUnlock(&pInfo->checkInfoLock); stDebug( diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 918a6b6cb1..de67d55703 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -118,7 +118,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, len); - (void) tEncodeStreamRetrieveReq(&encoder, req); + (void)tEncodeStreamRetrieveReq(&encoder, req); tEncoderClear(&encoder); SRpcMsg rpcMsg = {0}; @@ -130,7 +130,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r return code; } - stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, + stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), QID:0x%" PRIx64, pTask->id.idStr, pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId); } @@ -254,7 +254,7 @@ static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStre int32_t type = pTask->outputInfo.type; int32_t num = streamTaskGetNumOfDownstream(pTask); - if(type != TASK_OUTPUT__SHUFFLE_DISPATCH && type != TASK_OUTPUT__FIXED_DISPATCH) { + if (type != TASK_OUTPUT__SHUFFLE_DISPATCH && type != TASK_OUTPUT__FIXED_DISPATCH) { terrno = TSDB_CODE_INVALID_PARA; stError("s-task:%s invalid dispatch type:%d not dispatch data", pTask->id.idStr, type); return NULL; @@ -283,7 +283,7 @@ static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStre return NULL; } } - } else { // shuffle dispatch + } else { // shuffle dispatch int32_t numOfBlocks = taosArrayGetSize(pData->blocks); int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; @@ -470,7 +470,7 @@ static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t streamMutexLock(&pMsgInfo->lock); } - (void) taosArrayPush(pMsgInfo->pSendInfo, &entry); + (void)taosArrayPush(pMsgInfo->pSendInfo, &entry); if (lock) { streamMutexUnlock(&pMsgInfo->lock); @@ -591,7 +591,7 @@ static void doMonitorDispatchData(void* param, void* tmrId) { SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet; int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; - int32_t s = taosArrayGetSize(pTask->msgInfo.pSendInfo); + int32_t s = taosArrayGetSize(pTask->msgInfo.pSendInfo); SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, 0); if (pEntry != NULL) { setResendInfo(pEntry, now); @@ -617,7 +617,8 @@ static void doMonitorDispatchData(void* param, void* tmrId) { void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { int32_t vgId = pTask->pMeta->vgId; if (pTask->msgInfo.pRetryTmr != NULL) { - streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, "dispatch-monitor-tmr"); + streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, + "dispatch-monitor-tmr"); } else { pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer); } @@ -651,7 +652,8 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } } } else { - (void) buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); + (void)buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, + pDataBlock->info.parTbName); } snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, @@ -666,7 +668,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S // failed to put into name buffer, no need to do anything if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { - (void) tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); + (void)tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); } } @@ -848,7 +850,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { } if (++pTmrInfo->activeCounter < 50) { - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); + streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "chkpt-ready-monitor"); return; } @@ -907,7 +910,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { continue; } - (void) taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); + (void)taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, pTask->info.taskLevel, pInfo->upstreamTaskId); } @@ -931,8 +934,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { if (*pTaskId == pReadyInfo->upstreamTaskId) { // send msg again SRpcMsg msg = {0}; - int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, - checkpointId, &msg); + int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, + pReadyInfo->childId, checkpointId, &msg); if (code == TSDB_CODE_SUCCESS) { code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); if (code == TSDB_CODE_SUCCESS) { @@ -948,7 +951,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { } } - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); + streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "chkpt-ready-monitor"); streamMutexUnlock(&pActiveInfo->lock); } else { int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); @@ -1015,7 +1019,8 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { if (pTmrInfo->tmrHandle == NULL) { pTmrInfo->tmrHandle = taosTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer); } else { - streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); + streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, + "chkpt-ready-monitor"); } // mark the timer monitor checkpointId @@ -1087,8 +1092,8 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch payloadLen += sizeof(SRetrieveTableRsp); - (void) taosArrayPush(pReq->dataLen, &payloadLen); - (void) taosArrayPush(pReq->data, &buf); + (void)taosArrayPush(pReq->dataLen, &payloadLen); + (void)taosArrayPush(pReq->data, &buf); pReq->totalLen += dataStrLen; return 0; @@ -1169,7 +1174,7 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); tEncoderInit(&encoder, (uint8_t*)abuf, len); - (void) tEncodeStreamCheckpointSourceRsp(&encoder, &rsp); + (void)tEncodeStreamCheckpointSourceRsp(&encoder, &rsp); tEncoderClear(&encoder); initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len); @@ -1185,7 +1190,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId}; // todo retry until it success - (void) streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS); + (void)streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS); SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; streamMutexLock(&pActiveInfo->lock); @@ -1208,7 +1213,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId); } } else { - (void) taosArrayPush(pActiveInfo->pReadyMsgList, &info); + (void)taosArrayPush(pActiveInfo->pReadyMsgList, &info); stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1); } @@ -1217,7 +1222,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa } void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, - int32_t childId, SEpSet* pEpset, int64_t checkpointId) { + int32_t childId, SEpSet* pEpset, int64_t checkpointId) { pReadyInfo->upstreamTaskId = upstreamTaskId; pReadyInfo->upstreamNodeEpset = *pEpset; pReadyInfo->upstreamNodeId = upstreamNodeId; @@ -1246,7 +1251,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; streamMutexLock(&pActiveInfo->lock); - (void) taosArrayPush(pActiveInfo->pReadyMsgList, &info); + (void)taosArrayPush(pActiveInfo->pReadyMsgList, &info); int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); int32_t total = streamTaskGetNumOfUpstream(pTask); @@ -1282,7 +1287,7 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; - bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); + bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); clearBufferedDispatchMsg(pTask); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index cd69c9168c..a57e8557a6 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -18,10 +18,10 @@ // maximum allowed processed block batches. One block may include several submit blocks #define MAX_STREAM_EXEC_BATCH_NUM 32 #define STREAM_RESULT_DUMP_THRESHOLD 300 -#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1) // 1MiB result data -#define STREAM_SCAN_HISTORY_TIMESLICE 1000 // 1000 ms -#define MIN_INVOKE_INTERVAL 50 // 50ms -#define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec +#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1) // 1MiB result data +#define STREAM_SCAN_HISTORY_TIMESLICE 1000 // 1000 ms +#define MIN_INVOKE_INTERVAL 50 // 50ms +#define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); @@ -54,7 +54,7 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl // not handle error, if dispatch failed, try next time. // checkpoint trigger will be checked - (void) streamDispatchStreamBlock(pTask); + (void)streamDispatchStreamBlock(pTask); } return code; @@ -78,7 +78,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* } stDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, - SIZE_IN_MiB(size)); + SIZE_IN_MiB(size)); code = doOutputResultBlockImpl(pTask, pStreamBlocks); if (code != TSDB_CODE_SUCCESS) { // back pressure and record position @@ -99,7 +99,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to *totalSize = 0; int32_t size = 0; - int32_t numOfBlocks= 0; + int32_t numOfBlocks = 0; SArray* pRes = NULL; while (1) { @@ -129,7 +129,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); - (void) assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); + (void)assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; block.info.childId = pTask->info.selfChildId; @@ -140,8 +140,8 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to stError("s-task:%s failed to add retrieve block", pTask->id.idStr); } - stDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64 " dump results", pTask->id.idStr, - pTask->info.selfChildId, pRetrieveBlock->reqId); + stDebug("s-task:%s(child %d) retrieve process completed, QID:0x%" PRIx64 " dump results", pTask->id.idStr, + pTask->info.selfChildId, pRetrieveBlock->reqId); } break; @@ -174,7 +174,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to } stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, - pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size)); + pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size)); // current output should be dispatched to down stream nodes if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { @@ -251,7 +251,7 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t* } SSDataBlock block = {0}; - (void) assignOneDataBlock(&block, output); + (void)assignOneDataBlock(&block, output); block.info.childId = pTask->info.selfChildId; void* p = taosArrayPush(pRes, &block); @@ -259,7 +259,8 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t* stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr); } - (*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); + (*pSize) += + blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); numOfBlocks += 1; if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || (*pSize) >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { @@ -283,7 +284,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { const char* id = pTask->id.idStr; if (!pTask->hTaskInfo.operatorOpen) { - (void) qSetStreamOpOpen(exec); + (void)qSetStreamOpOpen(exec); pTask->hTaskInfo.operatorOpen = true; } @@ -315,13 +316,13 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { int32_t size = 0; streamScanHistoryDataImpl(pTask, pRes, &size, &finished); - if(streamTaskShouldStop(pTask)) { + if (streamTaskShouldStop(pTask)) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); } // dispatch the generated results, todo fix error - (void) handleScanhistoryResultBlocks(pTask, pRes, size); + (void)handleScanhistoryResultBlocks(pTask, pRes, size); if (finished) { return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0); @@ -346,11 +347,11 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { stError( "s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related " "fill-history task", - id, (int32_t) pTask->streamTaskId.taskId); + id, (int32_t)pTask->streamTaskId.taskId); // 1. free it and remove fill-history task from disk meta-store // todo: this function should never be failed. - (void) streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); + (void)streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); // 2. save to disk streamMetaWLock(pMeta); @@ -367,7 +368,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { id, streamTaskGetStatus(pTask).name, el, pStreamTask->id.idStr); } - ETaskStatus status = streamTaskGetStatus(pStreamTask).state; + ETaskStatus status = streamTaskGetStatus(pStreamTask).state; STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; // It must be halted for a source stream task, since when the related scan-history-data task start scan the history @@ -408,14 +409,14 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, pTimeWindow->ekey, p, pStreamTask->status.schedStatus); - (void) streamTaskResetTimewindowFilter(pStreamTask); + (void)streamTaskResetTimewindowFilter(pStreamTask); } else { stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr); } // NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec // 2. send msg to mnode to launch a checkpoint to keep the state for current stream - (void) streamTaskSendCheckpointReq(pStreamTask); + (void)streamTaskSendCheckpointReq(pStreamTask); // 3. assign the status to the value that will be kept in disk pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state; @@ -429,12 +430,12 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { static int32_t haltCallback(SStreamTask* pTask, void* param) { streamTaskOpenAllUpstreamInput(pTask); - (void) streamTaskSendCheckpointReq(pTask); + (void)streamTaskSendCheckpointReq(pTask); return TSDB_CODE_SUCCESS; } int32_t streamTransferStatePrepare(SStreamTask* pTask) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SStreamMeta* pMeta = pTask->pMeta; ASSERT(pTask->status.appendTranstateBlock == 1); @@ -466,7 +467,7 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) { // set input static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) { - void* pExecutor = pTask->exec.pExecutor; + void* pExecutor = pTask->exec.pExecutor; int32_t code = 0; const SStreamQueueItem* pItem = pInput; @@ -479,7 +480,7 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, - pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); + pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); ASSERT((*pVer) <= pSubmit->submit.ver); (*pVer) = pSubmit->submit.ver; @@ -497,7 +498,7 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int SArray* pBlockList = pMerged->submits; int32_t numOfBlocks = taosArrayGetSize(pBlockList); stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks, - pMerged->ver); + pMerged->ver); code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); ASSERT((*pVer) <= pMerged->ver); (*pVer) = pMerged->ver; @@ -549,7 +550,7 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) pBlock->srcVgId = pTask->pMeta->vgId; code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); if (code == 0) { - (void) streamDispatchStreamBlock(pTask); + (void)streamDispatchStreamBlock(pTask); } else { // todo put into queue failed, retry streamFreeQitem((SStreamQueueItem*)pBlock); } @@ -568,7 +569,7 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) } } -//static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } +// static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } static void setLastExecTs(SStreamTask* pTask, int64_t ts) { pTask->status.lastExecTs = ts; } static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) { @@ -581,7 +582,7 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type)); int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id); - if(code) { + if (code) { stError("s-task:%s failed to set input block, not exec for these blocks", id); return; } @@ -609,7 +610,7 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i if (ver != pInfo->processedVer) { stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64 - " ckpt:%" PRId64, + " ckpt:%" PRId64, id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer); pInfo->processedVer = ver; } @@ -625,10 +626,10 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB STaskId* pHTaskId = &pTask->hTaskInfo.id; SStreamTask* pHTask = NULL; - int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); - if (code == TSDB_CODE_SUCCESS) { // ignore the error code. - (void) streamTaskReleaseState(pHTask); - (void) streamTaskReloadState(pTask); + int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); + if (code == TSDB_CODE_SUCCESS) { // ignore the error code. + (void)streamTaskReleaseState(pHTask); + (void)streamTaskReloadState(pTask); stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, streamTaskGetStatus(pHTask).name); // todo execute qExecTask to fetch the reload-generated result, if this is stream is for session window query. @@ -707,7 +708,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { // dispatch checkpoint msg to all downstream tasks int32_t type = pInput->type; if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { - (void) streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); + (void)streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); continue; } @@ -744,14 +745,14 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { if (type != STREAM_INPUT__CHECKPOINT) { doStreamTaskExecImpl(pTask, pInput, numOfBlocks); streamFreeQitem(pInput); - } else { // todo other thread may change the status - // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. + } else { // todo other thread may change the status + // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state == TASK_STATUS__CK) { stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); - (void) streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue - } else { // todo refactor + (void)streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue + } else { // todo refactor int32_t code = 0; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { code = streamTaskSendCheckpointSourceRsp(pTask); @@ -804,7 +805,7 @@ void streamResumeTask(SStreamTask* pTask) { const char* id = pTask->id.idStr; while (1) { - (void) doStreamExecTask(pTask); + (void)doStreamExecTask(pTask); // check if continue streamMutexLock(&pTask->lock); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index b99c9f5f65..164ba558f2 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -1092,7 +1092,7 @@ static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* } // enqueue - stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, + stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), QID:0x%" PRIx64, pTask->id.idStr, pTask->pMeta->vgId, pTask->info.taskLevel, pReq->srcTaskId, pReq->srcNodeId, pReq->reqId); pData->type = STREAM_INPUT__DATA_RETRIEVE; diff --git a/source/libs/transport/inc/transLog.h b/source/libs/transport/inc/transLog.h index 121939787c..8f789821fd 100644 --- a/source/libs/transport/inc/transLog.h +++ b/source/libs/transport/inc/transLog.h @@ -32,12 +32,12 @@ extern "C" { #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }} #define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } } -#define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) -#define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", gtid:%s", __VA_ARGS__, buf); }} while (0) -#define tGError(param, ...) do { if (rpcDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tError(param ", gtid:%s", __VA_ARGS__, buf);} } while(0) -#define tGWarn(param, ...) do { if (rpcDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tWarn(param ", gtid:%s", __VA_ARGS__, buf); }} while(0) -#define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", gtid:%s", __VA_ARGS__, buf); }} while(0) -#define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", gtid:%s", __VA_ARGS__, buf); }} while(0) +#define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0) +#define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", QID:%s", __VA_ARGS__, buf); }} while (0) +#define tGError(param, ...) do { if (rpcDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tError(param ", QID:%s", __VA_ARGS__, buf);} } while(0) +#define tGWarn(param, ...) do { if (rpcDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tWarn(param ", QID:%s", __VA_ARGS__, buf); }} while(0) +#define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", QID:%s", __VA_ARGS__, buf); }} while(0) +#define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", QID:%s", __VA_ARGS__, buf); }} while(0) // clang-format on From 472d03f1186e5ed6bb76d3f4523974a79da7a172 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 22 Aug 2024 10:09:31 +0800 Subject: [PATCH 091/100] fix(query):adjust error log --- source/libs/executor/src/sysscanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index fc1159cc9d..242e4c0830 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1436,7 +1436,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { int64_t suid = mr.me.ctbEntry.suid; code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); pAPI->metaReaderFn.clearReader(&mr1); pAPI->metaReaderFn.clearReader(&mr); From aab3e16f49e53d378de9d44f66b79406fc40c873 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 22 Aug 2024 10:13:26 +0800 Subject: [PATCH 092/100] change QID log --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7135e8f070..f0ea99a0bb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1707,7 +1707,7 @@ int32_t doProcessMsgFromServer(void* param) { char tbuf[40] = {0}; TRACE_TO_STR(trace, tbuf); - tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, QID: %s", pMsg->info.handle, + tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, QID:%s", pMsg->info.handle, TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); if (pSendInfo->requestObjRefId != 0) { From a6b77da8f9c00d37e35c17e6da6adf5097908a72 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 20 Aug 2024 19:58:13 +0800 Subject: [PATCH 093/100] remove audit from taos.cfg in community package --- packaging/cfg/taos.cfg | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 6d25db843d..ebbab8697b 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -98,11 +98,6 @@ # enable/disable system monitor # monitor 1 -# enable/disable audit log -# audit 1 - -# enable/disable audit create table -# auditCreateTable 1 # The following parameter is used to limit the maximum number of lines in log files. # max number of lines per log filters From ccd95c8bc073f575829c3897fa8e7f53ddd2b71e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 22 Aug 2024 10:57:38 +0800 Subject: [PATCH 094/100] fix(util/lru): free cache if shards init failed --- source/util/src/tlrucache.c | 56 ++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 83e0e2d811..55531108f2 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -87,13 +87,13 @@ struct SLRUEntry { #define TAOS_LRU_ENTRY_REF(h) (++(h)->refs) static bool taosLRUEntryUnref(SLRUEntry *entry) { - ASSERT(entry->refs > 0); + // ASSERT(entry->refs > 0); --entry->refs; return entry->refs == 0; } static void taosLRUEntryFree(SLRUEntry *entry) { - ASSERT(entry->refs == 0); + // ASSERT(entry->refs == 0); if (entry->deleter) { (*entry->deleter)(entry->keyData, entry->keyLength, entry->value, entry->ud); @@ -129,7 +129,7 @@ static void taosLRUEntryTableApply(SLRUEntryTable *table, _taos_lru_table_func_t SLRUEntry *h = table->list[i]; while (h) { SLRUEntry *n = h->nextHash; - ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h)); + // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h)); func(h); h = n; } @@ -155,7 +155,7 @@ static int taosLRUEntryTableApplyF(SLRUEntryTable *table, _taos_lru_functor_t fu SLRUEntry *h = table->list[i]; while (h) { SLRUEntry *n = h->nextHash; - ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h)); + // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h)); ret = functor(h->keyData, h->keyLength, h->value, ud); if (ret) { return ret; @@ -205,7 +205,7 @@ static void taosLRUEntryTableResize(SLRUEntryTable *table) { ++count; } } - ASSERT(table->elems == count); + // ASSERT(table->elems == count); taosMemoryFree(table->list); table->list = newList; @@ -261,16 +261,16 @@ struct SLRUCacheShard { static void taosLRUCacheShardMaintainPoolSize(SLRUCacheShard *shard) { while (shard->highPriPoolUsage > shard->highPriPoolCapacity) { shard->lruLowPri = shard->lruLowPri->next; - ASSERT(shard->lruLowPri != &shard->lru); + // ASSERT(shard->lruLowPri != &shard->lru); TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(shard->lruLowPri, false); - ASSERT(shard->highPriPoolUsage >= shard->lruLowPri->totalCharge); + // ASSERT(shard->highPriPoolUsage >= shard->lruLowPri->totalCharge); shard->highPriPoolUsage -= shard->lruLowPri->totalCharge; } } static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) { - ASSERT(e->next == NULL && e->prev == NULL); + // ASSERT(e->next == NULL && e->prev == NULL); if (shard->highPriPoolRatio > 0 && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) { e->next = &shard->lru; @@ -297,7 +297,7 @@ static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) { } static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) { - ASSERT(e->next && e->prev); + // ASSERT(e->next && e->prev); if (shard->lruLowPri == e) { shard->lruLowPri = e->prev; @@ -306,10 +306,10 @@ static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) { e->prev->next = e->next; e->prev = e->next = NULL; - ASSERT(shard->lruUsage >= e->totalCharge); + // ASSERT(shard->lruUsage >= e->totalCharge); shard->lruUsage -= e->totalCharge; if (TAOS_LRU_ENTRY_IN_HIGH_POOL(e)) { - ASSERT(shard->highPriPoolUsage >= e->totalCharge); + // ASSERT(shard->highPriPoolUsage >= e->totalCharge); shard->highPriPoolUsage -= e->totalCharge; } } @@ -317,13 +317,13 @@ static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) { static void taosLRUCacheShardEvictLRU(SLRUCacheShard *shard, size_t charge, SArray *deleted) { while (shard->usage + charge > shard->capacity && shard->lru.next != &shard->lru) { SLRUEntry *old = shard->lru.next; - ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); + // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); taosLRUCacheShardLRURemove(shard, old); (void)taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); - ASSERT(shard->usage >= old->totalCharge); + // ASSERT(shard->usage >= old->totalCharge); shard->usage -= old->totalCharge; (void)taosArrayPush(deleted, &old); @@ -332,6 +332,9 @@ static void taosLRUCacheShardEvictLRU(SLRUCacheShard *shard, size_t charge, SArr static void taosLRUCacheShardSetCapacity(SLRUCacheShard *shard, size_t capacity) { SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); + if (!lastReferenceList) { + return; + } (void)taosThreadMutexLock(&shard->mutex); @@ -411,11 +414,11 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * if (old != NULL) { status = TAOS_LRU_STATUS_OK_OVERWRITTEN; - ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old)); + // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old)); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); if (!TAOS_LRU_ENTRY_HAS_REFS(old)) { taosLRUCacheShardLRURemove(shard, old); - ASSERT(shard->usage >= old->totalCharge); + // ASSERT(shard->usage >= old->totalCharge); shard->usage -= old->totalCharge; (void)taosArrayPush(lastReferenceList, &old); @@ -476,7 +479,7 @@ static LRUHandle *taosLRUCacheShardLookup(SLRUCacheShard *shard, const void *key (void)taosThreadMutexLock(&shard->mutex); e = taosLRUEntryTableLookup(&shard->table, key, keyLen, hash); if (e != NULL) { - ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e)); + // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e)); if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { taosLRUCacheShardLRURemove(shard, e); } @@ -495,12 +498,12 @@ static void taosLRUCacheShardErase(SLRUCacheShard *shard, const void *key, size_ SLRUEntry *e = taosLRUEntryTableRemove(&shard->table, key, keyLen, hash); if (e != NULL) { - ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e)); + // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e)); TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { taosLRUCacheShardLRURemove(shard, e); - ASSERT(shard->usage >= e->totalCharge); + // ASSERT(shard->usage >= e->totalCharge); shard->usage -= e->totalCharge; lastReference = true; } @@ -532,11 +535,11 @@ static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) { while (shard->lru.next != &shard->lru) { SLRUEntry *old = shard->lru.next; - ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); + // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); taosLRUCacheShardLRURemove(shard, old); (void)taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); - ASSERT(shard->usage >= old->totalCharge); + // ASSERT(shard->usage >= old->totalCharge); shard->usage -= old->totalCharge; (void)taosArrayPush(lastReferenceList, &old); @@ -557,7 +560,7 @@ static bool taosLRUCacheShardRef(SLRUCacheShard *shard, LRUHandle *handle) { SLRUEntry *e = (SLRUEntry *)handle; (void)taosThreadMutexLock(&shard->mutex); - ASSERT(TAOS_LRU_ENTRY_HAS_REFS(e)); + // ASSERT(TAOS_LRU_ENTRY_HAS_REFS(e)); TAOS_LRU_ENTRY_REF(e); (void)taosThreadMutexUnlock(&shard->mutex); @@ -578,7 +581,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b lastReference = taosLRUEntryUnref(e); if (lastReference && TAOS_LRU_ENTRY_IN_CACHE(e)) { if (shard->usage > shard->capacity || eraseIfLastRef) { - ASSERT(shard->lru.next == &shard->lru || eraseIfLastRef); + // ASSERT(shard->lru.next == &shard->lru || eraseIfLastRef); (void)taosLRUEntryTableRemove(&shard->table, e->keyData, e->keyLength, e->hash); TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); @@ -590,7 +593,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b } if (lastReference && e->value) { - ASSERT(shard->usage >= e->totalCharge); + // ASSERT(shard->usage >= e->totalCharge); shard->usage -= e->totalCharge; } @@ -628,7 +631,7 @@ static size_t taosLRUCacheShardGetPinnedUsage(SLRUCacheShard *shard) { (void)taosThreadMutexLock(&shard->mutex); - ASSERT(shard->usage >= shard->lruUsage); + // ASSERT(shard->usage >= shard->lruUsage); usage = shard->usage - shard->lruUsage; (void)taosThreadMutexUnlock(&shard->mutex); @@ -699,7 +702,8 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo for (int i = 0; i < numShards; ++i) { if (TSDB_CODE_SUCCESS != taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits)) - return NULL; + taosMemoryFree(cache); + return NULL; } cache->numShards = numShards; @@ -718,7 +722,7 @@ void taosLRUCacheCleanup(SLRUCache *cache) { if (cache) { if (cache->shards) { int numShards = cache->numShards; - ASSERT(numShards > 0); + // ASSERT(numShards > 0); for (int i = 0; i < numShards; ++i) { taosLRUCacheShardCleanup(&cache->shards[i]); } From 300b30d7c59df9ec1fd92b256b9d34da037d0d73 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 22 Aug 2024 11:06:27 +0800 Subject: [PATCH 095/100] fix(query):init block info's pointer --- source/common/src/tdatablock.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2181496635..b1e1dcadc2 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1670,6 +1670,8 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { int32_t code = 0; dst->info = src->info; + dst->info.pks[0].pData = NULL; + dst->info.pks[1].pData = NULL; dst->info.rows = 0; dst->info.capacity = 0; @@ -1707,6 +1709,8 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { uint32_t cap = dst->info.capacity; dst->info = src->info; + dst->info.pks[0].pData = NULL; + dst->info.pks[1].pData = NULL; dst->info.capacity = cap; return code; } @@ -1737,6 +1741,8 @@ int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc) { uint32_t cap = pDst->info.capacity; pDst->info = pSrc->info; + pDst->info.pks[0].pData = NULL; + pDst->info.pks[1].pData = NULL; code = copyPkVal(&pDst->info, &pSrc->info); if (code != TSDB_CODE_SUCCESS) { uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); @@ -1854,6 +1860,8 @@ int32_t blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx, SSDataBlo } pBlock->info = pDataBlock->info; + pBlock->info.pks[0].pData = NULL; + pBlock->info.pks[1].pData = NULL; pBlock->info.rows = 0; pBlock->info.capacity = 0; @@ -1916,20 +1924,20 @@ int32_t copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc) { // prepare the pk buffer if needed SValue* p = &pDst->pks[0]; - p->type = pDst->pks[0].type; - p->pData = taosMemoryCalloc(1, pDst->pks[0].nData); + p->type = pSrc->pks[0].type; + p->pData = taosMemoryCalloc(1, pSrc->pks[0].nData); QUERY_CHECK_NULL(p->pData, code, lino, _end, terrno); - p->nData = pDst->pks[0].nData; - memcpy(p->pData, pDst->pks[0].pData, p->nData); + p->nData = pSrc->pks[0].nData; + memcpy(p->pData, pSrc->pks[0].pData, p->nData); p = &pDst->pks[1]; - p->type = pDst->pks[1].type; - p->pData = taosMemoryCalloc(1, pDst->pks[1].nData); + p->type = pSrc->pks[1].type; + p->pData = taosMemoryCalloc(1, pSrc->pks[1].nData); QUERY_CHECK_NULL(p->pData, code, lino, _end, terrno); - p->nData = pDst->pks[1].nData; - memcpy(p->pData, pDst->pks[1].pData, p->nData); + p->nData = pSrc->pks[1].nData; + memcpy(p->pData, pSrc->pks[1].pData, p->nData); _end: if (code != TSDB_CODE_SUCCESS) { @@ -1951,6 +1959,8 @@ int32_t createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData, SSDataB } pDstBlock->info = pDataBlock->info; + pDstBlock->info.pks[0].pData = NULL; + pDstBlock->info.pks[1].pData = NULL; pDstBlock->info.rows = 0; pDstBlock->info.capacity = 0; From d334c2e8df370212aaabdd7abc344b95f887e514 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Aug 2024 11:07:31 +0800 Subject: [PATCH 096/100] enh: refactor some assert --- include/common/tdataformat.h | 2 +- include/common/trow.h | 5 +- include/common/tvariant.h | 4 +- source/common/src/tdataformat.c | 147 ++++++++++++++----------- source/common/src/tmsg.c | 27 +++-- source/common/src/tname.c | 6 +- source/common/src/tvariant.c | 9 +- source/libs/parser/src/parInsertUtil.c | 2 +- source/libs/tdb/src/db/tdbBtree.c | 6 - 9 files changed, 112 insertions(+), 96 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 0acb28fabd..c14704ae19 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -193,7 +193,7 @@ int32_t tColDataDecompress(void *input, SColDataCompressInfo *info, SColData *co // for stmt bind int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen); -void tColDataSortMerge(SArray *colDataArr); +int32_t tColDataSortMerge(SArray *colDataArr); // for raw block int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, diff --git a/include/common/trow.h b/include/common/trow.h index e43c67f931..8d30384c61 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -213,7 +213,6 @@ static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) { } static FORCE_INLINE int16_t tdKvRowColIdAt(STSRow *pRow, col_id_t idx) { - ASSERT(idx >= 0); if (idx == 0) { return PRIMARYKEY_TIMESTAMP_COL_ID; } @@ -288,9 +287,7 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT * */ -static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { - pBuilder->sver = sver; -} +static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { pBuilder->sver = sver; } int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen); int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 6f88eba027..9a0585dada 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -46,14 +46,14 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value); /** * non floating point integers -*/ + */ int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value); void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); void taosVariantDestroy(SVariant *pV); -void taosVariantAssign(SVariant *pDst, const SVariant *pSrc); +int32_t taosVariantAssign(SVariant *pDst, const SVariant *pSrc); int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5caf93de65..f726a5430e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -119,7 +119,6 @@ static FORCE_INLINE void tRowBuildScanAddValue(SRowBuildScanInfo *sinfo, SColVal bool isPK = ((pTColumn->flags & COL_IS_KEY) != 0); if (isPK) { - ASSERTS(sinfo->numOfPKs < TD_MAX_PK_COLS, "too many primary keys"); sinfo->tupleIndices[sinfo->numOfPKs].type = colVal->value.type; sinfo->tupleIndices[sinfo->numOfPKs].offset = IS_VAR_DATA_TYPE(pTColumn->type) ? sinfo->tupleVarSize + sinfo->tupleFixedSize : pTColumn->offset; @@ -149,9 +148,15 @@ static int32_t tRowBuildScan(SArray *colVals, const STSchema *schema, SRowBuildS int32_t numOfColVals = TARRAY_SIZE(colVals); SColVal *colValArray = (SColVal *)TARRAY_DATA(colVals); - ASSERT(numOfColVals > 0); - ASSERT(colValArray[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(colValArray[0].value.type == TSDB_DATA_TYPE_TIMESTAMP); + if (!(numOfColVals > 0)) { + return TSDB_CODE_INVALID_PARA; + } + if (!(colValArray[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID)) { + return TSDB_CODE_INVALID_PARA; + } + if (!(colValArray[0].value.type == TSDB_DATA_TYPE_TIMESTAMP)) { + return TSDB_CODE_INVALID_PARA; + } *sinfo = (SRowBuildScanInfo){ .tupleFixedSize = schema->flen, @@ -166,7 +171,10 @@ static int32_t tRowBuildScan(SArray *colVals, const STSchema *schema, SRowBuildS } if (colValArray[colValIndex].cid == schema->columns[i].colId) { - ASSERT(colValArray[colValIndex].value.type == schema->columns[i].type); + if (!(colValArray[colValIndex].value.type == schema->columns[i].type)) { + code = TSDB_CODE_INVALID_PARA; + goto _exit; + } if (COL_VAL_IS_VALUE(&colValArray[colValIndex])) { tRowBuildScanAddValue(sinfo, &colValArray[colValIndex], schema->columns + i); @@ -272,7 +280,6 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo (*ppRow)->ts = colValArray[0].value.val; if (sinfo->tupleFlag == HAS_NONE || sinfo->tupleFlag == HAS_NULL) { - ASSERT(sinfo->tupleRowSize == sizeof(SRow)); return 0; } @@ -285,7 +292,6 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo for (int32_t i = 0; i < sinfo->numOfPKs; i++) { primaryKeys += tPutPrimaryKeyIndex(primaryKeys, sinfo->tupleIndices + i); } - ASSERT(primaryKeys == bitmap); // bitmap + fixed + varlen int32_t numOfColVals = TARRAY_SIZE(aColVal); @@ -356,7 +362,9 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c (*ppRow)->len = sinfo->kvRowSize; (*ppRow)->ts = colValArray[0].value.val; - ASSERT(sinfo->flag != HAS_NONE && sinfo->flag != HAS_NULL); + if (!(sinfo->flag != HAS_NONE && sinfo->flag != HAS_NULL)) { + return TSDB_CODE_INVALID_PARA; + } uint8_t *primaryKeys = (*ppRow)->data; SKVIdx *indices = (SKVIdx *)(primaryKeys + sinfo->kvPKSize); @@ -367,7 +375,6 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c for (int32_t i = 0; i < sinfo->numOfPKs; i++) { primaryKeys += tPutPrimaryKeyIndex(primaryKeys, sinfo->kvIndices + i); } - ASSERT(primaryKeys == (uint8_t *)indices); int32_t numOfColVals = TARRAY_SIZE(aColVal); int32_t colValIndex = 1; @@ -504,8 +511,8 @@ _exit: } int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { - ASSERT(iCol < pTSchema->numOfCols); - ASSERT(pRow->sver == pTSchema->version); + if (!(iCol < pTSchema->numOfCols)) return TSDB_CODE_INVALID_PARA; + if (!(pRow->sver == pTSchema->version)) return TSDB_CODE_INVALID_PARA; STColumn *pTColumn = pTSchema->columns + iCol; @@ -786,7 +793,7 @@ struct SRowIter { }; int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { - ASSERT(pRow->sver == pTSchema->version); + if (!(pRow->sver == pTSchema->version)) return TSDB_CODE_INVALID_PARA; int32_t code = 0; @@ -839,7 +846,6 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { pIter->pv = pIter->pf + pTSchema->flen; break; default: - ASSERT(0); break; } } @@ -952,7 +958,6 @@ SColVal *tRowIterNext(SRowIter *pIter) { bv = GET_BIT2(pIter->pb, pIter->iTColumn - 1); break; default: - ASSERT(0); break; } @@ -1070,14 +1075,15 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * pv = pf + pTSchema->flen; break; default: - ASSERTS(0, "Invalid row flag"); return TSDB_CODE_INVALID_DATA_FMT; } while (pColData) { if (pTColumn) { if (pTColumn->colId == pColData->cid) { - ASSERT(pTColumn->type == pColData->type); + if (!(pTColumn->type == pColData->type)) { + return TSDB_CODE_INVALID_PARA; + } if (pb) { uint8_t bv; switch (pRow->flag) { @@ -1095,7 +1101,6 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * bv = GET_BIT2(pb, iTColumn - 1); break; default: - ASSERTS(0, "Invalid row flag"); return TSDB_CODE_INVALID_DATA_FMT; } @@ -1178,7 +1183,7 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo } else if (pRow->flag & KV_FLG_BIG) { pv = pKVIdx->idx + (pKVIdx->nCol << 2); } else { - ASSERT(0); + return TSDB_CODE_INVALID_PARA; } while (pColData) { @@ -1193,7 +1198,6 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo } else if (pRow->flag & KV_FLG_BIG) { pData = pv + ((uint32_t *)pKVIdx->idx)[iCol]; } else { - ASSERTS(0, "Invalid KV row format"); return TSDB_CODE_INVALID_DATA_FMT; } @@ -1256,8 +1260,8 @@ _exit: * flag < 0: backward update */ int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag) { - ASSERT(pRow->sver == pTSchema->version); - ASSERT(nColData > 0); + if (!(pRow->sver == pTSchema->version)) return TSDB_CODE_INVALID_PARA; + if (!(nColData > 0)) return TSDB_CODE_INVALID_PARA; if (pRow->flag == HAS_NONE) { return tRowNoneUpsertColData(aColData, nColData, flag); @@ -1277,8 +1281,6 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) { return; } - ASSERT(row->numOfPKs <= TD_MAX_PK_COLS); - SPrimaryKeyIndex indices[TD_MAX_PK_COLS]; uint8_t *data = row->data; @@ -1317,8 +1319,6 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) { } while (0) int32_t tValueCompare(const SValue *tv1, const SValue *tv2) { - ASSERT(tv1->type == tv2->type); - switch (tv1->type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -1476,7 +1476,6 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c printf("%s:%d type:%d vlen:%d, val:%" PRIi8 "\n", tag, ln, (int32_t)type, vlen, *(int8_t *)val); break; default: - ASSERT(0); break; } } @@ -1526,7 +1525,6 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tPutCStr(p ? p + n : p, pTagVal->pKey); } else { n += tPutI16v(p ? p + n : p, pTagVal->cid); - ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type @@ -1602,8 +1600,6 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { isLarge = 1; } - ASSERT(szTag <= INT16_MAX); - // build tag (*ppTag) = (STag *)taosMemoryCalloc(szTag, 1); if ((*ppTag) == NULL) { @@ -1806,8 +1802,14 @@ STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { pTSchema->version = version; // timestamp column - ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); + if (!(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP)) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + if (!(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID)) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } pTSchema->columns[0].colId = aSchema[0].colId; pTSchema->columns[0].type = aSchema[0].type; pTSchema->columns[0].flags = aSchema[0].flags; @@ -1910,7 +1912,9 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, pColData->nData += nData; } } else { - ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); + if (!(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal)) { + return TSDB_CODE_INVALID_PARA; + } code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; if (pData) { @@ -2259,7 +2263,9 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pDa // VALUE NONE NULL }; int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { - ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->value.type); + if (!(pColData->cid == pColVal->cid && pColData->type == pColVal->value.type)) { + return TSDB_CODE_INVALID_PARA; + } return tColDataAppendValueImpl[pColData->flag][pColVal->flag]( pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val, pColVal->value.nData); @@ -2569,8 +2575,8 @@ static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pDa // VALUE NONE NULL }; int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward) { - ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->value.type); - ASSERT(pColData->nVal > 0); + if (!(pColData->cid == pColVal->cid && pColData->type == pColVal->value.type)) return TSDB_CODE_INVALID_PARA; + if (!(pColData->nVal > 0)) return TSDB_CODE_INVALID_PARA; if (tColDataUpdateValueImpl[pColData->flag][pColVal->flag] == NULL) return 0; @@ -2687,7 +2693,6 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) { case (HAS_VALUE | HAS_NULL | HAS_NONE): return GET_BIT2(pColData->pBitMap, iVal); default: - ASSERTS(0, "not possible"); return 0; } } @@ -3053,7 +3058,9 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32 int32_t code = 0; if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { - ASSERT(pColData->type == pBind->buffer_type); + if (!(pColData->type == pBind->buffer_type)) { + return TSDB_CODE_INVALID_PARA; + } } if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type @@ -3521,15 +3528,21 @@ static void tColDataMerge(SColData *aColData, int32_t nColData) { } } -void tColDataSortMerge(SArray *colDataArr) { +int32_t tColDataSortMerge(SArray *colDataArr) { int32_t nColData = TARRAY_SIZE(colDataArr); SColData *aColData = (SColData *)TARRAY_DATA(colDataArr); - if (aColData[0].nVal <= 1) goto _exit; + if (!(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP)) { + return TSDB_CODE_INVALID_PARA; + } + if (!(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID)) { + return TSDB_CODE_INVALID_PARA; + } + if (!(aColData[0].flag == HAS_VALUE)) { + return TSDB_CODE_INVALID_PARA; + } - ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(aColData[0].flag == HAS_VALUE); + if (aColData[0].nVal <= 1) goto _exit; int8_t doSort = 0; int8_t doMerge = 0; @@ -3578,7 +3591,7 @@ void tColDataSortMerge(SArray *colDataArr) { } _exit: - return; + return 0; } static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) { @@ -3685,8 +3698,7 @@ int32_t tPutColData(uint8_t version, uint8_t *pBuf, SColData *pColData) { } else if (version == 1) { return tPutColDataVersion1(pBuf, pColData); } else { - ASSERT(0); - return -1; + return TSDB_CODE_INVALID_PARA; } } @@ -3696,8 +3708,7 @@ int32_t tGetColData(uint8_t version, uint8_t *pBuf, SColData *pColData) { } else if (version == 1) { return tGetColDataVersion1(pBuf, pColData); } else { - ASSERT(0); - return -1; + return TSDB_CODE_INVALID_PARA; } } @@ -3733,7 +3744,6 @@ static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, i CALC_SUM_MAX_MIN(*sum, *max, *min, val); break; default: - ASSERT(0); break; } } @@ -3765,7 +3775,6 @@ static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum CALC_SUM_MAX_MIN(*sum, *max, *min, val); break; default: - ASSERT(0); break; } } @@ -3797,7 +3806,6 @@ static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t CALC_SUM_MAX_MIN(*sum, *max, *min, val); break; default: - ASSERT(0); break; } } @@ -3829,7 +3837,6 @@ static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, in CALC_SUM_MAX_MIN(*sum, *max, *min, val); break; default: - ASSERT(0); break; } } @@ -3861,7 +3868,6 @@ static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum, CALC_SUM_MAX_MIN(*sum, *max, *min, val); break; default: - ASSERT(0); break; } } @@ -3893,7 +3899,6 @@ static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum, CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); break; default: - ASSERT(0); break; } } @@ -3925,7 +3930,6 @@ static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum, CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); break; default: - ASSERT(0); break; } } @@ -3957,7 +3961,6 @@ static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *su CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); break; default: - ASSERT(0); break; } } @@ -3989,7 +3992,6 @@ static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_ CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); break; default: - ASSERT(0); break; } } @@ -4021,7 +4023,6 @@ static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, i CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); break; default: - ASSERT(0); break; } } @@ -4053,7 +4054,6 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); break; default: - ASSERT(0); break; } } @@ -4092,7 +4092,6 @@ static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum } break; default: - ASSERT(0); break; } } @@ -4153,7 +4152,9 @@ int32_t tValueColumnAppend(SValueColumn *valCol, const SValue *value) { valCol->type = value->type; } - ASSERT(value->type == valCol->type); + if (!(value->type == valCol->type)) { + return TSDB_CODE_INVALID_PARA; + } if (IS_VAR_DATA_TYPE(value->type)) { if ((code = tBufferPutI32(&valCol->offsets, tBufferGetSize(&valCol->data)))) { @@ -4230,7 +4231,9 @@ int32_t tValueColumnGet(SValueColumn *valCol, int32_t idx, SValue *value) { int32_t tValueColumnCompress(SValueColumn *valCol, SValueColumnCompressInfo *info, SBuffer *output, SBuffer *assist) { int32_t code; - ASSERT(valCol->numOfValues > 0); + if (!(valCol->numOfValues > 0)) { + return TSDB_CODE_INVALID_PARA; + } (*info) = (SValueColumnCompressInfo){ .cmprAlg = info->cmprAlg, @@ -4344,7 +4347,7 @@ int32_t tValueColumnCompressInfoDecode(SBufferReader *reader, SValueColumnCompre if ((code = tBufferGetI32v(reader, &info->dataOriginalSize))) return code; if ((code = tBufferGetI32v(reader, &info->dataCompressedSize))) return code; } else { - ASSERT(0); + return TSDB_CODE_INVALID_PARA; } return 0; @@ -4360,7 +4363,9 @@ int32_t tCompressData(void *input, // input int32_t code; extraSizeNeeded = (info->cmprAlg == NO_COMPRESSION) ? info->originalSize : info->originalSize + COMP_OVERFLOW_BYTES; - ASSERT(outputSize >= extraSizeNeeded); + if (!(outputSize >= extraSizeNeeded)) { + return TSDB_CODE_INVALID_PARA; + } if (info->cmprAlg == NO_COMPRESSION) { (void)memcpy(output, input, info->originalSize); @@ -4442,10 +4447,14 @@ int32_t tDecompressData(void *input, // input ) { int32_t code; - ASSERT(outputSize >= info->originalSize); + if (!(outputSize >= info->originalSize)) { + return TSDB_CODE_INVALID_PARA; + } if (info->cmprAlg == NO_COMPRESSION) { - ASSERT(info->compressedSize == info->originalSize); + if (!(info->compressedSize == info->originalSize)) { + return TSDB_CODE_INVALID_PARA; + } (void)memcpy(output, input, info->compressedSize); } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { SBuffer local; @@ -4478,7 +4487,9 @@ int32_t tDecompressData(void *input, // input return TSDB_CODE_COMPRESS_ERROR; } - ASSERT(decompressedSize == info->originalSize); + if (!(decompressedSize == info->originalSize)) { + return TSDB_CODE_COMPRESS_ERROR; + } tBufferDestroy(&local); } else { DEFINE_VAR(info->cmprAlg); @@ -4512,7 +4523,9 @@ int32_t tDecompressData(void *input, // input return TSDB_CODE_COMPRESS_ERROR; } - ASSERT(decompressedSize == info->originalSize); + if (!(decompressedSize == info->originalSize)) { + return TSDB_CODE_COMPRESS_ERROR; + } tBufferDestroy(&local); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index f02144a1d1..58a7f8f448 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -103,7 +103,9 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { pIter->totalLen = htonl(pMsg->length); pIter->numOfBlocks = htonl(pMsg->numOfBlocks); - ASSERT(pIter->totalLen > 0); + if (!(pIter->totalLen > 0)) { + return terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; + } pIter->len = 0; pIter->pMsg = pMsg; if (pIter->totalLen <= sizeof(SSubmitReq)) { @@ -114,17 +116,21 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { } int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { - ASSERT(pIter->len >= 0); + if (!(pIter->len >= 0)) { + return terrno = TSDB_CODE_INVALID_MSG_LEN; + } if (pIter->len == 0) { pIter->len += sizeof(SSubmitReq); } else { if (pIter->len >= pIter->totalLen) { - ASSERT(0); + return terrno = TSDB_CODE_INVALID_MSG_LEN; } pIter->len += (sizeof(SSubmitBlk) + pIter->dataLen + pIter->schemaLen); - ASSERT(pIter->len > 0); + if (!(pIter->len > 0)) { + return terrno = TSDB_CODE_INVALID_MSG_LEN; + } } if (pIter->len > pIter->totalLen) { @@ -377,7 +383,9 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) } } - ASSERT(desc.subPlanNum == taosArrayGetSize(desc.subDesc)); + if (!(desc.subPlanNum == taosArrayGetSize(desc.subDesc))) { + return TSDB_CODE_INVALID_MSG; + } if (!taosArrayPush(pReq->query->queryDesc, &desc)) { return terrno; @@ -8476,7 +8484,7 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { } else if (pReq->type == TSDB_NORMAL_TABLE) { if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1; } else { - ASSERT(0); + return TSDB_CODE_INVALID_MSG; } // ENCODESQL @@ -8528,7 +8536,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { } else if (pReq->type == TSDB_NORMAL_TABLE) { if (tDecodeSSchemaWrapperEx(pCoder, &pReq->ntb.schemaRow) < 0) return -1; } else { - ASSERT(0); + return TSDB_CODE_INVALID_MSG; } // DECODESQL @@ -9311,7 +9319,6 @@ bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) { return pLeft->uid == pRight->uid; } else { uError("offset type:%d", pLeft->type); - ASSERT(0); } } return false; @@ -9727,7 +9734,9 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm // auto create table if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { - ASSERT(pSubmitTbData->pCreateTbReq); + if (!(pSubmitTbData->pCreateTbReq)) { + return TSDB_CODE_INVALID_MSG; + } if (tEncodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) return -1; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 491c203a58..8d0f324509 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -105,7 +105,6 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { size_t tnameLen = strlen(name->tname); if (tnameLen > 0) { - /*ASSERT(name->type == TSDB_TABLE_NAME_T);*/ dst[len] = TS_PATH_DELIMITER[0]; memcpy(dst + len + 1, name->tname, tnameLen); @@ -160,7 +159,10 @@ int32_t tNameGetFullDbName(const SName* name, char* dst) { bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } const char* tNameGetTableName(const SName* name) { - ASSERT(name != NULL && name->type == TSDB_TABLE_NAME_T); + if (!(name != NULL && name->type == TSDB_TABLE_NAME_T)) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } return &name->tname[0]; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index cd238fef25..3cb8d5f41e 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -459,8 +459,8 @@ void taosVariantDestroy(SVariant *pVar) { } } -void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { - if (pSrc == NULL || pDst == NULL) return; +int32_t taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { + if (pSrc == NULL || pDst == NULL) return 0; pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_VARBINARY || @@ -468,19 +468,20 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; char *p = taosMemoryRealloc(pDst->pz, len); - ASSERT(p); + if (!p) return terrno; (void)memset(p, 0, len); pDst->pz = p; (void)memcpy(pDst->pz, pSrc->pz, pSrc->nLen); pDst->nLen = pSrc->nLen; - return; + return 0; } if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) { pDst->i = pSrc->i; } + return 0; } int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 3eb69afa40..3553810a08 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -745,7 +745,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool taosArraySort(pTableCxt->pData->aCol, insColDataComp); - tColDataSortMerge(pTableCxt->pData->aCol); + code = tColDataSortMerge(pTableCxt->pData->aCol); } else { // skip the table has no data to insert // eg: import a csv without valid data diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f878766f77..5c34a8c23f 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -42,10 +42,6 @@ struct SBTree { #define TDB_BTREE_PAGE_IS_ROOT(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT) #define TDB_BTREE_PAGE_IS_LEAF(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF) #define TDB_BTREE_PAGE_IS_OVFL(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_OVFL) -#define TDB_BTREE_ASSERT_FLAG(flags) \ - ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ - TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0) || \ - TDB_FLAG_IS(flags, TDB_BTREE_OVFL)) #pragma pack(push, 1) typedef struct { @@ -443,14 +439,12 @@ int tdbBtreeInitPage(SPage *pPage, void *arg, int init) { // init page flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); - TDB_BTREE_ASSERT_FLAG(flags); tdbPageInit(pPage, leaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); } else { // zero page flags = ((SBtreeInitPageArg *)arg)->flags; leaf = flags & TDB_BTREE_LEAF; - TDB_BTREE_ASSERT_FLAG(flags); tdbPageZero(pPage, leaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); From acdb14aefbeaebeab716c37e0d7b522007b2091b Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 22 Aug 2024 12:53:55 +0800 Subject: [PATCH 097/100] opeCFile with w+b mode --- source/libs/executor/src/tsort.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 5bd11489c9..6e23700d52 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1350,7 +1350,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) { } if (code == TSDB_CODE_SUCCESS) { taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath); - pMemFile->pTdFile = taosOpenCFile(pMemFile->memFilePath, "w+"); + pMemFile->pTdFile = taosOpenCFile(pMemFile->memFilePath, "w+b"); if (pMemFile->pTdFile == NULL) { code = terrno = TAOS_SYSTEM_ERROR(errno); } From ba8332e859d06edfdd004fcf3c4bfd3f30547c96 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 22 Aug 2024 13:39:53 +0800 Subject: [PATCH 098/100] fix if statement --- source/util/src/tlrucache.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 55531108f2..9c783f0cf6 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -701,9 +701,10 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo size_t perShard = (capacity + (numShards - 1)) / numShards; for (int i = 0; i < numShards; ++i) { if (TSDB_CODE_SUCCESS != - taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits)) + taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits)) { taosMemoryFree(cache); - return NULL; + return NULL; + } } cache->numShards = numShards; From c5291c2d1f816bfa3092cfea83fc8b567a4946c8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 22 Aug 2024 13:49:38 +0800 Subject: [PATCH 099/100] fix: return code issue --- source/libs/executor/inc/tsort.h | 2 +- source/libs/executor/src/mergeoperator.c | 6 +++--- source/libs/executor/src/scanoperator.c | 19 ++++++++++++------- source/libs/executor/src/sortoperator.c | 18 ++++++++++-------- source/libs/executor/src/tsort.c | 19 ++++++++++++------- 5 files changed, 38 insertions(+), 26 deletions(-) diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 474f3eedbf..6f6384af5b 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -66,7 +66,7 @@ typedef struct SMsortComparParam { typedef struct SSortHandle SSortHandle; typedef struct STupleHandle STupleHandle; -typedef SSDataBlock* (*_sort_fetch_block_fn_t)(void* param); +typedef int32_t (*_sort_fetch_block_fn_t)(void* param, SSDataBlock** ppBlock); typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* param); /** diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 8bc7c2db50..3f85324f57 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -65,10 +65,10 @@ static SSDataBlock* doNonSortMerge1(SOperatorInfo* pOperator); static SSDataBlock* doColsMerge1(SOperatorInfo* pOperator); static int32_t doColsMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock); -SSDataBlock* sortMergeloadNextDataBlock(void* param) { +int32_t sortMergeloadNextDataBlock(void* param, SSDataBlock** ppBlock) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator); - return pBlock; + *ppBlock = pOperator->fpSet.getNextFn(pOperator); + return TSDB_CODE_SUCCESS; } int32_t openSortMergeOperator(SOperatorInfo* pOperator) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1a6bb8a5cc..b7b5e4fff9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -5414,7 +5414,7 @@ static void doGetBlockForTableMergeScan(SOperatorInfo* pOperator, bool* pFinishe return; } -static SSDataBlock* getBlockForTableMergeScan(void* param) { +static int32_t getBlockForTableMergeScan(void* param, SSDataBlock** ppBlock) { STableMergeScanSortSourceParam* source = param; SOperatorInfo* pOperator = source->pOperator; @@ -5422,6 +5422,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSDataBlock* pBlock = NULL; int64_t st = taosGetTimestampUs(); + int32_t code = TSDB_CODE_SUCCESS; while (true) { if (pInfo->rtnNextDurationBlocks) { @@ -5456,10 +5457,11 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { if (pInfo->bNextDurationBlockEvent || pInfo->bNewFilesetEvent) { if (!bSkipped) { - int32_t code = createOneDataBlock(pBlock, true, &pInfo->nextDurationBlocks[pInfo->numNextDurationBlocks]); + code = createOneDataBlock(pBlock, true, &pInfo->nextDurationBlocks[pInfo->numNextDurationBlocks]); if (code) { terrno = code; - return NULL; + *ppBlock = NULL; + return code; } ++pInfo->numNextDurationBlocks; @@ -5473,7 +5475,8 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { if (pInfo->bNewFilesetEvent) { pInfo->rtnNextDurationBlocks = true; - return NULL; + *ppBlock = NULL; + return code; } if (pInfo->bNextDurationBlockEvent) { @@ -5488,11 +5491,13 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - - return pBlock; + *ppBlock = pBlock; + + return code; } - return NULL; + *ppBlock = NULL; + return code; } int32_t generateSortByTsPkInfo(SArray* colMatchInfo, int32_t order, SArray** ppSortArray) { diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index cb15c3c836..e23988d357 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -337,10 +337,10 @@ static int32_t getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, return code; } -SSDataBlock* loadNextDataBlock(void* param) { +int32_t loadNextDataBlock(void* param, SSDataBlock** ppBlock) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator); - return pBlock; + *ppBlock = pOperator->fpSet.getNextFn(pOperator); + return TSDB_CODE_SUCCESS; } // todo refactor: merged with fetch fp @@ -609,30 +609,32 @@ typedef struct SGroupSortSourceParam { SGroupSortOperatorInfo* grpSortOpInfo; } SGroupSortSourceParam; -SSDataBlock* fetchNextGroupSortDataBlock(void* param) { +int32_t fetchNextGroupSortDataBlock(void* param, SSDataBlock** ppBlock) { + *ppBlock = NULL; + SGroupSortSourceParam* source = param; SGroupSortOperatorInfo* grpSortOpInfo = source->grpSortOpInfo; if (grpSortOpInfo->prefetchedSortInput) { SSDataBlock* block = grpSortOpInfo->prefetchedSortInput; grpSortOpInfo->prefetchedSortInput = NULL; - return block; + *ppBlock = block; } else { SOperatorInfo* childOp = source->childOpInfo; SSDataBlock* block = childOp->fpSet.getNextFn(childOp); if (block != NULL) { if (block->info.id.groupId == grpSortOpInfo->currGroupId) { grpSortOpInfo->childOpStatus = CHILD_OP_SAME_GROUP; - return block; + *ppBlock = block; } else { grpSortOpInfo->childOpStatus = CHILD_OP_NEW_GROUP; grpSortOpInfo->prefetchedSortInput = block; - return NULL; } } else { grpSortOpInfo->childOpStatus = CHILD_OP_FINISHED; - return NULL; } } + + return TSDB_CODE_SUCCESS; } int32_t beginSortGroup(SOperatorInfo* pOperator) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 5bd11489c9..61cbbab72e 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -619,7 +619,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 for (int32_t i = 0; i < pParam->numOfSources; ++i) { SSortSource* pSource = pParam->pSources[i]; - pSource->src.pBlock = pHandle->fetchfp(pSource->param); + TAOS_CHECK_RETURN(pHandle->fetchfp(pSource->param, &pSource->src.pBlock)); // set current source is done if (pSource->src.pBlock == NULL) { @@ -711,7 +711,7 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT } } else { int64_t st = taosGetTimestampUs(); - pSource->src.pBlock = pHandle->fetchfp(((SSortSource*)pSource)->param); + TAOS_CHECK_RETURN(pHandle->fetchfp(((SSortSource*)pSource)->param, &pSource->src.pBlock)); pSource->fetchUs += taosGetTimestampUs() - st; pSource->fetchNum++; if (pSource->src.pBlock == NULL) { @@ -2236,8 +2236,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { while (1) { bool bExtractedBlock = false; bool bSkipBlock = false; - - SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param); + SSDataBlock* pBlk = NULL; + + TAOS_CHECK_RETURN(pHandle->fetchfp(pSrc->param, &pBlk)); if (pBlk != NULL && pHandle->mergeLimit > 0) { SSDataBlock* p = NULL; code = getRowsBlockWithinMergeLimit(pHandle, mTableNumRows, pBlk, &bExtractedBlock, &bSkipBlock, &p); @@ -2390,7 +2391,8 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); while (1) { - SSDataBlock* pBlock = pHandle->fetchfp(source->param); + SSDataBlock* pBlock = NULL; + TAOS_CHECK_RETURN(pHandle->fetchfp(source->param, &pBlock)); if (pBlock == NULL) { break; } @@ -2701,7 +2703,8 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) { while (1) { // fetch data - SSDataBlock* pBlock = pHandle->fetchfp(source->param); + SSDataBlock* pBlock = NULL; + TAOS_CHECK_RETURN(pHandle->fetchfp(source->param, &pBlock)); if (NULL == pBlock) { break; } @@ -2828,7 +2831,9 @@ static int32_t tsortSingleTableMergeNextTuple(SSortHandle* pHandle, STupleHandle } SSortSource* source = *pSource; - SSDataBlock* pBlock = pHandle->fetchfp(source->param); + SSDataBlock* pBlock = NULL; + TAOS_CHECK_RETURN(pHandle->fetchfp(source->param, &pBlock)); + if (!pBlock || pBlock->info.rows == 0) { setCurrentSourceDone(source, pHandle); pHandle->tupleHandle.pBlock = NULL; From 1bc4521aa1b7488fadbcd85bcab16f5f02e64676 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 22 Aug 2024 14:32:34 +0800 Subject: [PATCH 100/100] fix(tdb/btree): use unsigned type for memcpy --- source/libs/tdb/src/db/tdbBtree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 5c34a8c23f..ff40616d70 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1133,7 +1133,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const memcpy(pCell + nLocal - sizeof(pgno), &pgno, sizeof(pgno)); - int lastKeyPageSpace = 0; + size_t lastKeyPageSpace = 0; // pack left key & val to ovpages do { // cal key to cpy