From 2d478d389697803254cbc4dd90b1517f66f4cd5c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Aug 2024 10:16:16 +0800 Subject: [PATCH 01/31] refactor: update logs. --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 82b0a729b0..09a6c7ca1e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -6458,8 +6458,8 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO _end: if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; - T_LONG_JMP(pTaskInfo->env, code); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + T_LONG_JMP(pTaskInfo->env, code); } setOperatorCompleted(pOperator); } From 9dd3b1b56b6aeb2ccccf8ab2abf698773e9d4b7a Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 26 Aug 2024 13:15:10 +0800 Subject: [PATCH 02/31] fix create tsma on Windows crash --- source/dnode/mnode/impl/src/mndSma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 303e86bb82..7467d9bba3 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1845,8 +1845,7 @@ static int32_t mndTSMAGenerateOutputName(const char* tsmaName, char* streamName, static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { #ifdef WINDOWS - terrno = TSDB_CODE_MND_INVALID_PLATFORM; - goto _OVER; + TAOS_RETURN(TSDB_CODE_MND_INVALID_PLATFORM); #endif SMnode * pMnode = pReq->info.node; int32_t code = -1; From 42a31d4ca1fe2eb27cbf5219e1a319cdc524c9ba Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 26 Aug 2024 15:32:50 +0800 Subject: [PATCH 03/31] fix(query):fix mem leak for fill --- source/libs/executor/src/streamfilloperator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index fac1cf48c7..453e64a151 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -1207,7 +1207,12 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols, (const SNodeListNode*)(pPhyFillNode->pValues)); - QUERY_CHECK_NULL(pFillSup->pAllColInfo, code, lino, _end, terrno); + if (pFillSup->pAllColInfo == NULL) { + code = terrno; + lino = __LINE__; + destroyExprInfo(noFillExprInfo, numOfNotFillCols); + goto _end; + } pFillSup->type = convertFillType(pPhyFillNode->mode); pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; From dc941cb704f62bdcd3c56b6b4d469c0233ecb6b8 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 26 Aug 2024 16:53:19 +0800 Subject: [PATCH 04/31] fix(query):fix mem leak for function --- source/libs/executor/src/executil.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 29ffd900f2..892f7948b4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1970,6 +1970,8 @@ int32_t createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, SExprInfo** // set the output buffer for the selectivity + tag query static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { int32_t num = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SqlFunctionCtx* p = NULL; SqlFunctionCtx** pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); @@ -1978,6 +1980,8 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu } SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + QUERY_CHECK_NULL(pSelectFuncs, code, lino, _end, terrno); + for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0) || @@ -1991,8 +1995,8 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu } else { int32_t tempRes = taosHashPut(pSelectFuncs, pName, strlen(pName), &num, sizeof(num)); if (tempRes != TSDB_CODE_SUCCESS && tempRes != TSDB_CODE_DUP_KEY) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(tempRes)); - return tempRes; + code = tempRes; + QUERY_CHECK_CODE(code, lino, _end); } p = &pCtx[i]; } @@ -2007,7 +2011,13 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu taosMemoryFreeClear(pValCtx); } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pValCtx); + taosHashCleanup(pSelectFuncs); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, From f4916211cacaab74f48f655eaab58e6c5be562f8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Aug 2024 16:56:48 +0800 Subject: [PATCH 05/31] enh: clear more asserts --- include/common/tmsg.h | 12 ++++++++---- include/util/tdef.h | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 5 ----- source/dnode/snode/src/snode.c | 8 +++++--- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 +--- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 4 +++- source/dnode/vnode/src/tsdb/tsdbWrite.c | 4 ++-- source/util/src/ttimer.c | 6 ++++-- source/util/src/tworker.c | 5 ++--- 10 files changed, 27 insertions(+), 25 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3dee29f864..010e538fb6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -707,7 +707,9 @@ static FORCE_INLINE SColCmprWrapper* tCloneSColCmprWrapper(const SColCmprWrapper } static FORCE_INLINE int32_t tInitDefaultSColCmprWrapperByCols(SColCmprWrapper* pCmpr, int32_t nCols) { - assert(!pCmpr->pColCmpr); + if (!(!pCmpr->pColCmpr)) { + return TSDB_CODE_INVALID_PARA; + } pCmpr->pColCmpr = (SColCmpr*)taosMemoryCalloc(nCols, sizeof(SColCmpr)); if (pCmpr->pColCmpr == NULL) { return terrno; @@ -718,7 +720,9 @@ static FORCE_INLINE int32_t tInitDefaultSColCmprWrapperByCols(SColCmprWrapper* p static FORCE_INLINE int32_t tInitDefaultSColCmprWrapper(SColCmprWrapper* pCmpr, SSchemaWrapper* pSchema) { pCmpr->nCols = pSchema->nCols; - assert(!pCmpr->pColCmpr); + if (!(!pCmpr->pColCmpr)) { + return TSDB_CODE_INVALID_PARA; + } pCmpr->pColCmpr = (SColCmpr*)taosMemoryCalloc(pCmpr->nCols, sizeof(SColCmpr)); if (pCmpr->pColCmpr == NULL) { return terrno; @@ -2813,8 +2817,8 @@ enum { TOPIC_SUB_TYPE__COLUMN, }; -#define DEFAULT_MAX_POLL_INTERVAL 3000000 -#define DEFAULT_SESSION_TIMEOUT 12000 +#define DEFAULT_MAX_POLL_INTERVAL 3000000 +#define DEFAULT_SESSION_TIMEOUT 12000 typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic diff --git a/include/util/tdef.h b/include/util/tdef.h index f087c28684..a750074953 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -40,7 +40,7 @@ extern const int32_t TYPE_BYTES[21]; #define LONG_BYTES sizeof(int64_t) #define FLOAT_BYTES sizeof(float) #define DOUBLE_BYTES sizeof(double) -#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) +#define POINTER_BYTES sizeof(void *) #define TSDB_KEYSIZE sizeof(TSKEY) #define TSDB_NCHAR_SIZE sizeof(TdUcs4) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index e599676cec..e85794b568 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -167,7 +167,6 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { SVnodeObj *pOld = NULL; (void)taosHashGetDup(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld); if (pOld) { - ASSERT(pOld->failed); vmFreeVnodeObj(&pOld); } int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); @@ -190,7 +189,6 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) vmReleaseVnode(pMgmt, pVnode); if (pVnode->failed) { - ASSERT(pVnode->pImpl == NULL); goto _closed; } dInfo("vgId:%d, pre close", pVnode->vgId); @@ -692,8 +690,6 @@ static void *vmRestoreVnodeInThread(void *param) { continue; } - ASSERT(pVnode->pImpl); - char stepDesc[TSDB_STEP_DESC_LEN] = {0}; snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been restored", pVnode->vgId, pMgmt->state.openVnodes, pMgmt->state.totalVnodes); @@ -764,7 +760,6 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) { dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno)); - ASSERT(errno == 0); } (void)taosThreadAttrDestroy(&thAttr); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index ce7c8aee7f..7fc079677b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -26,7 +26,9 @@ // clang-format on int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) { - ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0); + if (!(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0)) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = streamTaskInit(pTask, pSnode->pMeta, &pSnode->msgCb, nextProcessVer); if (code != TSDB_CODE_SUCCESS) { return code; @@ -135,7 +137,7 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return tqStreamTaskProcessRetrieveTriggerRsp(pSnode->pMeta, pMsg); default: sndError("invalid snode msg:%d", pMsg->msgType); - ASSERT(0); + return TSDB_CODE_INVALID_MSG; } return 0; } @@ -164,7 +166,7 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { case TDMT_STREAM_CONSEN_CHKPT: return tqStreamTaskProcessConsenChkptIdReq(pSnode->pMeta, pMsg); default: - ASSERT(0); + return TSDB_CODE_INVALID_MSG; } return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 21c5d33ec9..bc5fcfc27c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -1127,7 +1127,7 @@ int32_t tMergeTreeNext(SMergeTree *pMTree, bool *pHasNext) { (void)tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; } else { - ASSERT(c); + return TSDB_CODE_INTERNAL_ERROR; } } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 391b7f636d..a232891aad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1081,7 +1081,6 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo int32_t step = asc ? 1 : -1; // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit - // ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // 1. copy data in a batch model (void)memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); @@ -3828,7 +3827,6 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t return false; } - // ASSERT(key >= last->ts); if (key > last->ts) { return false; } else if (key == last->ts) { @@ -3891,7 +3889,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t } else if (key == pFirst->ts) { return pFirst->version >= ver; } else { - // ASSERT(0); + tsdbError("unexpected error, key:%" PRId64 ", first:%" PRId64, key, pFirst->ts); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 5d4dc94431..e0e428be7e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -1319,7 +1319,9 @@ static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const S return true; } } else { // it must be the last point - ASSERT(p->version == 0); + if (!(p->version == 0)) { + tsdbError("unexpected version:%d", p->version); + } } } } else { // (p->ts > pBlock->maxKey.ts) { diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index a35068163f..8104d64c99 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -33,8 +33,8 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 int32_t affectedrows = 0; int32_t numOfRows = 0; - if (ASSERTS(pTsdb->mem != NULL, "vgId:%d, mem is NULL", TD_VID(pTsdb->pVnode))) { - TAOS_RETURN(TSDB_CODE_INVALID_PTR); + if (pTsdb->mem == NULL) { + TAOS_RETURN(TSDB_CODE_INTERNAL_ERROR); } arrSize = taosArrayGetSize(pMsg->aSubmitTbData); diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 6ade226cd8..60edb2f045 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -159,7 +159,7 @@ static void lockTimerList(timer_list_t* list) { static void unlockTimerList(timer_list_t* list) { int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&(list->lockedBy), tid, 0) != tid) { - ASSERTS(false, "%" PRId64 " trying to unlock a timer list not locked by current thread.", tid); + uError("%" PRId64 " trying to unlock a timer list not locked by current thread.", tid); } } @@ -505,7 +505,9 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han } } - ASSERTS(timer->refCount == 1, "timer refCount=%d not expected 1", timer->refCount); + if (timer->refCount == 1) { + uError("timer refCount=%d not expected 1", timer->refCount); + } memset(timer, 0, sizeof(*timer)); *pTmrId = (tmr_h)doStartTimer(timer, fp, mseconds, param, ctrl); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index b2064d6787..60444e586f 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -494,7 +494,7 @@ int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) } } break; default: - assert(0); + return TSDB_CODE_INVALID_PARA; } return 0; } @@ -517,7 +517,7 @@ void tSingleWorkerCleanup(SSingleWorker *pWorker) { taosMemoryFree(pWorker->pool); break; default: - assert(0); + break; } } @@ -812,7 +812,6 @@ int32_t tQueryAutoQWorkerInit(SQueryAutoQWorkerPool *pool) { if (!pool->exitedWorkers) return TSDB_CODE_OUT_OF_MEMORY; pool->maxInUse = pool->max * 2 + 2; - if (!pool->pCb) { pool->pCb = taosMemoryCalloc(1, sizeof(SQueryAutoQWorkerPoolCB)); if (!pool->pCb) return TSDB_CODE_OUT_OF_MEMORY; From dc10857eec6b5d481ab860db9284e27a4747c47c Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 26 Aug 2024 09:05:57 +0000 Subject: [PATCH 06/31] fix/TD-31542-remove-assert-rename --- source/libs/monitorfw/inc/{taos_assert.h => taos_test.h} | 2 -- source/libs/monitorfw/src/taos_collector.c | 2 +- source/libs/monitorfw/src/taos_collector_registry.c | 2 +- source/libs/monitorfw/src/taos_counter.c | 2 +- source/libs/monitorfw/src/taos_gauge.c | 2 +- source/libs/monitorfw/src/taos_linked_list.c | 2 +- source/libs/monitorfw/src/taos_map.c | 2 +- source/libs/monitorfw/src/taos_metric.c | 2 +- source/libs/monitorfw/src/taos_metric_formatter.c | 3 +-- source/libs/monitorfw/src/taos_metric_formatter_custom.c | 2 +- source/libs/monitorfw/src/taos_metric_sample.c | 2 +- source/libs/monitorfw/src/taos_string_builder.c | 2 +- 12 files changed, 11 insertions(+), 14 deletions(-) rename source/libs/monitorfw/inc/{taos_assert.h => taos_test.h} (97%) diff --git a/source/libs/monitorfw/inc/taos_assert.h b/source/libs/monitorfw/inc/taos_test.h similarity index 97% rename from source/libs/monitorfw/inc/taos_assert.h rename to source/libs/monitorfw/inc/taos_test.h index b73e6572b6..b881b2cee2 100644 --- a/source/libs/monitorfw/inc/taos_assert.h +++ b/source/libs/monitorfw/inc/taos_test.h @@ -14,8 +14,6 @@ * limitations under the License. */ -#include - #ifndef TAOS_TEST_H #define TAOS_TEST_H diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c index 6ba34725b7..21ebb3f737 100644 --- a/source/libs/monitorfw/src/taos_collector.c +++ b/source/libs/monitorfw/src/taos_collector.c @@ -22,7 +22,7 @@ #include "taos_collector_registry.h" // Private -#include "taos_assert.h" +#include "taos_test.h" #include "taos_collector_t.h" #include "taos_log.h" #include "taos_map_i.h" diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index f028dabce8..94295bf9c0 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -24,7 +24,7 @@ #include "taos_collector_registry.h" // Private -#include "taos_assert.h" +#include "taos_test.h" #include "taos_collector_registry_t.h" #include "taos_collector_t.h" #include "taos_errors.h" diff --git a/source/libs/monitorfw/src/taos_counter.c b/source/libs/monitorfw/src/taos_counter.c index 5588286988..ef7d41cf2c 100644 --- a/source/libs/monitorfw/src/taos_counter.c +++ b/source/libs/monitorfw/src/taos_counter.c @@ -20,7 +20,7 @@ #include "taos_alloc.h" // Private -#include "taos_assert.h" +#include "taos_test.h" #include "taos_errors.h" #include "taos_log.h" #include "taos_metric_i.h" diff --git a/source/libs/monitorfw/src/taos_gauge.c b/source/libs/monitorfw/src/taos_gauge.c index 5b3eedeb84..00bf22c44d 100644 --- a/source/libs/monitorfw/src/taos_gauge.c +++ b/source/libs/monitorfw/src/taos_gauge.c @@ -20,7 +20,7 @@ #include "taos_alloc.h" // Private -#include "taos_assert.h" +#include "taos_test.h" #include "taos_errors.h" #include "taos_log.h" #include "taos_metric_i.h" diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c index eab1d3f5db..293f3f60a8 100644 --- a/source/libs/monitorfw/src/taos_linked_list.c +++ b/source/libs/monitorfw/src/taos_linked_list.c @@ -18,7 +18,7 @@ #include "taos_alloc.h" // Private -#include "taos_assert.h" +#include "taos_test.h" #include "taos_linked_list_i.h" #include "taos_linked_list_t.h" #include "taos_log.h" diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c index ba325ac517..2f5bf566c2 100644 --- a/source/libs/monitorfw/src/taos_map.c +++ b/source/libs/monitorfw/src/taos_map.c @@ -21,13 +21,13 @@ #include "taos_alloc.h" // Private -#include "taos_assert.h" #include "taos_errors.h" #include "taos_linked_list_i.h" #include "taos_linked_list_t.h" #include "taos_log.h" #include "taos_map_i.h" #include "taos_map_t.h" +#include "taos_test.h" #define TAOS_MAP_INITIAL_SIZE 32 diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c index 7d35dc2421..42564437d0 100644 --- a/source/libs/monitorfw/src/taos_metric.c +++ b/source/libs/monitorfw/src/taos_metric.c @@ -20,13 +20,13 @@ #include "taos_alloc.h" // Private -#include "taos_assert.h" #include "taos_errors.h" #include "taos_log.h" #include "taos_map_i.h" #include "taos_metric_formatter_i.h" #include "taos_metric_i.h" #include "taos_metric_sample_i.h" +#include "taos_test.h" char *taos_metric_type_map[4] = {"counter", "gauge", "histogram", "summary"}; diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c index 3d3b456aad..a20a8d919c 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter.c +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -20,7 +20,6 @@ #include "taos_alloc.h" // Private -#include "taos_assert.h" #include "taos_collector_t.h" #include "taos_linked_list_t.h" #include "taos_map_i.h" @@ -28,7 +27,7 @@ #include "taos_metric_sample_t.h" #include "taos_metric_t.h" #include "taos_string_builder_i.h" - +#include "taos_test.h" taos_metric_formatter_t *taos_metric_formatter_new() { taos_metric_formatter_t *self = (taos_metric_formatter_t *)taos_malloc(sizeof(taos_metric_formatter_t)); diff --git a/source/libs/monitorfw/src/taos_metric_formatter_custom.c b/source/libs/monitorfw/src/taos_metric_formatter_custom.c index 1cf8560d21..da05d09d9c 100644 --- a/source/libs/monitorfw/src/taos_metric_formatter_custom.c +++ b/source/libs/monitorfw/src/taos_metric_formatter_custom.c @@ -21,7 +21,7 @@ #include "taos_metric_sample_t.h" #include "tjson.h" #include "taos_monitor_util_i.h" -#include "taos_assert.h" +#include "taos_test.h" #include "tdef.h" #include "taos_collector_t.h" #include "taos_log.h" diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c index e45671bd79..e4b41d5475 100644 --- a/source/libs/monitorfw/src/taos_metric_sample.c +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -20,11 +20,11 @@ #include "taos_alloc.h" // Private -#include "taos_assert.h" #include "taos_errors.h" #include "taos_log.h" #include "taos_metric_sample_i.h" #include "taos_metric_sample_t.h" +#include "taos_test.h" #ifdef C11_ATOMIC #include diff --git a/source/libs/monitorfw/src/taos_string_builder.c b/source/libs/monitorfw/src/taos_string_builder.c index 5180a72d5c..6e3fe1d2e3 100644 --- a/source/libs/monitorfw/src/taos_string_builder.c +++ b/source/libs/monitorfw/src/taos_string_builder.c @@ -20,9 +20,9 @@ #include "taos_alloc.h" // Private -#include "taos_assert.h" #include "taos_string_builder_i.h" #include "taos_string_builder_t.h" +#include "taos_test.h" // The initial size of a string created via taos_string_builder #define TAOS_STRING_BUILDER_INIT_SIZE 32 From 6e53fa15dce46c7c5d4e00563e26db6b053e8db8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 26 Aug 2024 17:06:31 +0800 Subject: [PATCH 07/31] fix trans retry error --- source/dnode/mnode/impl/src/mndIndex.c | 14 +++++++------- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 8786c3a934..6dd963b0ba 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -89,7 +89,7 @@ static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName, } } - return -1; + return TSDB_CODE_MND_TAG_NOT_EXIST; } int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { @@ -188,7 +188,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb sdbRelease(pSdb, pVgroup); code = TSDB_CODE_MND_RETURN_VALUE_NULL; if (terrno != 0) code = terrno; - return -1; + return code; } STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -199,7 +199,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + return code; } sdbRelease(pSdb, pVgroup); } @@ -899,8 +899,8 @@ static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) { int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SIdxObj *pIdx = NULL; @@ -944,8 +944,8 @@ int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxOb } int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SIdxObj *pIdx = NULL; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0334990365..9270a3e681 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -808,7 +808,7 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) } if (pCol == NULL) { - code = TSDB_CODE_VND_COL_NOT_EXISTS; + code = 0; goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 75db6e2925..7b61ed3cdf 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -2327,6 +2327,7 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, pRsp->contLen = 0; if ((code = tDeserializeSDropIdxReq(pReq, len, &req))) { + pRsp->code = code; return code; } From 9c6dcda4d82df90c75d23a5d53d5aa6554640fd9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 26 Aug 2024 17:35:59 +0800 Subject: [PATCH 08/31] fix trans retry error --- source/dnode/vnode/src/meta/metaTable.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 9270a3e681..2e7efb004d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -808,7 +808,10 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) } if (pCol == NULL) { + metaError("vgId:%d, failed to drop index on %s.%s,since %s", TD_VID(pMeta->pVnode), pReq->stb, pReq->colName, + tstrerror(TSDB_CODE_VND_COL_NOT_EXISTS)); code = 0; + goto _err; } From e2481c4409aaa397ec64c8f929137ab0bff8aaa4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 26 Aug 2024 17:40:29 +0800 Subject: [PATCH 09/31] fix trans retry error --- source/dnode/mnode/impl/src/mndIndex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 6dd963b0ba..5ff577ae06 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -824,7 +824,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p } mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name); - mndTransSetDbName(pTrans, pDb->name, NULL); + mndTransSetDbName(pTrans, pDb->name, pStb->name); TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); mndTransSetSerial(pTrans); From f3bec3829078f025433a62f5fb5aa99ae8a3af35 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Mon, 26 Aug 2024 14:23:50 +0800 Subject: [PATCH 10/31] fix:[TD-31684] Modify GROUP/PARTITION BY clause to prioritize treating columns as column names before considering them as aliases --- source/libs/parser/src/parTranslater.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7e84ef8482..e5e47a6c9b 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1476,7 +1476,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p pCxt->errCode = getFuncInfo(pCxt, (SFunctionNode*)pFoundNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { if (fmIsVectorFunc(((SFunctionNode*)pFoundNode)->funcId)) { - pCxt->errCode = TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, (*pCol)->colName); return DEAL_RES_ERROR; } else if (fmIsPseudoColumnFunc(((SFunctionNode*)pFoundNode)->funcId)) { if ('\0' != (*pCol)->tableAlias[0]) { @@ -1500,6 +1500,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p nodesDestroyNode(*(SNode**)pCol); *(SNode**)pCol = (SNode*)pNew; if (QUERY_NODE_COLUMN == nodeType(pFoundNode)) { + pCxt->errCode = TSDB_CODE_SUCCESS; if ('\0' != (*pCol)->tableAlias[0]) { return translateColumnWithPrefix(pCxt, pCol); } else { @@ -1776,7 +1777,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { res = translateColumnWithPrefix(pCxt, pCol); } else { bool found = false; - if ((clauseSupportAlias(pCxt->currClause)) && + if ((pCxt->currClause == SQL_CLAUSE_ORDER_BY) && !(*pCol)->node.asParam) { res = translateColumnUseAlias(pCxt, pCol, &found); } @@ -5292,6 +5293,7 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST); } bool other; + pCxt->currClause = SQL_CLAUSE_GROUP_BY; int32_t code = translateClausePosition(pCxt, pSelect->pProjectionList, pSelect->pGroupByList, &other); if (TSDB_CODE_SUCCESS == code) { if (0 == LIST_LENGTH(pSelect->pGroupByList)) { @@ -5301,7 +5303,6 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { code = replaceGroupByAlias(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { - pCxt->currClause = SQL_CLAUSE_GROUP_BY; pSelect->timeLineResMode = TIME_LINE_NONE; code = translateExprList(pCxt, pSelect->pGroupByList); } From 5b68cc1dd37e2d7b0aee5e98e0e670657995b0f7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Aug 2024 19:20:30 +0800 Subject: [PATCH 11/31] fix: error log --- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index e0e428be7e..23ae33ee3b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -1320,7 +1320,7 @@ static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const S } } else { // it must be the last point if (!(p->version == 0)) { - tsdbError("unexpected version:%d", p->version); + tsdbError("unexpected version:%" PRId64, p->version); } } } From a3573aae67b3ee64dfb8ee5810b5998dcd96792b Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Mon, 26 Aug 2024 16:38:20 +0800 Subject: [PATCH 12/31] fix:[TD-31700] fix memory leak when error occurs in sclInitOperatorParams. --- source/libs/scalar/src/scalar.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9428f051aa..0ee1941717 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -594,7 +594,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal return TSDB_CODE_SUCCESS; _return: - taosMemoryFreeClear(paramList); + sclFreeParamList(paramList, paramNum); SCL_RET(code); } From 1db3bdb4df9a1f7434ef53d7ed4759152e8e48b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Aug 2024 09:32:48 +0800 Subject: [PATCH 13/31] enh: deal with return codes --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 15 ++++- source/dnode/mgmt/node_util/src/dmEps.c | 10 ++- source/dnode/vnode/src/inc/vnodeInt.h | 8 +-- source/dnode/vnode/src/meta/metaQuery.c | 7 +- source/dnode/vnode/src/meta/metaTable.c | 75 +++++++++++++++++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- source/util/src/tcache.c | 30 ++++++--- 7 files changed, 115 insertions(+), 34 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index fe438c4396..4cb6c5c724 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -57,7 +57,11 @@ void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { if (!pVnode->failed) { SVnodeLoadLite vload = {0}; if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) { - (void)taosArrayPush(pInfo->pVloads, &vload); + if (taosArrayPush(pInfo->pVloads, &vload) == NULL) { + taosArrayDestroy(pInfo->pVloads); + pInfo->pVloads = NULL; + break; + } } } pIter = taosHashIterate(pMgmt->hash, pIter); @@ -841,6 +845,9 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { arbHbRsp.dnodeId = pMgmt->pData->dnodeId; strncpy(arbHbRsp.arbToken, arbHbReq.arbToken, TSDB_ARB_TOKEN_SIZE); arbHbRsp.hbMembers = taosArrayInit(size, sizeof(SVArbHbRspMember)); + if (arbHbRsp.hbMembers == NULL) { + goto _OVER; + } for (int32_t i = 0; i < size; i++) { SVArbHbReqMember *pReqMember = taosArrayGet(arbHbReq.hbMembers, i); @@ -865,7 +872,11 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { continue; } - (void)taosArrayPush(arbHbRsp.hbMembers, &rspMember); + if (taosArrayPush(arbHbRsp.hbMembers, &rspMember) == NULL) { + dError("dnodeId:%d vgId:%d failed to push arb hb rsp member", arbHbReq.dnodeId, pReqMember->vgId); + vmReleaseVnode(pMgmt, pVnode); + goto _OVER; + } vmReleaseVnode(pMgmt, pVnode); } diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index af5530215b..315c4d7430 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -256,7 +256,9 @@ _OVER: SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; (void)taosGetFqdnPortFromEp(tsFirst, &dnodeEp.ep); - (void)taosArrayPush(pData->dnodeEps, &dnodeEp); + if (taosArrayPush(pData->dnodeEps, &dnodeEp) == NULL) { + return terrno; + } } if ((code = dmReadDnodePairs(pData)) != 0) { @@ -398,7 +400,11 @@ static void dmResetEps(SDnodeData *pData, SArray *dnodeEps) { for (int32_t i = 0; i < numOfEps; i++) { SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i); - (void)taosHashPut(pData->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); + int32_t code = taosHashPut(pData->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); + if (code) { + dError("dnode:%d, fqdn:%s port:%u isMnode:%d failed to put into hash, reason:%s", pDnodeEp->id, pDnodeEp->ep.fqdn, + pDnodeEp->ep.port, pDnodeEp->isMnode, tstrerror(code)); + } } pData->validMnodeEps = true; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 6f562cef4a..820abcaea6 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -156,7 +156,7 @@ int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); int32_t metaTrimTables(SMeta* pMeta); -void metaDropTables(SMeta* pMeta, SArray* tbUids); +int32_t metaDropTables(SMeta* pMeta, SArray* tbUids); int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs); @@ -435,9 +435,9 @@ typedef struct SVCommitSched { } SVCommitSched; typedef struct SVMonitorObj { - char strClusterId[TSDB_CLUSTER_ID_LEN]; - char strDnodeId[TSDB_NODE_ID_LEN]; - char strVgId[TSDB_VGROUP_ID_LEN]; + char strClusterId[TSDB_CLUSTER_ID_LEN]; + char strDnodeId[TSDB_NODE_ID_LEN]; + char strVgId[TSDB_VGROUP_ID_LEN]; } SVMonitorObj; typedef struct { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 1668f699ae..7daea64e18 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1486,7 +1486,12 @@ int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { taosHashInit(numOfElems / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < numOfElems; i++) { STUidTagInfo *pTagInfo = taosArrayGet(pUidTagInfo, i); - (void)taosHashPut(pSepecifiedUidMap, &pTagInfo->uid, sizeof(uint64_t), &i, sizeof(int32_t)); + int32_t code = taosHashPut(pSepecifiedUidMap, &pTagInfo->uid, sizeof(uint64_t), &i, sizeof(int32_t)); + if (code) { + metaCloseCtbCursor(pCur); + taosHashCleanup(pSepecifiedUidMap); + return code; + } } } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0334990365..5d87c02ab2 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -381,7 +381,11 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb break; } - (void)taosArrayPush(tbUidList, &(((SCtbIdxKey *)pKey)->uid)); + if (taosArrayPush(tbUidList, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { + tdbFree(pKey); + (void)tdbTbcClose(pCtbIdxc); + return terrno; + } } (void)tdbTbcClose(pCtbIdxc); @@ -505,7 +509,11 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; } - oStbEntry.pBuf = taosMemoryMalloc(nData); + if ((oStbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { + (void)tdbTbcClose(pTbDbc); + (void)tdbTbcClose(pUidIdxc); + return terrno; + } memcpy(oStbEntry.pBuf, pData, nData); tDecoderInit(&dc, oStbEntry.pBuf, nData); (void)metaDecodeEntry(&dc, &oStbEntry); @@ -527,6 +535,13 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { STsdb *pTsdb = pMeta->pVnode->pTsdb; SArray *uids = taosArrayInit(8, sizeof(int64_t)); + if (uids == NULL) { + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + (void)tdbTbcClose(pTbDbc); + (void)tdbTbcClose(pUidIdxc); + return terrno; + } if (deltaCol == 1) { int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId; int8_t col_type = pReq->schemaRow.pSchema[nCols - 1].type; @@ -1108,7 +1123,10 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi } if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) { - (void)taosArrayPush(tbUids, &uid); + if (taosArrayPush(tbUids, &uid) == NULL) { + rc = terrno; + goto _exit; + } if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { (void)tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); @@ -1125,11 +1143,15 @@ _exit: return rc; } -void metaDropTables(SMeta *pMeta, SArray *tbUids) { - if (taosArrayGetSize(tbUids) == 0) return; +int32_t metaDropTables(SMeta *pMeta, SArray *tbUids) { + int32_t code = 0; + if (taosArrayGetSize(tbUids) == 0) return TSDB_CODE_SUCCESS; int64_t nCtbDropped = 0; SSHashObj *suidHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (suidHash == NULL) { + return terrno; + } metaWLock(pMeta); for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { @@ -1137,7 +1159,8 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) { tb_uid_t suid = 0; int8_t sysTbl = 0; int type; - (void)metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); + code = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); + if (code) return code; if (!sysTbl && type == TSDB_CHILD_TABLE && suid != 0 && suidHash) { int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t)); if (pVal) { @@ -1145,7 +1168,8 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) { } else { nCtbDropped = 1; } - (void)tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t)); + code = tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t)); + if (code) return code; } /* if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { @@ -1170,6 +1194,7 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) { tSimpleHashCleanup(suidHash); pMeta->changed = true; + return 0; } static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { @@ -1210,7 +1235,10 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { tbFName[TSDB_TABLE_FNAME_LEN] = '\0'; int32_t ret = vnodeValidateTableHash(pMeta->pVnode, tbFName); if (ret < 0 && terrno == TSDB_CODE_VND_HASH_MISMATCH) { - (void)taosArrayPush(uidList, &me.uid); + if (taosArrayPush(uidList, &me.uid) == NULL) { + code = terrno; + break; + } } } tDecoderClear(&dc); @@ -1239,7 +1267,8 @@ int32_t metaTrimTables(SMeta *pMeta) { } metaInfo("vgId:%d, trim %ld tables", TD_VID(pMeta->pVnode), taosArrayGetSize(tbUids)); - metaDropTables(pMeta, tbUids); + code = metaDropTables(pMeta, tbUids); + if (code) goto end; end: taosArrayDestroy(tbUids); @@ -1867,11 +1896,19 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA } else { memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); } - (void)taosArrayPush(pTagArray, &val); + if (taosArrayPush(pTagArray, &val) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(pTagArray); + goto _err; + } } else { STagVal val = {.cid = pCol->colId}; if (tTagGet(pOldTag, &val)) { - (void)taosArrayPush(pTagArray, &val); + if (taosArrayPush(pTagArray, &val) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(pTagArray); + goto _err; + } } } } @@ -2238,6 +2275,9 @@ static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterT } SArray *tagIdxList = taosArrayInit(512, sizeof(SMetaPair)); + if (tagIdxList == NULL) { + goto _err; + } TBC *pTagIdxc = NULL; TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL)); @@ -2255,7 +2295,9 @@ static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterT } SMetaPair pair = {.key = pKey, nKey = nKey}; - (void)taosArrayPush(tagIdxList, &pair); + if (taosArrayPush(tagIdxList, &pair) == NULL) { + goto _err; + } } (void)tdbTbcClose(pTagIdxc); @@ -2797,7 +2839,14 @@ int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { SColCmprWrapper *p = &e.colCmpr; for (int32_t i = 0; i < p->nCols; i++) { SColCmpr *pCmpr = &p->pColCmpr[i]; - (void)taosHashPut(pColCmprObj, &pCmpr->id, sizeof(pCmpr->id), &pCmpr->alg, sizeof(pCmpr->alg)); + rc = taosHashPut(pColCmprObj, &pCmpr->id, sizeof(pCmpr->id), &pCmpr->alg, sizeof(pCmpr->alg)); + if (rc < 0) { + tDecoderClear(&dc); + tdbFree(pData); + metaULock(pMeta); + taosHashClear(pColCmprObj); + return rc; + } } } else { tDecoderClear(&dc); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 75db6e2925..a7ce9acbe5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -960,7 +960,9 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, } if (ttlReq.nUids > 0) { - metaDropTables(pVnode->pMeta, ttlReq.pTbUids); + int32_t code = metaDropTables(pVnode->pMeta, ttlReq.pTbUids); + if (code) return code; + (void)tqUpdateTbUidList(pVnode->pTq, ttlReq.pTbUids, false); } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index b7ca99d26a..c33e0ee501 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -154,6 +154,10 @@ static void *taosCacheTimedRefresh(void *handle); static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); + if (pCacheArrayList == NULL) { + uError("failed to allocate memory, reason:%s", strerror(errno)); + return; + } (void)taosThreadMutexInit(&guard, NULL); @@ -169,7 +173,11 @@ TdThread doRegisterCacheObj(SCacheObj *pCacheObj) { (void)taosThreadOnce(&cacheThreadInit, doInitRefreshThread); (void)taosThreadMutexLock(&guard); - (void)taosArrayPush(pCacheArrayList, &pCacheObj); + if (taosArrayPush(pCacheArrayList, &pCacheObj) != 0) { + uError("failed to add cache object into array, reason:%s", strerror(errno)); + (void)taosThreadMutexUnlock(&guard); + return cacheRefreshWorker; + } (void)taosThreadMutexUnlock(&guard); return cacheRefreshWorker; @@ -262,7 +270,7 @@ static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { pNode->pNext = pEntry->next; pEntry->next = pNode; pEntry->num += 1; - //A S S E R T((pEntry->next && pEntry->num > 0) || (NULL == pEntry->next && pEntry->num == 0)); + // A S S E R T((pEntry->next && pEntry->num > 0) || (NULL == pEntry->next && pEntry->num == 0)); } static void removeNodeInEntryList(SCacheEntry *pe, SCacheNode *prev, SCacheNode *pNode) { @@ -274,7 +282,7 @@ static void removeNodeInEntryList(SCacheEntry *pe, SCacheNode *prev, SCacheNode pNode->pNext = NULL; pe->num -= 1; - //A S S E R T((pe->next && pe->num > 0) || (NULL == pe->next && pe->num == 0)); + // A S S E R T((pe->next && pe->num > 0) || (NULL == pe->next && pe->num == 0)); } static FORCE_INLINE SCacheEntry *doFindEntry(SCacheObj *pCacheObj, const void *key, size_t keyLen) { @@ -499,7 +507,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { uDebug("cache:%s, data: %p acquired by data in cache, refcnt:%d", pCacheObj->name, ptNode->data, ref); // the data if referenced by at least one object, so the reference count must be greater than the value of 2. - //A S S E R T(ref >= 2); + // A S S E R T(ref >= 2); return data; } @@ -574,19 +582,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (ref == 1) { // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be // destroyed by refresh worker if decrease ref count before removing it from linked-list. - //A S S E R T(pNode->pTNodeHeader->pData == pNode); + // A S S E R T(pNode->pTNodeHeader->pData == pNode); __trashcan_wr_lock(pCacheObj); (void)doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); __trashcan_unlock(pCacheObj); ref = T_REF_DEC(pNode); - //A S S E R T(ref == 0); + // A S S E R T(ref == 0); doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); } else { ref = T_REF_DEC(pNode); - //A S S E R T(ref >= 0); + // A S S E R T(ref >= 0); } } else { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread @@ -608,7 +616,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { "others already, prev must in trashcan", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data, T_REF_VAL_GET(pNode)); - //A S S E R T(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL); + // A S S E R T(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL); } else { removeNodeInEntryList(pe, prev, p); uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, @@ -668,7 +676,7 @@ void doTraverseElems(SCacheObj *pCacheObj, bool (*fp)(void *param, SCacheNode *p } else { *pPre = next; pEntry->num -= 1; - //A S S E R T((pEntry->next && pEntry->num > 0) || (NULL == pEntry->next && pEntry->num == 0)); + // A S S E R T((pEntry->next && pEntry->num > 0) || (NULL == pEntry->next && pEntry->num == 0)); (void)atomic_sub_fetch_ptr(&pCacheObj->numOfElems, 1); pNode = next; @@ -734,7 +742,7 @@ SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pDat void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) { if (pNode->inTrashcan) { /* node is already in trash */ - //A S S E R T(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); + // A S S E R T(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); return; } @@ -780,7 +788,7 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { STrashElem *pElem = pCacheObj->pTrash; while (pElem) { T_REF_VAL_CHECK(pElem->pData); - //A S S E R T(pElem->next != pElem && pElem->prev != pElem); + // A S S E R T(pElem->next != pElem && pElem->prev != pElem); if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, From 8f03a041034c5080991f18d06325673915847cdb Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Tue, 27 Aug 2024 10:29:50 +0800 Subject: [PATCH 14/31] fix:[TD-31654] fix heap-buffer-overflow caused by unintialized variable. --- source/libs/scalar/src/sclfunc.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index ed55bde663..a1c80c7028 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1591,16 +1591,14 @@ int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * SColumnInfoData *pInputData[3]; SColumnInfoData *pOutputData = pOutput[0].columnData; int32_t outputLen; - int32_t numOfRows; + int32_t numOfRows = 0; pInputData[0] = pInput[0].columnData; pInputData[1] = pInput[1].columnData; pInputData[2] = pInput[2].columnData; for (int32_t i = 0; i < inputNum; ++i) { - if (pInput[i].numOfRows > numOfRows) { - numOfRows = pInput[i].numOfRows; - } + numOfRows = TMAX(numOfRows, pInput[i].numOfRows); } outputLen = pInputData[0]->info.bytes; @@ -1619,9 +1617,13 @@ int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * for (int32_t i = 0; i < inputNum; ++i) { if (colDataIsNull_s(pInputData[i], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) { colDataSetNULL(pOutputData, k); - continue; + hasNull = true; + break; } } + if (hasNull) { + continue; + } int32_t colIdx1 = (pInput[0].numOfRows == 1) ? 0 : k; int32_t colIdx2 = (pInput[1].numOfRows == 1) ? 0 : k; From 19e40e8bee4e6f3050484657ee41104b29a2198c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Aug 2024 10:41:38 +0800 Subject: [PATCH 15/31] fix(query): fix reader not release when error occuring. --- source/libs/executor/src/sysscanoperator.c | 311 +++++++++++---------- 1 file changed, 169 insertions(+), 142 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index d8a2331980..0a8d0b4cbd 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1336,6 +1336,171 @@ _end: return code; } +static int32_t doSetUserTableMetaInfo(SStoreMetaReader* pMetaReaderFn, SStoreMeta* pMetaFn, void* pVnode, + SMTbCursor* pCur, SMetaReader* pMReader, int64_t uid, const char* dbname, + int32_t vgId, SSDataBlock* p, int32_t rowIndex, const char* idStr) { + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t lino = 0; + int32_t code = pMetaReaderFn->getTableEntryByUid(pMReader, uid); + if (code < 0) { + qError("failed to get table meta, cname:%s, uid:%" PRId64 ", code:%s, %s", uid, tstrerror(terrno), idStr); + return code; + } + + STR_TO_VARSTR(n, pMReader->me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + + code = colDataSetVal(pColInfoData, rowIndex, n, false); + QUERY_CHECK_CODE(code, lino, _end); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, rowIndex, dbname, false); + QUERY_CHECK_CODE(code, lino, _end); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, rowIndex, (char*)&vgId, false); + QUERY_CHECK_CODE(code, lino, _end); + + int32_t tableType = pMReader->me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = pMReader->me.ctbEntry.btime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, rowIndex, (char*)&ts, false); + QUERY_CHECK_CODE(code, lino, _end); + + SMetaReader mr1 = {0}; + pMetaReaderFn->initReader(&mr1, pVnode, META_READER_NOLOCK, pMetaFn); + + int64_t suid = pMReader->me.ctbEntry.suid; + code = pMetaReaderFn->getTableEntryByUid(&mr1, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pCur->mr.me.name, suid, + tstrerror(code), idStr); + pMetaReaderFn->clearReader(&mr1); + QUERY_CHECK_CODE(code, lino, _end); + } + + pColInfoData = taosArrayGet(p->pDataBlock, 3); + if (pColInfoData == NULL) { + pMetaReaderFn->clearReader(&mr1); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + } + + code = colDataSetVal(pColInfoData, rowIndex, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); + if (code != 0) { + pMetaReaderFn->clearReader(&mr1); + QUERY_CHECK_CODE(code, lino, _end); + } + + // super table name + STR_TO_VARSTR(n, mr1.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + if (pColInfoData == NULL) { + pMetaReaderFn->clearReader(&mr1); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + } + + code = colDataSetVal(pColInfoData, rowIndex, n, false); + pMetaReaderFn->clearReader(&mr1); + QUERY_CHECK_CODE(code, lino, _end); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + if (pMReader->me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pMReader->me.ctbEntry.comment); + code = colDataSetVal(pColInfoData, rowIndex, comment, false); + QUERY_CHECK_CODE(code, lino, _end); + } else if (pMReader->me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + code = colDataSetVal(pColInfoData, rowIndex, comment, false); + QUERY_CHECK_CODE(code, lino, _end); + } else { + colDataSetNULL(pColInfoData, rowIndex); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, rowIndex, (char*)&pMReader->me.uid, false); + QUERY_CHECK_CODE(code, lino, _end); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, rowIndex, (char*)&pMReader->me.ctbEntry.ttlDays, false); + QUERY_CHECK_CODE(code, lino, _end); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, rowIndex, (char*)&pCur->mr.me.ntbEntry.btime, false); + QUERY_CHECK_CODE(code, lino, _end); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + + code = colDataSetVal(pColInfoData, rowIndex, (char*)&pCur->mr.me.ntbEntry.schemaRow.nCols, false); + QUERY_CHECK_CODE(code, lino, _end); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + colDataSetNULL(pColInfoData, rowIndex); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + if (pMReader->me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pMReader->me.ntbEntry.comment); + code = colDataSetVal(pColInfoData, rowIndex, comment, false); + QUERY_CHECK_CODE(code, lino, _end); + } else if (pMReader->me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + code = colDataSetVal(pColInfoData, rowIndex, comment, false); + QUERY_CHECK_CODE(code, lino, _end); + } else { + colDataSetNULL(pColInfoData, rowIndex); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, rowIndex, (char*)&pMReader->me.uid, false); + QUERY_CHECK_CODE(code, lino, _end); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, rowIndex, (char*)&pMReader->me.ntbEntry.ttlDays, false); + QUERY_CHECK_CODE(code, lino, _end); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + // impl later + } + +_end: + qError("%s failed at line %d since %s, %s", __func__, lino, tstrerror(code), idStr); + return code; +} + static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; @@ -1377,152 +1542,14 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); - ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid); - if (ret < 0) { - pAPI->metaReaderFn.clearReader(&mr); - continue; - } - STR_TO_VARSTR(n, mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - - code = colDataSetVal(pColInfoData, numOfRows, n, false); - QUERY_CHECK_CODE(code, lino, _end); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, dbname, false); - QUERY_CHECK_CODE(code, lino, _end); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); - QUERY_CHECK_CODE(code, lino, _end); - - int32_t tableType = mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = mr.me.ctbEntry.btime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); - QUERY_CHECK_CODE(code, lino, _end); - - SMetaReader mr1 = {0}; - pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); - - int64_t suid = mr.me.ctbEntry.suid; - code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, - suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - pAPI->metaReaderFn.clearReader(&mr1); - pAPI->metaReaderFn.clearReader(&mr); - T_LONG_JMP(pTaskInfo->env, terrno); - } - pColInfoData = taosArrayGet(p->pDataBlock, 3); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); - QUERY_CHECK_CODE(code, lino, _end); - - // super table name - STR_TO_VARSTR(n, mr1.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, n, false); - QUERY_CHECK_CODE(code, lino, _end); - pAPI->metaReaderFn.clearReader(&mr1); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - if (mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); - code = colDataSetVal(pColInfoData, numOfRows, comment, false); - QUERY_CHECK_CODE(code, lino, _end); - } else if (mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - code = colDataSetVal(pColInfoData, numOfRows, comment, false); - QUERY_CHECK_CODE(code, lino, _end); - } else { - colDataSetNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.uid, false); - QUERY_CHECK_CODE(code, lino, _end); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); - QUERY_CHECK_CODE(code, lino, _end); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false); - QUERY_CHECK_CODE(code, lino, _end); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - QUERY_CHECK_CODE(code, lino, _end); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - colDataSetNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - if (mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); - code = colDataSetVal(pColInfoData, numOfRows, comment, false); - QUERY_CHECK_CODE(code, lino, _end); - } else if (mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - code = colDataSetVal(pColInfoData, numOfRows, comment, false); - QUERY_CHECK_CODE(code, lino, _end); - } else { - colDataSetNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.uid, false); - QUERY_CHECK_CODE(code, lino, _end); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); - QUERY_CHECK_CODE(code, lino, _end); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - // impl later - } + doSetUserTableMetaInfo(&pAPI->metaReaderFn, &pAPI->metaFn, pInfo->readHandle.vnode, pInfo->pCur, &mr, *uid, dbname, + vgId, p, numOfRows, GET_TASKID(pTaskInfo)); pAPI->metaReaderFn.clearReader(&mr); - pColInfoData = taosArrayGet(p->pDataBlock, 9); + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 9); QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); From bcb1057e80a9d8f077c79c47c0839d17bf23fa76 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 27 Aug 2024 10:50:28 +0800 Subject: [PATCH 16/31] fix(query):clear meta reader --- source/libs/executor/src/executil.c | 1 + source/libs/executor/src/executor.c | 7 +++--- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/sysscanoperator.c | 25 +++++++++++++++++----- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 29ffd900f2..68bfec1fc0 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -435,6 +435,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, code = nodesCloneNode(pTagCond, &pTagCondTmp); if (TSDB_CODE_SUCCESS != code) { *pQualified = false; + pAPI->metaReaderFn.clearReader(&mr); return code; } STransTagExprCtx ctx = {.code = 0, .pReader = &mr}; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index f908ef5984..3ecaf61193 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -372,11 +372,11 @@ static int32_t filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); - QUERY_CHECK_NULL(qa, code, lino, _end, terrno); + QUERY_CHECK_NULL(qa, code, lino, _error, terrno); int32_t numOfUids = taosArrayGetSize(tableIdList); if (numOfUids == 0) { (*ppArrayRes) = qa; - goto _end; + goto _error; } STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; @@ -437,10 +437,11 @@ static int32_t filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } +_end: pAPI->metaReaderFn.clearReader(&mr); (*ppArrayRes) = qa; -_end: +_error: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 310992efed..aa6f3b2bde 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -640,6 +640,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, META_READER_LOCK, &pHandle->api.metaFn); + freeReader = true; code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -661,7 +662,6 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int QUERY_CHECK_CODE(code, lino, _end); val = *pVal; - freeReader = true; } else { pCache->cacheHit += 1; STableCachedVal* pVal = taosLRUCacheValue(pCache->pTableMetaEntryCache, h); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 9eb9d60226..59f3833987 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -655,16 +655,20 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow); if (smrSuperTable.me.stbEntry.schemaRow.pSchema) { - QUERY_CHECK_NULL(schemaWrapper, code, lino, _end, terrno); + if (schemaWrapper == NULL) { + code = terrno; + lino = __LINE__; + pAPI->metaReaderFn.clearReader(&smrSuperTable); + goto _end; + } } code = taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); if (code == TSDB_CODE_DUP_KEY) { code = TSDB_CODE_SUCCESS; } - QUERY_CHECK_CODE(code, lino, _end); - schemaRow = schemaWrapper; pAPI->metaReaderFn.clearReader(&smrSuperTable); + QUERY_CHECK_CODE(code, lino, _end); } } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { qDebug("sysTableScanUserCols cursor get normal table, %s", GET_TASKID(pTaskInfo)); @@ -789,11 +793,12 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { code = sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); - QUERY_CHECK_CODE(code, lino, _end); pAPI->metaReaderFn.clearReader(&smrSuperTable); pAPI->metaReaderFn.clearReader(&smrChildTable); + QUERY_CHECK_CODE(code, lino, _end); + if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo, pTaskInfo); numOfRows = 0; @@ -831,6 +836,8 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { pAPI->metaReaderFn.clearReader(&smrSuperTable); pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; + blockDataDestroy(dataBlock); + dataBlock = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -846,7 +853,15 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } else { code = sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); - QUERY_CHECK_CODE(code, lino, _end); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); + pInfo->pCur = NULL; + blockDataDestroy(dataBlock); + dataBlock = NULL; + T_LONG_JMP(pTaskInfo->env, terrno); + } } pAPI->metaReaderFn.clearReader(&smrSuperTable); } From c60ea644e3e762b23ec00470a12f4dff2d49dffb Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 27 Aug 2024 11:49:28 +0800 Subject: [PATCH 17/31] fix: return --- source/libs/function/src/functionMgt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index e50dbf8b14..f2df3c8e4d 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -118,7 +118,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { *reqStatus = -1; - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } const char* name = funcMgtBuiltins[funcId].name; From f32a17c72b86d6b090f2cef92723177c12ce4c1c Mon Sep 17 00:00:00 2001 From: lyh250-666 Date: Tue, 27 Aug 2024 11:54:41 +0800 Subject: [PATCH 18/31] Modify error code passing for taosMemoryCalloc function --- source/client/src/clientEnv.c | 4 +- source/client/src/clientHb.c | 7 ++-- source/client/src/clientImpl.c | 20 ++++++---- source/client/src/clientMain.c | 8 ++-- source/client/src/clientMsgHandler.c | 6 +-- source/client/src/clientRawBlockWrite.c | 4 +- source/client/src/clientSmlLine.c | 2 +- source/client/src/clientTmq.c | 50 ++++++++++++------------- 8 files changed, 52 insertions(+), 49 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 491a49778c..986927b85d 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -477,7 +477,7 @@ int32_t createTscObj(const char *user, const char *auth, const char *db, int32_t STscObj **pObj) { *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj)); if (NULL == *pObj) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*pObj)->pRequests = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); @@ -556,7 +556,7 @@ int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj (*pRequest)->inCallback = false; (*pRequest)->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); if (NULL == (*pRequest)->msgBuf) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _return; } (*pRequest)->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 70a519d8ae..9d6f106336 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -169,8 +169,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { int32_t code = 0; SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == vgInfo) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; + return terrno; } vgInfo->vgVersion = rsp->vgVersion; @@ -713,7 +712,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { if (NULL == hbBasic) { tscError("calloc %d failed", (int32_t)sizeof(SQueryHbReqBasic)); releaseTscObj(connKey->tscRid); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } hbBasic->connId = pTscObj->connId; @@ -1174,7 +1173,7 @@ static FORCE_INLINE void hbMgrInitHandle() { int32_t hbGatherAllInfo(SAppHbMgr *pAppHbMgr, SClientHbBatchReq **pBatchReq) { *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); (*pBatchReq)->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 039e17fd87..27a66f6df6 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -135,7 +135,7 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass if (pInst == NULL) { p = taosMemoryCalloc(1, sizeof(struct SAppInstInfo)); if (NULL == p) { - TSC_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + TSC_ERR_JRET(terrno); } p->mgmtEp = epSet; code = taosThreadMutexInit(&p->qnodeMutex, NULL); @@ -513,9 +513,11 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32 taosMemoryFree(pResInfo->userFields); } pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); + if(NULL == pResInfo->fields) return terrno; pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); - if (NULL == pResInfo->fields || NULL == pResInfo->userFields) { - return TSDB_CODE_OUT_OF_MEMORY; + if (NULL == pResInfo->userFields) { + taosMemoryFree(pResInfo->fields); + return terrno; } if (numOfCols != pResInfo->numOfCols) { tscError("numOfCols:%d != pResInfo->numOfCols:%d", numOfCols, pResInfo->numOfCols); @@ -1590,7 +1592,7 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo) { *pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (*pMsgSendInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*pMsgSendInfo)->msgType = TDMT_MND_CONNECT; @@ -1601,7 +1603,7 @@ static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInf (*pMsgSendInfo)->param = taosMemoryCalloc(1, sizeof(pRequest->self)); if (NULL == (*pMsgSendInfo)->param) { taosMemoryFree(*pMsgSendInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *(int64_t*)(*pMsgSendInfo)->param = pRequest->self; @@ -1977,7 +1979,11 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { pResInfo->convertBuf = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL || pResInfo->convertBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pResInfo->row); + taosMemoryFree(pResInfo->pCol); + taosMemoryFree(pResInfo->length); + taosMemoryFree(pResInfo->convertBuf); + return terrno; } } @@ -2125,7 +2131,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int taosMemoryFreeClear(pResultInfo->convertJson); pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); - if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pResultInfo->convertJson == NULL) return terrno; char* p1 = pResultInfo->convertJson; int32_t totalLen = 0; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 55401d7eb2..236f1ff739 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -965,7 +965,7 @@ int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) { int32_t code = TSDB_CODE_SUCCESS; SCatalogReq *pTarget = taosMemoryCalloc(1, sizeof(SCatalogReq)); if (pTarget == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } else { pTarget->pDbVgroup = taosArrayDup(pSrc->pDbVgroup, NULL); pTarget->pDbCfg = taosArrayDup(pSrc->pDbCfg, NULL); @@ -1174,7 +1174,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SS *pCxt = taosMemoryCalloc(1, sizeof(SParseContext)); if (*pCxt == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } **pCxt = (SParseContext){.requestId = pRequest->requestId, @@ -1208,7 +1208,7 @@ int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *p STscObj *pTscObj = pRequest->pTscObj; SSqlCallbackWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); if (pWrapper == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } else { pWrapper->pRequest = pRequest; pRequest->pWrapper = pWrapper; @@ -1229,7 +1229,7 @@ int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *p pWrapper->pCatalogReq = taosMemoryCalloc(1, sizeof(SCatalogReq)); if (pWrapper->pCatalogReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } else { pWrapper->pCatalogReq->forceUpdate = updateMetaForce; TSC_ERR_RET(qnodeRequired(pRequest, &pWrapper->pCatalogReq->qNodeRequired)); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 771a22b7e3..54020b77ef 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -201,7 +201,7 @@ End: SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - + if(pMsgSendInfo == NULL) return pMsgSendInfo; pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; pMsgSendInfo->param = pRequest; @@ -507,7 +507,7 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { int32_t code = 0; int32_t line = 0; SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - TSDB_CHECK_NULL(pBlock, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pBlock, code, line, END, terrno); pBlock->info.hasVarCol = true; pBlock->pDataBlock = taosArrayInit(SHOW_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); @@ -658,7 +658,7 @@ static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) { int32_t code = 0; int32_t line = 0; SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - TSDB_CHECK_NULL(pBlock, code, line, END, TSDB_CODE_OUT_OF_MEMORY); + TSDB_CHECK_NULL(pBlock, code, line, END, terrno); pBlock->info.hasVarCol = true; pBlock->pDataBlock = taosArrayInit(COMPACT_DB_RESULT_COLS, sizeof(SColumnInfoData)); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 7bd2b726e9..d184f8a559 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1900,7 +1900,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { SET_ERROR_MSG("calloc fields failed"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } for (int i = 0; i < pSW->nCols; i++) { @@ -2059,7 +2059,7 @@ static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, void* rspObj, tmq_ra len += sizeof(int8_t) + sizeof(int32_t); buf = taosMemoryCalloc(1, len); if (buf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAILED; } tEncoderInit(&encoder, buf, len); diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index f3efee0805..14a334e54c 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -104,7 +104,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } char* tmp = taosMemoryCalloc(pVal->length, 1); if (tmp == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 1db1df5d2e..5b1462bb5c 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -506,7 +506,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ((SMsgHead*)buf)->vgId = htonl(vgId); @@ -526,7 +526,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); if (pParam == NULL) { taosMemoryFree(buf); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pParam->params = pParamSet; @@ -540,7 +540,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse if (pMsgSendInfo == NULL) { taosMemoryFree(buf); taosMemoryFree(pParam); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pMsgSendInfo->msgInfo = (SDataBuf){.pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL}; @@ -581,7 +581,7 @@ static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, voi SMqCommitCbParamSet** ppParamSet) { SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pParamSet->refId = tmq->refId; @@ -1382,7 +1382,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFName == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto FAIL; } @@ -1414,7 +1414,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; taosMemoryFree(buf); goto FAIL; } @@ -1842,7 +1842,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl int32_t tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** ppRspObj) { SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); if (pRspObj == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pRspObj->resType = RES_TYPE__TMQ_META; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); @@ -1857,7 +1857,7 @@ int32_t tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqMetaRspObj** int32_t tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqBatchMetaRspObj** ppRspObj) { SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); if (pRspObj == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); @@ -1968,7 +1968,7 @@ int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, in SMqRspObj** ppRspObj) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); if (pRspObj == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pRspObj->common.resType = RES_TYPE__TMQ; (void)memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); @@ -1981,7 +1981,7 @@ int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pV SMqTaosxRspObj** ppRspObj) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); if (pRspObj == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pRspObj->common.resType = RES_TYPE__TMQ_METADATA; (void)memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); @@ -2007,8 +2007,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; + return terrno; } if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { @@ -2031,10 +2030,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pParam); taosMemoryFreeClear(msg); - return code; + return terrno; } sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; @@ -2955,7 +2953,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { pReq = taosMemoryCalloc(1, tlen); if (pReq == NULL) { tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", pTmq->consumerId, tlen); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { @@ -2968,7 +2966,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { if (pParam == NULL) { tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId); taosMemoryFree(pReq); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pParam->refId = pTmq->refId; @@ -2979,7 +2977,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { if (sendInfo == NULL) { taosMemoryFree(pReq); taosMemoryFree(pParam); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; @@ -3174,7 +3172,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ((SMsgHead*)buf)->vgId = htonl(vgId); @@ -3194,14 +3192,14 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(buf); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SMqCommittedParam* pParam = taosMemoryMalloc(sizeof(SMqCommittedParam)); if (pParam == NULL) { taosMemoryFree(buf); taosMemoryFree(sendInfo); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tsem2_init(&pParam->sem, 0, 0) != 0) { taosMemoryFree(buf); @@ -3392,7 +3390,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a if (*assignment == NULL) { tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, (*numOfAssignment) * sizeof(tmq_topic_assignment)); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } @@ -3420,7 +3418,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a if (needFetch) { pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon)); if (pCommon == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } @@ -3467,7 +3465,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a char* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { taosMemoryFree(pParam); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } @@ -3482,7 +3480,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a if (sendInfo == NULL) { taosMemoryFree(pParam); taosMemoryFree(msg); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto end; } @@ -3630,7 +3628,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ char* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tSerializeSMqSeekReq(msg, msgSize, &req) < 0) { @@ -3641,7 +3639,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(msg); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SMqSeekParam* pParam = taosMemoryMalloc(sizeof(SMqSeekParam)); From 809932b283c4ef2afa0b681aedded6b08bf259d0 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 27 Aug 2024 12:52:50 +0800 Subject: [PATCH 19/31] fix: reqStatus --- source/libs/function/src/functionMgt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index f2df3c8e4d..b1aecdfa8b 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -117,7 +117,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { - *reqStatus = -1; + *reqStatus = FUNC_DATA_REQUIRED_NOT_LOAD; return TSDB_CODE_SUCCESS; } From 3b1a1ed2cd849b49ce3e6b38ee14e3480b3e5882 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 27 Aug 2024 13:25:33 +0800 Subject: [PATCH 20/31] udf function required data load --- source/libs/function/src/functionMgt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index b1aecdfa8b..b9c8ce9ad3 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -117,7 +117,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { - *reqStatus = FUNC_DATA_REQUIRED_NOT_LOAD; + *reqStatus = FUNC_DATA_REQUIRED_DATA_LOAD; return TSDB_CODE_SUCCESS; } From afca8eb914b2ec268c500889036c79022b96da83 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Aug 2024 13:36:27 +0800 Subject: [PATCH 21/31] fix: error handle --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index bc5fcfc27c..f89e35dbb1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -1126,7 +1126,7 @@ int32_t tMergeTreeNext(SMergeTree *pMTree, bool *pHasNext) { if (c > 0) { (void)tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; - } else { + } else if (!c) { return TSDB_CODE_INTERNAL_ERROR; } } From 0207f9689af7a16cbbe07316fac5b2ffee1a25bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Aug 2024 13:45:39 +0800 Subject: [PATCH 22/31] fix(tsdb): fix race condition during set table list and drop tsdbreader. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 391b7f636d..ea50df1ac1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4588,6 +4588,8 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t STableBlockScanInfo** p = NULL; int32_t iter = 0; + (void)tsdbAcquireReader(pReader); + while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) { clearBlockScanInfo(*p); } @@ -4595,12 +4597,14 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t if (size < num) { code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); if (code) { + (void) tsdbReleaseReader(pReader); return code; } char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); if (p1 == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + (void) tsdbReleaseReader(pReader); + return terrno; } pReader->status.uidList.tableUidList = (uint64_t*)p1; @@ -4617,16 +4621,19 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t STableBlockScanInfo* pInfo = NULL; code = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i, &pInfo); if (code != TSDB_CODE_SUCCESS) { + (void) tsdbReleaseReader(pReader); return code; } code = initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader); if (code != TSDB_CODE_SUCCESS) { + (void) tsdbReleaseReader(pReader); return code; } } - return TDB_CODE_SUCCESS; + (void) tsdbReleaseReader(pReader); + return code; } uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; } From a99783ba66e7c86114c1a3c48296784ed786c350 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Aug 2024 13:46:30 +0800 Subject: [PATCH 23/31] fix(query): release lock when errors occuring. --- source/libs/executor/src/scanoperator.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 93a03bdffa..e0b78c20a1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -419,7 +419,11 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca size_t size = taosArrayGetSize(pBlock->pDataBlock); bool keep = false; code = doFilterByBlockSMA(pOperator->exprSupp.pFilterInfo, pBlock->pBlockAgg, size, pBlockInfo->rows, &keep); - QUERY_CHECK_CODE(code, lino, _end); + if (code) { + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); + qError("%s failed to do filter by block sma, code:%s", GET_TASKID(pTaskInfo), tstrerror(code)); + QUERY_CHECK_CODE(code, lino, _end); + } if (!keep) { qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, From 86da2cb91333632ee93da8b91de056e0957e9db6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Aug 2024 15:07:34 +0800 Subject: [PATCH 24/31] fix(query): fix syntax error. --- source/libs/executor/src/sysscanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 70cb6f4be4..75b96c25ca 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1361,7 +1361,7 @@ static int32_t doSetUserTableMetaInfo(SStoreMetaReader* pMetaReaderFn, SStoreMet int32_t lino = 0; int32_t code = pMetaReaderFn->getTableEntryByUid(pMReader, uid); if (code < 0) { - qError("failed to get table meta, cname:%s, uid:%" PRId64 ", code:%s, %s", uid, tstrerror(terrno), idStr); + qError("failed to get table meta, uid:%" PRId64 ", code:%s, %s", uid, tstrerror(terrno), idStr); return code; } From 76045b7874a87471d02caabc45b426582361a7b4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 27 Aug 2024 16:36:01 +0800 Subject: [PATCH 25/31] fix:[TD-31702]avoid remove slowlog when check if log out of logkeepdays --- source/os/src/osDir.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index c04add2f42..9f3908183e 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -294,6 +294,8 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { int32_t len = (int32_t)strlen(filename); if (len > 3 && strcmp(filename + len - 3, ".gz") == 0) { len -= 3; + }else{ + continue; } int64_t fileSec = 0; @@ -308,7 +310,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { (void)taosRemoveFile(filename); - // printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); + uInfo("file:%s is removed, days:%d keepDays:%d, sed:%"PRId64, filename, days, keepDays, fileSec); } else { // printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); } From b2c254817ba84ffebebe87c2cbbbef9bd128a710 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Aug 2024 17:07:10 +0800 Subject: [PATCH 26/31] fix(query): check return value. --- source/libs/executor/src/sysscanoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 75b96c25ca..77d302e4e3 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1560,10 +1560,11 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); - doSetUserTableMetaInfo(&pAPI->metaReaderFn, &pAPI->metaFn, pInfo->readHandle.vnode, &mr, *uid, dbname, vgId, p, + code = doSetUserTableMetaInfo(&pAPI->metaReaderFn, &pAPI->metaFn, pInfo->readHandle.vnode, &mr, *uid, dbname, vgId, p, numOfRows, GET_TASKID(pTaskInfo)); pAPI->metaReaderFn.clearReader(&mr); + QUERY_CHECK_CODE(code, lino, _end); SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 9); QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); From 4b872f849ce238fd5304da2ef23fc1cd44330876 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 27 Aug 2024 17:07:13 +0800 Subject: [PATCH 27/31] fmFuncDynDataRequired: only return state --- include/libs/function/functionMgt.h | 2 +- source/libs/executor/src/scanoperator.c | 5 ++--- source/libs/function/src/functionMgt.c | 14 +++++--------- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 905f1c2575..233e201ac3 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -295,7 +295,7 @@ typedef enum EFuncDataRequired { } EFuncDataRequired; EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); -int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus); +EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo); int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 310992efed..3a4bf9cfd9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -226,9 +226,8 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset); - int32_t reqStatus; - code = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo, &reqStatus); - if (code != TSDB_CODE_SUCCESS || reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { + EFuncDataRequired reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo); + if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { notLoadBlock = false; break; } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index b9c8ce9ad3..254a06426c 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -115,24 +115,20 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow); } -int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus) { +EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { - *reqStatus = FUNC_DATA_REQUIRED_DATA_LOAD; - return TSDB_CODE_SUCCESS; + return FUNC_DATA_REQUIRED_DATA_LOAD; } const char* name = funcMgtBuiltins[funcId].name; if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) { - *reqStatus = FUNC_DATA_REQUIRED_NOT_LOAD; - return TSDB_CODE_SUCCESS;; + return FUNC_DATA_REQUIRED_NOT_LOAD;; } if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) { - *reqStatus = FUNC_DATA_REQUIRED_DATA_LOAD; - return TSDB_CODE_SUCCESS; + return FUNC_DATA_REQUIRED_DATA_LOAD; } else { - *reqStatus = funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo); - return TSDB_CODE_SUCCESS; + return funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo); } } From d6580a2e651d58db599e7ef4e02b53a38a1d38dd Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 27 Aug 2024 17:11:50 +0800 Subject: [PATCH 28/31] fix(stream):fix mem leak for stream interval operator --- source/libs/executor/src/streamtimewindowoperator.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 08caf71eca..2d29bcd069 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -495,8 +495,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { nodesDestroyNode((SNode*)pInfo->pPhyNode); colDataDestroy(&pInfo->twAggSup.timeWindowData); cleanupExprSupp(&pInfo->scalarSupp); - tSimpleHashCleanup(pInfo->pUpdatedMap); - pInfo->pUpdatedMap = NULL; + if (pInfo->pUpdatedMap != NULL) { + tSimpleHashSetFreeFp(pInfo->pUpdatedMap, destroyFlusedppPos); + tSimpleHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; + } tSimpleHashCleanup(pInfo->pDeletedMap); blockDataDestroy(pInfo->pCheckpointRes); From 033ba15b4d920549ca345734e324a9e344b09a47 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 27 Aug 2024 17:45:06 +0800 Subject: [PATCH 29/31] fix:[TD-31672] test case error in windows because different format for %p --- utils/test/c/tmq_taosx_ci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 49cfa3dff8..3a79a3763c 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -1170,7 +1170,7 @@ void testDetailError() { int32_t code = tmq_write_raw((TAOS*)1, raw); ASSERT(code); const char* err = tmq_err2str(code); - char* tmp = strstr(err, "Invalid parameters,detail:taos:0x1 or data"); + char* tmp = strstr(err, "Invalid parameters,detail:taos:"); ASSERT(tmp != NULL); } From ef5c1b28f92f9302c9dce409def88ac2a70938ea Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 27 Aug 2024 19:17:31 +0800 Subject: [PATCH 30/31] fix: free ScalarParam --- source/libs/scalar/src/scalar.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 0ee1941717..cff73067b1 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -74,15 +74,17 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int code = colDataSetVal(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); if (code != TSDB_CODE_SUCCESS) { - return code; + goto _exit; } code = colInfoDataEnsureCapacity(out->columnData, 1, true); if (code != TSDB_CODE_SUCCESS) { - return code; + goto _exit; } code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); + +_exit: sclFreeParam(&in); return code; From a04defbcca87109e5667879503fa60a86da5f1a0 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Tue, 27 Aug 2024 19:25:41 +0800 Subject: [PATCH 31/31] add test case for TD_31684 --- tests/army/query/queryBugs.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/army/query/queryBugs.py b/tests/army/query/queryBugs.py index a7a6f35372..19459c9581 100644 --- a/tests/army/query/queryBugs.py +++ b/tests/army/query/queryBugs.py @@ -141,12 +141,42 @@ class TDTestCase(TBase): tdSql.checkRows(1) tdSql.checkData(0, 0, 102) + def FIX_TD_31684(self): + tdLog.info("check bug TD_31684 ...\n") + sqls = [ + "drop database if exists td_31684", + "create database td_31684 cachemodel 'both' stt_trigger 1;", + "use td_31684;", + "create table t1 (ts timestamp, id int, name int) ;", + "insert into t1 values(now, 1, 1);", + "insert into t1 values(now, 1, 2);", + "insert into t1 values(now, 2, 3);", + "insert into t1 values(now, 3, 4);" + ] + tdSql.executes(sqls) + sql1 = "select count(name) as total_name from t1 group by name" + sql2 = "select id as new_id, last(name) as last_name from t1 group by id order by new_id" + sql3 = "select id as new_id, count(id) as id from t1 group by id order by new_id" + tdSql.query(sql1) + tdSql.checkRows(4) + + tdSql.query(sql2) + tdSql.checkRows(3) + tdSql.checkData(0, 1, 2) + tdSql.checkData(1, 1, 3) + + tdSql.query(sql3) + tdSql.checkRows(3) + tdSql.checkData(0, 1, 2) + tdSql.checkData(2, 1, 1) + # run def run(self): tdLog.debug(f"start to excute {__file__}") # TD BUGS self.FIX_TD_30686() + self.FIX_TD_31684() # TS BUGS self.FIX_TS_5105()