From e5fde30635d998a6264e06c296c251615a68813a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 Jul 2024 17:37:48 +0800 Subject: [PATCH 01/43] fix: function return code validation --- include/os/osString.h | 1 + source/libs/executor/src/groupcacheoperator.c | 201 ++++++++++++------ 2 files changed, 140 insertions(+), 62 deletions(-) diff --git a/include/os/osString.h b/include/os/osString.h index e53aceb83a..05492763c2 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -61,6 +61,7 @@ typedef enum { M2C = 0, C2M } ConvType; #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) #define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) +#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src)) char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 28bccaedff..379cceeb03 100755 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -30,9 +30,9 @@ static void removeGroupCacheFile(SGroupCacheFileInfo* pFileInfo) { if (pFileInfo->fd.fd) { - taosCloseFile(&pFileInfo->fd.fd); + (void)taosCloseFile(&pFileInfo->fd.fd); pFileInfo->fd.fd = NULL; - taosThreadMutexDestroy(&pFileInfo->fd.mutex); + (void)taosThreadMutexDestroy(&pFileInfo->fd.mutex); } pFileInfo->deleted = true; } @@ -88,7 +88,7 @@ static void logGroupCacheExecInfo(SGroupCacheOperatorInfo* pGrpCacheOperator) { static void freeSGcSessionCtx(void* p) { SGcSessionCtx* pSession = p; if (pSession->semInit) { - tsem_destroy(&pSession->waitSem); + (void)tsem_destroy(&pSession->waitSem); } } @@ -148,6 +148,10 @@ void blockDataDeepCleanup(SSDataBlock* pDataBlock) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + if (NULL == p) { + qError("fail to get %dth col in dataBlock, numOfCols:%d", i, numOfCols); + continue; + } taosMemoryFreeClear(p->pData); if (IS_VAR_DATA_TYPE(p->info.type)) { taosMemoryFreeClear(p->varmeta.offset); @@ -196,10 +200,14 @@ static FORCE_INLINE int32_t initOpenCacheFile(SGroupCacheFileFd* pFileFd, char* TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE|TD_FILE_AUTO_DEL); //TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE); if (NULL == newFd) { - return TAOS_SYSTEM_ERROR(errno); + QRY_ERR_RET(TAOS_SYSTEM_ERROR(errno)); } pFileFd->fd = newFd; - taosThreadMutexInit(&pFileFd->mutex, NULL); + int32_t code = taosThreadMutexInit(&pFileFd->mutex, NULL); + if (code) { + qError("taosThreadMutexInit failed, code:%x", code); + QRY_ERR_RET(code); + } qTrace("file path %s created", filename); @@ -218,13 +226,16 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); if (NULL == pTmp) { - sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId); + (void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId); SGroupCacheFileInfo newFile = {0}; - taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)); + if (taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile))) { + QRY_ERR_RET(terrno); + } pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); if (NULL == pTmp) { - return TSDB_CODE_OUT_OF_MEMORY; + qError("fail to get file %d from pCacheFile", fileId); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } } @@ -240,14 +251,14 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S } } - taosThreadMutexLock(&pTmp->fd.mutex); + (void)taosThreadMutexLock(&pTmp->fd.mutex); *ppFd = &pTmp->fd; return TSDB_CODE_SUCCESS; } static FORCE_INLINE void releaseFdToFileCtx(SGroupCacheFileFd* pFd) { - taosThreadMutexUnlock(&pFd->mutex); + (void)taosThreadMutexUnlock(&pFd->mutex); } static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamCtx* pCtx, SGcBlkBufInfo* pHead) { @@ -275,7 +286,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC int64_t blkId = pHead->basic.blkId; pHead = pHead->next; - taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); continue; } @@ -295,7 +306,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC int64_t blkId = pHead->basic.blkId; pHead = pHead->next; - taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); continue; } @@ -321,7 +332,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC int64_t blkId = pHead->basic.blkId; pHead = pHead->next; - taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); + (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); } _return: @@ -330,7 +341,7 @@ _return: taosHashRelease(pGrpHash, pGroup); } - atomic_val_compare_exchange_32(&pGCache->blkCache.writeDownstreamId, pCtx->id, -1); + (void)atomic_val_compare_exchange_32(&pGCache->blkCache.writeDownstreamId, pCtx->id, -1); return code; } @@ -341,7 +352,8 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr } pBufInfo = taosHashGet(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId)); if (NULL == pBufInfo) { - return TSDB_CODE_OUT_OF_MEMORY; + qError("fail to get blk %d from pCache->pDirtyBlk", pBufInfo->basic.blkId); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } int32_t code = TSDB_CODE_SUCCESS; SGcBlkBufInfo* pWriteHead = NULL; @@ -424,7 +436,7 @@ static int32_t addBlkToBufCache(struct SOperatorInfo* pOperator, SSDataBlock* pB qError("group cache add block to cache failed, size:%" PRId64, bufSize); return TSDB_CODE_OUT_OF_MEMORY; } - blockDataToBuf(pBufInfo->pBuf, pBlock); + QRY_ERR_RET(blockDataToBuf(pBufInfo->pBuf, pBlock)); SGcFileCacheCtx* pFileCtx = pGCache->batchFetch ? &pCtx->fileCtx : &pGroup->pVgCtx->fileCtx; @@ -449,6 +461,10 @@ void blockDataDeepClear(SSDataBlock* pDataBlock) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + if (NULL == p) { + qError("fail to get %d col from pDataBlock, numOfCols:%d", i, numOfCols); + continue; + } p->pData = NULL; if (IS_VAR_DATA_TYPE(p->info.type)) { p->varmeta.offset = NULL; @@ -473,8 +489,9 @@ static int32_t buildGroupCacheBaseBlock(SSDataBlock** ppDst, SSDataBlock* pSrc) taosMemoryFree(*ppDst); return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info)); + TAOS_MEMCPY(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info)); blockDataDeepClear(*ppDst); + return TSDB_CODE_SUCCESS; } @@ -490,11 +507,17 @@ static int32_t acquireBaseBlockFromList(SGcDownstreamCtx* pCtx, SSDataBlock** pp return TSDB_CODE_SUCCESS; } -static void releaseBaseBlockToList(SGcDownstreamCtx* pCtx, SSDataBlock* pBlock) { +static int32_t releaseBaseBlockToList(SGcDownstreamCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + blockDataDeepCleanup(pBlock); taosWLockLatch(&pCtx->blkLock); - taosArrayPush(pCtx->pFreeBlock, &pBlock); + if (NULL == taosArrayPush(pCtx->pFreeBlock, &pBlock)) { + code = terrno; + } taosWUnLockLatch(&pCtx->blkLock); + + return code; } @@ -573,14 +596,15 @@ static int32_t retrieveBlkFromBufCache(SGroupCacheOperatorInfo* pGCache, SGroupC return code; } - taosHashPut(pCache->pReadBlk, &sessionId, sizeof(sessionId), ppRes, POINTER_BYTES); + QRY_ERR_RET(taosHashPut(pCache->pReadBlk, &sessionId, sizeof(sessionId), ppRes, POINTER_BYTES)); + return TSDB_CODE_SUCCESS; } static FORCE_INLINE void initGcVgroupCtx(SOperatorInfo* pOperator, SGcVgroupCtx* pVgCtx, int32_t downstreamId, int32_t vgId, SArray* pTbList) { pVgCtx->pTbList = pTbList; pVgCtx->id = vgId; - snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d", + (void)snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d", tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, downstreamId, vgId); pVgCtx->fileCtx.baseFilename[sizeof(pVgCtx->fileCtx.baseFilename) - 1] = 0; @@ -594,15 +618,27 @@ static int32_t addNewGroupToVgHash(SOperatorInfo* pOperator, SSHashObj* pHash, S if (NULL == pList) { return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pList, pNew); + if (NULL == taosArrayPush(pList, pNew)) { + QRY_ERR_RET(terrno); + } + SGcVgroupCtx vgCtx = {0}; initGcVgroupCtx(pOperator, &vgCtx, pNew->pGroup->downstreamIdx, pNew->vgId, pList); - tSimpleHashPut(pHash, &pNew->vgId, sizeof(pNew->vgId), &vgCtx, sizeof(vgCtx)); + QRY_ERR_RET(tSimpleHashPut(pHash, &pNew->vgId, sizeof(pNew->vgId), &vgCtx, sizeof(vgCtx))); + pNew->pGroup->pVgCtx = tSimpleHashGet(pHash, &pNew->vgId, sizeof(pNew->vgId)); + if (NULL == pNew->pGroup->pVgCtx) { + qError("fail to get vg %d ctx from vgHash", pNew->vgId); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + return TSDB_CODE_SUCCESS; } - taosArrayPush(pVgCtx->pTbList, pNew); + if (NULL == taosArrayPush(pVgCtx->pTbList, pNew)) { + QRY_ERR_RET(terrno); + } + return TSDB_CODE_SUCCESS; } @@ -620,6 +656,11 @@ static FORCE_INLINE int32_t appendNewGroupToDownstream(struct SOperatorInfo* pOp for (int32_t i = 0; i < num; ++i) { SGcNewGroupInfo* pNew = taosArrayGet(pCtx->pNewGrpList, i); + if (NULL == pNew) { + qError("fail to get vg %d SGcNewGroupInfo from pNewGrpList", i); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (!pGCache->batchFetch) { code = addNewGroupToVgHash(pOperator, pCtx->pVgTbHash, pNew); if (code) { @@ -672,7 +713,9 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p if (code) { return code; } - taosArrayPush(pGCache->pDownstreams[downstreamIdx].pFreeBlock, &pGCache->pDownstreams[downstreamIdx].pBaseBlock); + if (NULL == taosArrayPush(pGCache->pDownstreams[downstreamIdx].pFreeBlock, &pGCache->pDownstreams[downstreamIdx].pBaseBlock)) { + QRY_ERR_RET(terrno); + } } } @@ -681,26 +724,36 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p return code; } -static void notifyWaitingSessions(SArray* pWaitQueue) { +static int32_t notifyWaitingSessions(SArray* pWaitQueue) { if (NULL == pWaitQueue || taosArrayGetSize(pWaitQueue) <= 0) { - return; + return TSDB_CODE_SUCCESS; } int32_t n = taosArrayGetSize(pWaitQueue); for (int32_t i = 0; i < n; ++i) { SGcSessionCtx* pSession = taosArrayGetP(pWaitQueue, i); - tsem_post(&pSession->waitSem); + if (NULL == pSession) { + qError("fail to get %d SGcSessionCtx in pWaitQueue, total:%d", i, n); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + + QRY_ERR_RET(tsem_post(&pSession->waitSem)); } + + return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void handleGroupFetchDone(SGroupCacheData* pGroup) { +static FORCE_INLINE int32_t handleGroupFetchDone(SGroupCacheData* pGroup) { + int32_t code = TSDB_CODE_SUCCESS; pGroup->pBlock = NULL; atomic_store_8((int8_t*)&pGroup->fetchDone, true); - taosThreadMutexLock(&pGroup->mutex); - notifyWaitingSessions(pGroup->waitQueue); + (void)taosThreadMutexLock(&pGroup->mutex); + code = notifyWaitingSessions(pGroup->waitQueue); taosArrayClear(pGroup->waitQueue); - taosThreadMutexUnlock(&pGroup->mutex); + (void)taosThreadMutexUnlock(&pGroup->mutex); + + return code; } static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int32_t downstreamId, int32_t vgId) { @@ -714,14 +767,15 @@ static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); if (NULL == pTmp) { - sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId); + (void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId); SGroupCacheFileInfo newFile = {0}; newFile.groupNum = 1; - taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)); + QRY_ERR_RET(taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile))); pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); if (NULL == pTmp) { - return TSDB_CODE_OUT_OF_MEMORY; + qError("fail to get file %d in pCacheFile", fileId); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } } else { pTmp->groupNum++; @@ -746,7 +800,7 @@ static int32_t handleVgroupTableFetchDone(SGcDownstreamCtx* pCtx, SGroupCacheDat if (NULL == pNew || pNew->uid == uid) { break; } - handleGroupFetchDone(pNew->pGroup); + QRY_ERR_RET(handleGroupFetchDone(pNew->pGroup)); } groupCacheSwitchNewFile(&pGroup->pVgCtx->fileCtx, pGroup->downstreamIdx, pGroup->vgId, true); @@ -764,15 +818,21 @@ static int32_t handleVgroupTableFetchDone(SGcDownstreamCtx* pCtx, SGroupCacheDat } -static FORCE_INLINE void initNewGroupData(SGcDownstreamCtx* pCtx, SGroupCacheData* pGroup, int32_t downstreamIdx, int32_t vgId, bool batchFetch, bool needCache) { - taosThreadMutexInit(&pGroup->mutex, NULL); +static FORCE_INLINE int32_t initNewGroupData(SGcDownstreamCtx* pCtx, SGroupCacheData* pGroup, int32_t downstreamIdx, int32_t vgId, bool batchFetch, bool needCache) { + QRY_ERR_RET(taosThreadMutexInit(&pGroup->mutex, NULL)); + pGroup->downstreamIdx = downstreamIdx; pGroup->vgId = vgId; pGroup->fileId = -1; pGroup->blkList.pList = taosArrayInit(10, sizeof(SGcBlkBufBasic)); + if (NULL == pGroup->blkList.pList) { + QRY_ERR_RET(terrno); + } pGroup->startOffset = -1; pGroup->needCache = needCache; pGroup->pVgCtx = tSimpleHashGet(pCtx->pVgTbHash, &pGroup->vgId, sizeof(pGroup->vgId)); + + return TSDB_CODE_SUCCESS; } static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* pParam, SGroupCacheData** ppGrp, int32_t vgId, int64_t uid) { @@ -801,7 +861,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* if (NULL == *ppGrp) { return TSDB_CODE_OUT_OF_MEMORY; } - initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache); + QRY_ERR_RET(initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache)); qDebug("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache); @@ -829,7 +889,9 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* static int32_t addBlkToGroupCache(bool batchFetch, SGroupCacheData* pGroup, SGcBlkBufInfo* pNewBlk, int64_t* pIdx) { taosWLockLatch(&pGroup->blkList.lock); - taosArrayPush(pGroup->blkList.pList, &pNewBlk->basic); + if (NULL == taosArrayPush(pGroup->blkList.pList, &pNewBlk->basic)) { + QRY_ERR_RET(terrno); + } *pIdx = taosArrayGetSize(pGroup->blkList.pList) - 1; taosWUnLockLatch(&pGroup->blkList.lock); @@ -889,7 +951,7 @@ static int32_t handleGroupCacheRetrievedBlk(struct SOperatorInfo* pOperator, SSD pGroup->pBlock = pBlock; } - notifyWaitingSessions(pGroup->waitQueue); + QRY_ERR_RET(notifyWaitingSessions(pGroup->waitQueue)); if (pGroup == pSession->pGroupData) { if (pGroup->needCache) { pSession->lastBlkId = newBlkIdx; @@ -909,7 +971,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes SHashObj* pGrpHash = pGCache->globalGrp ? pGCache->pGrpHash : pCtx->pGrpHash; SGroupCacheData* pGroup = NULL; while (NULL != (pGroup = taosHashIterate(pGrpHash, pGroup))) { - handleGroupFetchDone(pGroup); + QRY_ERR_RET(handleGroupFetchDone(pGroup)); } pCtx->fetchDone = true; } else { @@ -920,7 +982,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes uidNum = taosArrayGetSize(pVgCtx->pTbList); for (int32_t i = 0; i < uidNum; ++i) { SGcNewGroupInfo* pNew = taosArrayGet(pVgCtx->pTbList, i); - handleGroupFetchDone(pNew->pGroup); + QRY_ERR_RET(handleGroupFetchDone(pNew->pGroup)); } taosArrayClear(pVgCtx->pTbList); } @@ -961,8 +1023,8 @@ static int32_t getCacheBlkFromDownstreamOperator(struct SOperatorInfo* pOperator } SGcSessionCtx* pWaitCtx = *ppWaitCtx; pWaitCtx->newFetch = true; - taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId)); - tsem_post(&pWaitCtx->waitSem); + (void)taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId)); + QRY_ERR_RET(tsem_post(&pWaitCtx->waitSem)); return code; } @@ -1028,36 +1090,47 @@ static int32_t groupCacheSessionWait(struct SOperatorInfo* pOperator, SGcDownstr SGroupCacheOperatorInfo* pGCache = pOperator->info; SGroupCacheData* pGroup = pSession->pGroupData; int32_t code = TSDB_CODE_SUCCESS; + bool inLock = true; if (NULL == pGroup->waitQueue) { pGroup->waitQueue = taosArrayInit(1, POINTER_BYTES); if (NULL == pGroup->waitQueue) { - taosThreadMutexUnlock(&pSession->pGroupData->mutex); - return TSDB_CODE_OUT_OF_MEMORY; + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } - taosArrayPush(pGroup->waitQueue, &pSession); + if (NULL == taosArrayPush(pGroup->waitQueue, &pSession)) { + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } if (!pSession->semInit) { - tsem_init(&pSession->waitSem, 0, 0); + QRY_ERR_JRET(tsem_init(&pSession->waitSem, 0, 0)); pSession->semInit = true; } - taosThreadMutexUnlock(&pSession->pGroupData->mutex); + (void)taosThreadMutexUnlock(&pSession->pGroupData->mutex); + inLock = false; - taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES); + QRY_ERR_JRET(taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES)); - tsem_wait(&pSession->waitSem); + (void)tsem_wait(&pSession->waitSem); if (pSession->newFetch) { pSession->newFetch = false; return getCacheBlkFromDownstreamOperator(pOperator, pCtx, sessionId, pSession, ppRes); } - taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId)); + (void)taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId)); bool got = false; return getBlkFromSessionCacheImpl(pOperator, sessionId, pSession, ppRes, &got); + +_return: + + if (inLock) { + (void)taosThreadMutexUnlock(&pSession->pGroupData->mutex); + } + + return code; } @@ -1077,7 +1150,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s if ((atomic_load_64(&pCtx->fetchSessionId) == sessionId) || (-1 == atomic_val_compare_exchange_64(&pCtx->fetchSessionId, -1, sessionId))) { if (locked) { - taosThreadMutexUnlock(&pSession->pGroupData->mutex); + (void)taosThreadMutexUnlock(&pSession->pGroupData->mutex); locked = false; } @@ -1095,7 +1168,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s break; } - taosThreadMutexLock(&pSession->pGroupData->mutex); + (void)taosThreadMutexLock(&pSession->pGroupData->mutex); locked = true; }; @@ -1103,7 +1176,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s _return: if (locked) { - taosThreadMutexUnlock(&pSession->pGroupData->mutex); + (void)taosThreadMutexUnlock(&pSession->pGroupData->mutex); } return code; @@ -1165,6 +1238,10 @@ static int32_t initGroupCacheSession(struct SOperatorInfo* pOperator, SOperatorP } *ppSession = taosHashGet(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); + if (NULL == *ppSession) { + qError("fail to get session %d from pSessions", pGcParam->sessionId); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } return TSDB_CODE_SUCCESS; } @@ -1187,15 +1264,15 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock } else if (pSession->pGroupData->needCache) { SSDataBlock** ppBlock = taosHashGet(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); if (ppBlock) { - releaseBaseBlockToList(pCtx, *ppBlock); - taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); + QRY_ERR_RET(releaseBaseBlockToList(pCtx, *ppBlock)); + (void)taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); } } code = getBlkFromSessionCache(pOperator, pGcParam->sessionId, pSession, ppRes); if (NULL == *ppRes) { qDebug("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows); - taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); + (void)taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); } else { pSession->resRows += (*ppRes)->info.rows; qDebug("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows); @@ -1241,7 +1318,7 @@ static void freeRemoveGroupCacheData(void* p) { taosArrayDestroy(pGroup->waitQueue); taosArrayDestroy(pGroup->blkList.pList); - taosThreadMutexDestroy(&pGroup->mutex); + (void)taosThreadMutexDestroy(&pGroup->mutex); qTrace("group removed"); } @@ -1271,7 +1348,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) { int32_t defaultVg = 0; SGcVgroupCtx vgCtx = {0}; initGcVgroupCtx(pOperator, &vgCtx, pCtx->id, defaultVg, NULL); - tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx)); + QRY_ERR_RET(tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx))); } pCtx->pNewGrpList = taosArrayInit(10, sizeof(SGcNewGroupInfo)); @@ -1302,7 +1379,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) { return TSDB_CODE_OUT_OF_MEMORY; } - snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d", + (void)snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d", tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pCtx->id); pCtx->fileCtx.baseFilename[sizeof(pCtx->fileCtx.baseFilename) - 1] = 0; pCtx->fileCtx.baseNameLen = strlen(pCtx->fileCtx.baseFilename); From 558d22dff449f1bb7194070a2e4288cc2670a126 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 Jul 2024 18:27:49 +0800 Subject: [PATCH 02/43] fix: function return code validation --- source/libs/executor/src/hashjoin.c | 4 +- source/libs/executor/src/hashjoinoperator.c | 84 +++++++++++++++----- source/libs/executor/src/mergejoin.c | 74 +++++++++++------ source/libs/executor/src/mergejoinoperator.c | 8 +- 4 files changed, 124 insertions(+), 46 deletions(-) diff --git a/source/libs/executor/src/hashjoin.c b/source/libs/executor/src/hashjoin.c index c22b331a16..d4a84afea2 100755 --- a/source/libs/executor/src/hashjoin.c +++ b/source/libs/executor/src/hashjoin.c @@ -90,7 +90,7 @@ int32_t hLeftJoinHandleSeqRowRemains(struct SOperatorInfo* pOperator, SHJoinOper while (!allFetched) { hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched); if (pJoin->midBlk->info.rows > 0) { - doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL); + HJ_ERR_RET(doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL)); if (pJoin->midBlk->info.rows > 0) { pCtx->readMatch = true; HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk)); @@ -170,7 +170,7 @@ int32_t hLeftJoinHandleSeqProbeRows(struct SOperatorInfo* pOperator, SHJoinOpera while (!allFetched) { hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched); if (pJoin->midBlk->info.rows > 0) { - doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL); + HJ_ERR_RET(doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL)); if (pJoin->midBlk->info.rows > 0) { pCtx->readMatch = true; HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk)); diff --git a/source/libs/executor/src/hashjoinoperator.c b/source/libs/executor/src/hashjoinoperator.c index 2fe2ccc56f..bbbdff6c17 100755 --- a/source/libs/executor/src/hashjoinoperator.c +++ b/source/libs/executor/src/hashjoinoperator.c @@ -238,10 +238,12 @@ static int32_t hJoinInitValColsInfo(SHJoinTableCtx* pTable, SNodeList* pList) { if (NULL == pTable->valVarCols) { pTable->valVarCols = taosArrayInit(pTable->valNum, sizeof(int32_t)); if (NULL == pTable->valVarCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } - taosArrayPush(pTable->valVarCols, &i); + if (NULL == taosArrayPush(pTable->valVarCols, &i)) { + return terrno; + } } pTable->valCols[i].bytes = pColNode->node.resType.bytes; if (!pTable->valCols[i].keyCol && !pTable->valCols[i].vardata) { @@ -332,7 +334,7 @@ static int32_t hJoinInitTableInfo(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* return code; } - memcpy(&pTable->inputStat, pStat, sizeof(*pStat)); + TAOS_MEMCPY(&pTable->inputStat, pStat, sizeof(*pStat)); HJ_ERR_RET(hJoinInitPrimExprCtx(pTable->primExpr, &pTable->primCtx, pTable)); @@ -416,7 +418,9 @@ static FORCE_INLINE int32_t hJoinAddPageToBufs(SArray* pRowBufs) { return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pRowBufs, &page); + if (NULL == taosArrayPush(pRowBufs, &page)) { + return terrno; + } return TSDB_CODE_SUCCESS; } @@ -464,12 +468,21 @@ static void hJoinDestroyKeyHash(SSHashObj** ppHash) { *ppHash = NULL; } -static FORCE_INLINE char* hJoinRetrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow) { +static FORCE_INLINE int32_t hJoinRetrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow, char** ppData) { + *ppData = NULL; + if ((uint16_t)-1 == pRow->pageId) { - return NULL; + return TSDB_CODE_SUCCESS; } SBufPageInfo *pPage = taosArrayGet(pRowBufs, pRow->pageId); - return pPage->data + pRow->offset; + if (NULL == pPage) { + qError("fail to get %d page, total:%d", pRow->pageId, (int32_t)taosArrayGetSize(pRowBufs)); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + + *ppData = pPage->data + pRow->offset; + + return TSDB_CODE_SUCCESS; } static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum, SBufRowInfo* pStart, SSDataBlock* pRes) { @@ -479,9 +492,11 @@ static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum int32_t probeIdx = 0; SBufRowInfo* pRow = pStart; int32_t code = 0; + char* pData = NULL; for (int32_t r = 0; r < rowNum; ++r) { - char* pData = hJoinRetrieveColDataFromRowBufs(pJoin->pRowBufs, pRow); + HJ_ERR_RET(hJoinRetrieveColDataFromRowBufs(pJoin->pRowBufs, pRow, &pData)); + char* pValData = pData + pBuild->valBitMapSize; char* pKeyData = pProbe->keyData; buildIdx = buildValIdx = probeIdx = 0; @@ -544,7 +559,7 @@ int32_t hJoinCopyNMatchRowsToBlock(SHJoinOperatorInfo* pJoin, SSDataBlock* pRes, SColumnInfoData* pSrc = taosArrayGet(pJoin->ctx.pProbeData->pDataBlock, pProbe->valCols[probeIdx].srcSlot); SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pProbe->valCols[probeIdx].dstSlot); - colDataAssignNRows(pDst, pRes->info.rows, pSrc, startIdx, rows); + QRY_ERR_RET(colDataAssignNRows(pDst, pRes->info.rows, pSrc, startIdx, rows)); probeIdx++; } @@ -606,11 +621,11 @@ bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *p } if (pTable->keyCols[i].vardata) { pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx]; - memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData)); + TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, varDataTLen(pData)); bufLen += varDataTLen(pData); } else { pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx; - memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes); + TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes); bufLen += pTable->keyCols[i].bytes; } } @@ -627,6 +642,10 @@ bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *p static int32_t hJoinSetKeyColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable) { for (int32_t i = 0; i < pTable->keyNum; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot); + if (NULL == pCol) { + qError("fail to get %d col, total:%d", pTable->keyCols[i].srcSlot, (int32_t)taosArrayGetSize(pBlock->pDataBlock)); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) { qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata); return TSDB_CODE_INVALID_PARA; @@ -654,6 +673,10 @@ static int32_t hJoinSetValColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable) continue; } SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->valCols[i].srcSlot); + if (NULL == pCol) { + qError("fail to get %d col, total:%d", pTable->valCols[i].srcSlot, (int32_t)taosArrayGetSize(pBlock->pDataBlock)); + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (pTable->valCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) { qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->valCols[i].srcSlot, pCol->info.type, pTable->valCols[i].vardata); return TSDB_CODE_INVALID_PARA; @@ -683,7 +706,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32 char *pData = NULL; size_t bufLen = pTable->valBitMapSize; - memset(pTable->valData, 0, pTable->valBitMapSize); + TAOS_MEMSET(pTable->valData, 0, pTable->valBitMapSize); for (int32_t i = 0, m = 0; i < pTable->valNum; ++i) { if (pTable->valCols[i].keyCol) { continue; @@ -693,7 +716,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32 colDataSetNull_f(pTable->valData, m); } else { pData = pTable->valCols[i].data + pTable->valCols[i].offset[rowIdx]; - memcpy(pTable->valData + bufLen, pData, varDataTLen(pData)); + TAOS_MEMCPY(pTable->valData + bufLen, pData, varDataTLen(pData)); bufLen += varDataTLen(pData); } } else { @@ -701,7 +724,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32 colDataSetNull_f(pTable->valData, m); } else { pData = pTable->valCols[i].data + pTable->valCols[i].bytes * rowIdx; - memcpy(pTable->valData + bufLen, pData, pTable->valCols[i].bytes); + TAOS_MEMCPY(pTable->valData + bufLen, pData, pTable->valCols[i].bytes); bufLen += pTable->valCols[i].bytes; } } @@ -1009,7 +1032,11 @@ static SSDataBlock* hJoinMainProcess(struct SOperatorInfo* pOperator) { } if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) { - doFilter(pRes, pJoin->pFinFilter, NULL); + code = doFilter(pRes, pJoin->pFinFilter, NULL); + if (code) { + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } } if (pRes->info.rows > 0) { return pRes; @@ -1037,7 +1064,11 @@ static SSDataBlock* hJoinMainProcess(struct SOperatorInfo* pOperator) { } if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) { - doFilter(pRes, pJoin->pFinFilter, NULL); + code = doFilter(pRes, pJoin->pFinFilter, NULL); + if (code) { + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } } if (pRes->info.rows > 0) { @@ -1118,13 +1149,25 @@ static uint32_t hJoinGetFinBlkCapacity(SHJoinOperatorInfo* pJoin, SHashJoinPhysi int32_t hJoinInitResBlocks(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinNode) { pJoin->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); + if (NULL == pJoin->finBlk) { + QRY_ERR_RET(terrno); + } ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0); - blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode)); + int32_t code = blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode)); + if (TSDB_CODE_SUCCESS != code) { + QRY_ERR_RET(terrno); + } if (NULL != pJoin->pPreFilter) { pJoin->midBlk = createOneDataBlock(pJoin->finBlk, false); - blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity); + if (NULL == pJoin->finBlk) { + QRY_ERR_RET(terrno); + } + code = blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity); + if (TSDB_CODE_SUCCESS != code) { + QRY_ERR_RET(terrno); + } } pJoin->blkThreshold = pJoin->finBlk->info.capacity * HJOIN_BLK_THRESHOLD_RATIO; @@ -1151,8 +1194,8 @@ SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t n setOperatorInfo(pOperator, "HashJoinOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0]); - hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1]); + HJ_ERR_JRET(hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0])); + HJ_ERR_JRET(hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1])); hJoinSetBuildAndProbeTable(pInfo, pJoinNode); @@ -1182,6 +1225,7 @@ SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t n return pOperator; _return: + if (pInfo != NULL) { destroyHashJoinOperator(pInfo); } diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index 5f0a2eadfb..1c625df52f 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -357,7 +357,7 @@ static int32_t mOuterJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, bool* contLoop) do { startRowIdx = build->grpRowIdx; - mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); + (void)mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); if (pCtx->midBlk->info.rows > 0) { if (build->rowBitmapSize > 0) { @@ -1283,7 +1283,7 @@ static int32_t mSemiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p do { blockDataCleanup(pCtx->midBlk); - mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); + (void)mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); if (pCtx->midBlk->info.rows > 0) { MJ_ERR_RET(mJoinFilterAndKeepSingleRow(pCtx->midBlk, pCtx->pJoin->pPreFilter)); @@ -1358,7 +1358,7 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) { build->pHashCurGrp = *(SArray**)pGrp; ASSERT(1 == taosArrayGetSize(build->pHashCurGrp)); build->grpRowIdx = 0; - mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); + (void)mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); ASSERT(build->grpRowIdx < 0); } @@ -1616,7 +1616,7 @@ static int32_t mAntiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p do { blockDataCleanup(pCtx->midBlk); - mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); + (void)mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); if (pCtx->midBlk->info.rows > 0) { MJ_ERR_RET(mJoinFilterAndNoKeepRows(pCtx->midBlk, pCtx->pJoin->pPreFilter)); @@ -2102,7 +2102,9 @@ int32_t mAsofBackwardHandleGrpRemains(SMJoinWindowCtx* pCtx) { return (pCtx->lastEqGrp) ? mAsofBackwardDumpUpdateEqRows(pCtx, pCtx->pJoin, false, true) : mAsofBackwardDumpGrpCache(pCtx); } -static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) { +static int32_t mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) { + *newBlock = false; + bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe); bool buildGot = false; @@ -2116,7 +2118,7 @@ static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* mJoinSetDone(pOperator); } - return false; + return TSDB_CODE_SUCCESS; } break; @@ -2124,13 +2126,18 @@ static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* if (buildGot && NULL == pCtx->cache.outBlk) { pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false); - blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit); + if (NULL == pCtx->cache.outBlk) { + MJ_ERR_RET(terrno); + } + MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit)); } pCtx->probeGrp.blk = pJoin->probe->blk; pCtx->buildGrp.blk = pJoin->build->blk; - return true; + *newBlock = true; + + return TSDB_CODE_SUCCESS; } @@ -2142,6 +2149,7 @@ SSDataBlock* mAsofBackwardJoinDo(struct SOperatorInfo* pOperator) { int64_t buildTs = 0; SColumnInfoData* pBuildCol = NULL; SColumnInfoData* pProbeCol = NULL; + bool newBlock = false; blockDataCleanup(pCtx->finBlk); @@ -2154,7 +2162,8 @@ SSDataBlock* mAsofBackwardJoinDo(struct SOperatorInfo* pOperator) { } do { - if (!mAsofBackwardRetrieve(pOperator, pJoin, pCtx)) { + MJ_ERR_JRET(mAsofBackwardRetrieve(pOperator, pJoin, pCtx, &newBlock)); + if (!newBlock) { if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) { continue; } @@ -2271,7 +2280,7 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) { pGrp->readIdx = 0; //pGrp->endIdx = pGrp->blk->info.rows - 1; } else { - taosArrayPop(pCache->grps); + (void)taosArrayPop(pCache->grps); pGrp = taosArrayGet(pCache->grps, 0); ASSERT(pGrp->blk == pCache->outBlk); //pGrp->endIdx = pGrp->blk->info.rows - pGrp->beginIdx; @@ -2479,7 +2488,9 @@ int32_t mAsofForwardSkipBuildGrp(SMJoinWindowCtx* pCtx, SMJoinOperatorInfo* pJoi return TSDB_CODE_SUCCESS; } -static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) { +static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) { + *newBlock = false; + bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe); bool buildGot = false; @@ -2516,7 +2527,10 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p if (buildGot && pJoin->build->newBlk) { if (NULL == pCtx->cache.outBlk) { pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false); - blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit); + if (NULL == pCtx->cache.outBlk) { + MJ_ERR_RET(terrno); + } + MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit)); } MJOIN_PUSH_BLK_TO_CACHE(&pCtx->cache, pJoin->build->blk); @@ -2537,6 +2551,7 @@ SSDataBlock* mAsofForwardJoinDo(struct SOperatorInfo* pOperator) { int64_t buildTs = 0; SColumnInfoData* pBuildCol = NULL; SColumnInfoData* pProbeCol = NULL; + bool newBlock = false; blockDataCleanup(pCtx->finBlk); @@ -2549,7 +2564,8 @@ SSDataBlock* mAsofForwardJoinDo(struct SOperatorInfo* pOperator) { } do { - if (!mAsofForwardRetrieve(pOperator, pJoin, pCtx)) { + MJ_ERR_JRET(mAsofForwardRetrieve(pOperator, pJoin, pCtx, &newBlock)); + if (!newBlock) { if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) { continue; } @@ -2643,7 +2659,7 @@ static FORCE_INLINE void mWinJoinPopFrontGroup(SMJoinWindowCtx* pCtx, SMJoinGrpR if (pGrp->blk == pCtx->cache.outBlk) { blockDataCleanup(pGrp->blk); } else if (pGrp->clonedBlk) { - blockDataDestroy(pGrp->blk); + (void)blockDataDestroy(pGrp->blk); } taosArrayPopFrontBatch(pCtx->cache.grps, 1); @@ -2667,6 +2683,10 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) { pGrp->beginIdx = 0; pGrp->readIdx = 0; } + + if (NULL == pGrp->blk) { + MJ_ERR_RET(terrno); + } pGrp->clonedBlk = true; } @@ -2674,14 +2694,16 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) { return TSDB_CODE_SUCCESS; } -static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) { +static int32_t mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) { + *newBlock = false; + bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe); bool buildGot = false; do { if (probeGot || MJOIN_DS_NEED_INIT(pOperator, pJoin->build)) { if (NULL == pJoin->build->blk) { - mWinJoinCloneCacheBlk(pCtx); + MJ_ERR_RET(mWinJoinCloneCacheBlk(pCtx)); } buildGot = mJoinRetrieveBlk(pJoin, &pJoin->build->blkRowIdx, &pJoin->build->blk, pJoin->build); @@ -2692,7 +2714,7 @@ static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin mJoinSetDone(pOperator); } - return false; + return TSDB_CODE_SUCCESS; } if (buildGot && pCtx->forwardRowsAcq) { @@ -2709,8 +2731,9 @@ static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin } while (true); pCtx->probeGrp.blk = pJoin->probe->blk; - - return true; + *newBlock = true; + + return TSDB_CODE_SUCCESS; } int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) { @@ -3220,6 +3243,7 @@ SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int64_t probeTs = 0; SColumnInfoData* pProbeCol = NULL; + bool newBlock = false; blockDataCleanup(pCtx->finBlk); @@ -3232,7 +3256,8 @@ SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator) { } do { - if (!mWinJoinRetrieve(pOperator, pJoin, pCtx)) { + MJ_ERR_JRET(mWinJoinRetrieve(pOperator, pJoin, pCtx, &newBlock)); + if (!newBlock) { if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) { continue; } @@ -3367,7 +3392,7 @@ int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* p } pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)); + MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode))); pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO; @@ -3407,11 +3432,14 @@ int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJ pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0); - blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)); + MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode))); if (pJoin->pFPreFilter) { pCtx->midBlk = createOneDataBlock(pCtx->finBlk, false); - blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity); + if (NULL == pCtx->midBlk) { + MJ_ERR_RET(terrno); + } + MJ_ERR_RET(blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity)); } pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO; diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 2e2101231b..9db2c7d300 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -1147,6 +1147,7 @@ void mJoinResetForBuildTable(SMJoinTableCtx* pTable) { int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, bool restart) { SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); SMJoinGrpRows* pGrp = NULL; + int32_t code = TSDB_CODE_SUCCESS; if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) { return TSDB_CODE_SUCCESS; @@ -1207,7 +1208,12 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol if (0 == pGrp->beginIdx && pTable->multiEqGrpRows && 0 == pTable->eqRowLimit) { pGrp->blk = createOneDataBlock(pTable->blk, true); - taosArrayPush(pTable->createdBlks, &pGrp->blk); + if (NULL == pGrp->blk) { + MJ_ERR_JRET(terrno); + } + if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) { + MJ_ERR_JRET(terrno); + } } else { if (!pTable->multiEqGrpRows) { pGrp->endIdx = pGrp->beginIdx; From cb9162609ecc52941ccfa7e5a61ea939ca589ce9 Mon Sep 17 00:00:00 2001 From: dmchen Date: Sun, 21 Jul 2024 03:16:12 +0000 Subject: [PATCH 03/43] fix/TD-30989 --- source/dnode/mnode/impl/src/mndShow.c | 47 +- source/dnode/mnode/impl/src/mndSma.c | 643 ++++++++++++++----------- source/dnode/mnode/impl/src/mndSnode.c | 182 ++++--- 3 files changed, 483 insertions(+), 389 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 444ed3d750..10b54e2afb 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -31,17 +31,18 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow); static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq); int32_t mndInitShow(SMnode *pMnode) { + int32_t code = 0; SShowMgmt *pMgmt = &pMnode->showMgmt; pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show"); if (pMgmt->cache == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc show cache since %s", terrstr()); - return -1; + TAOS_RETURN(code); } mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq); - return 0; + TAOS_RETURN(code); } void mndCleanupShow(SMnode *pMnode) { @@ -212,6 +213,7 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { } static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { + int32_t code = 0; SMnode *pMnode = pReq->info.node; SShowMgmt *pMgmt = &pMnode->showMgmt; SShowObj *pShow = NULL; @@ -220,10 +222,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { int32_t rowsRead = 0; mDebug("mndProcessRetrieveSysTableReq start"); SRetrieveTableReq retrieveReq = {0}; - if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } + TAOS_CHECK_RETURN(tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq)); mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb); @@ -232,17 +231,17 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { if (pMeta == NULL) { pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); if (pMeta == NULL) { - terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + code = TSDB_CODE_PAR_TABLE_NOT_EXIST; mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); - return -1; + TAOS_RETURN(code); } } pShow = mndCreateShowObj(pMnode, &retrieveReq); if (pShow == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; mError("failed to process show-meta req since %s", terrstr()); - return -1; + TAOS_RETURN(code); } pShow->pMeta = pMeta; @@ -250,9 +249,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } else { pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); if (pShow == NULL) { - terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; + code = TSDB_CODE_MND_INVALID_SHOWOBJ; mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); - return -1; + TAOS_RETURN(code); } } @@ -264,9 +263,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; if (retrieveFp == NULL) { mndReleaseShowObj(pShow, false); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + code = TSDB_CODE_MSG_NOT_PROCESSED; mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); - return -1; + TAOS_RETURN(code); } mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); @@ -275,14 +274,16 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } else { memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); } - if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { - return -1; + code = -1; + if (retrieveReq.db[0] && + (code = mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db)) != 0) { + TAOS_RETURN(code); } if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) { if(strcmp(pReq->info.conn.user, "root") != 0){ mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; + code = TSDB_CODE_MND_NO_RIGHTS; + TAOS_RETURN(code); } } @@ -308,11 +309,11 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } else { rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead); if (rowsRead < 0) { - terrno = rowsRead; + code = rowsRead; mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); mndReleaseShowObj(pShow, true); blockDataDestroy(pBlock); - return -1; + TAOS_RETURN(code); } pBlock->info.rows = rowsRead; @@ -325,10 +326,10 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { mndReleaseShowObj(pShow, false); - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); blockDataDestroy(pBlock); - return -1; + TAOS_RETURN(code); } pRsp->handle = htobe64(pShow->id); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 946df84a0f..1a59b35cf1 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -405,48 +405,74 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, } static int32_t mndSetCreateSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + int32_t code = 0; SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetCreateSmaUndoLogs(SMnode* pMnode, STrans* pTrans, SSmaObj* pSma) { + int32_t code = 0; SSdbRaw * pUndoRaw = mndSmaActionEncode(pSma); - if (!pUndoRaw) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; + if (!pUndoRaw) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED)); + TAOS_RETURN(code); } static int32_t mndSetCreateSmaCommitLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSmaActionEncode(pSma); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetCreateSmaVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { + int32_t code = 0; SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_UPDATE) != 0) return -1; - return 0; + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pVgRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_UPDATE)); + TAOS_RETURN(code); } static int32_t mndSetCreateSmaVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { + int32_t code = 0; SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; - return 0; + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pVgRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_READY)); + TAOS_RETURN(code); } static int32_t mndSetUpdateSmaStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + int32_t code = 0; SStbObj stbObj = {0}; taosRLockLatch(&pStb->lock); memcpy(&stbObj, pStb, sizeof(SStbObj)); @@ -462,18 +488,27 @@ static int32_t mndSetUpdateSmaStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb stbObj.smaVer++; SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SSmaObj *pSma) { + int32_t code = 0; SVnodeGid *pVgid = pVgroup->vnodeGid + 0; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; + if (pDnode == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -481,12 +516,10 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, // todo add sma info here SNode *pAst = NULL; - if (nodesStringToNode(pSma->ast, &pAst) < 0) { - return -1; - } - if (qExtractResultSchema(pAst, &pSma->schemaRow.nCols, &pSma->schemaRow.pSchema) != 0) { + TAOS_CHECK_RETURN(nodesStringToNode(pSma->ast, &pAst)); + if ((code = qExtractResultSchema(pAst, &pSma->schemaRow.nCols, &pSma->schemaRow.pSchema)) != 0) { nodesDestroyNode(pAst); - return -1; + TAOS_RETURN(code); } nodesDestroyNode(pAst); pSma->schemaRow.version = 1; @@ -496,7 +529,7 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, pSma->schemaTag.version = 1; pSma->schemaTag.pSchema = taosMemoryCalloc(1, sizeof(SSchema)); if (!pSma->schemaTag.pSchema) { - return -1; + TAOS_RETURN(-1); } pSma->schemaTag.pSchema[0].type = TSDB_DATA_TYPE_BIGINT; pSma->schemaTag.pSchema[0].bytes = TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]; @@ -506,14 +539,20 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, int32_t smaContLen = 0; void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); - if (pSmaReq == NULL) return -1; + if (pSmaReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } pVgroup->pTsma = pSmaReq; int32_t contLen = 0; void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); if (pReq == NULL) { taosMemoryFreeClear(pSmaReq); - return -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } action.mTraceId = pTrans->mTraceId; @@ -522,10 +561,10 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFreeClear(pSmaReq); taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } action.pCont = pSmaReq; @@ -533,12 +572,12 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, action.msgType = TDMT_VND_CREATE_SMA; action.acceptableCode = TSDB_CODE_TSMA_ALREADY_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFreeClear(pSmaReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static void mndDestroySmaObj(SSmaObj *pSmaObj) { @@ -550,10 +589,11 @@ static void mndDestroySmaObj(SSmaObj *pSmaObj) { static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCreate, SDbObj *pDb, SStbObj *pStb, const char *streamName) { + int32_t code = 0; if (pDb->cfg.replications > 1) { - terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + code = TSDB_CODE_MND_INVALID_SMA_OPTION; mError("sma:%s, failed to create since not support multiple replicas", pCreate->name); - return -1; + TAOS_RETURN(code); } SSmaObj smaObj = {0}; memcpy(smaObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); @@ -623,25 +663,25 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.conf.triggerParam = TSDB_MAX_ROLLUP_MAX_DELAY; } - if (mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg) != 0) { + if ((code = mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg)) != 0) { mError("sma:%s, failed to create since %s", smaObj.name, terrstr()); - return -1; + TAOS_RETURN(code); } smaObj.dstVgId = streamObj.fixedSinkVg.vgId; streamObj.fixedSinkVgId = smaObj.dstVgId; SNode *pAst = NULL; if (nodesStringToNode(streamObj.ast, &pAst) < 0) { - terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + code = TSDB_CODE_MND_INVALID_SMA_OPTION; mError("sma:%s, failed to create since parse ast error", smaObj.name); - return -1; + TAOS_RETURN(code); } // extract output schema from ast if (qExtractResultSchema(pAst, (int32_t *)&streamObj.outputSchema.nCols, &streamObj.outputSchema.pSchema) != 0) { - terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + code = TSDB_CODE_MND_INVALID_SMA_OPTION; mError("sma:%s, failed to create since extract result schema error", smaObj.name); - return -1; + TAOS_RETURN(code); } SQueryPlan *pPlan = NULL; @@ -655,39 +695,43 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea }; if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { - terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + code = TSDB_CODE_MND_INVALID_SMA_OPTION; mError("sma:%s, failed to create since create query plan error", smaObj.name); - return -1; + TAOS_RETURN(code); } // save physcial plan if (nodesNodeToString((SNode *)pPlan, false, &streamObj.physicalPlan, NULL) != 0) { - terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + code = TSDB_CODE_MND_INVALID_SMA_OPTION; mError("sma:%s, failed to create since save physcial plan error", smaObj.name); - return -1; + TAOS_RETURN(code); } if (pAst != NULL) nodesDestroyNode(pAst); nodesDestroyNode((SNode *)pPlan); - int32_t code = -1; + code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-sma"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); mndTransSetSerial(pTrans); mInfo("trans:%d, used to create sma:%s stream:%s", pTrans->id, pCreate->name, streamObj.name); - if (mndAddNewVgPrepareAction(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; - if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; - if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; - if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; - if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; - if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; - if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; - if (mndScheduleStream(pMnode, &streamObj, 1685959190000, NULL) != 0) goto _OVER; - if (mndPersistStream(pTrans, &streamObj) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddNewVgPrepareAction(pMnode, pTrans, &streamObj.fixedSinkVg), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndScheduleStream(pMnode, &streamObj, 1685959190000, NULL), NULL, _OVER); + TAOS_CHECK_GOTO(mndPersistStream(pTrans, &streamObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); mInfo("sma:%s, uid:%" PRIi64 " create on stb:%" PRIi64 ", dstSuid:%" PRIi64 " dstTb:%s dstVg:%d", pCreate->name, smaObj.uid, smaObj.stbUid, smaObj.dstTbUid, smaObj.dstTbName, smaObj.dstVgId); @@ -698,35 +742,34 @@ _OVER: tFreeStreamObj(&streamObj); mndDestroySmaObj(&smaObj); mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { - terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; - if (pCreate->name[0] == 0) return -1; - if (pCreate->stb[0] == 0) return -1; - if (pCreate->igExists < 0 || pCreate->igExists > 1) return -1; - if (pCreate->intervalUnit < 0) return -1; - if (pCreate->slidingUnit < 0) return -1; - if (pCreate->timezone < 0) return -1; - if (pCreate->interval < 0) return -1; - if (pCreate->offset < 0) return -1; - if (pCreate->sliding < 0) return -1; - if (pCreate->exprLen < 0) return -1; - if (pCreate->tagsFilterLen < 0) return -1; - if (pCreate->sqlLen < 0) return -1; - if (pCreate->astLen < 0) return -1; - if (pCreate->exprLen != 0 && strlen(pCreate->expr) + 1 != pCreate->exprLen) return -1; - if (pCreate->tagsFilterLen != 0 && strlen(pCreate->tagsFilter) + 1 != pCreate->tagsFilterLen) return -1; - if (pCreate->sqlLen != 0 && strlen(pCreate->sql) + 1 != pCreate->sqlLen) return -1; - if (pCreate->astLen != 0 && strlen(pCreate->ast) + 1 != pCreate->astLen) return -1; + int32_t code = TSDB_CODE_MND_INVALID_SMA_OPTION; + if (pCreate->name[0] == 0) TAOS_RETURN(code); + if (pCreate->stb[0] == 0) TAOS_RETURN(code); + if (pCreate->igExists < 0 || pCreate->igExists > 1) TAOS_RETURN(code); + if (pCreate->intervalUnit < 0) TAOS_RETURN(code); + if (pCreate->slidingUnit < 0) TAOS_RETURN(code); + if (pCreate->timezone < 0) TAOS_RETURN(code); + if (pCreate->interval < 0) TAOS_RETURN(code); + if (pCreate->offset < 0) TAOS_RETURN(code); + if (pCreate->sliding < 0) TAOS_RETURN(code); + if (pCreate->exprLen < 0) TAOS_RETURN(code); + if (pCreate->tagsFilterLen < 0) TAOS_RETURN(code); + if (pCreate->sqlLen < 0) TAOS_RETURN(code); + if (pCreate->astLen < 0) TAOS_RETURN(code); + if (pCreate->exprLen != 0 && strlen(pCreate->expr) + 1 != pCreate->exprLen) TAOS_RETURN(code); + if (pCreate->tagsFilterLen != 0 && strlen(pCreate->tagsFilter) + 1 != pCreate->tagsFilterLen) TAOS_RETURN(code); + if (pCreate->sqlLen != 0 && strlen(pCreate->sql) + 1 != pCreate->sqlLen) TAOS_RETURN(code); + if (pCreate->astLen != 0 && strlen(pCreate->ast) + 1 != pCreate->astLen) TAOS_RETURN(code); SName smaName = {0}; if (tNameFromString(&smaName, pCreate->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE) < 0) return -1; if (*(char *)tNameGetTableName(&smaName) == 0) return -1; - terrno = 0; - return 0; + TAOS_RETURN(code); } static void mndGetStreamNameFromSmaName(char *streamName, char *smaName) { @@ -746,22 +789,20 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { int64_t mTraceId = TRACE_GET_ROOTID(&pReq->info.traceId); - if (tDeserializeSMCreateSmaReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSMCreateSmaReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER); + #ifdef WINDOWS terrno = TSDB_CODE_MND_INVALID_PLATFORM; goto _OVER; #endif mInfo("sma:%s, start to create", createReq.name); - if (mndCheckCreateSmaReq(&createReq) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckCreateSmaReq(&createReq), NULL, _OVER); pStb = mndAcquireStb(pMnode, createReq.stb); if (pStb == NULL) { mError("sma:%s, failed to create since stb:%s not exist", createReq.name, createReq.stb); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; goto _OVER; } @@ -771,11 +812,11 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { pStream = mndAcquireStream(pMnode, streamName); if (pStream != NULL) { mError("sma:%s, failed to create since stream:%s already exist", createReq.name, streamName); - terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; + code = TSDB_CODE_MND_STREAM_ALREADY_EXIST; goto _OVER; } SSIdx idx = {0}; - if (mndAcquireGlobalIdx(pMnode, createReq.name, SDB_SMA, &idx) == 0) { + if ((code = mndAcquireGlobalIdx(pMnode, createReq.name, SDB_SMA, &idx)) == 0) { pSma = idx.pIdx; } else { goto _OVER; @@ -787,20 +828,18 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { code = 0; goto _OVER; } else { - terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + code = TSDB_CODE_MND_SMA_ALREADY_EXIST; goto _OVER; } } pDb = mndAcquireDbBySma(pMnode, createReq.name); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + code = TSDB_CODE_MND_DB_NOT_SELECTED; goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER); code = mndCreateSma(pMnode, pReq, &createReq, pDb, pStb, streamName); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -816,49 +855,74 @@ _OVER: mndReleaseDb(pMnode, pDb); tFreeSMCreateSmaReq(&createReq); - return code; + TAOS_RETURN(code); } static int32_t mndSetDropSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + int32_t code = 0; SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + return -1; + } + TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)); return 0; } static int32_t mndSetDropSmaCommitLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSmaActionEncode(pSma); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + return -1; + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)); return 0; } static int32_t mndSetDropSmaVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { + int32_t code = 0; SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPING) != 0) return -1; + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + return -1; + } + TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pVgRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPING)); return 0; } static int32_t mndSetDropSmaVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { + int32_t code = 0; SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED) != 0) return -1; + if (pVgRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + return -1; + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pVgRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED)); return 0; } static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + int32_t code = 0; SVnodeGid *pVgid = pVgroup->vnodeGid + 0; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; + if (pDnode == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -866,19 +930,23 @@ static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SD int32_t contLen = 0; void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; + if (pReq == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_VND_NOT_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *pSma) { @@ -888,17 +956,29 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p STrans *pTrans = NULL; pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); - if (pVgroup == NULL) goto _OVER; + if (pVgroup == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } pStb = mndAcquireStb(pMnode, pSma->stb); - if (pStb == NULL) goto _OVER; + if (pStb == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-sma"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); mndTransSetSerial(pTrans); @@ -910,26 +990,26 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p sdbRelease(pMnode->pSdb, pStream); goto _OVER; } else { - if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) { + if ((code = mndStreamSetDropAction(pMnode, pTrans, pStream)) < 0) { mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); sdbRelease(pMnode->pSdb, pStream); goto _OVER; } // drop stream - if (mndPersistTransLog(pStream, pTrans, SDB_STATUS_DROPPED) < 0) { + if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_DROPPED)) < 0) { mError("stream:%s, failed to drop log since %s", pStream->name, terrstr()); sdbRelease(pMnode->pSdb, pStream); goto _OVER; } } - if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER; - if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; - if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; - if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; - if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; - if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetDropSmaRedoLogs(pMnode, pTrans, pSma), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropSmaCommitLogs(pMnode, pTrans, pSma), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; @@ -938,7 +1018,7 @@ _OVER: mndReleaseStream(pMnode, pStream); mndReleaseVgroup(pMnode, pVgroup); mndReleaseStb(pMnode, pStb); - return code; + TAOS_RETURN(code); } int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { @@ -955,20 +1035,24 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p if (pSma->stbUid == pStb->uid) { mndTransSetSerial(pTrans); pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); - if (pVgroup == NULL) goto _OVER; + if (pVgroup == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } char streamName[TSDB_TABLE_FNAME_LEN] = {0}; mndGetStreamNameFromSmaName(streamName, pSma->name); SStreamObj *pStream = mndAcquireStream(pMnode, streamName); if (pStream != NULL && pStream->smaId == pSma->uid) { - if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) { + if ((code = mndStreamSetDropAction(pMnode, pTrans, pStream)) < 0) { mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); mndReleaseStream(pMnode, pStream); goto _OVER; } - if (mndPersistTransLog(pStream, pTrans, SDB_STATUS_DROPPED) < 0) { + if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_DROPPED)) < 0) { mndReleaseStream(pMnode, pStream); goto _OVER; } @@ -976,9 +1060,9 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p mndReleaseStream(pMnode, pStream); } - if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; - if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; - if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropSmaCommitLogs(pMnode, pTrans, pSma), NULL, _OVER); mndReleaseVgroup(pMnode, pVgroup); pVgroup = NULL; } @@ -992,10 +1076,11 @@ _OVER: sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pSma); mndReleaseVgroup(pMnode, pVgroup); - return code; + TAOS_RETURN(code); } int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -1005,17 +1090,17 @@ int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { if (pIter == NULL) break; if (pSma->dbUid == pDb->uid) { - if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) { + if ((code = mndSetDropSmaCommitLogs(pMnode, pTrans, pSma)) != 0) { sdbRelease(pSdb, pSma); sdbCancelFetch(pSdb, pSma); - return -1; + TAOS_RETURN(code); } } sdbRelease(pSdb, pSma); } - return 0; + TAOS_RETURN(code); } static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { @@ -1025,10 +1110,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { SSmaObj *pSma = NULL; SMDropSmaReq dropReq = {0}; - if (tDeserializeSMDropSmaReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSMDropSmaReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER); mInfo("sma:%s, start to drop", dropReq.name); @@ -1044,20 +1126,18 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { code = 0; goto _OVER; } else { - terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + code = TSDB_CODE_MND_SMA_NOT_EXIST; goto _OVER; } } pDb = mndAcquireDbBySma(pMnode, dropReq.name); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + code = TSDB_CODE_MND_DB_NOT_SELECTED; goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER); code = mndDropSma(pMnode, pReq, pDb, pSma); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1069,7 +1149,7 @@ _OVER: mndReleaseSma(pMnode, pSma); mndReleaseDb(pMnode, pDb); - return code; + TAOS_RETURN(code); } static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { @@ -1149,7 +1229,8 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool SVgObj *pVg = mndAcquireVgroup(pMnode, pSma->dstVgId); if (pVg == NULL) { - code = -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; sdbRelease(pSdb, pSma); sdbCancelFetch(pSdb, pIter); return code; @@ -1158,8 +1239,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool info.expr = taosMemoryMalloc(pSma->exprLen + 1); if (info.expr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pSma); sdbCancelFetch(pSdb, pIter); return code; @@ -1169,8 +1249,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool info.expr[pSma->exprLen] = 0; if (NULL == taosArrayPush(rsp->pIndex, &info)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(info.expr); sdbRelease(pSdb, pSma); sdbCancelFetch(pSdb, pIter); @@ -1183,7 +1262,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool sdbRelease(pSdb, pSma); } - return code; + TAOS_RETURN(code); } static int32_t mndProcessGetSmaReq(SRpcMsg *pReq) { @@ -1193,10 +1272,7 @@ static int32_t mndProcessGetSmaReq(SRpcMsg *pReq) { SUserIndexRsp rsp = {0}; bool exist = false; - if (tDeserializeSUserIndexReq(pReq->pCont, pReq->contLen, &indexReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSUserIndexReq(pReq->pCont, pReq->contLen, &indexReq), NULL, _OVER); code = mndGetSma(pMnode, &indexReq, &rsp, &exist); if (code) { @@ -1205,14 +1281,12 @@ static int32_t mndProcessGetSmaReq(SRpcMsg *pReq) { if (!exist) { // TODO GET INDEX FROM FULLTEXT - code = -1; - terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + code = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; } else { int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -1229,7 +1303,7 @@ _OVER: mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); } - return code; + TAOS_RETURN(code); } static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { @@ -1239,15 +1313,11 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { STableIndexRsp rsp = {0}; bool exist = false; - if (tDeserializeSTableIndexReq(pReq->pCont, pReq->contLen, &indexReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSTableIndexReq(pReq->pCont, pReq->contLen, &indexReq), NULL, _OVER); rsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); if (NULL == rsp.pIndex) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -1257,14 +1327,12 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { } if (!exist) { - code = -1; - terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + code = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; } else { int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -1282,7 +1350,7 @@ _OVER: } tFreeSerializeSTableIndexRsp(&rsp); - return code; + TAOS_RETURN(code); } static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { @@ -1500,27 +1568,37 @@ static void mndCreateTSMABuildDropStreamReq(SCreateTSMACxt* pCxt) { } static int32_t mndSetUpdateDbTsmaVersionPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { + int32_t code = 0; SSdbRaw *pRedoRaw = mndDbActionEncode(pOld); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) { sdbFreeRaw(pRedoRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - return 0; + TAOS_RETURN(code); } static int32_t mndSetUpdateDbTsmaVersionCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndDbActionEncode(pNew); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { sdbFreeRaw(pCommitRaw); - return -1; + TAOS_RETURN(code); } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - return 0; + TAOS_RETURN(code); } static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { @@ -1532,11 +1610,11 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pCxt->pRpcReq, "create-tsma"); if (!pTrans) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } mndTransSetDbName(pTrans, pCxt->pDb->name, NULL); - if (mndTransCheckConflict(pCxt->pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pCxt->pMnode, pTrans), NULL, _OVER); mndTransSetSerial(pTrans); mInfo("trans:%d, used to create tsma:%s stream:%s", pTrans->id, pCxt->pCreateSmaReq->name, @@ -1548,12 +1626,12 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { createStreamRedoAction.contLen = tSerializeSCMCreateStreamReq(0, 0, pCxt->pCreateStreamReq); createStreamRedoAction.pCont = taosMemoryCalloc(1, createStreamRedoAction.contLen); if (!createStreamRedoAction.pCont) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } if (createStreamRedoAction.contLen != tSerializeSCMCreateStreamReq(createStreamRedoAction.pCont, createStreamRedoAction.contLen, pCxt->pCreateStreamReq)) { mError("sma: %s, failed to create due to create stream req encode failure", pCxt->pCreateSmaReq->name); - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -1563,12 +1641,12 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { createStreamUndoAction.contLen = tSerializeSMDropStreamReq(0, 0, pCxt->pDropStreamReq); createStreamUndoAction.pCont = taosMemoryCalloc(1, createStreamUndoAction.contLen); if (!createStreamUndoAction.pCont) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } if (createStreamUndoAction.contLen != tSerializeSMDropStreamReq(createStreamUndoAction.pCont, createStreamUndoAction.contLen, pCxt->pDropStreamReq)) { mError("sma: %s, failed to create due to drop stream req encode failure", pCxt->pCreateSmaReq->name); - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -1581,37 +1659,37 @@ static int32_t mndCreateTSMATxnPrepare(SCreateTSMACxt* pCxt) { dropStbUndoAction.contLen = tSerializeSMDropStbReq(0, 0, &dropStbReq); dropStbUndoAction.pCont = taosMemoryCalloc(1, dropStbUndoAction.contLen); if (!dropStbUndoAction.pCont) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } if (dropStbUndoAction.contLen != tSerializeSMDropStbReq(dropStbUndoAction.pCont, dropStbUndoAction.contLen, &dropStbReq)) { mError("sma: %s, failed to create due to drop stb req encode failure", pCxt->pCreateSmaReq->name); - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } SDbObj newDb = {0}; memcpy(&newDb, pCxt->pDb, sizeof(SDbObj)); newDb.tsmaVersion++; - if (mndSetUpdateDbTsmaVersionPrepareLogs(pCxt->pMnode, pTrans, pCxt->pDb, &newDb) != 0) goto _OVER; - if (mndSetUpdateDbTsmaVersionCommitLogs(pCxt->pMnode, pTrans, pCxt->pDb, &newDb) != 0) goto _OVER; - if (mndSetCreateSmaRedoLogs(pCxt->pMnode, pTrans, pCxt->pSma) != 0) goto _OVER; - if (mndSetCreateSmaUndoLogs(pCxt->pMnode, pTrans, pCxt->pSma) != 0) goto _OVER; - if (mndSetCreateSmaCommitLogs(pCxt->pMnode, pTrans, pCxt->pSma) != 0) goto _OVER; - if (mndTransAppendRedoAction(pTrans, &createStreamRedoAction) != 0) goto _OVER; - if (mndTransAppendUndoAction(pTrans, &createStreamUndoAction) != 0) goto _OVER; - if (mndTransAppendUndoAction(pTrans, &dropStbUndoAction) != 0) goto _OVER; - if (mndTransPrepare(pCxt->pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetUpdateDbTsmaVersionPrepareLogs(pCxt->pMnode, pTrans, pCxt->pDb, &newDb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetUpdateDbTsmaVersionCommitLogs(pCxt->pMnode, pTrans, pCxt->pDb, &newDb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSmaRedoLogs(pCxt->pMnode, pTrans, pCxt->pSma), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSmaUndoLogs(pCxt->pMnode, pTrans, pCxt->pSma), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSmaCommitLogs(pCxt->pMnode, pTrans, pCxt->pSma), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransAppendRedoAction(pTrans, &createStreamRedoAction), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransAppendUndoAction(pTrans, &createStreamUndoAction), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransAppendUndoAction(pTrans, &dropStbUndoAction), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pCxt->pMnode, pTrans), NULL, _OVER); code = TSDB_CODE_SUCCESS; _OVER: mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) { - int32_t code; + int32_t code = 0; SSmaObj sma = {0}; SCMCreateStreamReq createStreamReq = {0}; SMDropStreamReq dropStreamReq = {0}; @@ -1620,9 +1698,8 @@ static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) { initSMAObj(pCxt); SNodeList* pProjects = NULL; - terrno = nodesStringToList(pCxt->pCreateSmaReq->expr, &pProjects); - if (TSDB_CODE_SUCCESS != terrno) { - code = -1; + code = nodesStringToList(pCxt->pCreateSmaReq->expr, &pProjects); + if (TSDB_CODE_SUCCESS != code) { goto _OVER; } pCxt->pProjects = pProjects; @@ -1631,16 +1708,14 @@ static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) { if (pCxt->pCreateSmaReq->pVgroupVerList) { pCxt->pCreateStreamReq->pVgroupVerList = taosArrayDup(pCxt->pCreateSmaReq->pVgroupVerList, NULL); if (!pCxt->pCreateStreamReq->pVgroupVerList) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } } if (LIST_LENGTH(pProjects) > 0) { createStreamReq.pCols = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SField)); if (!createStreamReq.pCols) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } } @@ -1648,8 +1723,7 @@ static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) { mndCreateTSMABuildCreateStreamReq(pCxt); mndCreateTSMABuildDropStreamReq(pCxt); - if (TSDB_CODE_SUCCESS != mndCreateTSMATxnPrepare(pCxt)) { - code = -1; + if (TSDB_CODE_SUCCESS != (code = mndCreateTSMATxnPrepare(pCxt))) { goto _OVER; } else { mInfo("sma:%s, uid:%" PRIi64 " create on stb:%" PRIi64 " dstTb:%s dstVg:%d", pCxt->pCreateSmaReq->name, sma.uid, @@ -1663,7 +1737,7 @@ _OVER: pCxt->pCreateStreamReq = NULL; if (pProjects) nodesDestroyList(pProjects); pCxt->pProjects = NULL; - return code; + TAOS_RETURN(code); } static void mndTSMAGenerateOutputName(const char* tsmaName, char* streamName, char* targetStbName) { @@ -1689,24 +1763,23 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { SMCreateSmaReq createReq = {0}; if (sdbGetSize(pMnode->pSdb, SDB_SMA) >= tsMaxTsmaNum) { - terrno = TSDB_CODE_MND_MAX_TSMA_NUM_EXCEEDED; + code = TSDB_CODE_MND_MAX_TSMA_NUM_EXCEEDED; goto _OVER; } if (tDeserializeSMCreateSmaReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } mInfo("start to create tsma: %s", createReq.name); - if (mndCheckCreateSmaReq(&createReq)) - goto _OVER; + if ((code = mndCheckCreateSmaReq(&createReq)) != 0) goto _OVER; if (createReq.normSourceTbUid == 0) { pStb = mndAcquireStb(pMnode, createReq.stb); if (!pStb && !createReq.recursiveTsma) { mError("tsma:%s, failed to create since stb:%s not exist", createReq.name, createReq.stb); - terrno = TSDB_CODE_MND_STB_NOT_EXIST; + code = TSDB_CODE_MND_STB_NOT_EXIST; goto _OVER; } } @@ -1722,13 +1795,13 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { goto _OVER; } if (pSma) { - terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + code = TSDB_CODE_MND_SMA_ALREADY_EXIST; goto _OVER; } SStbObj *pTargetStb = mndAcquireStb(pMnode, streamTargetStbFullName); if (pTargetStb) { - terrno = TSDB_CODE_TDB_STB_ALREADY_EXIST; + code = TSDB_CODE_TDB_STB_ALREADY_EXIST; mError("tsma: %s, failed to create since output stable already exists: %s", createReq.name, streamTargetStbFullName); goto _OVER; @@ -1737,25 +1810,23 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { pStream = mndAcquireStream(pMnode, streamName); if (pStream != NULL) { mError("tsma:%s, failed to create since stream:%s already exist", createReq.name, streamName); - terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + code = TSDB_CODE_MND_SMA_ALREADY_EXIST; goto _OVER; } pDb = mndAcquireDbBySma(pMnode, createReq.name); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + code = TSDB_CODE_MND_DB_NOT_SELECTED; goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER); if (createReq.recursiveTsma) { pBaseTsma = sdbAcquire(pMnode->pSdb, SDB_SMA, createReq.baseTsmaName); if (!pBaseTsma) { mError("base tsma: %s not found when creating recursive tsma", createReq.baseTsmaName); - terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + code = TSDB_CODE_MND_SMA_NOT_EXIST; goto _OVER; } if (!pStb) { @@ -1791,7 +1862,7 @@ _OVER: mndReleaseDb(pMnode, pDb); tFreeSMCreateSmaReq(&createReq); - return code; + TAOS_RETURN(code); } static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { @@ -1799,7 +1870,7 @@ static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { STransAction dropStreamRedoAction = {0}; STrans *pTrans = mndTransCreate(pCxt->pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pCxt->pRpcReq, "drop-tsma"); if (!pTrans) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } SMDropStreamReq dropStreamReq = {0}; @@ -1814,13 +1885,13 @@ static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { dropStreamRedoAction.contLen = tSerializeSMDropStreamReq(0, 0, pCxt->pDropStreamReq); dropStreamRedoAction.pCont = taosMemoryCalloc(1, dropStreamRedoAction.contLen); if (!dropStreamRedoAction.pCont) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } if (dropStreamRedoAction.contLen != tSerializeSMDropStreamReq(dropStreamRedoAction.pCont, dropStreamRedoAction.contLen, pCxt->pDropStreamReq)) { mError("tsma: %s, failed to drop due to drop stream req encode failure", pCxt->pDropSmaReq->name); - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -1838,30 +1909,30 @@ static int32_t mndDropTSMA(SCreateTSMACxt* pCxt) { dropStbRedoAction.contLen = tSerializeSMDropStbReq(0, 0, &dropStbReq); dropStbRedoAction.pCont = taosMemoryCalloc(1, dropStbRedoAction.contLen); if (!dropStbRedoAction.pCont) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } if (dropStbRedoAction.contLen != tSerializeSMDropStbReq(dropStbRedoAction.pCont, dropStbRedoAction.contLen, &dropStbReq)) { mError("tsma: %s, failedto drop due to drop stb req encode failure", pCxt->pDropSmaReq->name); - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } SDbObj newDb = {0}; memcpy(&newDb, pCxt->pDb, sizeof(SDbObj)); newDb.tsmaVersion++; - if (mndSetUpdateDbTsmaVersionPrepareLogs(pCxt->pMnode, pTrans, pCxt->pDb, &newDb) != 0) goto _OVER; - if (mndSetUpdateDbTsmaVersionCommitLogs(pCxt->pMnode, pTrans, pCxt->pDb, &newDb) != 0) goto _OVER; - if (mndSetDropSmaRedoLogs(pCxt->pMnode, pTrans, pCxt->pSma) != 0) goto _OVER; - if (mndSetDropSmaCommitLogs(pCxt->pMnode, pTrans, pCxt->pSma) != 0) goto _OVER; - if (mndTransAppendRedoAction(pTrans, &dropStreamRedoAction) != 0) goto _OVER; - if (mndTransAppendRedoAction(pTrans, &dropStbRedoAction) != 0) goto _OVER; - if (mndTransPrepare(pCxt->pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetUpdateDbTsmaVersionPrepareLogs(pCxt->pMnode, pTrans, pCxt->pDb, &newDb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetUpdateDbTsmaVersionCommitLogs(pCxt->pMnode, pTrans, pCxt->pDb, &newDb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropSmaRedoLogs(pCxt->pMnode, pTrans, pCxt->pSma), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropSmaCommitLogs(pCxt->pMnode, pTrans, pCxt->pSma), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransAppendRedoAction(pTrans, &dropStreamRedoAction), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransAppendRedoAction(pTrans, &dropStbRedoAction), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pCxt->pMnode, pTrans), NULL, _OVER); code = TSDB_CODE_SUCCESS; _OVER: tFreeMDropStreamReq(pCxt->pDropStreamReq); mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static bool hasRecursiveTsmasBasedOnMe(SMnode* pMnode, const SSmaObj* pSma) { @@ -1887,7 +1958,7 @@ static int32_t mndProcessDropTSMAReq(SRpcMsg* pReq) { SDbObj * pDb = NULL; SMnode * pMnode = pReq->info.node; if (tDeserializeSMDropSmaReq(pReq->pCont, pReq->contLen, &dropReq) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -1903,21 +1974,21 @@ static int32_t mndProcessDropTSMAReq(SRpcMsg* pReq) { goto _OVER; } if (!pSma) { - terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + code = TSDB_CODE_MND_SMA_NOT_EXIST; goto _OVER; } pDb = mndAcquireDbBySma(pMnode, dropReq.name); if (!pDb) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + code = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if ((code = mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb)) != 0) { goto _OVER; } if (hasRecursiveTsmasBasedOnMe(pMnode, pSma)) { - terrno = TSDB_CODE_MND_INVALID_DROP_TSMA; + code = TSDB_CODE_MND_INVALID_DROP_TSMA; goto _OVER; } @@ -1939,7 +2010,7 @@ _OVER: mndReleaseStb(pMnode, pStb); mndReleaseSma(pMnode, pSma); mndReleaseDb(pMnode, pDb); - return code; + TAOS_RETURN(code); } static int32_t mndRetrieveTSMA(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { @@ -2104,7 +2175,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa funcInfo.funcId = pFuncNode->funcId; funcInfo.colId = ((SColumnNode *)pFuncNode->pParameterList->pHead->pNode)->colId; if (!taosArrayPush(pInfo->pFuncs, &funcInfo)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(pInfo->pFuncs); nodesDestroyNode(pNode); return code; @@ -2136,7 +2207,7 @@ int32_t dumpTSMAInfoFromSmaObj(const SSmaObj* pSma, const SStbObj* pDestStb, STa } } } - return code; + TAOS_RETURN(code); } // @note remember to mndReleaseSma(*ppOut) @@ -2181,30 +2252,30 @@ static int32_t mndGetTSMA(SMnode *pMnode, char *tsmaFName, STableTSMAInfoRsp *rs STableTSMAInfo *pTsma = taosMemoryCalloc(1, sizeof(STableTSMAInfo)); if (!pTsma) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pMnode->pSdb, pSma); mndReleaseStb(pMnode, pDstStb); - return code; + TAOS_RETURN(code); } - terrno = mndGetDeepestBaseForTsma(pMnode, pSma, &pBaseTsma); - if (terrno == 0) { - terrno = dumpTSMAInfoFromSmaObj(pSma, pDstStb, pTsma, pBaseTsma); + code = mndGetDeepestBaseForTsma(pMnode, pSma, &pBaseTsma); + if (code == 0) { + code = dumpTSMAInfoFromSmaObj(pSma, pDstStb, pTsma, pBaseTsma); } mndReleaseStb(pMnode, pDstStb); sdbRelease(pMnode->pSdb, pSma); if (pBaseTsma) mndReleaseSma(pMnode, pBaseTsma); if (terrno) { tFreeTableTSMAInfo(pTsma); - return code; + TAOS_RETURN(code); } if (NULL == taosArrayPush(rsp->pTsmas, &pTsma)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; tFreeTableTSMAInfo(pTsma); } *exist = true; } - return 0; + TAOS_RETURN(code); } typedef bool (*tsmaFilter)(const SSmaObj* pSma, void* param); @@ -2248,17 +2319,17 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt STableTSMAInfo *pTsma = taosMemoryCalloc(1, sizeof(STableTSMAInfo)); if (!pTsma) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; mndReleaseStb(pMnode, pStb); sdbRelease(pSdb, pSma); sdbCancelFetch(pSdb, pIter); - return code; + TAOS_RETURN(code); } pTsma->streamUid = streamId; - terrno = mndGetDeepestBaseForTsma(pMnode, pSma, &pBaseTsma); - if (terrno == 0) { - terrno = dumpTSMAInfoFromSmaObj(pSma, pStb, pTsma, pBaseTsma); + code = mndGetDeepestBaseForTsma(pMnode, pSma, &pBaseTsma); + if (code == 0) { + code = dumpTSMAInfoFromSmaObj(pSma, pStb, pTsma, pBaseTsma); } mndReleaseStb(pMnode, pStb); sdbRelease(pSdb, pSma); @@ -2266,13 +2337,13 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt if (terrno) { tFreeTableTSMAInfo(pTsma); sdbCancelFetch(pSdb, pIter); - return code; + TAOS_RETURN(code); } if (NULL == taosArrayPush(pRsp->pTsmas, &pTsma)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; tFreeTableTSMAInfo(pTsma); sdbCancelFetch(pSdb, pIter); - return code; + TAOS_RETURN(code); } *exist = true; } @@ -2304,15 +2375,11 @@ static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq) { bool exist = false; SMnode * pMnode = pReq->info.node; - if (tDeserializeTableTSMAInfoReq(pReq->pCont, pReq->contLen, &tsmaReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeTableTSMAInfoReq(pReq->pCont, pReq->contLen, &tsmaReq), NULL, _OVER); rsp.pTsmas = taosArrayInit(4, POINTER_BYTES); if (NULL == rsp.pTsmas) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -2326,14 +2393,12 @@ static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq) { } if (!exist) { - code = -1; - terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + code = TSDB_CODE_MND_SMA_NOT_EXIST; } else { int32_t contLen = tSerializeTableTSMAInfoRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -2347,7 +2412,7 @@ static int32_t mndProcessGetTbTSMAReq(SRpcMsg *pReq) { _OVER: tFreeTableTSMAInfoRsp(&rsp); - return code; + TAOS_RETURN(code); } static int32_t mkNonExistTSMAInfo(const STSMAVersion *pTsmaVer, STableTSMAInfo **ppTsma) { @@ -2377,8 +2442,8 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t hbRsp.pTsmas = taosArrayInit(numOfTsmas, POINTER_BYTES); if (!hbRsp.pTsmas) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } for (int32_t i = 0; i < numOfTsmas; ++i) { @@ -2390,17 +2455,17 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t snprintf(tsmaFName, sizeof(tsmaFName), "%s.%s", pTsmaVer->dbFName, pTsmaVer->name); SSmaObj* pSma = mndAcquireSma(pMnode, tsmaFName); if (!pSma) { - terrno = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); - if (terrno) goto _OVER; + code = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); + if (code) goto _OVER; taosArrayPush(hbRsp.pTsmas, &pTsmaInfo); continue; } if (pSma->uid != pTsmaVer->tsmaId) { mDebug("tsma: %s.%" PRIx64 " tsmaId mismatch with current %" PRIx64, tsmaFName, pTsmaVer->tsmaId, pSma->uid); - terrno = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); + code = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); mndReleaseSma(pMnode, pSma); - if (terrno) goto _OVER; + if (code) goto _OVER; taosArrayPush(hbRsp.pTsmas, &pTsmaInfo); continue; } else if (pSma->version == pTsmaVer->version) { @@ -2411,9 +2476,9 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t SStbObj* pDestStb = mndAcquireStb(pMnode, pSma->dstTbName); if (!pDestStb) { mInfo("tsma: %s.%" PRIx64 " dest stb: %s not found, maybe dropped", tsmaFName, pTsmaVer->tsmaId, pSma->dstTbName); - terrno = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); + code = mkNonExistTSMAInfo(pTsmaVer, &pTsmaInfo); mndReleaseSma(pMnode, pSma); - if (terrno) goto _OVER; + if (code) goto _OVER; taosArrayPush(hbRsp.pTsmas, &pTsmaInfo); continue; } @@ -2422,15 +2487,15 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t STableTSMAInfo * pInfo = NULL; pInfo = taosMemoryCalloc(1, sizeof(STableTSMAInfo)); if (!pInfo) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; mndReleaseSma(pMnode, pSma); mndReleaseStb(pMnode, pDestStb); goto _OVER; } SSmaObj* pBaseSma = NULL; - terrno = mndGetDeepestBaseForTsma(pMnode, pSma, &pBaseSma); - if (terrno == 0) terrno = dumpTSMAInfoFromSmaObj(pSma, pDestStb, pInfo, pBaseSma); + code = mndGetDeepestBaseForTsma(pMnode, pSma, &pBaseSma); + if (code == 0) code = dumpTSMAInfoFromSmaObj(pSma, pDestStb, pInfo, pBaseSma); mndReleaseStb(pMnode, pDestStb); mndReleaseSma(pMnode, pSma); @@ -2445,13 +2510,13 @@ int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t rspLen = tSerializeTSMAHbRsp(NULL, 0, &hbRsp); if (rspLen < 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } pRsp = taosMemoryMalloc(rspLen); if (!pRsp) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; rspLen = 0; goto _OVER; } @@ -2462,5 +2527,5 @@ _OVER: tFreeTSMAHbRsp(&hbRsp); *ppRsp = pRsp; *pRspLen = rspLen; - return code; + TAOS_RETURN(code); } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 4243ccb77c..4bb8ab92fb 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -170,38 +170,54 @@ static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew } static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { + int32_t code = 0; SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; - return 0; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING)); + TAOS_RETURN(code); } static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) { + int32_t code = 0; SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; + if (pUndoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED)); + TAOS_RETURN(code); } static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - return 0; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); + TAOS_RETURN(code); } static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + int32_t code = 0; SDCreateSnodeReq createReq = {0}; createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); @@ -212,23 +228,24 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.msgType = TDMT_DND_CREATE_SNODE; action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + int32_t code = 0; SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); @@ -239,12 +256,12 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.msgType = TDMT_DND_DROP_SNODE; action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; - if (mndTransAppendUndoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { @@ -256,23 +273,27 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, snodeObj.updateTime = snodeObj.createdTime; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mndTransSetSerial(pTrans); mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); - if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) goto _OVER; - if (mndSetCreateSnodeUndoLogs(pTrans, &snodeObj) != 0) goto _OVER; - if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) goto _OVER; - if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER; - if (mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetCreateSnodeRedoLogs(pTrans, &snodeObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSnodeUndoLogs(pTrans, &snodeObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSnodeCommitLogs(pTrans, &snodeObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; _OVER: mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { @@ -282,32 +303,27 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SMCreateSnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER); mInfo("snode:%d, start to create", createReq.dnodeId); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE), NULL, _OVER); -// pObj = mndAcquireSnode(pMnode, createReq.dnodeId); -// if (pObj != NULL) { -// terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; -// goto _OVER; -// } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { -// goto _OVER; -// } + // pObj = mndAcquireSnode(pMnode, createReq.dnodeId); + // if (pObj != NULL) { + // terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; + // goto _OVER; + // } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { + // goto _OVER; + // } if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){ - terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; + code = TSDB_CODE_MND_SNODE_ALREADY_EXIST; goto _OVER; } pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); if (pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + code = TSDB_CODE_MND_DNODE_NOT_EXIST; goto _OVER; } @@ -317,40 +333,51 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); - return -1; + TAOS_RETURN(code); } // mndReleaseSnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); tFreeSMCreateQnodeReq(&createReq); - return code; + TAOS_RETURN(code); } static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { + int32_t code = 0; SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; - return 0; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)); + TAOS_RETURN(code); } static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)); + TAOS_RETURN(code); } static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + int32_t code = 0; SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); @@ -361,20 +388,20 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.msgType = TDMT_DND_DROP_SNODE; action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) { if (pObj == NULL) return 0; - if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) return -1; - if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) return -1; + TAOS_CHECK_RETURN(mndSetDropSnodeRedoLogs(pTrans, pObj)); + TAOS_CHECK_RETURN(mndSetDropSnodeCommitLogs(pTrans, pObj)); if (!force) { - if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1; + TAOS_CHECK_RETURN(mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj)); } return 0; } @@ -383,18 +410,22 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); - if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; _OVER: mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { @@ -403,23 +434,20 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { SSnodeObj *pObj = NULL; SMDropSnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER); mInfo("snode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE), NULL, _OVER); if (dropReq.dnodeId <= 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } pObj = mndAcquireSnode(pMnode, dropReq.dnodeId); if (pObj == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; goto _OVER; } @@ -434,7 +462,7 @@ _OVER: mndReleaseSnode(pMnode, pObj); tFreeSMCreateQnodeReq(&dropReq); - return code; + TAOS_RETURN(code); } static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { From e96718b43bc7c50f23e9f315323acfe131cbf6ac Mon Sep 17 00:00:00 2001 From: dmchen Date: Sun, 21 Jul 2024 05:14:16 +0000 Subject: [PATCH 04/43] fix/TD-30989 --- source/dnode/mnode/impl/src/mndStb.c | 930 ++++++++++++++------------- 1 file changed, 488 insertions(+), 442 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 73542bbb1e..efbe906d79 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -625,87 +625,99 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, } int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { + int32_t code = 0; if (pCreate->igExists < 0 || pCreate->igExists > 1) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } if (pCreate->numOfColumns < TSDB_MIN_COLUMNS || pCreate->numOfTags + pCreate->numOfColumns > TSDB_MAX_COLUMNS) { - terrno = TSDB_CODE_PAR_INVALID_COLUMNS_NUM; - return -1; + code = TSDB_CODE_PAR_INVALID_COLUMNS_NUM; + TAOS_RETURN(code); } if (pCreate->numOfTags <= 0 || pCreate->numOfTags > TSDB_MAX_TAGS) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } SField *pField = taosArrayGet(pCreate->pColumns, 0); if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) { - terrno = TSDB_CODE_PAR_INVALID_FIRST_COLUMN; - return -1; + code = TSDB_CODE_PAR_INVALID_FIRST_COLUMN; + TAOS_RETURN(code); } for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { SFieldWithOptions *pField1 = taosArrayGet(pCreate->pColumns, i); if (pField1->type >= TSDB_DATA_TYPE_MAX) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } if (pField1->bytes <= 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } if (pField1->name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } } for (int32_t i = 0; i < pCreate->numOfTags; ++i) { SField *pField1 = taosArrayGet(pCreate->pTags, i); if (pField1->type >= TSDB_DATA_TYPE_MAX) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } if (pField1->bytes <= 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } if (pField1->name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } } - return 0; + TAOS_RETURN(code); } static int32_t mndSetCreateStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + int32_t code = 0; SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { - sdbFreeRaw(pRedoRaw); - return -1; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) { + sdbFreeRaw(pRedoRaw); + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - sdbFreeRaw(pCommitRaw); - return -1; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { + sdbFreeRaw(pCommitRaw); + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; @@ -723,7 +735,9 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } STransAction action = {0}; @@ -734,25 +748,28 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.msgType = TDMT_VND_CREATE_STB; action.acceptableCode = TSDB_CODE_TDB_STB_ALREADY_EXIST; action.retryCode = TSDB_CODE_TDB_STB_NOT_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + TAOS_RETURN(code); } sdbRelease(pSdb, pVgroup); } - return 0; + TAOS_RETURN(code); } int32_t mndSetForceDropCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SStbObj *pStb) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; int32_t contLen; void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen, NULL, 0); if (pReq == NULL) { - return -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } STransAction action = {0}; @@ -763,15 +780,16 @@ int32_t mndSetForceDropCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SVgO action.msgType = TDMT_VND_CREATE_STB; action.acceptableCode = TSDB_CODE_TDB_STB_ALREADY_EXIST; action.retryCode = TSDB_CODE_TDB_STB_NOT_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; @@ -789,8 +807,8 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } STransAction action = {0}; @@ -799,16 +817,16 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; action.acceptableCode = TSDB_CODE_TDB_STB_NOT_EXIST; - if (mndTransAppendUndoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + TAOS_RETURN(code); } sdbRelease(pSdb, pVgroup); } - return 0; + TAOS_RETURN(code); } static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { @@ -822,6 +840,7 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { } int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) { + int32_t code = 0; memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN); memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); pDst->createdTime = taosGetTimestampMs(); @@ -850,8 +869,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat if (pDst->commentLen > 0) { pDst->comment = taosMemoryCalloc(pDst->commentLen + 1, 1); if (pDst->comment == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } memcpy(pDst->comment, pCreate->pComment, pDst->commentLen + 1); } @@ -860,8 +879,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat if (pDst->ast1Len > 0) { pDst->pAst1 = taosMemoryCalloc(pDst->ast1Len, 1); if (pDst->pAst1 == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } memcpy(pDst->pAst1, pCreate->pAst1, pDst->ast1Len); } @@ -870,8 +889,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat if (pDst->ast2Len > 0) { pDst->pAst2 = taosMemoryCalloc(pDst->ast2Len, 1); if (pDst->pAst2 == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } memcpy(pDst->pAst2, pCreate->pAst2, pDst->ast2Len); } @@ -879,13 +898,13 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema)); pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema)); if (pDst->pColumns == NULL || pDst->pTags == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } if (pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(code); } for (int32_t i = 0; i < pDst->numOfColumns; ++i) { @@ -921,7 +940,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pColCmpr->id = pSchema->colId; pColCmpr->alg = pField->compress; } - return 0; + TAOS_RETURN(code); } static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) { SName name = {0}; @@ -936,16 +955,20 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea char fullIdxName[TSDB_INDEX_FNAME_LEN * 2] = {0}; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); - if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb), NULL, _OVER); SSchema *pSchema = &(stbObj.pTags[0]); mndGenIdxNameForFirstTag(fullIdxName, pDb->name, stbObj.name, pSchema->name); SSIdx idx = {0}; if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) { - terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; mndReleaseIdx(pMnode, idx.pIdx); goto _OVER; } @@ -960,16 +983,16 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea idxObj.stbUid = stbObj.uid; idxObj.dbUid = stbObj.dbUid; - if (mndSetCreateIdxCommitLogs(pMnode, pTrans, &idxObj) < 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetCreateIdxCommitLogs(pMnode, pTrans, &idxObj), NULL, _OVER); - if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; _OVER: mndTransDrop(pTrans); mndStbActionDelete(pMnode->pSdb, &stbObj); - return code; + TAOS_RETURN(code); } int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { @@ -1130,6 +1153,7 @@ static bool mndValidateSchema(SSchema *pSchemas, int32_t nSchema, SArray *pField } static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq *createReq) { + int32_t code = 0; taosRLockLatch(&pStb->lock); memcpy(pDst, pStb, sizeof(SStbObj)); taosRUnLockLatch(&pStb->lock); @@ -1143,13 +1167,13 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq pDst->pCmpr = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SColCmpr)); if (pDst->pColumns == NULL || pDst->pTags == NULL || pDst->pCmpr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } if (pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(code); } for (int32_t i = 0; i < pDst->numOfColumns; ++i) { @@ -1205,13 +1229,13 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { bool isAlter = false; if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } mInfo("stb:%s, start to create", createReq.name); if (mndCheckCreateStbReq(&createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -1241,17 +1265,17 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { mInfo("stb:%s, schema version is only increased by 1 number, do alter operation", createReq.name); } else { mError("stb:%s, schema version increase more than 1 number, error is returned", createReq.name); - terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER; + code = TSDB_CODE_MND_INVALID_SCHEMA_VER; goto _OVER; } } else { mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid, origin tagVer:%d colVer:%d", createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); - terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER; + code = TSDB_CODE_MND_INVALID_SCHEMA_VER; goto _OVER; } } else { - terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; + code = TSDB_CODE_MND_STB_ALREADY_EXIST; goto _OVER; } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { @@ -1265,33 +1289,32 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { pDb = mndAcquireDbByStb(pMnode, createReq.name); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + code = TSDB_CODE_MND_DB_NOT_SELECTED; goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if ((code = mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb)) != 0) { goto _OVER; } int32_t numOfStbs = -1; - if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) { + if ((code = mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs)) != 0) { goto _OVER; } if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { - terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; + code = TSDB_CODE_MND_SINGLE_STB_MODE_DB; goto _OVER; } - if ((terrno = grantCheck(TSDB_GRANT_STABLE)) < 0) { - code = -1; + if ((code = grantCheck(TSDB_GRANT_STABLE)) < 0) { goto _OVER; } if (isAlter) { bool needRsp = false; SStbObj pDst = {0}; - if (mndBuildStbFromAlter(pStb, &pDst, &createReq) != 0) { + if ((code = mndBuildStbFromAlter(pStb, &pDst, &createReq)) != 0) { taosMemoryFreeClear(pDst.pTags); taosMemoryFreeClear(pDst.pColumns); taosMemoryFreeClear(pDst.pCmpr); @@ -1328,27 +1351,28 @@ _OVER: mndReleaseDb(pMnode, pDb); tFreeSMCreateStbReq(&createReq); - return code; + TAOS_RETURN(code); } static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { + int32_t code = 0; if (pAlter->commentLen >= 0) return 0; if (pAlter->ttl != 0) return 0; if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } for (int32_t i = 0; i < pAlter->numOfFields; ++i) { SField *pField = taosArrayGet(pAlter->pFields, i); if (pField->name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } } - return 0; + TAOS_RETURN(code); } int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { @@ -1356,19 +1380,19 @@ int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema)); pNew->pCmpr = taosMemoryCalloc(pNew->numOfColumns, sizeof(SColCmpr)); if (pNew->pTags == NULL || pNew->pColumns == NULL || pNew->pCmpr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pNew->pColumns, pOld->pColumns, sizeof(SSchema) * pOld->numOfColumns); memcpy(pNew->pTags, pOld->pTags, sizeof(SSchema) * pOld->numOfTags); memcpy(pNew->pCmpr, pOld->pCmpr, sizeof(SColCmpr) * pOld->numOfColumns); - return 0; + TAOS_RETURN(0); } static int32_t mndUpdateStbCommentAndTTL(const SStbObj *pOld, SStbObj *pNew, char *pComment, int32_t commentLen, int32_t ttl) { + int32_t code = 0; if (commentLen > 0) { pNew->commentLen = commentLen; pNew->comment = taosMemoryCalloc(1, commentLen + 1); @@ -1386,48 +1410,49 @@ static int32_t mndUpdateStbCommentAndTTL(const SStbObj *pOld, SStbObj *pNew, cha pNew->ttl = ttl; } - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; + if ((code = mndAllocStbSchemas(pOld, pNew)) != 0) { + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ntags) { + int32_t code = 0; if (pOld->numOfTags + ntags > TSDB_MAX_TAGS) { - terrno = TSDB_CODE_MND_TOO_MANY_TAGS; - return -1; + code = TSDB_CODE_MND_TOO_MANY_TAGS; + TAOS_RETURN(code); } if (pOld->numOfColumns + ntags + pOld->numOfTags > TSDB_MAX_COLUMNS) { - terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS; - return -1; + code = TSDB_CODE_MND_TOO_MANY_COLUMNS; + TAOS_RETURN(code); } if (!mndValidateSchema(pOld->pTags, pOld->numOfTags, pFields, TSDB_MAX_TAGS_LEN)) { - terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH; - return -1; + code = TSDB_CODE_PAR_INVALID_TAGS_LENGTH; + TAOS_RETURN(code); } pNew->numOfTags = pNew->numOfTags + ntags; - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; + if ((code = mndAllocStbSchemas(pOld, pNew)) != 0) { + TAOS_RETURN(code); } if (pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ntags) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(code); } for (int32_t i = 0; i < ntags; i++) { SField *pField = taosArrayGet(pFields, i); if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { - terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + TAOS_RETURN(code); } if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) { - terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_TAG_ALREADY_EXIST; + TAOS_RETURN(code); } SSchema *pSchema = &pNew->pTags[pOld->numOfTags + i]; @@ -1441,10 +1466,11 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p } pNew->tagVer++; - return 0; + TAOS_RETURN(code); } static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -1461,11 +1487,11 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pTopic->ast, &pAst) != 0) { - terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; + code = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; mError("topic:%s, create ast error", pTopic->name); sdbRelease(pSdb, pTopic); sdbCancelFetch(pSdb, pIter); - return -1; + TAOS_RETURN(code); } SNodeList *pNodeList = NULL; @@ -1481,13 +1507,13 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, goto NEXT; } if (pCol->colId > 0 && pCol->colId == colId) { - terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; + code = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; mError("topic:%s, check colId:%d conflicted", pTopic->name, pCol->colId); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pTopic); - return -1; + TAOS_RETURN(code); } mInfo("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); } @@ -1497,10 +1523,11 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, nodesDestroyNode(pAst); nodesDestroyList(pNodeList); } - return 0; + TAOS_RETURN(code); } static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -1510,11 +1537,11 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName SNode *pAst = NULL; if (nodesStringToNode(pStream->ast, &pAst) != 0) { - terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; + code = TSDB_CODE_MND_INVALID_STREAM_OPTION; mError("stream:%s, create ast error", pStream->name); sdbRelease(pSdb, pStream); sdbCancelFetch(pSdb, pIter); - return -1; + TAOS_RETURN(code); } SNodeList *pNodeList = NULL; @@ -1528,13 +1555,13 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName goto NEXT; } if (pCol->colId > 0 && pCol->colId == colId) { - terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; + code = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; mError("stream:%s, check colId:%d conflicted", pStream->name, pCol->colId); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); sdbRelease(pSdb, pStream); sdbCancelFetch(pSdb, pIter); - return -1; + TAOS_RETURN(code); } mInfo("stream:%s, check colId:%d passed", pStream->name, pCol->colId); } @@ -1544,10 +1571,11 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName nodesDestroyNode(pAst); nodesDestroyList(pNodeList); } - return 0; + TAOS_RETURN(code); } static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -1560,11 +1588,11 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pSma->ast, &pAst) != 0) { - terrno = TSDB_CODE_SDB_INVALID_DATA_CONTENT; + code = TSDB_CODE_SDB_INVALID_DATA_CONTENT; mError("tsma:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d failed since parse AST err", pSma->name, stbFullName, suid, colId); sdbCancelFetch(pSdb, pIter); - return -1; + TAOS_RETURN(code); } SNodeList *pNodeList = NULL; @@ -1579,13 +1607,13 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, goto NEXT; } if ((pCol->colId) > 0 && (pCol->colId == colId)) { - terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA; + code = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA; mError("tsma:%s, check colId:%d conflicted", pSma->name, pCol->colId); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); sdbRelease(pSdb, pSma); sdbCancelFetch(pSdb, pIter); - return -1; + TAOS_RETURN(code); } mInfo("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); } @@ -1595,38 +1623,28 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, nodesDestroyNode(pAst); nodesDestroyList(pNodeList); } - return 0; + TAOS_RETURN(code); } int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { - if (mndCheckAlterColForTopic(pMnode, stbFullName, suid, colId) < 0) { - return -1; - } - if (mndCheckAlterColForStream(pMnode, stbFullName, suid, colId) < 0) { - return -1; - } - - if (mndCheckAlterColForTSma(pMnode, stbFullName, suid, colId) < 0) { - return -1; - } - return 0; + TAOS_CHECK_RETURN(mndCheckAlterColForTopic(pMnode, stbFullName, suid, colId)); + TAOS_CHECK_RETURN(mndCheckAlterColForStream(pMnode, stbFullName, suid, colId)); + TAOS_CHECK_RETURN(mndCheckAlterColForTSma(pMnode, stbFullName, suid, colId)); + TAOS_RETURN(0); } static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *tagName) { + int32_t code = 0; int32_t tag = mndFindSuperTableTagIndex(pOld, tagName); if (tag < 0) { - terrno = TSDB_CODE_MND_TAG_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_TAG_NOT_EXIST; + TAOS_RETURN(code); } col_id_t colId = pOld->pTags[tag].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId)); - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); memmove(pNew->pTags + tag, pNew->pTags + tag + 1, sizeof(SSchema) * (pNew->numOfTags - tag - 1)); pNew->numOfTags--; @@ -1637,13 +1655,14 @@ static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj // return -1; // } mInfo("stb:%s, start to drop tag %s", pNew->name, tagName); - return 0; + TAOS_RETURN(code); } static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, SArray *pFields) { + int32_t code = 0; if ((int32_t)taosArrayGetSize(pFields) != 2) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } SField *pField0 = taosArrayGet(pFields, 0); @@ -1654,48 +1673,43 @@ static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj * int32_t tag = mndFindSuperTableTagIndex(pOld, oldTagName); if (tag < 0) { - terrno = TSDB_CODE_MND_TAG_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_TAG_NOT_EXIST; + TAOS_RETURN(code); } col_id_t colId = pOld->pTags[tag].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId)); if (mndFindSuperTableTagIndex(pOld, newTagName) >= 0) { - terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_TAG_ALREADY_EXIST; + TAOS_RETURN(code); } if (mndFindSuperTableColumnIndex(pOld, newTagName) >= 0) { - terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + TAOS_RETURN(code); } - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); SSchema *pSchema = (SSchema *)(pNew->pTags + tag); memcpy(pSchema->name, newTagName, TSDB_COL_NAME_LEN); pNew->tagVer++; mInfo("stb:%s, start to modify tag %s to %s", pNew->name, oldTagName, newTagName); - return 0; + TAOS_RETURN(code); } static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) { + int32_t code = 0; int32_t tag = mndFindSuperTableTagIndex(pOld, pField->name); if (tag < 0) { - terrno = TSDB_CODE_MND_TAG_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_TAG_NOT_EXIST; + TAOS_RETURN(code); } col_id_t colId = pOld->pTags[tag].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId)); uint32_t nLen = 0; for (int32_t i = 0; i < pOld->numOfTags; ++i) { @@ -1703,32 +1717,30 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj } if (nLen > TSDB_MAX_TAGS_LEN) { - terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH; - return -1; + code = TSDB_CODE_PAR_INVALID_TAGS_LENGTH; + TAOS_RETURN(code); } - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); SSchema *pTag = pNew->pTags + tag; if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_VARBINARY || pTag->type == TSDB_DATA_TYPE_NCHAR || pTag->type == TSDB_DATA_TYPE_GEOMETRY)) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } if (pField->bytes <= pTag->bytes) { - terrno = TSDB_CODE_MND_INVALID_ROW_BYTES; - return -1; + code = TSDB_CODE_MND_INVALID_ROW_BYTES; + TAOS_RETURN(code); } pTag->bytes = pField->bytes; pNew->tagVer++; mInfo("stb:%s, start to modify tag len %s to %d", pNew->name, pField->name, pField->bytes); - return 0; + TAOS_RETURN(code); } static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, SArray *pField, @@ -1741,22 +1753,17 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj * int32_t code = 0; int32_t idx = mndFindSuperTableColumnIndex(pOld, p->name); if (idx == -1) { - terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_COLUMN_NOT_EXIST; + TAOS_RETURN(code); } SSchema *pTarget = &pOld->pColumns[idx]; col_id_t colId = pTarget->colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId)); - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); code = validColCmprByType(pTarget->type, p->bytes); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return code; + TAOS_RETURN(code); } int8_t updated = 0; @@ -1772,55 +1779,54 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj * } if (updated == 0) { - terrno = TSDB_CODE_MND_COLUMN_COMPRESS_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_COLUMN_COMPRESS_ALREADY_EXIST; + TAOS_RETURN(code); } else if (updated == -1) { - terrno = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; - return -1; + code = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; + TAOS_RETURN(code); } pNew->colVer++; - return 0; + TAOS_RETURN(code); } static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols, int8_t withCompress) { + int32_t code = 0; if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) { - terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS; - return -1; + code = TSDB_CODE_MND_TOO_MANY_COLUMNS; + TAOS_RETURN(code); } - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) != 0) { - return -1; + if ((code = grantCheck(TSDB_GRANT_TIMESERIES)) != 0) { + TAOS_RETURN(code); } if (!mndValidateSchema(pOld->pColumns, pOld->numOfColumns, pFields, TSDB_MAX_BYTES_PER_ROW)) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - return -1; + code = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + TAOS_RETURN(code); } pNew->numOfColumns = pNew->numOfColumns + ncols; - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); if (pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ncols) { - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; + code = TSDB_CODE_OUT_OF_RANGE; + TAOS_RETURN(code); } for (int32_t i = 0; i < ncols; i++) { if (withCompress) { SFieldWithOptions *pField = taosArrayGet(pFields, i); if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { - terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + TAOS_RETURN(code); } if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) { - terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_TAG_ALREADY_EXIST; + TAOS_RETURN(code); } SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i]; @@ -1837,13 +1843,13 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray } else { SField *pField = taosArrayGet(pFields, i); if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { - terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + TAOS_RETURN(code); } if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) { - terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; - return -1; + code = TSDB_CODE_MND_TAG_ALREADY_EXIST; + TAOS_RETURN(code); } SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i]; @@ -1861,34 +1867,31 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray } pNew->colVer++; - return 0; + TAOS_RETURN(code); } static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *colName) { + int32_t code = 0; int32_t col = mndFindSuperTableColumnIndex(pOld, colName); if (col < 0) { - terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_COLUMN_NOT_EXIST; + TAOS_RETURN(code); } if (col == 0) { - terrno = TSDB_CODE_MND_INVALID_STB_ALTER_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_ALTER_OPTION; + TAOS_RETURN(code); } if (pOld->numOfColumns == 2) { - terrno = TSDB_CODE_MND_INVALID_STB_ALTER_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_ALTER_OPTION; + TAOS_RETURN(code); } col_id_t colId = pOld->pColumns[col].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId)); - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); int32_t sz = pNew->numOfColumns - col - 1; memmove(pNew->pColumns + col, pNew->pColumns + col + 1, sizeof(SSchema) * sz); @@ -1897,14 +1900,15 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb pNew->colVer++; mInfo("stb:%s, start to drop col %s", pNew->name, colName); - return 0; + TAOS_RETURN(code); } static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) { + int32_t code = 0; int32_t col = mndFindSuperTableColumnIndex(pOld, pField->name); if (col < 0) { - terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_COLUMN_NOT_EXIST; + TAOS_RETURN(code); } col_id_t colId = pOld->pColumns[col].colId; @@ -1915,63 +1919,70 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO } if (nLen > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_MND_INVALID_ROW_BYTES; - return -1; + code = TSDB_CODE_MND_INVALID_ROW_BYTES; + TAOS_RETURN(code); } - if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId)); - if (mndAllocStbSchemas(pOld, pNew) != 0) { - return -1; - } + TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); SSchema *pCol = pNew->pColumns + col; if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_VARBINARY || pCol->type == TSDB_DATA_TYPE_NCHAR || pCol->type == TSDB_DATA_TYPE_GEOMETRY)) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_STB_OPTION; + TAOS_RETURN(code); } if (pField->bytes <= pCol->bytes) { - terrno = TSDB_CODE_MND_INVALID_ROW_BYTES; - return -1; + code = TSDB_CODE_MND_INVALID_ROW_BYTES; + TAOS_RETURN(code); } pCol->bytes = pField->bytes; pNew->colVer++; mInfo("stb:%s, start to modify col len %s to %d", pNew->name, pField->name, pField->bytes); - return 0; + TAOS_RETURN(code); } static int32_t mndSetAlterStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + int32_t code = 0; SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { - sdbFreeRaw(pRedoRaw); - return -1; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; + if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) { + sdbFreeRaw(pRedoRaw); + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetAlterStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - sdbFreeRaw(pCommitRaw); - return -1; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) { + sdbFreeRaw(pCommitRaw); + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, void *alterOriData, int32_t alterOriDataLen) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; @@ -1989,27 +2000,30 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_STB; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + TAOS_RETURN(code); } sdbRelease(pSdb, pVgroup); } - return 0; + TAOS_RETURN(code); } static int32_t mndSetAlterStbRedoActions2(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, void *alterOriData, int32_t alterOriDataLen) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; @@ -2027,39 +2041,42 @@ static int32_t mndSetAlterStbRedoActions2(SMnode *pMnode, STrans *pTrans, SDbObj if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_CREATE_INDEX; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + TAOS_RETURN(code); } sdbRelease(pSdb, pVgroup); } - return 0; + TAOS_RETURN(code); } static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { + int32_t code = 0; taosRLockLatch(&pStb->lock); int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; pRsp->pSchemas = taosMemoryCalloc(totalCols, sizeof(SSchema)); if (pRsp->pSchemas == NULL) { taosRUnLockLatch(&pStb->lock); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } pRsp->pSchemaExt = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchemaExt)); if (pRsp->pSchemaExt == NULL) { taosRUnLockLatch(&pStb->lock); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); @@ -2102,18 +2119,19 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa } taosRUnLockLatch(&pStb->lock); - return 0; + TAOS_RETURN(code); } static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableCfgRsp *pRsp) { + int32_t code = 0; taosRLockLatch(&pStb->lock); int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; pRsp->pSchemas = taosMemoryCalloc(totalCols, sizeof(SSchema)); if (pRsp->pSchemas == NULL) { taosRUnLockLatch(&pStb->lock); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); @@ -2166,30 +2184,31 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, } taosRUnLockLatch(&pStb->lock); - return 0; + TAOS_RETURN(code); } static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion *pStbVer, bool *schema, bool *sma) { + int32_t code = 0; char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName); SDbObj *pDb = mndAcquireDb(pMnode, pStbVer->dbFName); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; + code = TSDB_CODE_MND_DB_NOT_SELECTED; + TAOS_RETURN(code); } if (pDb->uid != pStbVer->dbId) { mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; + code = TSDB_CODE_MND_DB_NOT_SELECTED; + TAOS_RETURN(code); } SStbObj *pStb = mndAcquireStb(pMnode, tbFName); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - return -1; + code = TSDB_CODE_PAR_TABLE_NOT_EXIST; + TAOS_RETURN(code); } taosRLockLatch(&pStb->lock); @@ -2214,54 +2233,56 @@ static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion *pStbVer, bo } static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { + int32_t code = 0; char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); SDbObj *pDb = mndAcquireDb(pMnode, dbFName); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; + code = TSDB_CODE_MND_DB_NOT_SELECTED; + TAOS_RETURN(code); } SStbObj *pStb = mndAcquireStb(pMnode, tbFName); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - return -1; + code = TSDB_CODE_PAR_TABLE_NOT_EXIST; + TAOS_RETURN(code); } - int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); + code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); - return code; + TAOS_RETURN(code); } static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { + int32_t code = 0; char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); SDbObj *pDb = mndAcquireDb(pMnode, dbFName); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; + code = TSDB_CODE_MND_DB_NOT_SELECTED; + TAOS_RETURN(code); } SStbObj *pStb = mndAcquireStb(pMnode, tbFName); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - return -1; + code = TSDB_CODE_PAR_TABLE_NOT_EXIST; + TAOS_RETURN(code); } - int32_t code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp); + code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); - return code; + TAOS_RETURN(code); } static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, int32_t *pLen) { - int32_t ret; + int32_t code = 0; SEncoder ec = {0}; uint32_t contLen = 0; SMAlterStbRsp alterRsp = {0}; @@ -2270,20 +2291,20 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, i alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); if (NULL == alterRsp.pMeta) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } - ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, alterRsp.pMeta); - if (ret) { + code = mndBuildStbSchemaImp(pDb, pObj, name.tname, alterRsp.pMeta); + if (code) { tFreeSMAlterStbRsp(&alterRsp); - return ret; + return code; } - tEncodeSize(tEncodeSMAlterStbRsp, &alterRsp, contLen, ret); - if (ret) { + tEncodeSize(tEncodeSMAlterStbRsp, &alterRsp, contLen, code); + if (code) { tFreeSMAlterStbRsp(&alterRsp); - return ret; + return code; } void *cont = taosMemoryMalloc(contLen); @@ -2296,18 +2317,22 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, i *pCont = cont; *pLen = contLen; - return 0; + TAOS_RETURN(code); } int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, void **pCont, int32_t *pLen) { - int32_t ret = -1; + int32_t code = -1; SDbObj *pDb = mndAcquireDb(pMnode, dbFName); if (NULL == pDb) { - return -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } SStbObj *pObj = mndAcquireStb(pMnode, stbFName); if (NULL == pObj) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; goto _OVER; } @@ -2319,18 +2344,18 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, vo stbRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); if (NULL == stbRsp.pMeta) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, stbRsp.pMeta); - if (ret) { + code = mndBuildStbSchemaImp(pDb, pObj, name.tname, stbRsp.pMeta); + if (code) { tFreeSMCreateStbRsp(&stbRsp); goto _OVER; } - tEncodeSize(tEncodeSMCreateStbRsp, &stbRsp, contLen, ret); - if (ret) { + tEncodeSize(tEncodeSMCreateStbRsp, &stbRsp, contLen, code); + if (code) { tFreeSMCreateStbRsp(&stbRsp); goto _OVER; } @@ -2345,7 +2370,7 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, vo *pCont = cont; *pLen = contLen; - ret = 0; + code = 0; _OVER: if (pObj) { @@ -2356,53 +2381,61 @@ _OVER: mndReleaseDb(pMnode, pDb); } - return ret; + TAOS_RETURN(code); } static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void *alterOriData, int32_t alterOriDataLen) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "alter-stb"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); if (needRsp) { void *pCont = NULL; int32_t contLen = 0; - if (mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen), NULL, _OVER); mndTransSetRpcRsp(pTrans, pCont, contLen); } - if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; - if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; - if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; _OVER: mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void *alterOriData, int32_t alterOriDataLen, const SMAlterStbReq *pAlter) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "alter-stb"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); if (needRsp) { void *pCont = NULL; int32_t contLen = 0; - if (mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen), NULL, _OVER); mndTransSetRpcRsp(pTrans, pCont, contLen); } @@ -2413,16 +2446,16 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) { exist = true; } - if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; - if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb), NULL, _OVER); if (exist == true) { - if (mndSetDropIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; - if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetDropIdxPrepareLogs(pMnode, pTrans, &idxObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj), NULL, _OVER); } - if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); } else if (pAlter->alterType == TSDB_ALTER_TABLE_UPDATE_TAG_NAME) { SIdxObj idxObj = {0}; @@ -2436,24 +2469,24 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO exist = true; } - if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; - if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb), NULL, _OVER); if (exist == true) { memcpy(idxObj.colName, nTagName, strlen(nTagName)); idxObj.colName[strlen(nTagName)] = 0; - if (mndSetAlterIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; - if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetAlterIdxPrepareLogs(pMnode, pTrans, &idxObj), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj), NULL, _OVER); } - if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); } code = 0; _OVER: mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { @@ -2530,7 +2563,7 @@ _OVER: if (pAlter->commentLen > 0) { taosMemoryFreeClear(stbObj.comment); } - return code; + TAOS_RETURN(code); } static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { @@ -2541,7 +2574,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { SMAlterStbReq alterReq = {0}; if (tDeserializeSMAlterStbReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -2550,17 +2583,17 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { pDb = mndAcquireDbByStb(pMnode, alterReq.name); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + code = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } pStb = mndAcquireStb(pMnode, alterReq.name); if (pStb == NULL) { - terrno = TSDB_CODE_MND_STB_NOT_EXIST; + code = TSDB_CODE_MND_STB_NOT_EXIST; goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if ((code = mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb)) != 0) { goto _OVER; } @@ -2581,34 +2614,45 @@ _OVER: mndReleaseDb(pMnode, pDb); tFreeSMAltertbReq(&alterReq); - return code; + TAOS_RETURN(code); } static int32_t mndSetDropStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + int32_t code = 0; SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { - sdbFreeRaw(pRedoRaw); - return -1; + if (pRedoRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); } - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) { + sdbFreeRaw(pRedoRaw); + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); - if (pCommitRaw == NULL) return -1; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { sdbFreeRaw(pCommitRaw); - return -1; + TAOS_RETURN(code); } - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)); - return 0; + TAOS_RETURN(code); } static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; @@ -2626,8 +2670,8 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } STransAction action = {0}; @@ -2636,42 +2680,47 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; action.acceptableCode = TSDB_CODE_TDB_STB_NOT_EXIST; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); - return -1; + TAOS_RETURN(code); } sdbRelease(pSdb, pVgroup); } - return 0; + TAOS_RETURN(code); } static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "drop-stb"); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } mInfo("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); - if (mndSetDropStbPrepareLogs(pMnode, pTrans, pStb) != 0) goto _OVER; - if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; - if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; - if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; - if (mndDropSmasByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; - if (mndUserRemoveStb(pMnode, pTrans, pStb->name) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndSetDropStbPrepareLogs(pMnode, pTrans, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropStbCommitLogs(pMnode, pTrans, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndDropIdxsByStb(pMnode, pTrans, pDb, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndDropSmasByStb(pMnode, pTrans, pDb, pStb), NULL, _OVER); + TAOS_CHECK_GOTO(mndUserRemoveStb(pMnode, pTrans, pStb->name), NULL, _OVER); + TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; _OVER: mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, int64_t suid) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -2683,7 +2732,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, if (pTopic->stbUid == suid) { sdbRelease(pSdb, pTopic); sdbCancelFetch(pSdb, pIter); - return -1; + TAOS_RETURN(-1); } } @@ -2694,11 +2743,11 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pTopic->ast, &pAst) != 0) { - terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + code = TSDB_CODE_MND_INVALID_TOPIC_OPTION; mError("topic:%s, create ast error", pTopic->name); sdbRelease(pSdb, pTopic); sdbCancelFetch(pSdb, pIter); - return -1; + TAOS_RETURN(code); } SNodeList *pNodeList = NULL; @@ -2712,7 +2761,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, nodesDestroyNode(pAst); nodesDestroyList(pNodeList); sdbCancelFetch(pSdb, pIter); - return -1; + TAOS_RETURN(-1); } else { goto NEXT; } @@ -2722,10 +2771,11 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, nodesDestroyNode(pAst); nodesDestroyList(pNodeList); } - return 0; + TAOS_RETURN(code); } static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, int64_t suid) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -2736,16 +2786,16 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, if (pStream->targetStbUid == suid) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStream); - return -1; + TAOS_RETURN(-1); } SNode *pAst = NULL; if (nodesStringToNode(pStream->ast, &pAst) != 0) { - terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; + code = TSDB_CODE_MND_INVALID_STREAM_OPTION; mError("stream:%s, create ast error", pStream->name); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStream); - return -1; + TAOS_RETURN(code); } SNodeList *pNodeList = NULL; @@ -2759,7 +2809,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, sdbRelease(pSdb, pStream); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); - return -1; + TAOS_RETURN(-1); } else { goto NEXT; } @@ -2769,7 +2819,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, nodesDestroyNode(pAst); nodesDestroyList(pNodeList); } - return 0; + TAOS_RETURN(code); } static int32_t mndProcessDropTtltbRsp(SRpcMsg *pRsp) { return 0; } @@ -2783,10 +2833,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { SStbObj *pStb = NULL; SMDropStbReq dropReq = {0}; - if (tDeserializeSMDropStbReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSMDropStbReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER); mInfo("stb:%s, start to drop", dropReq.name); @@ -2797,7 +2844,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { code = 0; goto _OVER; } else { - terrno = TSDB_CODE_MND_STB_NOT_EXIST; + code = TSDB_CODE_MND_STB_NOT_EXIST; goto _OVER; } } @@ -2809,21 +2856,21 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { pDb = mndAcquireDbByStb(pMnode, dropReq.name); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + code = TSDB_CODE_MND_DB_NOT_SELECTED; goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if ((code = mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb)) != 0) { goto _OVER; } if (mndCheckDropStbForTopic(pMnode, dropReq.name, pStb->uid) < 0) { - terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED; + code = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED; goto _OVER; } if (mndCheckDropStbForStream(pMnode, dropReq.name, pStb->uid) < 0) { - terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; + code = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; goto _OVER; } @@ -2843,7 +2890,7 @@ _OVER: mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); tFreeSMDropStbReq(&dropReq); - return code; + TAOS_RETURN(code); } static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { @@ -2853,40 +2900,35 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { STableMetaRsp metaRsp = {0}; SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; + if (pUser == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } bool sysinfo = pUser->sysInfo; - if (tDeserializeSTableInfoReq(pReq->pCont, pReq->contLen, &infoReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSTableInfoReq(pReq->pCont, pReq->contLen, &infoReq), NULL, _OVER); if (0 == strcmp(infoReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { mInfo("information_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildInsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, sysinfo, &metaRsp) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndBuildInsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, sysinfo, &metaRsp), NULL, _OVER); } else if (0 == strcmp(infoReq.dbFName, TSDB_PERFORMANCE_SCHEMA_DB)) { mInfo("performance_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildPerfsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndBuildPerfsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp), NULL, _OVER); } else { mInfo("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp), NULL, _OVER); } int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); if (rspLen < 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -2904,7 +2946,7 @@ _OVER: mndReleaseUser(pMnode, pUser); tFreeSTableMetaRsp(&metaRsp); - return code; + TAOS_RETURN(code); } static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { @@ -2913,39 +2955,30 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { STableCfgReq cfgReq = {0}; STableCfgRsp cfgRsp = {0}; - if (tDeserializeSTableCfgReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } + TAOS_CHECK_GOTO(tDeserializeSTableCfgReq(pReq->pCont, pReq->contLen, &cfgReq), NULL, _OVER); char dbName[TSDB_DB_NAME_LEN] = {0}; mndExtractShortDbNameFromDbFullName(cfgReq.dbFName, dbName); if (0 == strcmp(dbName, TSDB_INFORMATION_SCHEMA_DB)) { mInfo("information_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); - if (mndBuildInsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndBuildInsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp), NULL, _OVER); } else if (0 == strcmp(dbName, TSDB_PERFORMANCE_SCHEMA_DB)) { mInfo("performance_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); - if (mndBuildPerfsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndBuildPerfsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp), NULL, _OVER); } else { mInfo("stb:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); - if (mndBuildStbCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { - goto _OVER; - } + TAOS_CHECK_GOTO(mndBuildStbCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp), NULL, _OVER); } int32_t rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp); if (rspLen < 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -2962,23 +2995,24 @@ _OVER: } tFreeSTableCfgRsp(&cfgRsp); - return code; + TAOS_RETURN(code); } int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { + int32_t code = 0; SSTbHbRsp hbRsp = {0}; hbRsp.pMetaRsp = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); if (hbRsp.pMetaRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } hbRsp.pIndexRsp = taosArrayInit(numOfStbs, sizeof(STableIndexRsp)); if (NULL == hbRsp.pIndexRsp) { taosArrayDestroy(hbRsp.pMetaRsp); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } for (int32_t i = 0; i < numOfStbs; ++i) { @@ -3024,8 +3058,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t STableIndexRsp indexRsp = {0}; indexRsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); if (NULL == indexRsp.pIndex) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); @@ -3046,30 +3080,31 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t int32_t rspLen = tSerializeSSTbHbRsp(NULL, 0, &hbRsp); if (rspLen < 0) { tFreeSSTbHbRsp(&hbRsp); - terrno = TSDB_CODE_INVALID_MSG; - return -1; + code = TSDB_CODE_INVALID_MSG; + TAOS_RETURN(code); } void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { tFreeSSTbHbRsp(&hbRsp); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TAOS_RETURN(code); } tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); tFreeSSTbHbRsp(&hbRsp); *ppRsp = pRsp; *pRspLen = rspLen; - return 0; + TAOS_RETURN(code); } int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { + int32_t code = 0; SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; + code = TSDB_CODE_MND_DB_NOT_SELECTED; + TAOS_RETURN(code); } int32_t numOfStbs = 0; @@ -3088,7 +3123,7 @@ int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { *pNumOfStbs = numOfStbs; mndReleaseDb(pMnode, pDb); - return 0; + TAOS_RETURN(code); } void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) { @@ -4046,29 +4081,40 @@ static int32_t mndSetDropTbsRedoActions(SMnode *pMnode, STrans *pTrans, const SV } static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx *pCtx) { + int32_t code = 0; SMnode *pMnode = pRsp->info.node; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pRsp, "drop-tbs"); mndTransSetChangeless(pTrans); - if (pTrans == NULL) goto _OVER; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } - if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); void *pIter = taosHashIterate(pCtx->pVgMap, NULL); while (pIter) { const SVDropTbVgReqs *pVgReqs = pIter; int32_t len = 0; void *p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len); - if (!p || mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len) != 0) { + if (!p) { + taosHashCancelIterate(pCtx->pVgMap, pIter); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OVER; + } + if ((code = mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len)) != 0) { taosHashCancelIterate(pCtx->pVgMap, pIter); goto _OVER; } pIter = taosHashIterate(pCtx->pVgMap, pIter); } - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER; _OVER: mndTransDrop(pTrans); - return terrno; + TAOS_RETURN(code); } static int32_t mndProcessDropTbWithTsma(SRpcMsg *pReq) { @@ -4079,23 +4125,23 @@ static int32_t mndProcessDropTbWithTsma(SRpcMsg *pReq) { SMDropTbsReq dropReq = {0}; bool locked = false; if (tDeserializeSMDropTbsReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OVER; } SMndDropTbsWithTsmaCtx *pCtx = NULL; - terrno = mndInitDropTbsWithTsmaCtx(&pCtx); - if (terrno) goto _OVER; + code = mndInitDropTbsWithTsmaCtx(&pCtx); + if (code) goto _OVER; for (int32_t i = 0; i < dropReq.pVgReqs->size; ++i) { SMDropTbReqsOnSingleVg *pReq = taosArrayGet(dropReq.pVgReqs, i); - terrno = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId); - if (terrno) goto _OVER; + code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId); + if (code) goto _OVER; } if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0) code = 0; _OVER: tFreeSMDropTbsReq(&dropReq); if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx); - return code; + TAOS_RETURN(code); } static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupInfo *pVgInfo, char *name, tb_uid_t suid, @@ -4140,7 +4186,7 @@ _end: taosArrayDestroy(pInfo->dbInfo.dbVgInfos); pInfo->dbInfo.dbVgInfos = NULL; } - return code; + TAOS_RETURN(code); } int32_t vgHashValCmp(const void *lp, const void *rp) { @@ -4249,7 +4295,7 @@ static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) { SVFetchTtlExpiredTbsRsp rsp = {0}; SMndDropTbsWithTsmaCtx *pCtx = NULL; if (pRsp->code != TSDB_CODE_SUCCESS) { - terrno = pRsp->code; + code = pRsp->code; goto _end; } if (pRsp->contLen == 0) { @@ -4258,18 +4304,18 @@ static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) { } tDecoderInit(&decoder, pRsp->pCont, pRsp->contLen); - terrno = tDecodeVFetchTtlExpiredTbsRsp(&decoder, &rsp); - if (terrno) goto _end; + code = tDecodeVFetchTtlExpiredTbsRsp(&decoder, &rsp); + if (code) goto _end; - terrno = mndInitDropTbsWithTsmaCtx(&pCtx); - if (terrno) goto _end; + code = mndInitDropTbsWithTsmaCtx(&pCtx); + if (code) goto _end; - terrno = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId); - if (terrno) goto _end; + code = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId); + if (code) goto _end; if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0) code = 0; _end: if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx); tDecoderClear(&decoder); tFreeFetchTtlExpiredTbsRsp(&rsp); - return code; + TAOS_RETURN(code); } From ebe0074da42821fa6ca6e84e1b561d4e50546d2e Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 22 Jul 2024 00:06:47 +0800 Subject: [PATCH 05/43] fix: interp crash --- include/libs/function/functionMgt.h | 2 + source/libs/executor/src/executorInt.c | 3 +- source/libs/executor/src/timesliceoperator.c | 17 +- source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 10 + source/libs/function/src/builtinsimpl.c | 35 ++++ source/libs/function/src/functionMgt.c | 7 + source/libs/parser/src/parTranslater.c | 29 ++- source/libs/planner/src/planLogicCreater.c | 2 +- tests/system-test/2-query/interp.py | 195 +++++++++++++++++++ 10 files changed, 294 insertions(+), 7 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 86db6640c5..9fb4c80bb4 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -138,6 +138,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_CACHE_LAST_ROW, FUNCTION_TYPE_CACHE_LAST, FUNCTION_TYPE_TABLE_COUNT, + FUNCTION_TYPE_SELECT_TAG, // distributed splitting functions FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, @@ -256,6 +257,7 @@ bool fmIsConstantResFunc(SFunctionNode* pFunc); bool fmIsSkipScanCheckFunc(int32_t funcId); bool fmIsPrimaryKeyFunc(int32_t funcId); bool fmIsProcessByRowFunc(int32_t funcId); +bool fmIsSelectTagFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType, int32_t pkBytes); SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 0476a7981c..3b4e0792a6 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -603,7 +603,8 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { - if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_key") == 0) { + if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_key") == 0 || + strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_tag") == 0) { // for groupkey along with functions that output multiple lines(e.g. Histogram) // need to match groupkey result for each output row of that function. if (pCtx[j].resultInfo->numOfRes != 0) { diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index cdcc702629..5a448520f1 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -233,6 +233,11 @@ static bool isGroupKeyFunc(SExprInfo* pExprInfo) { return (functionType == FUNCTION_TYPE_GROUP_KEY); } +static bool isSelectTagFunc(SExprInfo* pExprInfo) { + int32_t functionType = pExprInfo->pExpr->_function.functionType; + return (functionType == FUNCTION_TYPE_SELECT_TAG); +} + static bool getIgoreNullRes(SExprSupp* pExprSup) { for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[i]; @@ -296,7 +301,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); continue; } else if (!isInterpFunc(pExprInfo)) { - if (isGroupKeyFunc(pExprInfo)) { + if (isGroupKeyFunc(pExprInfo) || isSelectTagFunc(pExprInfo)) { if (pSrcBlock != NULL) { int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); @@ -308,7 +313,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp char* v = colDataGetData(pSrc, index); colDataSetVal(pDst, pResBlock->info.rows, v, false); - } else { + } else if(!isSelectTagFunc(pExprInfo)){ // use stored group key SGroupKeys* pkey = pSliceInfo->pPrevGroupKey; if (pkey->isNull == false) { @@ -316,6 +321,14 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { colDataSetNULL(pDst, rows); } + } else { + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); + if (pkey->isNull == false) { + colDataSetVal(pDst, rows, pkey->pData, false); + } else { + colDataSetNULL(pDst, rows); + } } } continue; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 7615584f8c..f3ebb2359b 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -254,6 +254,7 @@ bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t groupKeyFunction(SqlFunctionCtx* pCtx); int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t selectValueFunciton(SqlFunctionCtx* pCtx); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 5058ebb8c3..75d6047c07 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -4108,6 +4108,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = md5Function, .finalizeFunc = NULL }, + { + .name = "_select_tag", + .type = FUNCTION_TYPE_SELECT_TAG, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .translateFunc = translateGroupKey, + .getEnvFunc = getGroupKeyFuncEnv, + .initFunc = functionSetup, + .processFunc = groupKeyFunction, + .finalizeFunc = groupKeyFinalize, + }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index aab5a52776..7e2f631816 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6650,6 +6650,41 @@ _group_key_over: return TSDB_CODE_SUCCESS; } +int32_t selectValueFunciton(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + int32_t startIndex = pInput->startRowIndex; + + // escape rest of data blocks to avoid first entry to be overwritten. + if (pInfo->hasResult) { + goto _group_key_over; + } + + if (pInputCol->pData == NULL || colDataIsNull_s(pInputCol, startIndex)) { + pInfo->isNull = true; + pInfo->hasResult = true; + goto _group_key_over; + } + + char* data = colDataGetData(pInputCol, startIndex); + if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { + memcpy(pInfo->data, data, + (pInputCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(data) : varDataTLen(data)); + } else { + memcpy(pInfo->data, data, pInputCol->info.bytes); + } + pInfo->hasResult = true; + +_group_key_over: + + SET_VAL(pResInfo, 1, 1); + return TSDB_CODE_SUCCESS; +} + int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index b99e67697c..9cb7d59a2c 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -268,6 +268,13 @@ bool fmIsGroupKeyFunc(int32_t funcId) { return FUNCTION_TYPE_GROUP_KEY == funcMgtBuiltins[funcId].type; } +bool fmIsSelectTagFunc(int32_t funcId) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return false; + } + return FUNCTION_TYPE_SELECT_TAG == funcMgtBuiltins[funcId].type; +} + bool fmIsBlockDistFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c2278649e1..f4b318422a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3162,6 +3162,25 @@ static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } +static EDealRes rewriteExprToSelectTagFunc(STranslateContext* pCxt, SNode** pNode) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + strcpy(pFunc->functionName, "_select_tag"); + strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); + strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias); + pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + *pNode = (SNode*)pFunc; + pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); + } + + return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); +} + static bool isWindowJoinProbeTablePrimCol(SSelectStmt* pSelect, SNode* pNode) { if (QUERY_NODE_COLUMN != nodeType(pNode)) { return false; @@ -3494,10 +3513,13 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { } } if (partionByTbname && - ((QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) || - (QUERY_NODE_FUNCTION == nodeType(*pNode) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)*pNode)->funcType))) { + (QUERY_NODE_FUNCTION == nodeType(*pNode) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)*pNode)->funcType)) { return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); } + if (partionByTbname && + ((QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG))) { + return rewriteExprToSelectTagFunc(pCxt->pTranslateCxt, pNode); + } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { pCxt->existCol = true; } @@ -4951,7 +4973,8 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { } static EDealRes needFillImpl(SNode* pNode, void* pContext) { - if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType) { + if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType + && FUNCTION_TYPE_SELECT_TAG != ((SFunctionNode*)pNode)->funcType) { *(bool*)pContext = true; return DEAL_RES_END; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 0cba28e254..89a1da3585 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -891,7 +891,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt } static bool isInterpFunc(int32_t funcId) { - return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId); + return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId) || fmIsSelectTagFunc(funcId); } static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 1cb95b59c5..bb96f6342c 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -38,6 +38,200 @@ class TDTestCase: tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(value, 2)") tdSql.checkRows(0) + + def ts5181(self): + tdSql.execute("create database db1 keep 36500") + tdSql.execute("use db1") + + tdSql.execute("CREATE STABLE db1.`stb1` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `v1` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium') TAGS (`t1` INT)") + + tdSql.execute("insert into db1.ttt_10000 using db1.stb1 tags( 44400 ) values('2024-02-19 16:05:17.649', 22300 ); ") + tdSql.execute("insert into db1.ttt_10000 using db1.stb1 tags( 44400 ) values('2024-02-19 16:05:48.818', 22300 ); ") + tdSql.execute("insert into db1.ttt_10 using db1.stb1 tags( 40 ) values('2024-02-19 16:25:36.013', 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-19 16:39:50.385' , 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-19 16:43:51.742' , 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-20 08:35:13.518' , 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-20 08:58:42.255' , 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-21 09:57:49.477' , 20 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 ) values('2024-02-21 09:58:21.882' , 20 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 ) values('2024-02-26 16:08:31.675' , 20 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 ) values('2024-02-26 16:11:43.445' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:12:30.276' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:07.188' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:07.653' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:07.879' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.083' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.273' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.429' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.599' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.775' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.940' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:09.110' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:09.254' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:09.409' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:34.750' , 12 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:49.820' , 12 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:59.551' , NULL ); ") + tdSql.execute("insert into db1.ttt_2 using db1.stb1 tags( 2 ) values('2024-02-19 15:26:39.644' , 2 ); ") + tdSql.execute("insert into db1.ttt_2 using db1.stb1 tags( 2 ) values('2024-02-19 15:26:40.433' , 2 ); ") + tdSql.execute("insert into db1.ttt_3 using db1.stb1 tags( 3 ) values('2024-02-19 15:27:22.613' , 1 ); ") + tdSql.execute("insert into db1.ttt_13 using db1.stb1 tags( 3 ) values('2024-02-19 15:27:39.719' , 1 ); ") + tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 ) values('2024-02-19 15:28:36.235' , 222 ); ") + tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 ) values('2024-02-19 15:28:59.310' , 222 ); ") + tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 ) values('2024-02-19 15:29:18.897' , 222 ); ") + tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 ) values('2024-02-19 15:50:24.682' , 223 ); ") + tdSql.execute("insert into db1.ttt_4 using db1.stb1 tags( 3 ) values('2024-02-19 15:31:19.945' , 222 ); ") + tdSql.execute("insert into db1.ttt_a using db1.stb1 tags( 3 ) values('2024-02-19 15:31:37.915' , 4 ); ") + tdSql.execute("insert into db1.ttt_axxxx using db1.stb1 tags( NULL ) values('2024-02-19 15:31:58.953' , 4 ); ") + tdSql.execute("insert into db1.ttt_axxx using db1.stb1 tags( 56 ) values('2024-02-19 15:32:22.323' , NULL ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-19 15:36:44.625' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-19 15:38:41.479' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-19 15:57:23.249' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-19 16:04:20.465' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-26 16:11:29.364' , 5444 ); ") + tdSql.execute("insert into db1.ttt_123 using db1.stb1 tags( 0 ) values('2024-02-19 15:44:52.136' , 223 ); ") + tdSql.execute("insert into db1.ttt_145 using db1.stb1 tags( 0 ) values('2024-02-19 15:50:28.580' , 223 ); ") + tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 ) values('2024-02-19 15:50:32.493' , 223 ); ") + tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 ) values('2024-02-19 15:57:36.866' , 223 ); ") + tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 ) values('2024-02-19 16:04:52.794' , 221113 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-27 08:47:11.366' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-28 09:35:46.474' , 5444 ); ") + + + tdSql.query("select *,tbname from db1.stb1 ;") + tdSql.checkRows(51) + + tdSql.query("select _irowts as ts,interp(v1),t1,tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) fill(prev)") + tdSql.checkRows(4) + + tdSql.query("select _irowts as ts,interp(v1),t1,tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) fill(prev) order by tbname") + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 2, 2) + tdSql.checkData(3, 2, 3) + tdSql.checkData(0, 3, "ttt_13") + tdSql.checkData(1, 3, "ttt_14") + tdSql.checkData(2, 3, "ttt_2") + tdSql.checkData(3, 3, "ttt_3") + + tdSql.query("select _irowts as ts,interp(v1),t1,tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(value, 0) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, "ttt_123") + tdSql.checkData(1, 2, 3) + tdSql.checkData(1, 3, "ttt_13") + + tdSql.query("select _irowts as ts,interp(v1),tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(value, 0) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 2, "ttt_123") + tdSql.checkData(1, 2, "ttt_13") + + tdSql.query("select _irowts as ts,interp(v1),t1,tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(NULL) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, "ttt_123") + tdSql.checkData(1, 2, 3) + tdSql.checkData(1, 3, "ttt_13") + + tdSql.query("select _irowts as ts,interp(v1),t1 from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(NULL) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 2, 0) + tdSql.checkData(1, 2, 3) + + tdSql.query("select _irowts as ts,interp(v1), tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(NULL) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 2, "ttt_123") + tdSql.checkData(1, 2, "ttt_13") + + tdSql.query("select _irowts as ts,interp(v1),t1,tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(NULL) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, "ttt_123") + tdSql.checkData(1, 2, 3) + tdSql.checkData(1, 3, "ttt_13") + + tdSql.query("select _irowts as ts,interp(v1),t1,tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(LINEAR) order by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 2, 3) + tdSql.checkData(0, 3, "ttt_14") + + tdSql.query("select _irowts as ts,interp(v1), tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(LINEAR) order by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 2, "ttt_14") + + tdSql.query("select _irowts as ts,interp(v1),t1 from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(LINEAR) order by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 2, 3) + + tdSql.query("select _irowts as ts,interp(v1),t1, tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(NEXT) order by tbname") + tdSql.checkRows(9) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, "ttt_123") + + tdSql.query("select _irowts as ts,interp(v1),t1 from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(NEXT) order by tbname") + tdSql.checkRows(9) + tdSql.checkData(0, 2, 0) + + tdSql.query("select _irowts as ts,interp(v1),tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(NEXT) order by tbname") + tdSql.checkRows(9) + tdSql.checkData(0, 2, "ttt_123") + + tdSql.query("select _irowts as ts,interp(v1),t1, tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(NULL_F) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, "ttt_123") + + tdSql.query("select _irowts as ts,interp(v1),t1, tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(VALUE_F, 5) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 1, 5) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, "ttt_123") def run(self): dbname = "db" @@ -5683,6 +5877,7 @@ class TDTestCase: tdSql.checkData(0, 1, None) self.interp_on_empty_table() + self.ts5181() def stop(self): From b72792e20e50b7b407af0296a04dcd069e0659b5 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 22 Jul 2024 08:48:26 +0800 Subject: [PATCH 06/43] unuesd code --- source/libs/function/inc/builtinsimpl.h | 1 - source/libs/function/src/builtinsimpl.c | 35 ------------------------- 2 files changed, 36 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index f3ebb2359b..7615584f8c 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -254,7 +254,6 @@ bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t groupKeyFunction(SqlFunctionCtx* pCtx); int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); -int32_t selectValueFunciton(SqlFunctionCtx* pCtx); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7e2f631816..aab5a52776 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6650,41 +6650,6 @@ _group_key_over: return TSDB_CODE_SUCCESS; } -int32_t selectValueFunciton(SqlFunctionCtx* pCtx) { - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - SInputColumnInfoData* pInput = &pCtx->input; - SColumnInfoData* pInputCol = pInput->pData[0]; - - int32_t startIndex = pInput->startRowIndex; - - // escape rest of data blocks to avoid first entry to be overwritten. - if (pInfo->hasResult) { - goto _group_key_over; - } - - if (pInputCol->pData == NULL || colDataIsNull_s(pInputCol, startIndex)) { - pInfo->isNull = true; - pInfo->hasResult = true; - goto _group_key_over; - } - - char* data = colDataGetData(pInputCol, startIndex); - if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { - memcpy(pInfo->data, data, - (pInputCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(data) : varDataTLen(data)); - } else { - memcpy(pInfo->data, data, pInputCol->info.bytes); - } - pInfo->hasResult = true; - -_group_key_over: - - SET_VAL(pResInfo, 1, 1); - return TSDB_CODE_SUCCESS; -} - int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); From 72015266743e56ab3c86197604a1b7e60c5d32d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Jul 2024 09:13:39 +0800 Subject: [PATCH 07/43] fix(stream): check return value. --- include/libs/stream/tstream.h | 11 +- source/common/src/tdatablock.c | 2 +- source/common/src/tmisce.c | 1 - source/dnode/vnode/src/inc/tq.h | 4 +- source/dnode/vnode/src/sma/smaTimeRange.c | 7 +- source/dnode/vnode/src/tq/tqSink.c | 294 ++++++++++----- source/dnode/vnode/src/tq/tqStreamTask.c | 63 +++- source/dnode/vnode/src/tqCommon/tqCommon.c | 138 +++++--- source/libs/stream/inc/streamInt.h | 6 - source/libs/stream/src/streamBackendRocksdb.c | 3 +- source/libs/stream/src/streamMeta.c | 335 +++++++++++------- source/libs/stream/src/streamSnapshot.c | 180 +++++++--- source/libs/stream/src/streamStartHistory.c | 160 +++++---- source/libs/stream/src/streamTask.c | 108 +++--- source/libs/stream/src/streamTimer.c | 5 +- source/libs/stream/src/streamUpdate.c | 1 - 16 files changed, 882 insertions(+), 436 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index dff212b15c..5f322be99b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -686,7 +686,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); // checkpoint related -int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId); +void streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId); int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId); int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId); bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId); @@ -770,9 +770,9 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta); int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask); // timer -tmr_h streamTimerGetInstance(); -void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, - const char* pMsg); +int32_t streamTimerGetInstance(tmr_h* pTmr); +void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, + const char* pMsg); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); @@ -809,6 +809,9 @@ void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp); int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp); int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask); +void streamMutexLock(TdThreadMutex *pMutex); +void streamMutexUnlock(TdThreadMutex *pMutex); +void streamMutexDestroy(TdThreadMutex *pMutex); #ifdef __cplusplus } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 43bac56f93..947fe93712 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2491,7 +2491,7 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; int32_t code = buildChildTableName(&rname); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { return code; } taosArrayDestroy(tags); diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 7b349e91b0..154fcc3f6b 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -150,7 +150,6 @@ int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) { cap -= nwrite; for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) { - int32_t ret = 0; if (_i == pEpSet->numOfEps - 1) { ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); } else { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index d3582ab8f3..56a3c00fee 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -155,8 +155,8 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_ SSubmitTbData* pTableData, int64_t earlyTs, const char* id); int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id); -SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, - SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule); +int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock, + SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq); #define TQ_ERR_GO_TO_END(c) \ do { \ diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index d0913081ac..c1a870e253 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -196,7 +196,12 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE}; int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1; - tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true); + + code = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true, &tbData.pCreateTbReq); + if (code) { + smaError("failed to build create-table req, code:%d", code); + continue; + } { uint64_t groupId = pDataBlock->info.id.groupId; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 3efc653f64..cd6e87b9fc 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -18,8 +18,6 @@ #include "tmsg.h" #include "tq.h" -#define MAX_CACHE_TABLE_INFO_NUM 10240 - typedef struct STableSinkInfo { uint64_t uid; tstr name; @@ -43,7 +41,7 @@ static int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, con static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid); static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags); -static SArray* createDefaultTagColName(); +static int32_t createDefaultTagColName(SArray** pList); static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, int64_t gid, bool newSubTableRule); static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo); @@ -95,7 +93,10 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p SSingleDeleteReq req = {.startTs = skey, .endTs = ekey}; strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); - taosArrayPush(deleteReq->deleteReqs, &req); + void* p = taosArrayPush(deleteReq->deleteReqs, &req); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } if (originName) name = originName; taosMemoryFreeClear(name); @@ -149,14 +150,20 @@ static bool tqGetTableInfo(SSHashObj* pTableInfoMap, uint64_t groupId, STableSin static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { void* buf = NULL; int32_t tlen = 0; - encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen); + + int32_t code = encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen); + if (code) { + tqError("vgId:%d failed to encode create table msg, create table failed, code:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen}; - if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg); + if (code) { tqError("failed to put into write-queue since %s", terrstr()); } - return TSDB_CODE_SUCCESS; + return code; } int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) { @@ -166,18 +173,36 @@ int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const // set super table name SName name = {0}; - tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); + + int32_t code = tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (code == 0) { + pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); + if (pCreateTableReq->ctb.stbName == NULL) { // ignore this error code + tqError("failed to duplicate the stb name:%s, failed to init create-table msg and create req table", stbFullName); + } + } pCreateTableReq->ctb.tagNum = numOfTags; - return TSDB_CODE_SUCCESS; + return code; } -SArray* createDefaultTagColName() { +int32_t createDefaultTagColName(SArray** pColNameList) { + *pColNameList = NULL; + SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN); - char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; - taosArrayPush(pTagColNameList, tagNameStr); - return pTagColNameList; + if (pTagColNameList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; + void* p = taosArrayPush(pTagColNameList, tagNameStr); + if (p == NULL) { + taosArrayDestroy(pTagColNameList); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pColNameList = pTagColNameList; + return TSDB_CODE_SUCCESS; } void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, @@ -201,18 +226,20 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int64_t suid) { - STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; - int32_t rows = pDataBlock->info.rows; - SArray* tagArray = taosArrayInit(4, sizeof(STagVal)); + STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; + int32_t rows = pDataBlock->info.rows; + SArray* tagArray = taosArrayInit(4, sizeof(STagVal)); + const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; - tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows); + tqDebug("s-task:%s build create %d table(s) msg", id, rows); int32_t code = 0; SVCreateTbBatchReq reqs = {0}; SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); if (NULL == reqs.pArray) { - tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno)); + tqError("s-task:%s failed to init create table msg, code:%s", id, tstrerror(terrno)); goto _end; } @@ -222,15 +249,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1); - initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags); + code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags); + if (code) { + tqError("s-task:%s vgId:%d failed to init create table msg", id, vgId); + continue; + } taosArrayClear(tagArray); if (size == 2) { STagVal tagVal = { .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; - taosArrayPush(tagArray, &tagVal); - pCreateTbReq->ctb.tagName = createDefaultTagColName(); + void* p = taosArrayPush(tagArray, &tagVal); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + code = createDefaultTagColName(&pCreateTbReq->ctb.tagName); + if (code) { + return code; + } } else { for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); @@ -245,14 +283,17 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S } else { memcpy(&tagVal.i64, pData, pTagData->info.bytes); } - taosArrayPush(tagArray, &tagVal); + void* p = taosArrayPush(tagArray, &tagVal); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; } } - tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); + code = tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); taosArrayDestroy(tagArray); tagArray = NULL; - if (pCreateTbReq->ctb.pTag == NULL) { + + if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) { tdDestroySVCreateTbReq(pCreateTbReq); code = TSDB_CODE_OUT_OF_MEMORY; goto _end; @@ -270,22 +311,34 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid, pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); - taosArrayPush(reqs.pArray, pCreateTbReq); + void* p = taosArrayPush(reqs.pArray, pCreateTbReq); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } STableSinkInfo* pInfo = NULL; bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo); if (!alreadyCached) { code = doCreateSinkInfo(pCreateTbReq->name, &pInfo); - doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr); + if (code) { + tqError("vgId:%d failed to create sink tableInfo for table:%s, s-task:%s", vgId, pCreateTbReq->name, id); + continue; + } + + code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, id); + if (code) { + tqError("vgId:%d failed to put sink tableInfo:%s into cache, s-task:%s", vgId, pCreateTbReq->name, id); + } } - tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); + tqDebug("s-task:%s build create table:%s msg complete", id, pCreateTbReq->name); } reqs.nReqs = taosArrayGetSize(reqs.pArray); code = tqPutReqToQueue(pVnode, &reqs); if (code != TSDB_CODE_SUCCESS) { - tqError("s-task:%s failed to send create table msg", pTask->id.idStr); + tqError("s-task:%s failed to send create table msg", id); } _end: @@ -348,15 +401,26 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c SRow* pOldRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k); if (pNewRow->ts < pOldRow->ts) { - taosArrayPush(pFinal, &pNewRow); + void* p = taosArrayPush(pFinal, &pNewRow); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } j += 1; } else if (pNewRow->ts > pOldRow->ts) { - taosArrayPush(pFinal, &pOldRow); + void* p = taosArrayPush(pFinal, &pOldRow); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + k += 1; } else { // check for the existance of primary key if (pNewRow->numOfPKs == 0) { - taosArrayPush(pFinal, &pNewRow); + void* p = taosArrayPush(pFinal, &pNewRow); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + k += 1; j += 1; tRowDestroy(pOldRow); @@ -369,7 +433,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c int32_t ret = tRowKeyCompare(&kNew, &kOld); if (ret <= 0) { - taosArrayPush(pFinal, &pNewRow); + void* p = taosArrayPush(pFinal, &pNewRow); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + j += 1; if (ret == 0) { @@ -377,7 +445,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c tRowDestroy(pOldRow); } } else { - taosArrayPush(pFinal, &pOldRow); + void* p = taosArrayPush(pFinal, &pOldRow); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + k += 1; } } @@ -386,12 +458,18 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c while (j < newLen) { SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pNew->aRowP, j++); - taosArrayPush(pFinal, &pRow); + void* p = taosArrayPush(pFinal, &pRow); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } while (k < oldLen) { SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k++); - taosArrayPush(pFinal, &pRow); + void* p = taosArrayPush(pFinal, &pRow); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } taosArrayDestroy(pNew->aRowP); @@ -425,34 +503,40 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam return true; } -SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, - SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule) { +int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock, + SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq) { + *pReq = NULL; + SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); if (pCreateTbReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } taosArrayClear(pTagArray); - initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1); - - STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; - taosArrayPush(pTagArray, &tagVal); - - tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); - - if (pCreateTbReq->ctb.pTag == NULL) { - tdDestroySVCreateTbReq(pCreateTbReq); - taosMemoryFreeClear(pCreateTbReq); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + int32_t code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1); + if (code != 0) { + return code; } - pCreateTbReq->ctb.tagName = createDefaultTagColName(); + STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; + void* p = taosArrayPush(pTagArray, &tagVal); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + code = tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); + if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) { + tdDestroySVCreateTbReq(pCreateTbReq); + taosMemoryFreeClear(pCreateTbReq); + return code; + } + + code = createDefaultTagColName(&pCreateTbReq->ctb.tagName); // set table name setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule); - return pCreateTbReq; + *pReq = pCreateTbReq; + return code; } int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) { @@ -555,7 +639,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat break; } SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); - taosArrayPush(pVals, &cv); + void* p = taosArrayPush(pVals, &cv); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); if (colDataIsNull_s(pColData, j)) { @@ -566,7 +653,11 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat } SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); - taosArrayPush(pVals, &cv); + void* p = taosArrayPush(pVals, &cv); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + dataIndex++; } else { void* colData = colDataGetData(pColData, j); @@ -574,12 +665,18 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat SValue sv = (SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)}; SColVal cv = COL_VAL_VALUE(pCol->colId, sv); - taosArrayPush(pVals, &cv); + void* p = taosArrayPush(pVals, &cv); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { SValue sv = {.type = pCol->type}; memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); SColVal cv = COL_VAL_VALUE(pCol->colId, sv); - taosArrayPush(pVals, &cv); + void* p = taosArrayPush(pVals, &cv); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } dataIndex++; } @@ -596,7 +693,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat } ASSERT(pRow); - taosArrayPush(pTableData->aRowP, &pRow); + void* p = taosArrayPush(pTableData->aRowP, &pRow); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } taosArrayDestroy(pVals); @@ -665,6 +765,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; int32_t vgId = TD_VID(pVnode); STableSinkInfo* pTableSinkInfo = NULL; + int32_t code = 0; bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, groupId, &pTableSinkInfo); @@ -686,7 +787,11 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat } else { // this groupId has not been kept in cache yet if (dstTableName[0] == 0) { memset(dstTableName, 0, TSDB_TABLE_NAME_LEN); - buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); + code = buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); + if (code) { + tqDebug("s-task:%s failed to build auto create table-name:%s, groupId:0x%" PRId64, id, dstTableName, groupId); + return code; + } } else { if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { @@ -699,8 +804,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat } } - int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo); - tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); + code = doCreateSinkInfo(dstTableName, &pTableSinkInfo); + if (code == 0) { + tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); + } else { + tqDebug("s-task:%s failed to build new sinkTableInfo, dstTable:%s", id, dstTableName); + return code; + } } if (alreadyCached) { @@ -731,20 +841,20 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; - pTableData->pCreateTbReq = + code = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, - pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); + (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1), + &pTableData->pCreateTbReq); taosArrayDestroy(pTagArray); - if (pTableData->pCreateTbReq == NULL) { - tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, - tstrerror(terrno)); + if (code) { + tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, tstrerror(code)); taosMemoryFree(pTableSinkInfo); - return terrno; + return code; } pTableSinkInfo->uid = 0; - doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); + code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); } else { metaReaderClear(&mr); @@ -765,12 +875,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat pTableSinkInfo->uid = mr.me.uid; metaReaderClear(&mr); - doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); + code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); } } } - return TDB_CODE_SUCCESS; + return code; } int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, @@ -864,14 +974,21 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (tbData.pCreateTbReq != NULL) { tdDestroySVCreateTbReq(tbData.pCreateTbReq); - doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id); + (void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id); tbData.pCreateTbReq = NULL; } continue; } - taosArrayPush(submitReq.aSubmitTbData, &tbData); + void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData); + if (p == NULL) { + tqDebug("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id); + } + code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1); + if (code) { // failed and continue + tqDebug("vgId:%d, s-task:%s submit msg failed, data lost", vgId, id); + } } } } else { @@ -918,16 +1035,24 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (tbData.pCreateTbReq != NULL) { tdDestroySVCreateTbReq(tbData.pCreateTbReq); - doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id); + (void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id); tbData.pCreateTbReq = NULL; } continue; } - taosArrayPush(submitReq.aSubmitTbData, &tbData); + void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData); + if (p == NULL) { + tqError("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id); + continue; + } int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; - taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); + code = taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); + if (code) { + tqError("vgId:%d, s-task:%s failed to put group into index map, code:%s", vgId, id, tstrerror(code)); + continue; + } } else { code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id); if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { @@ -951,7 +1076,10 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { taosHashCleanup(pTableIndexMap); if (hasSubmit) { - doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks); + code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks); + if (code) { // failed and continue + tqError("vgId:%d failed to build and send submit msg", vgId); + } } else { tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); @@ -989,7 +1117,11 @@ int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char } int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId)); - tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId); + if (code == 0) { + tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId); + } else { + tqError("s-task:%s failed to remove table meta from hashmap, groupId:%" PRId64, id, groupId); + } return code; } @@ -1019,10 +1151,14 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, len); - tEncodeSBatchDeleteReq(&encoder, &deleteReq); + code = tEncodeSBatchDeleteReq(&encoder, &deleteReq); tEncoderClear(&encoder); taosArrayDestroy(deleteReq.deleteReqs); + if (code) { + return code; + } + ((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode); SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)}; diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index b4829d1dd8..fa6a6bdffc 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -37,7 +37,12 @@ int32_t tqScanWal(STQ* pTq) { // check all tasks int32_t numOfTasks = 0; bool shouldIdle = true; - doScanWalForAllTasks(pMeta, &shouldIdle); + + int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle); + if (code) { + tqError("vgId:%d failed to start all tasks, try next time", vgId); + return code; + } streamMetaWLock(pMeta); int32_t times = (--pMeta->scanInfo.scanCounter); @@ -51,9 +56,13 @@ int32_t tqScanWal(STQ* pTq) { if (times > 0) { tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION); - tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION); + code = tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION); + if (code) { + tqError("vgId:%d sched scan wal in %dms failed, ignore this failure", vgId, SCAN_WAL_IDLE_DURATION); + } } - return 0; + + return code; } typedef struct SBuildScanWalMsgParam { @@ -69,28 +78,44 @@ static void doStartScanWal(void* param, void* tmrId) { tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks, pTq->pVnode->restored); - /*int32_t code = */ streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); + int32_t code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); taosMemoryFree(pParam); + + if (code) { + tqError("vgId:%d failed sched task to scan wal", vgId); + } } int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { SStreamMeta* pMeta = pTq->pStreamMeta; + int32_t code = 0; + int32_t vgId = TD_VID(pTq->pVnode); SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam)); + if (pParam == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pParam->pTq = pTq; pParam->numOfTasks = numOfTasks; - tmr_h pTimer = streamTimerGetInstance(); - ASSERT(pTimer); + tmr_h pTimer = NULL; + code = streamTimerGetInstance(&pTimer); + if (code) { + tqError("vgId:%d failed to get tmr ctrl during sched scan wal", vgId); + return code; + } if (pMeta->scanInfo.scanTimer == NULL) { pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer); } else { - taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); + code = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); + if (code) { + tqError("vgId:%d failed to start scan wal in:%dms", vgId, idleDuration); + } } - return TSDB_CODE_SUCCESS; + return code; } int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { @@ -207,7 +232,11 @@ bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) { double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0; qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs", id, pTask->step2Range.minVer, maxVer, el); - /*int32_t code = */streamTaskPutTranstateIntoInputQ(pTask); + int32_t code = streamTaskPutTranstateIntoInputQ(pTask); + if (code) { + qError("s-task:%s failed to put trans-state into inputQ", id); + } + return true; } else { qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64 @@ -290,8 +319,12 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems break; } } else { - walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer); + code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); + if (code) { + tqError("s-task:%s failed to seek ver to:%"PRId64 " in wal", id, pTask->chkInfo.nextProcessVer); + } + break; } } @@ -347,25 +380,25 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX; - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state != TASK_STATUS__READY) { tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name); - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pStreamMeta, pTask); continue; } bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems); - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); if ((numOfItems > 0) || hasNewData) { noDataInWal = false; code = streamTrySchedExec(pTask); if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pStreamMeta, pTask); - return -1; + return code; } } @@ -378,5 +411,5 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { } taosArrayDestroy(pTaskList); - return 0; + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 3c6100a8f4..778d79f8f6 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -160,6 +160,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; int64_t st = taosGetTimestampMs(); bool updated = false; + int32_t code = 0; SStreamTaskNodeUpdateMsg req = {0}; @@ -258,26 +259,40 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } } - // save + // stream do update the nodeEp info, write it into stream meta. if (updated) { tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId); - streamMetaSaveTask(pMeta, pTask); + code = streamMetaSaveTask(pMeta, pTask); + if (code) { + tqError("s-task:%s vgId:%d failed to save task, code:%s", idstr, vgId, tstrerror(code)); + } + if (ppHTask != NULL) { - streamMetaSaveTask(pMeta, *ppHTask); + code = streamMetaSaveTask(pMeta, *ppHTask); + if (code) { + tqError("s-task:%s vgId:%d failed to save related history task, code:%s", idstr, vgId, tstrerror(code)); + } } } else { tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId); } - streamTaskStop(pTask); + code = streamTaskStop(pTask); + if (code) { + tqError("s-task:%s vgId:%d failed to stop task, code:%s", idstr, vgId, tstrerror(code)); + } + if (ppHTask != NULL) { - streamTaskStop(*ppHTask); + code = streamTaskStop(*ppHTask); + if (code) { + tqError("s-task:%s vgId:%d failed to stop related history task, code:%s", idstr, vgId, tstrerror(code)); + } } // keep info streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st); - rsp.code = 0; + rsp.code = TSDB_CODE_SUCCESS; // possibly only handle the stream task. int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); @@ -305,13 +320,16 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM #if 0 taosMSleep(5000);// for test purpose, to trigger the leader election #endif - tqStreamTaskStartAsync(pMeta, cb, true); + code = tqStreamTaskStartAsync(pMeta, cb, true); + if (code) { + tqError("vgId:%d async start all tasks, failed, code:%s", vgId, tstrerror(code)); + } } } streamMetaWUnLock(pMeta); taosArrayDestroy(req.pNodeList); - return rsp.code; + return rsp.code; // always return true } int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { @@ -518,7 +536,7 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask); - if (pTask == NULL) { + if (code != 0) { tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } @@ -526,11 +544,17 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId, pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); - streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId); + code = streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId); streamMetaReleaseTask(pMeta, pTask); + if (code) { + return code; + } { // send checkpoint ready rsp SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg)); + if (pReadyRsp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pReadyRsp->upstreamTaskId = req.upstreamTaskId; pReadyRsp->upstreamNodeId = req.upstreamNodeId; @@ -607,7 +631,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve SStreamTask* p = NULL; code = streamMetaAcquireTask(pMeta, streamId, taskId, &p); if ((p != NULL) && (p->info.fillHistory == 0)) { - tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId); + code = tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId); } if (p != NULL) { @@ -631,6 +655,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + int32_t code = 0; int32_t vgId = pMeta->vgId; STaskId hTaskId = {0}; tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); @@ -649,8 +674,12 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen } streamTaskSetRemoveBackendFiles(pTask); - streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); + code = streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); streamMetaReleaseTask(pMeta, pTask); + + if (code) { + tqError("s-task:0x%x failed to clear related fill-history info, still exists", pReq->taskId); + } } streamMetaWUnLock(pMeta); @@ -746,7 +775,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { if (isLeader && !tsDisableStream) { streamMetaWUnLock(pMeta); - streamMetaStartAllTasks(pMeta); + code = streamMetaStartAllTasks(pMeta); } else { streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId); pMeta->startInfo.restartCount = 0; @@ -765,16 +794,16 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t vgId = pMeta->vgId; if (type == STREAM_EXEC_T_START_ONE_TASK) { - streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); + (void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); return 0; } else if (type == STREAM_EXEC_T_START_ALL_TASKS) { - streamMetaStartAllTasks(pMeta); + (void) streamMetaStartAllTasks(pMeta); return 0; } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { - restartStreamTasks(pMeta, isLeader); + (void) restartStreamTasks(pMeta, isLeader); return 0; } else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) { - streamMetaStopAllTasks(pMeta); + (void) streamMetaStopAllTasks(pMeta); return 0; } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) { int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); @@ -783,7 +812,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); - if (pTask != NULL) { + if (pTask != NULL || (code != 0)) { char* pStatus = NULL; if (streamTaskReadyToRun(pTask, &pStatus)) { int64_t execTs = pTask->status.lastExecTs; @@ -804,12 +833,12 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); - if (pTask != NULL) { // even in halt status, the data in inputQ must be processed + if (pTask != NULL || (code != 0)) { // even in halt status, the data in inputQ must be processed char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, p, pTask->chkInfo.nextProcessVer); - streamExecTask(pTask); + (void) streamExecTask(pTask); } else { int8_t status = streamTaskSetSchedStatusInactive(pTask); tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, @@ -829,6 +858,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { STaskStartInfo* pStartInfo = &pMeta->startInfo; int32_t vgId = pMeta->vgId; bool scanWal = false; + int32_t code = 0; streamMetaWLock(pMeta); if (pStartInfo->startAllTasks == 1) { @@ -844,8 +874,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { pStartInfo->restartCount); streamMetaWUnLock(pMeta); - restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); - return TSDB_CODE_SUCCESS; + return restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); } else { if (pStartInfo->restartCount == 0) { tqDebug("vgId:%d start all tasks completed in callbackFn, restartCount is 0", pMeta->vgId); @@ -862,10 +891,10 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { if (scanWal && (vgId != SNODE_HANDLE)) { tqDebug("vgId:%d start scan wal for executing tasks", vgId); - tqScanWalAsync(pMeta->ahandle, true); + code = tqScanWalAsync(pMeta->ahandle, true); } - return TSDB_CODE_SUCCESS; + return code; } int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { @@ -873,7 +902,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); - if (pTask == NULL) { + if (pTask == NULL || (code != 0)) { tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, pReq->taskId); return TSDB_CODE_SUCCESS; @@ -881,7 +910,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); streamTaskClearCheckInfo(pTask, true); // clear flag set during do checkpoint, and open inputQ for all upstream tasks @@ -904,7 +933,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; @@ -929,11 +958,10 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready", pTask->id.idStr, (int32_t)pReq->downstreamTaskId); - streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_STREAM_TASK_IVLD_STATUS); streamMetaReleaseTask(pMeta, pTask); - - return TSDB_CODE_SUCCESS; + return code; } SStreamTaskState pState = streamTaskGetStatus(pTask); @@ -948,7 +976,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) // re-send the lost checkpoint-trigger msg to downstream task tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, (int32_t)pReq->downstreamTaskId, checkpointId, transId); - streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_SUCCESS); } else { // not send checkpoint-trigger yet, wait int32_t recv = 0, total = 0; @@ -962,7 +990,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) "sending checkpoint-source/trigger", pTask->id.idStr, recv, total); } - streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); } } else { // upstream not recv the checkpoint-source/trigger till now @@ -971,12 +999,12 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "upstream sending checkpoint-source/trigger", pTask->id.idStr); - streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); } streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; + return code; } int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { @@ -994,9 +1022,9 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); - streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); + code = streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; + return code; } int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { @@ -1042,6 +1070,8 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t bool fromVnode) { SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; int32_t vgId = pMeta->vgId; + int32_t code = 0; + if (pTask == NULL) { return -1; } @@ -1065,9 +1095,9 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { pTask->hTaskInfo.operatorOpen = false; - streamStartScanHistoryAsync(pTask, igUntreated); + code = streamStartScanHistoryAsync(pTask, igUntreated); } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { - tqScanWalAsync((STQ*)handle, false); + code = tqScanWalAsync((STQ*)handle, false); } else { streamTrySchedExec(pTask); } @@ -1076,7 +1106,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t }*/ streamMetaReleaseTask(pMeta, pTask); - return 0; + return code; } int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) { @@ -1091,10 +1121,10 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name); - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); if (code != 0) { @@ -1105,10 +1135,10 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m SStreamTask* pHTask = NULL; code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); if (pHTask) { - taosThreadMutexLock(&pHTask->lock); + streamMutexLock(&pHTask->lock); SStreamTaskState p = streamTaskGetStatus(pHTask); tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name); - taosThreadMutexUnlock(&pHTask->lock); + streamMutexUnlock(&pHTask->lock); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); } @@ -1139,15 +1169,15 @@ int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask); - if (pTask == NULL) { + if (pTask == NULL || (code != 0)) { tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", pRsp->downstreamNodeId, pRsp->downstreamTaskId); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } - streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId); + code = streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId); streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; + return code; } int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { @@ -1173,11 +1203,11 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); - if (pTask == NULL) { + if (pTask == NULL || (code != 0)) { tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, req.taskId); - streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); - return TSDB_CODE_SUCCESS; + (void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); + return code; } // discard the rsp, since it is expired. @@ -1193,13 +1223,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { tqDebug("s-task:%s vgId:%d checkpointId:%" PRId64 " restore to consensus-checkpointId:%" PRId64 " from mnode", pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId); - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId); if (pTask->chkInfo.consensusTransId >= req.transId) { tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId); - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; } @@ -1215,14 +1245,14 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { } pTask->chkInfo.consensusTransId = req.transId; - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); if (pMeta->role == NODE_ROLE_LEADER) { - /*code = */ tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId); + code = tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId); } else { tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr); } streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; + return code; } \ No newline at end of file diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 2fe86817e3..93d2edd639 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -231,14 +231,8 @@ void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t up int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, int64_t checkpointId, SRpcMsg* pMsg); -typedef int32_t (*__stream_async_exec_fn_t)(void* param); - -int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code); void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); -void streamMutexLock(TdThreadMutex *pMutex); -void streamMutexUnlock(TdThreadMutex *pMutex); -void streamMutexDestroy(TdThreadMutex *pMutex); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 74fff23c6b..869877c9a8 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1540,7 +1540,7 @@ int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId) // compatible with previous version *processId = -1; code = 0; - stError("failed to open file to load extra info, file:%s, reason:%s", pDst, tstrerror(TAOS_SYSTEM_ERROR(errno))); + stWarn("failed to open file to load extra info, file:%s, reason:%s", pDst, tstrerror(TAOS_SYSTEM_ERROR(errno))); goto _EXIT; } @@ -2308,6 +2308,7 @@ _EXIT: taosMemoryFree(cfHandle); return code; } + void* taskDbAddRef(void* pTaskDb) { STaskDbWrapper* pBackend = pTaskDb; return taosAcquireRef(taskDbWrapperId, pBackend->refId); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 42d2f86dac..2ce82d8011 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -45,7 +45,7 @@ typedef struct STaskInitTs { SMetaRefMgt gMetaRefMgt; -void metaRefMgtInit(); +int32_t metaRefMgtInit(); void metaRefMgtCleanup(); int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid); @@ -56,9 +56,14 @@ static void streamMetaEnvInit() { streamMetaId = taosOpenRef(64, streamMetaCloseImpl); - metaRefMgtInit(); - int32_t code = streamTimerInit(); - if (code != 0) { + int32_t code = metaRefMgtInit(); + if (code) { + stError("failed to init stream meta mgmt env, start failed"); + return; + } + + code = streamTimerInit(); + if (code) { stError("failed to init stream meta env, start failed"); } } @@ -66,17 +71,29 @@ static void streamMetaEnvInit() { void streamMetaInit() { (void) taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaCleanup() { - taosCloseRef(streamBackendId); - taosCloseRef(streamBackendCfWrapperId); - taosCloseRef(streamMetaId); + (void) taosCloseRef(streamBackendId); + (void) taosCloseRef(streamBackendCfWrapperId); + (void) taosCloseRef(streamMetaId); metaRefMgtCleanup(); streamTimerCleanUp(); } -void metaRefMgtInit() { - taosThreadMutexInit(&(gMetaRefMgt.mutex), NULL); - gMetaRefMgt.pTable = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); +int32_t metaRefMgtInit() { + int32_t code = taosThreadMutexInit(&(gMetaRefMgt.mutex), NULL); + if (code) { + return code; + } + + if (code == 0) { + gMetaRefMgt.pTable = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + } + + if (gMetaRefMgt.pTable == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } else { + return code; + } } void metaRefMgtCleanup() { @@ -96,20 +113,34 @@ void metaRefMgtCleanup() { } int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { + int32_t code = 0; + void* p = NULL; + streamMutexLock(&gMetaRefMgt.mutex); - void* p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId)); + p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId)); if (p == NULL) { SArray* list = taosArrayInit(8, sizeof(void*)); - taosArrayPush(list, &rid); - taosHashPut(gMetaRefMgt.pTable, &vgId, sizeof(vgId), &list, sizeof(void*)); + p = taosArrayPush(list, &rid); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + code = taosHashPut(gMetaRefMgt.pTable, &vgId, sizeof(vgId), &list, sizeof(void*)); + if (code) { + stError("vgId:%d failed to put into metaRef table, rid:%" PRId64, (int32_t) vgId, *rid); + return code; + } } else { SArray* list = *(SArray**)p; - taosArrayPush(list, &rid); + void* px = taosArrayPush(list, &rid); + if (px == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } streamMutexUnlock(&gMetaRefMgt.mutex); - return 0; + return code; } int32_t streamMetaOpenTdb(SStreamMeta* pMeta) { @@ -141,19 +172,25 @@ enum STREAM_STATE_VER { }; int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { - int8_t ret = STREAM_STATA_COMPATIBLE; - TBC* pCur = NULL; - - if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream - return ret; - } - + int8_t ret = STREAM_STATA_COMPATIBLE; + TBC* pCur = NULL; + int32_t code = 0; void* pKey = NULL; int32_t kLen = 0; void* pVal = NULL; int32_t vLen = 0; - tdbTbcMoveToFirst(pCur); + if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream + return ret; + } + + code = tdbTbcMoveToFirst(pCur); + if (code) { + (void) tdbTbcClose(pCur); + stError("vgId:%d failed to open stream meta file cursor, not perform compatible check", pMeta->vgId); + return ret; + } + while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { if (pVal == NULL || vLen == 0) { break; @@ -178,7 +215,7 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { tdbFree(pKey); tdbFree(pVal); - tdbTbcClose(pCur); + (void) tdbTbcClose(pCur); return ret; } @@ -244,7 +281,11 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) streamMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); if ((ppBackend != NULL) && (*ppBackend != NULL)) { - taskDbAddRef(*ppBackend); + void* p = taskDbAddRef(*ppBackend); + if (p == NULL) { + stError("s-task:0x%x failed to ref backend", pTask->id.taskId); + return TSDB_CODE_FAILED; + } STaskDbWrapper* pBackend = *ppBackend; pBackend->pMeta = pMeta; @@ -278,7 +319,10 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) if (processVer != -1) pTask->chkInfo.processedVer = processVer; - taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); + int32_t code = taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); + if (code) { + stError("s-task:0x%x failed to put taskDb backend, code:out of memory", pTask->id.taskId); + } streamMutexUnlock(&pMeta->backendMutex); stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); @@ -290,7 +334,10 @@ void streamMetaRemoveDB(void* arg, char* key) { SStreamMeta* pMeta = arg; streamMutexLock(&pMeta->backendMutex); - taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key)); + int32_t code = taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key)); + if (code) { + stError("vgId:%d failed to remove key:%s in taskDbUnique map", pMeta->vgId, key); + } streamMutexUnlock(&pMeta->backendMutex); } @@ -398,12 +445,22 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, } #endif - taosThreadRwlockInit(&pMeta->lock, &attr); - taosThreadRwlockAttrDestroy(&attr); + code = taosThreadRwlockInit(&pMeta->lock, &attr); + if (code) { + goto _err; + } + + code = taosThreadRwlockAttrDestroy(&attr); + if (code) { + goto _err; + } int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); - metaRefMgtAdd(pMeta->vgId, pRid); + code = metaRefMgtAdd(pMeta->vgId, pRid); + if (code) { + goto _err; + } code = createMetaHbInfo(pRid, &pMeta->pHbInfo); if (code != TSDB_CODE_SUCCESS) { @@ -416,7 +473,8 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, if (pMeta->bkdChkptMgt == NULL) { goto _err; } - taosThreadMutexInit(&pMeta->backendMutex, NULL); + + code = taosThreadMutexInit(&pMeta->backendMutex, NULL); *p = pMeta; return code; @@ -425,9 +483,9 @@ _err: taosMemoryFree(pMeta->path); if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap); if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList); - if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb); - if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb); - if (pMeta->db) tdbClose(pMeta->db); + if (pMeta->pTaskDb) (void)tdbTbClose(pMeta->pTaskDb); + if (pMeta->pCheckpointDb) (void)tdbTbClose(pMeta->pCheckpointDb); + if (pMeta->db) (void) tdbClose(pMeta->db); if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo); if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks); if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet); @@ -473,7 +531,7 @@ void streamMetaClear(SStreamMeta* pMeta) { // release the ref by timer if (p->info.delaySchedParam != 0 && p->info.fillHistory == 0) { // one more ref in timer stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", p->id.idStr, p->refCnt); - taosTmrStop(p->schedInfo.pDelayTimer); + (void) taosTmrStop(p->schedInfo.pDelayTimer); p->info.delaySchedParam = 0; streamMetaReleaseTask(pMeta, p); } @@ -481,7 +539,11 @@ void streamMetaClear(SStreamMeta* pMeta) { streamMetaReleaseTask(pMeta, p); } - taosRemoveRef(streamBackendId, pMeta->streamBackendRid); + int32_t code = taosRemoveRef(streamBackendId, pMeta->streamBackendRid); + if (code) { + stError("vgId:%d remove stream backend Ref failed, rid:%"PRId64, pMeta->vgId, pMeta->streamBackendRid); + } + taosHashClear(pMeta->pTasksMap); taosArrayClear(pMeta->pTaskList); @@ -502,14 +564,7 @@ void streamMetaClose(SStreamMeta* pMeta) { if (pMeta == NULL) { return; } - - // int64_t rid = *(int64_t*)pMeta->pRid; - // if (taosTmrStop(pMeta->hbInfo.hbTmr)) { - // taosMemoryFree(pMeta->pRid); - // } else { - // // do nothing, stop by timer thread - // } - taosRemoveRef(streamMetaId, pMeta->rid); + (void) taosRemoveRef(streamMetaId, pMeta->rid); } void streamMetaCloseImpl(void* arg) { @@ -525,10 +580,11 @@ void streamMetaCloseImpl(void* arg) { streamMetaClear(pMeta); streamMetaWUnLock(pMeta); - tdbAbort(pMeta->db, pMeta->txn); - tdbTbClose(pMeta->pTaskDb); - tdbTbClose(pMeta->pCheckpointDb); - tdbClose(pMeta->db); + // already log the error, ignore here + (void) tdbAbort(pMeta->db, pMeta->txn); + (void) tdbTbClose(pMeta->pTaskDb); + (void) tdbTbClose(pMeta->pCheckpointDb); + (void) tdbClose(pMeta->db); taosArrayDestroy(pMeta->pTaskList); taosArrayDestroy(pMeta->chkpSaved); @@ -552,7 +608,7 @@ void streamMetaCloseImpl(void* arg) { bkdMgtDestroy(pMeta->bkdChkptMgt); pMeta->role = NODE_ROLE_UNINIT; - taosThreadRwlockDestroy(&pMeta->lock); + (void) taosThreadRwlockDestroy(&pMeta->lock); taosMemoryFree(pMeta); stDebug("vgId:%d end to close stream meta", vgId); @@ -568,9 +624,10 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { if (code < 0) { return -1; } + buf = taosMemoryCalloc(1, len); if (buf == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) { @@ -579,13 +636,19 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); - tEncodeStreamTask(&encoder, pTask); + code = tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); + if (code) { + stError("s-task:%s vgId:%d task meta encode failed, code:%s", pTask->id.idStr, vgId, tstrerror(code)); + return TSDB_CODE_INVALID_MSG; + } + int64_t id[2] = {pTask->id.streamId, pTask->id.taskId}; code = tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn); if (code != TSDB_CODE_SUCCESS) { + code = terrno; stError("s-task:%s vgId:%d task meta save to disk failed, code:%s", pTask->id.idStr, vgId, tstrerror(terrno)); } else { stDebug("s-task:%s vgId:%d task meta save to disk", pTask->id.idStr, vgId); @@ -612,33 +675,44 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pTaskId) { int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded) { *pAdded = false; + int32_t code = 0; STaskId id = streamTaskGetTaskId(pTask); void* p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (p != NULL) { - return 0; + stDebug("s-task:%" PRIx64 " already exist in meta, no need to register", id.taskId); + return code; } - if (pMeta->buildTaskFn(pMeta->ahandle, pTask, ver) < 0) { - return -1; + if ((code = pMeta->buildTaskFn(pMeta->ahandle, pTask, ver)) != 0) { + return code; } - taosArrayPush(pMeta->pTaskList, &pTask->id); - taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES); - - if (streamMetaSaveTask(pMeta, pTask) < 0) { - return -1; + p = taosArrayPush(pMeta->pTaskList, &pTask->id); + if (p == NULL) { + stError("s-task:0x%"PRIx64" failed to register task into meta-list, code: out of memory", id.taskId); + return TSDB_CODE_OUT_OF_MEMORY; } - if (streamMetaCommit(pMeta) < 0) { - return -1; + code = taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES); + if (code) { + stError("s-task:0x%"PRIx64" failed to register task into meta-list, code: out of memory", id.taskId); + return code; + } + + if ((code = streamMetaSaveTask(pMeta, pTask)) != 0) { + return code; + } + + if ((code = streamMetaCommit(pMeta)) != 0) { + return code; } if (pTask->info.fillHistory == 0) { - atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); + (void) atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); } *pAdded = true; - return 0; + return code; } int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) { @@ -703,7 +777,7 @@ static void doRemoveIdFromList(SArray* pTaskList, int32_t num, SStreamTaskId* id static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - streamTaskSendCheckpointSourceRsp(pTask); + (void) streamTaskSendCheckpointSourceRsp(pTask); } return 0; } @@ -726,7 +800,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t } // handle the dropping event - streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); + (void) streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); } else { stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); streamMetaWUnLock(pMeta); @@ -762,12 +836,12 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t pTask = *ppTask; // it is an fill-history task, remove the related stream task's id that points to it if (pTask->info.fillHistory == 0) { - atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); + (void) atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); } - taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); + (void) taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); doRemoveIdFromList(pMeta->pTaskList, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id); - streamMetaRemoveTask(pMeta, &id); + (void) streamMetaRemoveTask(pMeta, &id); ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList)); streamMetaWUnLock(pMeta); @@ -775,7 +849,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t ASSERT(pTask->status.timerActive == 0); if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt); - taosTmrStop(pTask->schedInfo.pDelayTimer); + (void) taosTmrStop(pTask->schedInfo.pDelayTimer); pTask->info.delaySchedParam = 0; streamMetaReleaseTask(pMeta, pTask); } @@ -820,9 +894,11 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) { int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { int64_t checkpointId = 0; + int32_t code = 0; TBC* pCur = NULL; if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { + stError("failed to open stream meta file, the latest checkpointId is 0, vgId:%d", pMeta->vgId); return checkpointId; } @@ -832,7 +908,13 @@ int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { int32_t vLen = 0; SDecoder decoder; - tdbTbcMoveToFirst(pCur); + code = tdbTbcMoveToFirst(pCur); + if (code) { + (void) tdbTbcClose(pCur); + stError("failed to open stream meta file cursor, the latest checkpointId is 0, vgId:%d", pMeta->vgId); + return checkpointId; + } + while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { if (pVal == NULL || vLen == 0) { break; @@ -851,8 +933,8 @@ int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { tdbFree(pKey); tdbFree(pVal); - tdbTbcClose(pCur); + (void)tdbTbcClose(pCur); return checkpointId; } @@ -864,23 +946,34 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { void* pVal = NULL; int32_t vLen = 0; SDecoder decoder; + int32_t vgId = 0; + int32_t code = 0; + SArray* pRecycleList = NULL; if (pMeta == NULL) { return; } - SArray* pRecycleList = taosArrayInit(4, sizeof(STaskId)); - int32_t vgId = pMeta->vgId; + pRecycleList = taosArrayInit(4, sizeof(STaskId)); + + vgId = pMeta->vgId; stInfo("vgId:%d load stream tasks from meta files", vgId); - int32_t code = tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL); + code = tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL); if (code != TSDB_CODE_SUCCESS) { stError("vgId:%d failed to open stream meta, code:%s, not load any stream tasks", vgId, tstrerror(terrno)); taosArrayDestroy(pRecycleList); return; } - tdbTbcMoveToFirst(pCur); + code = tdbTbcMoveToFirst(pCur); + if (code) { + stError("vgId:%d failed to open stream meta cursor, code:%s, not load any stream tasks", vgId, tstrerror(terrno)); + taosArrayDestroy(pRecycleList); + (void) tdbTbcClose(pCur); + return; + } + while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { if (pVal == NULL || vLen == 0) { break; @@ -910,7 +1003,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { tFreeStreamTask(pTask); STaskId id = streamTaskGetTaskId(pTask); - taosArrayPush(pRecycleList, &id); + (void) taosArrayPush(pRecycleList, &id); int32_t total = taosArrayGetSize(pRecycleList); stDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total); @@ -931,7 +1024,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { continue; } - taosArrayPush(pMeta->pTaskList, &pTask->id); + (void) taosArrayPush(pMeta->pTaskList, &pTask->id); } else { // todo this should replace the existed object put by replay creating stream task msg from mnode stError("s-task:0x%x already added into table meta by replaying WAL, need check", pTask->id.taskId); @@ -941,17 +1034,17 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) != 0) { stError("s-task:0x%x failed to put into hashTable, code:%s, continue", pTask->id.taskId, tstrerror(terrno)); - taosArrayPop(pMeta->pTaskList); + (void) taosArrayPop(pMeta->pTaskList); tFreeStreamTask(pTask); continue; } if (pTask->info.fillHistory == 0) { - atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); + (void) atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); } if (streamTaskShouldPause(pTask)) { - atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); + (void) atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); } ASSERT(pTask->status.downstreamReady == 0); @@ -967,7 +1060,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { if (taosArrayGetSize(pRecycleList) > 0) { for (int32_t i = 0; i < taosArrayGetSize(pRecycleList); ++i) { STaskId* pId = taosArrayGet(pRecycleList, i); - streamMetaRemoveTask(pMeta, pId); + (void) streamMetaRemoveTask(pMeta, pId); } } @@ -995,7 +1088,7 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->status.timerActive >= 1) { stDebug("s-task:%s in timer, blocking tasks in vgId:%d restart, set closing again", pTask->id.idStr, pMeta->vgId); - streamTaskStop(pTask); + (void) streamTaskStop(pTask); inTimer = true; } } @@ -1028,7 +1121,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { SStreamTask* pTask = *(SStreamTask**)pIter; stDebug("vgId:%d s-task:%s set task closing flag", vgId, pTask->id.idStr); - streamTaskStop(pTask); + (void) streamTaskStop(pTask); } streamMetaWUnLock(pMeta); @@ -1047,7 +1140,16 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { void streamMetaStartHb(SStreamMeta* pMeta) { int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); - metaRefMgtAdd(pMeta->vgId, pRid); + if (pRid == NULL) { + stError("vgId:%d failed to prepare the metaHb to mnode, hbMsg will not started, code: out of memory", pMeta->vgId); + return; + } + + int32_t code = metaRefMgtAdd(pMeta->vgId, pRid); + if (code) { + return; + } + *pRid = pMeta->rid; streamMetaHbToMnode(pRid, NULL); } @@ -1066,7 +1168,7 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo, int32_t vgId) { void streamMetaRLock(SStreamMeta* pMeta) { // stTrace("vgId:%d meta-rlock", pMeta->vgId); - taosThreadRwlockRdlock(&pMeta->lock); + (void) taosThreadRwlockRdlock(&pMeta->lock); } void streamMetaRUnLock(SStreamMeta* pMeta) { @@ -1081,30 +1183,13 @@ void streamMetaRUnLock(SStreamMeta* pMeta) { void streamMetaWLock(SStreamMeta* pMeta) { // stTrace("vgId:%d meta-wlock", pMeta->vgId); - taosThreadRwlockWrlock(&pMeta->lock); + (void) taosThreadRwlockWrlock(&pMeta->lock); // stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } void streamMetaWUnLock(SStreamMeta* pMeta) { // stTrace("vgId:%d meta-wunlock", pMeta->vgId); - taosThreadRwlockUnlock(&pMeta->lock); -} - -static void execHelper(struct SSchedMsg* pSchedMsg) { - __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; - int32_t code = execFn(pSchedMsg->thandle); - if (code != 0 && pSchedMsg->msg != NULL) { - *(int32_t*)pSchedMsg->msg = code; - } -} - -int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code) { - SSchedMsg schedMsg = {0}; - schedMsg.fp = execHelper; - schedMsg.ahandle = fn; - schedMsg.thandle = param; - schedMsg.msg = code; - return taosScheduleTask(pMeta->qHandle, &schedMsg); + (void) taosThreadRwlockUnlock(&pMeta->lock); } int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { @@ -1192,10 +1277,10 @@ static int32_t prepareBeforeStartTasks(SStreamMeta* pMeta, SArray** pList, int64 taosHashClear(pMeta->startInfo.pFailedTaskSet); pMeta->startInfo.startTs = now; - streamMetaResetTaskStatus(pMeta); + int32_t code = streamMetaResetTaskStatus(pMeta); streamMetaWUnLock(pMeta); - return TSDB_CODE_SUCCESS; + return code; } // restore the checkpoint id by negotiating the latest consensus checkpoint id @@ -1230,7 +1315,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); - streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); + (void) streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); continue; } @@ -1253,7 +1338,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); - streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); + (void) streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); continue; } @@ -1271,7 +1356,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", pTask->id.idStr); - streamLaunchFillHistoryTask(pTask); + (void) streamLaunchFillHistoryTask(pTask); // todo: how about retry launch fill-history task? } (void) streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true); @@ -1337,7 +1422,7 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { continue; } - streamTaskStop(pTask); + (void) streamTaskStop(pTask); streamMetaReleaseTask(pMeta, pTask); } @@ -1377,7 +1462,7 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask); if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId); - streamMetaAddFailedTask(pMeta, streamId, taskId); + (void) streamMetaAddFailedTask(pMeta, streamId, taskId); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } @@ -1468,7 +1553,10 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready}; - taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs)); + int32_t code = taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs)); + if (code) { + + } int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta); int32_t numOfRecv = taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet); @@ -1488,14 +1576,14 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 streamMetaResetStartInfo(pStartInfo, pMeta->vgId); streamMetaWUnLock(pMeta); - pStartInfo->completeFn(pMeta); + code = pStartInfo->completeFn(pMeta); } else { streamMetaWUnLock(pMeta); stDebug("vgId:%d recv check downstream results, s-task:0x%x succ:%d, received:%d, total:%d", pMeta->vgId, taskId, ready, numOfRecv, numOfTotal); } - return TSDB_CODE_SUCCESS; + return code; } int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta) { @@ -1569,19 +1657,26 @@ void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SSt int64_t startTs) { const char* id = pTask->id.idStr; int32_t vgId = pTask->pMeta->vgId; + int32_t code = 0; // keep the already updated info STaskUpdateEntry entry = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId, .transId = transId}; - taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); + code = taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); + if (code != 0) { + stError("s-task:%s failed to put updateTask into update list", id); + } int64_t el = taosGetTimestampMs() - startTs; if (pHTask != NULL) { STaskUpdateEntry hEntry = {.streamId = pHTask->id.streamId, .taskId = pHTask->id.taskId, .transId = transId}; - taosHashPut(pMeta->updateInfo.pTasks, &hEntry, sizeof(hEntry), NULL, 0); - - stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask/hTask closed, elapsed:%" PRId64 - " ms", - id, vgId, transId, el); + code = taosHashPut(pMeta->updateInfo.pTasks, &hEntry, sizeof(hEntry), NULL, 0); + if (code != 0) { + stError("s-task:%s failed to put updateTask into update list", id); + } else { + stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask/hTask closed, elapsed:%" PRId64 + " ms", + id, vgId, transId, el); + } } else { stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms", id, vgId, transId, el); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 02e4ed8d8b..3c27210a23 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -130,7 +130,6 @@ int32_t streamGetFileSize(char* path, char* name, int64_t* sz) { int32_t ret = 0; char* fullname = taosMemoryCalloc(1, strlen(path) + 32); - sprintf(fullname, "%s%s%s", path, TD_DIRSEP, name); ret = taosStatFile(fullname, sz, NULL, NULL); @@ -185,48 +184,89 @@ void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { } int32_t snapFileGenMeta(SBackendSnapFile2* pSnapFile) { + void* p = NULL; SBackendFileItem item = {0}; item.ref = 1; + // current item.name = pSnapFile->pCurrent; item.type = ROCKSDB_CURRENT_TYPE; - streamGetFileSize(pSnapFile->path, item.name, &item.size); - taosArrayPush(pSnapFile->pFileList, &item); + int32_t code = streamGetFileSize(pSnapFile->path, item.name, &item.size); + if (code) { + stError("failed to get file size"); + return code; + } + + p = taosArrayPush(pSnapFile->pFileList, &item); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } // mainfest item.name = pSnapFile->pMainfest; item.type = ROCKSDB_MAINFEST_TYPE; - streamGetFileSize(pSnapFile->path, item.name, &item.size); - taosArrayPush(pSnapFile->pFileList, &item); + code = streamGetFileSize(pSnapFile->path, item.name, &item.size); + if (code) { + return code; + } + + p = taosArrayPush(pSnapFile->pFileList, &item); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } // options item.name = pSnapFile->pOptions; item.type = ROCKSDB_OPTIONS_TYPE; - streamGetFileSize(pSnapFile->path, item.name, &item.size); - taosArrayPush(pSnapFile->pFileList, &item); + code = streamGetFileSize(pSnapFile->path, item.name, &item.size); + if (code) { + return code; + } + + p = taosArrayPush(pSnapFile->pFileList, &item); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + // sst for (int32_t i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { char* sst = taosArrayGetP(pSnapFile->pSst, i); item.name = sst; item.type = ROCKSDB_SST_TYPE; - streamGetFileSize(pSnapFile->path, item.name, &item.size); - taosArrayPush(pSnapFile->pFileList, &item); + code = streamGetFileSize(pSnapFile->path, item.name, &item.size); + if (code) { + return code; + } + + p = taosArrayPush(pSnapFile->pFileList, &item); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + // meta item.name = pSnapFile->pCheckpointMeta; item.type = ROCKSDB_CHECKPOINT_META_TYPE; if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) { - taosArrayPush(pSnapFile->pFileList, &item); + p = taosArrayPush(pSnapFile->pFileList, &item); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } item.name = pSnapFile->pCheckpointSelfcheck; item.type = ROCKSDB_CHECKPOINT_SELFCHECK_TYPE; if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) { - taosArrayPush(pSnapFile->pFileList, &item); + p = taosArrayPush(pSnapFile->pFileList, &item); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + return 0; } + int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { int32_t code = 0; TdDirPtr pDir = taosOpenDir(pSnapFile->path); @@ -288,12 +328,18 @@ int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { code = TSDB_CODE_OUT_OF_MEMORY; break; } - taosArrayPush(pSnapFile->pSst, &sst); + + void* p = taosArrayPush(pSnapFile->pSst, &sst); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } } - taosCloseDir(&pDir); - return code; + + return taosCloseDir(&pDir); } + int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) { int32_t code = 0; int32_t nBytes = 0; @@ -359,13 +405,16 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) { } taosArrayDestroy(pSnap->pFileList); taosArrayDestroy(pSnap->pSst); - taosCloseFile(&pSnap->fd); - - return; + int32_t code = taosCloseFile(&pSnap->fd); + if (code) { + stError("failed to close snapshot fd"); + } } + int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) { - // impl later int32_t code = 0; + SArray* pDbSnapSet = NULL; + SArray* pSnapInfoSet = taosArrayInit(4, sizeof(SStreamTaskSnap)); if (pSnapInfoSet == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -374,15 +423,13 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta code = streamCreateTaskDbSnapInfo(pMeta, path, pSnapInfoSet); if (code != 0) { stError("failed to do task db snap info, reason:%s", tstrerror(code)); - taosArrayDestroy(pSnapInfoSet); - return code; + goto _err; } - SArray* pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); + pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); if (pDbSnapSet == NULL) { - taosArrayDestroy(pSnapInfoSet); code = TSDB_CODE_OUT_OF_MEMORY; - return code; + goto _err; } for (int32_t i = 0; i < taosArrayGetSize(pSnapInfoSet); i++) { @@ -391,16 +438,24 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta SBackendSnapFile2 snapFile = {0}; code = streamBackendSnapInitFile(path, pSnap, &snapFile); ASSERT(code == 0); - taosArrayPush(pDbSnapSet, &snapFile); + + void* p = taosArrayPush(pDbSnapSet, &snapFile); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } pHandle->pDbSnapSet = pDbSnapSet; pHandle->pSnapInfoSet = pSnapInfoSet; pHandle->currIdx = 0; pHandle->pMeta = pMeta; - return 0; + + return code; _err: + taosArrayDestroy(pSnapInfoSet); + taosArrayDestroy(pDbSnapSet); streamSnapHandleDestroy(pHandle); return code; } @@ -414,7 +469,8 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { } taosArrayDestroy(handle->pDbSnapSet); } - streamDestroyTaskDbSnapInfo(handle->pMeta, handle->pSnapInfoSet); + + (void) streamDestroyTaskDbSnapInfo(handle->pMeta, handle->pSnapInfoSet); if (handle->pSnapInfoSet) { for (int32_t i = 0; i < taosArrayGetSize(handle->pSnapInfoSet); i++) { SStreamTaskSnap* pSnap = taosArrayGet(handle->pSnapInfoSet, i); @@ -422,8 +478,8 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { } taosArrayDestroy(handle->pSnapInfoSet); } + taosMemoryFree(handle->metaPath); - return; } int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* path, SStreamSnapReader** ppReader) { @@ -506,14 +562,22 @@ _NEXT: item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); pSnapFile->offset += nread; if (pSnapFile->offset >= item->size || nread < kBlockSize) { - taosCloseFile(&pSnapFile->fd); + code = taosCloseFile(&pSnapFile->fd); + if (code) { + stError("failed to close snapshot fd"); + } + pSnapFile->offset = 0; pSnapFile->currFileIdx += 1; } } else { stDebug("%s no data read, close file no.%d, move to next file, open and read", STREAM_STATE_TRANSFER, pSnapFile->currFileIdx); - taosCloseFile(&pSnapFile->fd); + code = taosCloseFile(&pSnapFile->fd); + if (code) { + stError("failed to close snapshot fd"); + } + pSnapFile->offset = 0; pSnapFile->currFileIdx += 1; @@ -577,14 +641,22 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path pHandle->pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); if (pHandle->pDbSnapSet == NULL) { - streamSnapWriterClose(pWriter, 0); + int32_t c = streamSnapWriterClose(pWriter, 0); // not override the error code, and igore this error code + if (c) { + stError("failed close snaphost writer"); + } + code = TSDB_CODE_OUT_OF_MEMORY; return code; } SBackendSnapFile2 snapFile = {0}; if (taosArrayPush(pHandle->pDbSnapSet, &snapFile) == NULL) { - streamSnapWriterClose(pWriter, 0); + int32_t c = streamSnapWriterClose(pWriter, 0); + if (c) { + stError("failed close snaphost writer"); + } + code = TSDB_CODE_OUT_OF_MEMORY; return code; } @@ -614,46 +686,62 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t pHdr->name, tstrerror(code)); } } + if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) { int64_t bytes = taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); if (bytes != pHdr->size) { code = TAOS_SYSTEM_ERROR(errno); stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); - return code; + goto _err; } else { stInfo("succ to write data %s", pItem->name); } pSnapFile->offset += bytes; } else { - taosCloseFile(&pSnapFile->fd); + code = taosCloseFile(&pSnapFile->fd); + if (code) { + stError("failed to close snapshot fd"); + } + pSnapFile->offset = 0; pSnapFile->currFileIdx += 1; SBackendFileItem item = {0}; item.name = taosStrdup(pHdr->name); item.type = pHdr->type; + if (item.name == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - taosArrayPush(pSnapFile->pFileList, &item); + void* p = taosArrayPush(pSnapFile->pFileList, &item); + if (p == NULL) { // can NOT goto _err here. + return TSDB_CODE_OUT_OF_MEMORY; + } - SBackendFileItem* pItem = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); - pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + SBackendFileItem* pItem2 = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); + pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem2->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pSnapFile->fd == NULL) { code = TAOS_SYSTEM_ERROR(errno); stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, TD_DIRSEP, pHdr->name, tstrerror(code)); + return code; } + // open fd again, let's close fd during handle errors. if (taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset) != pHdr->size) { code = TAOS_SYSTEM_ERROR(errno); stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); - return code; + goto _err; } - stInfo("succ to write data %s", pItem->name); + + stInfo("succ to write data %s", pItem2->name); pSnapFile->offset += pHdr->size; } - code = 0; -_EXIT: + return TSDB_CODE_SUCCESS; + +_err: + (void) taosCloseFile(&pSnapFile->fd); return code; } @@ -688,7 +776,10 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa item.name = taosStrdup((char*)ROCKSDB_CURRENT); item.type = ROCKSDB_CURRENT_TYPE; - taosArrayPush(pDbSnapFile->pFileList, &item); + void* p = taosArrayPush(pDbSnapFile->pFileList, &item); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pDbSnapFile->inited = 1; return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile); @@ -697,9 +788,12 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile); } else { SBackendSnapFile2 snapFile = {0}; - taosArrayPush(pHandle->pDbSnapSet, &snapFile); - pHandle->currIdx += 1; + void* p = taosArrayPush(pHandle->pDbSnapSet, &snapFile); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pHandle->currIdx += 1; return streamSnapWrite(pWriter, pData, nData); } } diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 1efb2af381..db0784d572 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -29,19 +29,19 @@ typedef struct SLaunchHTaskInfo { STaskId hTaskId; } SLaunchHTaskInfo; -static int32_t streamSetParamForScanHistory(SStreamTask* pTask); -static void streamTaskSetRangeStreamCalc(SStreamTask* pTask); -static int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated); -static SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, - int32_t hTaskId); -static void tryLaunchHistoryTask(void* param, void* tmrId); -static void doExecScanhistoryInFuture(void* param, void* tmrId); -static int32_t doStartScanHistoryTask(SStreamTask* pTask); -static int32_t streamTaskStartScanHistory(SStreamTask* pTask); -static void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask); -static int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask); -static void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now); -static void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now); +static int32_t streamSetParamForScanHistory(SStreamTask* pTask); +static int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask); +static void initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated); +static int32_t createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, int32_t hTaskId, + SLaunchHTaskInfo** pInfo); +static void tryLaunchHistoryTask(void* param, void* tmrId); +static void doExecScanhistoryInFuture(void* param, void* tmrId); +static int32_t doStartScanHistoryTask(SStreamTask* pTask); +static int32_t streamTaskStartScanHistory(SStreamTask* pTask); +static void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask); +static int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask); +static void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now); +static void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now); static int32_t streamTaskSetReady(SStreamTask* pTask) { int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); @@ -65,22 +65,19 @@ static int32_t streamTaskSetReady(SStreamTask* pTask) { int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { SStreamScanHistoryReq req; + int32_t code = 0; initScanHistoryReq(pTask, &req, igUntreated); int32_t len = sizeof(SStreamScanHistoryReq); void* serializedReq = rpcMallocCont(len); if (serializedReq == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(serializedReq, &req, len); SRpcMsg rpcMsg = {.contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_SCAN_HISTORY}; - if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { - /*ASSERT(0);*/ - } - - return 0; + return tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg); } int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) { @@ -109,8 +106,8 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) pTask->schedHistoryInfo.pTimer = taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); } else { - taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, - &pTask->schedHistoryInfo.pTimer); + streamTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, + &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); } return TSDB_CODE_SUCCESS; @@ -135,9 +132,19 @@ int32_t streamTaskStartScanHistory(SStreamTask* pTask) { int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { const char* id = pTask->id.idStr; + int32_t code = 0; - streamTaskSetReady(pTask); - streamTaskSetRangeStreamCalc(pTask); + code = streamTaskSetReady(pTask); + if (code) { + stError("s-task:%s failed to set task status ready", id); + return code; + } + + code = streamTaskSetRangeStreamCalc(pTask); + if (code) { + stError("s-task:%s failed to set the time range for stream task", id); + return code; + } SStreamTaskState p = streamTaskGetStatus(pTask); ASSERT(p.state == TASK_STATUS__READY); @@ -155,19 +162,23 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p.name, schedStatus); } - return TSDB_CODE_SUCCESS; + return code; } int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) { // set the state to be ready - streamTaskSetReady(pTask); - streamTaskSetRangeStreamCalc(pTask); + int32_t code = streamTaskSetReady(pTask); + if (code == 0) { + code = streamTaskSetRangeStreamCalc(pTask); + } - SStreamTaskState p = streamTaskGetStatus(pTask); - ASSERT((p.state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1)); + if (code == 0) { + SStreamTaskState p = streamTaskGetStatus(pTask); + ASSERT((p.state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1)); - stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name); - streamTaskStartScanHistory(pTask); + stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name); + code = streamTaskStartScanHistory(pTask); + } // NOTE: there will be an deadlock if launch fill history here. // start the related fill-history task, when current task is ready @@ -175,7 +186,7 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) { // streamLaunchFillHistoryTask(pTask); // } - return TSDB_CODE_SUCCESS; + return code; } // common @@ -212,8 +223,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, pStatus.name); - (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); - return -1; // todo set the correct error code + return streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); } stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", idStr, hStreamId, hTaskId); @@ -257,12 +267,11 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated) { +void initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated) { pReq->msgHead.vgId = pTask->info.nodeId; pReq->streamId = pTask->id.streamId; pReq->taskId = pTask->id.taskId; pReq->igUntreated = igUntreated; - return 0; } void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask) { @@ -281,7 +290,10 @@ void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask) { } // check if downstream tasks have been ready - streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_INIT_SCANHIST); + int32_t code = streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_INIT_SCANHIST); + if (code) { + stError("s-task:%s handle event init_scanhist failed", pTask->id.idStr); + } } void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now) { @@ -316,7 +328,8 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); - taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer); + streamTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, + pTask->pMeta->vgId, " start-history-task-tmr"); } } @@ -367,7 +380,8 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { pHTaskInfo->tickCount -= 1; if (pHTaskInfo->tickCount > 0) { - taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer); + streamTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer, + pTask->pMeta->vgId, " start-history-task-tmr"); streamMetaReleaseTask(pMeta, pTask); return; } @@ -417,21 +431,21 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { taosMemoryFree(pInfo); } -SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, int32_t hTaskId) { - SLaunchHTaskInfo* pInfo = taosMemoryCalloc(1, sizeof(SLaunchHTaskInfo)); - if (pInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +int32_t createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, int32_t hTaskId, + SLaunchHTaskInfo** pInfo) { + *pInfo = taosMemoryCalloc(1, sizeof(SLaunchHTaskInfo)); + if ((*pInfo) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } - pInfo->id.streamId = pTaskId->streamId; - pInfo->id.taskId = pTaskId->taskId; + (*pInfo)->id.streamId = pTaskId->streamId; + (*pInfo)->id.taskId = pTaskId->taskId; - pInfo->hTaskId.streamId = hStreamId; - pInfo->hTaskId.taskId = hTaskId; + (*pInfo)->hTaskId.streamId = hStreamId; + (*pInfo)->hTaskId.taskId = hTaskId; - pInfo->pMeta = pMeta; - return pInfo; + (*pInfo)->pMeta = pMeta; + return TSDB_CODE_SUCCESS; } int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { @@ -440,16 +454,18 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { const char* idStr = pTask->id.idStr; int64_t hStreamId = pTask->hTaskInfo.id.streamId; int32_t hTaskId = pTask->hTaskInfo.id.taskId; + SLaunchHTaskInfo* pInfo = NULL; + ASSERT(hTaskId != 0); stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId); - STaskId id = streamTaskGetTaskId(pTask); - SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId); - if (pInfo == NULL) { + STaskId id = streamTaskGetTaskId(pTask); + int32_t code = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId, &pInfo); + if (code) { stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr); - (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); - return terrno; + (void)streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); + return code; } // set the launch time info @@ -475,7 +491,8 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { } else { // timer exists ASSERT(pTask->status.timerActive >= 1); stDebug("s-task:%s set timer active flag, task timer not null", idStr); - taosTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer); + streamTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer, + pTask->pMeta->vgId, " start-history-task-tmr"); } return TSDB_CODE_SUCCESS; @@ -510,7 +527,7 @@ bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t nextProcessVe } } -void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { +int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask) { SDataRange* pRange = &pTask->dataRange; if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) { @@ -523,10 +540,12 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { "window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 "-%" PRId64, pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer); } + + return TSDB_CODE_SUCCESS; } else { ASSERT(pTask->info.fillHistory == 0); if (pTask->info.taskLevel >= TASK_LEVEL__AGG) { - return; + return TSDB_CODE_SUCCESS; } stDebug("s-task:%s level:%d related fill-history task exists, stream task timeWindow:%" PRId64 " - %" PRId64 @@ -536,7 +555,7 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { SVersionRange verRange = pRange->range; STimeWindow win = pRange->window; - streamSetParamForStreamScannerStep2(pTask, &verRange, &win); + return streamSetParamForStreamScannerStep2(pTask, &verRange, &win); } } @@ -554,7 +573,10 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) { } if (pTask->schedHistoryInfo.numOfTicks <= 0) { - streamStartScanHistoryAsync(pTask, 0); + int32_t code = streamStartScanHistoryAsync(pTask, 0); + if (code) { + stError("s-task:%s async start history task failed", pTask->id.idStr); + } int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s fill-history:%d start scan-history data, out of tmr, ref:%d", pTask->id.idStr, @@ -563,18 +585,26 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) { // release the task. streamMetaReleaseTask(pTask->pMeta, pTask); } else { - taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, - &pTask->schedHistoryInfo.pTimer); + streamTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, + &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr"); } } int32_t doStartScanHistoryTask(SStreamTask* pTask) { + int32_t code = 0; SVersionRange* pRange = &pTask->dataRange.range; + if (pTask->info.fillHistory) { - streamSetParamForScanHistory(pTask); + code = streamSetParamForScanHistory(pTask); + if (code) { + return code; + } } - streamSetParamForStreamScannerStep1(pTask, pRange, &pTask->dataRange.window); - int32_t code = streamStartScanHistoryAsync(pTask, 0); - return code; + code = streamSetParamForStreamScannerStep1(pTask, pRange, &pTask->dataRange.window); + if (code) { + return code; + } + + return streamStartScanHistoryAsync(pTask, 0); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 7c2d0b3556..0110a9825c 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -29,20 +29,20 @@ static void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo); static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) { int32_t childId = taosArrayGetSize(pArray); pTask->info.selfChildId = childId; - taosArrayPush(pArray, &pTask); - return 0; + void* p = taosArrayPush(pArray, &pTask); + return (p == NULL)? TSDB_CODE_OUT_OF_MEMORY:TSDB_CODE_SUCCESS; } static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEpSet, bool* pUpdated) { char buf[512] = {0}; if (pTask->info.nodeId == nodeId) { // execution task should be moved away bool isEqual = isEpsetEqual(&pTask->info.epSet, pEpSet); - epsetToStr(pEpSet, buf, tListLen(buf)); + (void)epsetToStr(pEpSet, buf, tListLen(buf)); if (!isEqual) { (*pUpdated) = true; char tmp[512] = {0}; - epsetToStr(&pTask->info.epSet, tmp, tListLen(tmp)); + (void) epsetToStr(&pTask->info.epSet, tmp, tListLen(tmp)); // only for log file, ignore errors epsetAssign(&pTask->info.epSet, pEpSet); stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s, old:%s", pTask->id.taskId, nodeId, buf, tmp); @@ -127,7 +127,10 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL; pTask->taskCheckInfo.pList = taosArrayInit(4, sizeof(SDownstreamStatusInfo)); - taosThreadMutexInit(&pTask->taskCheckInfo.checkInfoLock, NULL); + code = taosThreadMutexInit(&pTask->taskCheckInfo.checkInfoLock, NULL); + if (code) { + return code; + } if (fillHistory) { ASSERT(hasFillhistory); @@ -135,7 +138,7 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool epsetAssign(&(pTask->info.mnodeEpset), pEpset); - addToTaskset(pTaskList, pTask); + code = addToTaskset(pTaskList, pTask); *p = pTask; return code; @@ -221,17 +224,17 @@ void tFreeStreamTask(SStreamTask* pTask) { } if (pTask->schedInfo.pDelayTimer != NULL) { - taosTmrStop(pTask->schedInfo.pDelayTimer); + (void) taosTmrStop(pTask->schedInfo.pDelayTimer); pTask->schedInfo.pDelayTimer = NULL; } if (pTask->hTaskInfo.pTimer != NULL) { - /*bool ret = */ taosTmrStop(pTask->hTaskInfo.pTimer); + (void) taosTmrStop(pTask->hTaskInfo.pTimer); pTask->hTaskInfo.pTimer = NULL; } if (pTask->msgInfo.pRetryTmr != NULL) { - /*bool ret = */ taosTmrStop(pTask->msgInfo.pRetryTmr); + (void) taosTmrStop(pTask->msgInfo.pRetryTmr); pTask->msgInfo.pRetryTmr = NULL; } @@ -394,10 +397,12 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i return terrno; } - taosThreadMutexInit(&pTask->msgInfo.lock, NULL); + code = taosThreadMutexInit(&pTask->msgInfo.lock, NULL); + if (code) { + return code; + } TdThreadMutexAttr attr = {0}; - code = taosThreadMutexAttrInit(&attr); if (code != 0) { stError("s-task:%s initElapsed mutex attr failed, code:%s", pTask->id.idStr, tstrerror(code)); @@ -410,8 +415,16 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i return code; } - taosThreadMutexInit(&pTask->lock, &attr); - taosThreadMutexAttrDestroy(&attr); + code = taosThreadMutexInit(&pTask->lock, &attr); + if (code) { + return code; + } + + code = taosThreadMutexAttrDestroy(&attr); + if (code) { + return code; + } + streamTaskOpenAllUpstreamInput(pTask); STaskOutputInfo* pOutputInfo = &pTask->outputInfo; @@ -424,7 +437,11 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i // 2MiB per second for sink task // 50 times sink operator per second - streamTaskInitTokenBucket(pOutputInfo->pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr); + code = streamTaskInitTokenBucket(pOutputInfo->pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr); + if (code) { + return code; + } + pOutputInfo->pNodeEpsetUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset)); if (pOutputInfo->pNodeEpsetUpdateList == NULL) { stError("s-task:%s failed to prepare downstreamUpdateList, code:%s", pTask->id.idStr, @@ -474,13 +491,13 @@ int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstre pTask->upstreamInfo.pList = taosArrayInit(4, POINTER_BYTES); } - taosArrayPush(pTask->upstreamInfo.pList, &pEpInfo); - return TSDB_CODE_SUCCESS; + void* p = taosArrayPush(pTask->upstreamInfo.pList, &pEpInfo); + return (p == NULL)? TSDB_CODE_OUT_OF_MEMORY:TSDB_CODE_SUCCESS; } void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated) { char buf[512] = {0}; - epsetToStr(pEpSet, buf, tListLen(buf)); + (void) epsetToStr(pEpSet, buf, tListLen(buf)); // ignore error since it is only for log file. int32_t numOfUpstream = taosArrayGetSize(pTask->upstreamInfo.pList); for (int32_t i = 0; i < numOfUpstream; ++i) { @@ -491,7 +508,7 @@ void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpS *pUpdated = true; char tmp[512] = {0}; - epsetToStr(&pInfo->epSet, tmp, tListLen(tmp)); + (void) epsetToStr(&pInfo->epSet, tmp, tListLen(tmp)); epsetAssign(&pInfo->epSet, pEpSet); stDebug("s-task:0x%x update the upstreamInfo taskId:0x%x(nodeId:%d) newEpset:%s old:%s", pTask->id.taskId, @@ -526,7 +543,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated) { char buf[512] = {0}; - epsetToStr(pEpSet, buf, tListLen(buf)); + (void) epsetToStr(pEpSet, buf, tListLen(buf)); // ignore the error since only for log files. int32_t id = pTask->id.taskId; int8_t type = pTask->outputInfo.type; @@ -542,7 +559,7 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE if (!isEqual) { *pUpdated = true; char tmp[512] = {0}; - epsetToStr(&pVgInfo->epSet, tmp, tListLen(tmp)); + (void) epsetToStr(&pVgInfo->epSet, tmp, tListLen(tmp)); epsetAssign(&pVgInfo->epSet, pEpSet); stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s old:%s", id, pVgInfo->taskId, @@ -562,7 +579,7 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE *pUpdated = true; char tmp[512] = {0}; - epsetToStr(&pDispatcher->epSet, tmp, tListLen(tmp)); + (void) epsetToStr(&pDispatcher->epSet, tmp, tListLen(tmp)); epsetAssign(&pDispatcher->epSet, pEpSet); stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s old:%s", id, pDispatcher->taskId, @@ -580,8 +597,16 @@ int32_t streamTaskStop(SStreamTask* pTask) { int64_t st = taosGetTimestampMs(); const char* id = pTask->id.idStr; - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP); - qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP); + if (code) { + stError("failed to handle STOP event, s-task:%s", id); + } + + code = qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + if (code) { + stError("s-task:%s failed to kill task related query handle", id); + } + while (!streamTaskIsIdle(pTask)) { stDebug("s-task:%s level:%d wait for task to be idle and then close, check again in 100ms", id, pTask->info.taskLevel); @@ -590,7 +615,7 @@ int32_t streamTaskStop(SStreamTask* pTask) { int64_t el = taosGetTimestampMs() - st; stDebug("vgId:%d s-task:%s is closed in %" PRId64 " ms", vgId, id, el); - return 0; + return code; } bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList) { @@ -607,7 +632,10 @@ bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList) { bool updated = false; for (int32_t i = 0; i < taosArrayGetSize(pNodeList); ++i) { SNodeUpdateInfo* pInfo = taosArrayGet(pNodeList, i); - doUpdateTaskEpset(pTask, pInfo->nodeId, &pInfo->newEp, &updated); + int32_t code = doUpdateTaskEpset(pTask, pInfo->nodeId, &pInfo->newEp, &updated); + if (code) { + stError("s-task:0x%x failed to update the task nodeEp epset, code:%s", pTask->id.taskId, tstrerror(code)); + } } return updated; @@ -704,10 +732,11 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { } int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { + int32_t code = 0; SStreamMeta* pMeta = pTask->pMeta; STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; if (pTask->info.fillHistory == 0) { - return TSDB_CODE_SUCCESS; + return code; } SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId)); @@ -725,11 +754,11 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { (*ppStreamTask)->status.taskStatus = TASK_STATUS__READY; } - streamMetaSaveTask(pMeta, *ppStreamTask); + code = streamMetaSaveTask(pMeta, *ppStreamTask); streamMutexUnlock(&(*ppStreamTask)->lock); } - return TSDB_CODE_SUCCESS; + return code; } int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, int64_t resetRelHalt) { @@ -797,8 +826,7 @@ int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpoin initRpcMsg(&msg, TDMT_MND_STREAM_CHKPT_REPORT, buf, tlen); stDebug("s-task:%s vgId:%d build and send task checkpoint-report to mnode", id, vgId); - tmsgSendReq(&pTask->info.mnodeEpset, &msg); - return 0; + return tmsgSendReq(&pTask->info.mnodeEpset, &msg); } STaskId streamTaskGetTaskId(const SStreamTask* pTask) { @@ -880,6 +908,7 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) { static int32_t taskPauseCallback(SStreamTask* pTask, void* param) { SStreamMeta* pMeta = pTask->pMeta; + int32_t code = 0; int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); stInfo("vgId:%d s-task:%s pause stream task. paused task num:%d", pMeta->vgId, pTask->id.idStr, num); @@ -887,15 +916,15 @@ static int32_t taskPauseCallback(SStreamTask* pTask, void* param) { // in case of fill-history task, stop the tsdb file scan operation. if (pTask->info.fillHistory == 1) { void* pExecutor = pTask->exec.pExecutor; - qKillTask(pExecutor, TSDB_CODE_SUCCESS); + code = qKillTask(pExecutor, TSDB_CODE_SUCCESS); } stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr); - return TSDB_CODE_SUCCESS; + return code; } void streamTaskPause(SStreamTask* pTask) { - streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_PAUSE, taskPauseCallback, NULL); + (void) streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_PAUSE, taskPauseCallback, NULL); } void streamTaskResume(SStreamTask* pTask) { @@ -949,8 +978,7 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CHKPT, buf, tlen); stDebug("s-task:%s vgId:%d build and send task checkpoint req", id, vgId); - tmsgSendReq(&pTask->info.mnodeEpset, &msg); - return 0; + return tmsgSendReq(&pTask->info.mnodeEpset, &msg); } void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) { @@ -1044,7 +1072,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) { void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { pTask->status.removeBackendFiles = true; } -int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) { +void streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) { if (pTransId != NULL) { *pTransId = pTask->chkInfo.pActiveInfo->transId; } @@ -1052,8 +1080,6 @@ int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTr if (pCheckpointId != NULL) { *pCheckpointId = pTask->chkInfo.pActiveInfo->activeId; } - - return TSDB_CODE_SUCCESS; } int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId) { @@ -1084,7 +1110,7 @@ int32_t streamTaskCreateActiveChkptInfo(SActiveCheckpointInfo** pRes) { pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskDownstreamReadyInfo)); *pRes = pInfo; - return TSDB_CODE_SUCCESS; + return code; } void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { @@ -1101,12 +1127,12 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { pInfo->pCheckpointReadyRecvList = NULL; if (pInfo->pChkptTriggerTmr != NULL) { - taosTmrStop(pInfo->pChkptTriggerTmr); + (void) taosTmrStop(pInfo->pChkptTriggerTmr); pInfo->pChkptTriggerTmr = NULL; } if (pInfo->pSendReadyMsgTmr != NULL) { - taosTmrStop(pInfo->pSendReadyMsgTmr); + (void) taosTmrStop(pInfo->pSendReadyMsgTmr); pInfo->pSendReadyMsgTmr = NULL; } diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 931de397cc..fb1740ae0a 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -35,8 +35,9 @@ void streamTimerCleanUp() { streamTimer = NULL; } -tmr_h streamTimerGetInstance() { - return streamTimer; +int32_t streamTimerGetInstance(tmr_h* pTmr) { + *pTmr = streamTimer; + return TSDB_CODE_SUCCESS; } void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index e395b3066b..fc7b1c80dc 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -36,7 +36,6 @@ static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_ int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) { return compareInt64Val(pTs1, pTs2); - ; } int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) { From 68ecda7b078a28476e0c70942eac454d3dddf14a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Jul 2024 09:37:12 +0800 Subject: [PATCH 08/43] fix(stream): check return value. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 43 ++++++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 778d79f8f6..c6139387b9 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -412,13 +412,13 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); if (pTask) { - streamProcessDispatchRsp(pTask, pRsp, pMsg->code); + code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; + return code; } else { tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId); terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; - return terrno; + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } } @@ -426,15 +426,21 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { char* msgStr = pMsg->pCont; char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; SDecoder decoder; SStreamRetrieveReq req; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeStreamRetrieveReq(&decoder, &req); + code = tDecodeStreamRetrieveReq(&decoder, &req); tDecoderClear(&decoder); + if (code) { + tqError("vgId:%d failed to decode retrieve msg, quit handling it", pMeta->vgId); + return code; + } + SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask); + code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask); if (pTask == NULL) { tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, req.dstTaskId); @@ -464,6 +470,7 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { char* msgStr = pMsg->pCont; char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; SStreamTaskCheckReq req; SStreamTaskCheckRsp rsp = {0}; @@ -471,9 +478,14 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeStreamTaskCheckReq(&decoder, &req); + code = tDecodeStreamTaskCheckReq(&decoder, &req); tDecoderClear(&decoder); + if (code) { + tqError("vgId:%d decode check msg failed, not handle this msg", pMeta->vgId); + return code; + } + streamTaskProcessCheckMsg(pMeta, &req, &rsp); return streamTaskSendCheckRsp(pMeta, req.upstreamNodeId, &rsp, &pMsg->info, req.upstreamTaskId); } @@ -687,11 +699,17 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen // drop the related fill-history task firstly if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); - streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); + code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); + if (code) { + tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + } } // drop the stream task now - streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); + code = streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); + if (code) { + tqDebug("s-task:0x%x vgId:%d drop failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + } // commit the update streamMetaWLock(pMeta); @@ -703,12 +721,13 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen } streamMetaWUnLock(pMeta); - return 0; + return 0; // always return success } int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) { SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg; + int32_t code = 0; int32_t vgId = pMeta->vgId; tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId); @@ -718,7 +737,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask != NULL && (*ppTask) != NULL) { - streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq); + code = streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq); } else { // failed to get the task. int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); tqError( @@ -729,7 +748,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored streamMetaWUnLock(pMeta); // always return success when handling the requirement issued by mnode during transaction. - return TSDB_CODE_SUCCESS; + return code; } static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { @@ -1099,7 +1118,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { code = tqScanWalAsync((STQ*)handle, false); } else { - streamTrySchedExec(pTask); + code = streamTrySchedExec(pTask); } } /*else { ASSERT(status != TASK_STATUS__UNINIT); From 5acd35bf71cc59104a02a63be1bc16c2c7fa3b13 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Jul 2024 09:37:48 +0800 Subject: [PATCH 09/43] fix(stream): check return value. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c6139387b9..081382aa7d 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -411,7 +411,7 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); - if (pTask) { + if (pTask && (code == 0)) { code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pMeta, pTask); return code; From 06ca010059497ef2bac295ea87e26c574de6cd7c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Jul 2024 10:03:05 +0800 Subject: [PATCH 10/43] fix(stream): fix syntax error. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 081382aa7d..f1abbeb77f 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -708,7 +708,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen // drop the stream task now code = streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); if (code) { - tqDebug("s-task:0x%x vgId:%d drop failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + tqDebug("s-task:0x%x vgId:%d drop task failed", pReq->taskId, vgId); } // commit the update From ebe1d09d7491cb797996e6cbe659eefe3dbc3749 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Jul 2024 10:07:18 +0800 Subject: [PATCH 11/43] fix(stream): check return value. --- source/dnode/mnode/impl/inc/mndStream.h | 24 ++++---- source/dnode/mnode/impl/src/mndSma.c | 31 ++++++----- source/dnode/mnode/impl/src/mndStream.c | 62 +++++++++++++-------- source/dnode/mnode/impl/src/mndStreamHb.c | 20 ++++++- source/dnode/mnode/impl/src/mndStreamUtil.c | 24 +++++--- 5 files changed, 102 insertions(+), 59 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 0b6b6a9ef2..0a93848843 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -96,20 +96,20 @@ typedef struct STaskChkptInfo { int8_t dropHTask; }STaskChkptInfo; -int32_t mndInitStream(SMnode *pMnode); -void mndCleanupStream(SMnode *pMnode); -SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName); -void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); -int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); -int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream); -int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId); -int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt); -bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock); -int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId); +int32_t mndInitStream(SMnode *pMnode); +void mndCleanupStream(SMnode *pMnode); +int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream); +void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); +int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); +int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream); +int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId); +int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt); +bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock); +int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId); int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams); int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream); -SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); +int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList); void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t retryCode, int32_t acceptCode); @@ -146,7 +146,7 @@ void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInf int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); -SCheckpointConsensusInfo *mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks); +int32_t mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo); void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo); void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo); int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 946df84a0f..e4ac503d3b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -768,8 +768,8 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { char streamName[TSDB_TABLE_FNAME_LEN] = {0}; mndGetStreamNameFromSmaName(streamName, createReq.name); - pStream = mndAcquireStream(pMnode, streamName); - if (pStream != NULL) { + code = mndAcquireStream(pMnode, streamName, &pStream); + if (pStream != NULL || code != 0) { mError("sma:%s, failed to create since stream:%s already exist", createReq.name, streamName); terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; goto _OVER; @@ -905,8 +905,10 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p char streamName[TSDB_TABLE_FNAME_LEN] = {0}; mndGetStreamNameFromSmaName(streamName, pSma->name); - SStreamObj *pStream = mndAcquireStream(pMnode, streamName); - if (pStream == NULL || pStream->smaId != pSma->uid) { + SStreamObj *pStream = NULL; + + code = mndAcquireStream(pMnode, streamName, &pStream); + if (pStream == NULL || pStream->smaId != pSma->uid || code != 0) { sdbRelease(pMnode->pSdb, pStream); goto _OVER; } else { @@ -960,8 +962,9 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p char streamName[TSDB_TABLE_FNAME_LEN] = {0}; mndGetStreamNameFromSmaName(streamName, pSma->name); - SStreamObj *pStream = mndAcquireStream(pMnode, streamName); - if (pStream != NULL && pStream->smaId == pSma->uid) { + SStreamObj *pStream = NULL; + code = mndAcquireStream(pMnode, streamName, &pStream); + if ((pStream != NULL && pStream->smaId == pSma->uid) || code != 0) { if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); mndReleaseStream(pMnode, pStream); @@ -1734,8 +1737,8 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { goto _OVER; } - pStream = mndAcquireStream(pMnode, streamName); - if (pStream != NULL) { + code = mndAcquireStream(pMnode, streamName, &pStream); + if (pStream != NULL || code != 0) { mError("tsma:%s, failed to create since stream:%s already exist", createReq.name, streamName); terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; goto _OVER; @@ -2215,7 +2218,7 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt SSmaObj * pBaseTsma = NULL; SSdb * pSdb = pMnode->pSdb; void * pIter = NULL; - SStreamObj * pStreamObj = NULL; + SStreamObj * pStream = NULL; SStbObj * pStb = NULL; while (1) { @@ -2237,14 +2240,16 @@ static int32_t mndGetSomeTsmas(SMnode* pMnode, STableTSMAInfoRsp* pRsp, tsmaFilt char streamName[TSDB_TABLE_FNAME_LEN] = {0}; tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); sprintf(streamName, "%d.%s", smaName.acctId, smaName.tname); - pStreamObj = mndAcquireStream(pMnode, streamName); - if (!pStreamObj) { + pStream = NULL; + + code = mndAcquireStream(pMnode, streamName, &pStream); + if (!pStream || (code != 0)) { sdbRelease(pSdb, pSma); continue; } - int64_t streamId = pStreamObj->uid; - mndReleaseStream(pMnode, pStreamObj); + int64_t streamId = pStream->uid; + mndReleaseStream(pMnode, pStream); STableTSMAInfo *pTsma = taosMemoryCalloc(1, sizeof(STableTSMAInfo)); if (!pTsma) { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d57dc6e52e..fa4b9b64f4 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -251,13 +251,13 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream return 0; } -SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) { - SSdb *pSdb = pMnode->pSdb; - SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName); - if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { +int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream) { + SSdb *pSdb = pMnode->pSdb; + (*pStream) = sdbAcquire(pSdb, SDB_STREAM, streamName); + if ((*pStream) == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; } - return pStream; + return terrno; } void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { @@ -706,7 +706,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { char *sql = NULL; int32_t sqlLen = 0; const char *pMsg = "create stream tasks on dnodes"; - + int32_t code = 0; terrno = TSDB_CODE_SUCCESS; SCMCreateStreamReq createReq = {0}; @@ -726,8 +726,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - pStream = mndAcquireStream(pMnode, createReq.name); - if (pStream != NULL) { + code = mndAcquireStream(pMnode, createReq.name, &pStream); + if (pStream != NULL || code != 0) { if (createReq.igExists) { mInfo("stream:%s, already exist, ignore exist is set", createReq.name); goto _OVER; @@ -1077,7 +1077,12 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { } bool allReady = true; - SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); + SArray *pNodeSnapshot = NULL; + + int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot); + if (code) { + mError("failed to get the vgroup snapshot, ignore it and continue"); + } if (!allReady) { mWarn("not all vnodes ready, quit from vnodes status check"); taosArrayDestroy(pNodeSnapshot); @@ -1289,6 +1294,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; + int32_t code = 0; SMDropStreamReq dropReq = {0}; if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { @@ -1299,8 +1305,8 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { mDebug("recv drop stream:%s msg", dropReq.name); - pStream = mndAcquireStream(pMnode, dropReq.name); - if (pStream == NULL) { + code = mndAcquireStream(pMnode, dropReq.name, &pStream); + if (pStream == NULL || code != 0) { if (dropReq.igNotExists) { mInfo("stream:%s not exist, ignore not exist is set, drop stream exec done with success", dropReq.name); sdbRelease(pMnode->pSdb, pStream); @@ -1364,7 +1370,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return -1; } - int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid); + code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid); // drop all tasks if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) { @@ -1906,6 +1912,7 @@ static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter) { static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; + int32_t code = 0; SMPauseStreamReq pauseReq = {0}; if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) { @@ -1913,9 +1920,8 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { return -1; } - pStream = mndAcquireStream(pMnode, pauseReq.name); - - if (pStream == NULL) { + code = mndAcquireStream(pMnode, pauseReq.name, &pStream); + if (pStream == NULL || code != 0) { if (pauseReq.igNotExists) { mInfo("stream:%s, not exist, not pause stream", pauseReq.name); return 0; @@ -2000,7 +2006,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { return -1; } - int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid); + code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid); // if nodeUpdate happened, not send pause trans if (mndStreamSetPauseAction(pMnode, pTrans, pStream) < 0) { @@ -2039,6 +2045,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; + int32_t code = 0; if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { return -1; @@ -2050,9 +2057,8 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { return -1; } - pStream = mndAcquireStream(pMnode, resumeReq.name); - - if (pStream == NULL) { + code = mndAcquireStream(pMnode, resumeReq.name, &pStream); + if (pStream == NULL || code != 0) { if (resumeReq.igNotExists) { mInfo("stream:%s not exist, not resume stream", resumeReq.name); sdbRelease(pMnode->pSdb, pStream); @@ -2089,7 +2095,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { return -1; } - int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid); + code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid); // set the resume action if (mndStreamSetResumeAction(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) { @@ -2348,7 +2354,13 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { } bool allReady = true; - SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); + SArray *pNodeSnapshot = NULL; + + code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot); + if (code) { + mError("failed to take the vgroup snapshot, ignore it and continue"); + } + if (!allReady) { taosArrayDestroy(pNodeSnapshot); atomic_store_32(&mndNodeCheckSentinel, 0); @@ -2790,8 +2802,14 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) { mDebug("start to process consensus-checkpointId in tmr"); bool allReady = true; - SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); + SArray *pNodeSnapshot = NULL; + + int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot); taosArrayDestroy(pNodeSnapshot); + if (code) { + mError("failed to get the vgroup snapshot, ignore it and continue"); + } + if (!allReady) { mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process"); taosArrayDestroy(pStreamList); diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index bc10ec211d..5f0434e3d0 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -228,6 +228,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SStreamHbMsg req = {0}; SArray *pFailedChkpt = NULL; SArray *pOrphanTasks = NULL; + int32_t code = 0; if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) { @@ -296,8 +297,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SStreamObj *pStream = mndGetStreamObj(pMnode, p->id.streamId); int32_t numOfTasks = mndGetNumOfStreamTasks(pStream); - SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks); - mndAddConsensusTasks(pInfo, &cp); + SCheckpointConsensusInfo *pInfo = NULL; + + code = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks, &pInfo); + if (code == 0) { + mndAddConsensusTasks(pInfo, &cp); + } else { + mError("failed to get consensus checkpoint-info"); + } + mndReleaseStream(pMnode, pStream); } @@ -338,9 +346,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { // kill the checkpoint trans and then set all tasks status to be normal if (taosArrayGetSize(pFailedChkpt) > 0) { bool allReady = true; + if (pMnode != NULL) { - SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); + SArray *p = NULL; + + int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &p); taosArrayDestroy(p); + if (code) { + mError("failed to get the vgroup snapshot, ignore it and continue"); + } } else { allReady = false; } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 843c024286..3a84856ae0 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -80,11 +80,12 @@ void destroyStreamTaskIter(SStreamTaskIter* pIter) { taosMemoryFree(pIter); } -SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { +int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; SVgObj *pVgroup = NULL; int32_t replica = -1; // do the replica check + int32_t code = 0; *allReady = true; SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry)); @@ -157,7 +158,8 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { sdbRelease(pSdb, pObj); } - return pVgroupList; + *pList = pVgroupList; + return code; } SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) { @@ -935,10 +937,10 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i return TSDB_CODE_ACTION_IN_PROGRESS; } -SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks) { - void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); +int32_t mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo) { + *pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); if (pInfo != NULL) { - return (SCheckpointConsensusInfo*)pInfo; + return 0; } SCheckpointConsensusInfo p = { @@ -947,10 +949,14 @@ SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, .streamId = streamId, }; - taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p)); - - void* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId)); - return pChkptInfo; + int32_t code = taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p)); + if (code == 0) { + void *pChkptInfo = (SCheckpointConsensusInfo *)taosHashGet(pHash, &streamId, sizeof(streamId)); + *pInfo = pChkptInfo; + } else { + *pInfo = NULL; + } + return code; } // no matter existed or not, add the request into info list anyway, since we need to send rsp mannually From 45348b15be25cd23b8bfc01f8713b68111fe5422 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Jul 2024 11:06:24 +0800 Subject: [PATCH 12/43] fix: return code validation --- source/libs/executor/inc/mergejoin.h | 11 +- source/libs/executor/src/groupcacheoperator.c | 8 +- source/libs/executor/src/mergejoin.c | 262 ++++++++++++++- source/libs/executor/src/mergejoinoperator.c | 316 ++++++++++++++---- 4 files changed, 503 insertions(+), 94 deletions(-) diff --git a/source/libs/executor/inc/mergejoin.h b/source/libs/executor/inc/mergejoin.h index f6f1cf26f3..64db1a57a0 100755 --- a/source/libs/executor/inc/mergejoin.h +++ b/source/libs/executor/inc/mergejoin.h @@ -432,6 +432,15 @@ typedef struct SMJoinOperatorInfo { } \ } while (0) +#define MJ_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + terrno = _code; \ + } \ + return _code; \ + } while (0) + void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin); @@ -459,7 +468,7 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable); int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable); int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp); -bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build); +int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont); int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond); int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx); int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp); diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 379cceeb03..e9a6c14141 100755 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -149,7 +149,7 @@ void blockDataDeepCleanup(SSDataBlock* pDataBlock) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); if (NULL == p) { - qError("fail to get %dth col in dataBlock, numOfCols:%d", i, numOfCols); + qError("fail to get %dth col in dataBlock, numOfCols:%d", i, (int32_t)numOfCols); continue; } taosMemoryFreeClear(p->pData); @@ -352,7 +352,7 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr } pBufInfo = taosHashGet(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId)); if (NULL == pBufInfo) { - qError("fail to get blk %d from pCache->pDirtyBlk", pBufInfo->basic.blkId); + qError("fail to get blk %" PRId64 " from pCache->pDirtyBlk", pBufInfo->basic.blkId); QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } int32_t code = TSDB_CODE_SUCCESS; @@ -462,7 +462,7 @@ void blockDataDeepClear(SSDataBlock* pDataBlock) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); if (NULL == p) { - qError("fail to get %d col from pDataBlock, numOfCols:%d", i, numOfCols); + qError("fail to get %d col from pDataBlock, numOfCols:%d", i, (int32_t)numOfCols); continue; } p->pData = NULL; @@ -1239,7 +1239,7 @@ static int32_t initGroupCacheSession(struct SOperatorInfo* pOperator, SOperatorP *ppSession = taosHashGet(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); if (NULL == *ppSession) { - qError("fail to get session %d from pSessions", pGcParam->sessionId); + qError("fail to get session %" PRId64 " from pSessions", pGcParam->sessionId); QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); } diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index 1c625df52f..2c5121855a 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -72,9 +72,15 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) { if ((!pCtx->seqWinGrp) && 0 == cache->grpIdx && probeRows * buildTotalRows <= rowsLeft) { SMJoinGrpRows* pFirstBuild = taosArrayGet(cache->grps, 0); + if (NULL == pFirstBuild) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (pFirstBuild->readIdx == pFirstBuild->beginIdx) { for (; cache->grpIdx < buildGrpNum; ++cache->grpIdx) { SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); buildGrp->readIdx = buildGrp->beginIdx; } @@ -89,6 +95,9 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) { probeGrp->endIdx = probeGrp->readIdx; for (; cache->grpIdx < buildGrpNum && rowsLeft > 0; ++cache->grpIdx) { SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); @@ -134,7 +143,8 @@ static int32_t mOuterJoinHashFullCart(SMJoinMergeCtx* pCtx) { SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); if (build->grpRowIdx >= 0) { - bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); + bool contLoop = false; + MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop)); if (build->grpRowIdx < 0) { probeGrp->readIdx++; } @@ -171,7 +181,8 @@ static int32_t mOuterJoinHashFullCart(SMJoinMergeCtx* pCtx) { } build->grpRowIdx = 0; - bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); + bool contLoop = false; + MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop)); if (!contLoop) { if (build->grpRowIdx < 0) { probeGrp->readIdx++; @@ -193,15 +204,27 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t probeRows = GRP_REMAIN_ROWS(probeGrp); int32_t probeEndIdx = probeGrp->endIdx; if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) { SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0); + if (NULL == pFirstBuild) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (pFirstBuild->readIdx == pFirstBuild->beginIdx) { for (; build->grpIdx < buildGrpNum; ++build->grpIdx) { SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); buildGrp->readIdx = buildGrp->beginIdx; } @@ -215,6 +238,9 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) { probeGrp->endIdx = probeGrp->readIdx; for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); @@ -254,6 +280,10 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t probeEndIdx = probeGrp->endIdx; int32_t rowsLeft = pCtx->midBlk->info.capacity; @@ -274,6 +304,10 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (startRowIdx < 0) { startRowIdx = buildGrp->readIdx; } @@ -351,13 +385,17 @@ static int32_t mOuterJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, bool* contLoop) SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + int32_t startRowIdx = 0; //blockDataCleanup(pCtx->midBlk); do { startRowIdx = build->grpRowIdx; - (void)mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); + MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL)); if (pCtx->midBlk->info.rows > 0) { if (build->rowBitmapSize > 0) { @@ -406,6 +444,10 @@ static int32_t mOuterJoinHashSeqCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + bool contLoop = false; if (build->grpRowIdx >= 0) { @@ -500,7 +542,15 @@ static bool mLeftJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoi if (buildGot) { SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId); + if (NULL == pProbeCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId); + if (NULL == pBuildCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) { pJoin->build->blkRowIdx = pJoin->build->blk->info.rows; buildGot = false; @@ -649,15 +699,27 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t probeRows = GRP_REMAIN_ROWS(probeGrp); int32_t probeEndIdx = probeGrp->endIdx; if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) { SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0); + if (NULL == pFirstBuild) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (pFirstBuild->readIdx == pFirstBuild->beginIdx) { for (; build->grpIdx < buildGrpNum; ++build->grpIdx) { SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); buildGrp->readIdx = buildGrp->beginIdx; } @@ -671,6 +733,9 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) { probeGrp->endIdx = probeGrp->readIdx; for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); @@ -711,9 +776,13 @@ static int32_t mInnerJoinHashCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (build->grpRowIdx >= 0) { - bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); + bool contLoop = false; + MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop)); if (build->grpRowIdx < 0) { probeGrp->readIdx++; } @@ -734,7 +803,8 @@ static int32_t mInnerJoinHashCart(SMJoinMergeCtx* pCtx) { if (NULL != pGrp) { build->pHashCurGrp = *pGrp; build->grpRowIdx = 0; - bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); + bool contLoop = false; + MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop)); if (!contLoop) { if (build->grpRowIdx < 0) { probeGrp->readIdx++; @@ -772,7 +842,15 @@ static bool mInnerJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJo if (buildGot) { SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId); + if (NULL == pProbeCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId); + if (NULL == pBuildCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) { pJoin->build->blkRowIdx = pJoin->build->blk->info.rows; buildGot = false; @@ -903,6 +981,10 @@ static int32_t mFullJoinMergeCart(SMJoinMergeCtx* pCtx) { static FORCE_INLINE int32_t mFullJoinOutputHashRow(SMJoinMergeCtx* pCtx, SMJoinHashGrpRows* pGrpRows, int32_t idx) { SMJoinGrpRows grp = {0}; SMJoinRowPos* pPos = taosArrayGet(pGrpRows->pRows, idx); + if (NULL == pPos) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + grp.blk = pPos->pBlk; grp.readIdx = pPos->pos; grp.endIdx = pPos->pos; @@ -1042,6 +1124,10 @@ static int32_t mFullJoinHandleMergeGrpRemains(SMJoinMergeCtx* pCtx) { grpDone = false; SMJoinGrpRows* pGrpRows = taosArrayGet(build->eqGrps, pNMatch->grpIdx); + if (NULL == pGrpRows) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (pGrpRows->allRowsMatch) { continue; } @@ -1283,7 +1369,7 @@ static int32_t mSemiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p do { blockDataCleanup(pCtx->midBlk); - (void)mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); + MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL)); if (pCtx->midBlk->info.rows > 0) { MJ_ERR_RET(mJoinFilterAndKeepSingleRow(pCtx->midBlk, pCtx->pJoin->pPreFilter)); @@ -1312,6 +1398,9 @@ static int32_t mSemiJoinHashSeqCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } size_t bufLen = 0; int32_t probeEndIdx = probeGrp->endIdx; @@ -1343,6 +1432,10 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + size_t bufLen = 0; for (; !GRP_DONE(probeGrp) && !BLK_IS_FULL(pCtx->finBlk); ++probeGrp->readIdx) { @@ -1358,7 +1451,7 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) { build->pHashCurGrp = *(SArray**)pGrp; ASSERT(1 == taosArrayGetSize(build->pHashCurGrp)); build->grpRowIdx = 0; - (void)mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); + MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, NULL)); ASSERT(build->grpRowIdx < 0); } @@ -1372,6 +1465,10 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SMJoinGrpRows* buildGrp = NULL; int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t probeEndIdx = probeGrp->endIdx; @@ -1387,6 +1484,9 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { blockDataCleanup(pCtx->midBlk); for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); @@ -1448,6 +1548,10 @@ static int32_t mSemiJoinMergeFullCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, 0); + if (NULL == buildGrp || NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + int32_t probeRows = GRP_REMAIN_ROWS(probeGrp); int32_t probeEndIdx = probeGrp->endIdx; @@ -1584,6 +1688,10 @@ static int32_t mAntiJoinHashFullCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + size_t bufLen = 0; int32_t probeEndIdx = probeGrp->endIdx; @@ -1616,7 +1724,7 @@ static int32_t mAntiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p do { blockDataCleanup(pCtx->midBlk); - (void)mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); + MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL)); if (pCtx->midBlk->info.rows > 0) { MJ_ERR_RET(mJoinFilterAndNoKeepRows(pCtx->midBlk, pCtx->pJoin->pPreFilter)); @@ -1642,6 +1750,10 @@ static int32_t mAntiJoinHashSeqCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + size_t bufLen = 0; int32_t probeEndIdx = probeGrp->endIdx; @@ -1682,6 +1794,10 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); + if (NULL == probeGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SMJoinGrpRows* buildGrp = NULL; int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t probeEndIdx = probeGrp->endIdx; @@ -1697,6 +1813,10 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { blockDataCleanup(pCtx->midBlk); for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); rowsLeft -= GRP_REMAIN_ROWS(buildGrp); @@ -1889,6 +2009,9 @@ int32_t mAsofBackwardAddEqRowsToCache(struct SOperatorInfo* pOperator, SMJoinWin grp.blk = pTable->blk; SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) { return TSDB_CODE_SUCCESS; @@ -2249,6 +2372,10 @@ int32_t mAsofForwardTrimCacheBlk(SMJoinWindowCtx* pCtx) { } SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (pGrp->blk == pCtx->cache.outBlk && pCtx->pJoin->build->blkRowIdx > 0) { MJ_ERR_RET(blockDataTrimFirstRows(pGrp->blk, pCtx->pJoin->build->blkRowIdx)); pCtx->pJoin->build->blkRowIdx = 0; @@ -2271,6 +2398,10 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) { int32_t grpNum = taosArrayGetSize(pCache->grps); if (grpNum >= 1) { SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, grpNum - 1); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (pGrp->blk != pCache->outBlk) { int32_t beginIdx = (1 == grpNum) ? build->blkRowIdx : 0; MJ_ERR_RET(blockDataMergeNRows(pCache->outBlk, pGrp->blk, beginIdx, pGrp->blk->info.rows - beginIdx)); @@ -2282,6 +2413,10 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) { } else { (void)taosArrayPop(pCache->grps); pGrp = taosArrayGet(pCache->grps, 0); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + ASSERT(pGrp->blk == pCache->outBlk); //pGrp->endIdx = pGrp->blk->info.rows - pGrp->beginIdx; } @@ -2320,16 +2455,20 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) { return TSDB_CODE_SUCCESS; } -void mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) { +int32_t mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) { int32_t grpNum = taosArrayGetSize(pCtx->cache.grps); if (grpNum <= 0) { - return; + return TSDB_CODE_SUCCESS; } SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (1 == grpNum) { pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows - pGrp->beginIdx, pCtx->jLimit) - 1; - return; + return TSDB_CODE_SUCCESS; } ASSERT(pCtx->jLimit > (pGrp->blk->info.rows - pGrp->beginIdx)); @@ -2338,7 +2477,13 @@ void mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) { int64_t remainRows = pCtx->jLimit - (pGrp->endIdx - pGrp->beginIdx + 1); pGrp = taosArrayGet(pCtx->cache.grps, 1); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows, remainRows) - 1; + + return TSDB_CODE_SUCCESS; } int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) { @@ -2347,13 +2492,16 @@ int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) { MJ_ERR_RET(mAsofForwardChkFillGrpCache(pCtx)); } - mAsofForwardUpdateBuildGrpEndIdx(pCtx); + MJ_ERR_RET(mAsofForwardUpdateBuildGrpEndIdx(pCtx)); return mWinJoinDumpGrpCache(pCtx); } int32_t mAsofForwardSkipEqRows(SMJoinWindowCtx* pCtx, SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk) { SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) { *wholeBlk = false; @@ -2513,6 +2661,10 @@ static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo if (buildGot) { SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId); SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId); + if (NULL == pProbeCol || NULL == pBuildCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) { pJoin->build->blkRowIdx = pJoin->build->blk->info.rows; MJOIN_POP_TB_BLK(&pCtx->cache); @@ -2674,6 +2826,10 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) { } SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGetLast(pGrpArray); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (!pGrp->clonedBlk) { if (0 == pGrp->beginIdx) { pGrp->blk = createOneDataBlock(pGrp->blk, true); @@ -2720,6 +2876,10 @@ static int32_t mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJ if (buildGot && pCtx->forwardRowsAcq) { SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId); SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId); + if (NULL == pProbeCol || NULL == pBuildCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) { pJoin->build->blkRowIdx = pJoin->build->blk->info.rows; buildGot = false; @@ -2739,6 +2899,10 @@ static int32_t mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) { SSDataBlock* pBlk = build->blk; SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (pCtx->ascTs) { if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) < pCtx->winBeginTs) { *winEnd = false; @@ -2759,6 +2923,9 @@ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, if (*((int64_t*)pCol->pData + build->blkRowIdx) <= pCtx->winEndTs) { SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx}; SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); + if (NULL == pGrp) { + MJ_ERR_RET(terrno); + } pGrp->readIdx = pGrp->beginIdx; pGrp->endIdx = pGrp->beginIdx; @@ -2795,7 +2962,10 @@ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, if (*((int64_t*)pCol->pData + build->blkRowIdx) >= pCtx->winBeginTs) { SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx}; SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); - + if (NULL == pGrp) { + MJ_ERR_RET(terrno); + } + pGrp->readIdx = pGrp->beginIdx; pGrp->endIdx = pGrp->beginIdx; @@ -2864,7 +3034,15 @@ int32_t mWinJoinMoveAscWinBegin(SMJoinWindowCtx* pCtx) { int32_t grpNum = taosArrayGetSize(pCache->grps); for (int32_t i = 0; i < grpNum; ++i) { SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) < pCtx->winBeginTs) { mWinJoinPopFrontGroup(pCtx, pGrp); grpNum--; @@ -2900,6 +3078,7 @@ int32_t mWinJoinMoveAscWinBegin(SMJoinWindowCtx* pCtx) { pCache->grps = pCache->grpsQueue; pCache->rowNum = 1; pCache->grpsQueue = NULL; + continue; } @@ -2916,7 +3095,15 @@ int32_t mWinJoinMoveDescWinBegin(SMJoinWindowCtx* pCtx) { int32_t grpNum = taosArrayGetSize(pCache->grps); for (int32_t i = 0; i < grpNum; ++i) { SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) > pCtx->winEndTs) { mWinJoinPopFrontGroup(pCtx, pGrp); @@ -2953,6 +3140,7 @@ int32_t mWinJoinMoveDescWinBegin(SMJoinWindowCtx* pCtx) { pCache->grps = pCache->grpsQueue; pCache->rowNum = 1; pCache->grpsQueue = NULL; + continue; } @@ -2986,6 +3174,10 @@ void mWinJoinRemoveOverflowGrp(SMJoinWindowCtx* pCtx) { int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) { SSDataBlock* pBlk = build->blk; SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx}; if (pCtx->ascTs) { @@ -3001,6 +3193,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) <= pCtx->winEndTs) { SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); + if (NULL == pGrp) { + MJ_ERR_RET(terrno); + } pGrp->readIdx = pGrp->beginIdx; pGrp->endIdx = pBlk->info.rows - 1; @@ -3028,6 +3223,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S } SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); + if (NULL == pGrp) { + MJ_ERR_RET(terrno); + } pGrp->readIdx = pGrp->beginIdx; pGrp->endIdx = build->blkRowIdx - 1; @@ -3048,6 +3246,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) >= pCtx->winBeginTs) { SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); + if (NULL == pGrp) { + MJ_ERR_RET(terrno); + } pGrp->readIdx = pGrp->beginIdx; pGrp->endIdx = pBlk->info.rows - 1; @@ -3070,7 +3271,10 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S } SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); - + if (NULL == pGrp) { + MJ_ERR_RET(terrno); + } + pGrp->readIdx = pGrp->beginIdx; pGrp->endIdx = build->blkRowIdx - 1; @@ -3130,7 +3334,15 @@ int32_t mWinJoinMoveAscWinEnd(SMJoinWindowCtx* pCtx) { } SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) <= pCtx->winEndTs) { pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1; if (pCache->rowNum >= pCtx->jLimit) { @@ -3173,7 +3385,15 @@ int32_t mWinJoinMoveDescWinEnd(SMJoinWindowCtx* pCtx) { } SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) >= pCtx->winBeginTs) { pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1; pGrp->endIdx = pGrp->blk->info.rows - 1; @@ -3217,6 +3437,10 @@ int32_t mWinJoinTrimDumpGrpCache(SMJoinWindowCtx* pCtx) { int32_t buildGrpNum = taosArrayGetSize(cache->grps); for (int32_t i = 0; i < buildGrpNum && skipRows > 0; ++i) { SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, i); + if (NULL == buildGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (skipRows >= GRP_REMAIN_ROWS(buildGrp)) { skipRows -= GRP_REMAIN_ROWS(buildGrp); mWinJoinPopFrontGroup(pCtx, buildGrp); @@ -3318,7 +3542,7 @@ int32_t mJoinInitWindowCache(SMJoinWinCache* pCache, SMJoinOperatorInfo* pJoin, pCache->grps = taosArrayInit(2, sizeof(SMJoinGrpRows)); if (NULL == pCache->grps) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } //taosArrayReserve(pTable->eqGrps, 1); @@ -3392,6 +3616,10 @@ int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* p } pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); + if (NULL == pCtx->finBlk) { + MJ_ERR_RET(terrno); + } + MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode))); pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO; @@ -3430,6 +3658,10 @@ int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJ } pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); + if (NULL == pCtx->finBlk) { + MJ_ERR_RET(terrno); + } + ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0); MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode))); diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 9db2c7d300..8d94841847 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -30,6 +30,9 @@ int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, SMJoinGrpRows* pGrp) { SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } pGrp->beginIdx = pTable->blkRowIdx; pGrp->readIdx = pTable->blkRowIdx; @@ -59,12 +62,16 @@ int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBl } -void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) { +int32_t mJoinTrimKeepFirstRow(SSDataBlock* pBlock) { int32_t bmLen = BitmapLen(pBlock->info.rows); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); + if (NULL == pDst) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + // it is a reserved column for scalar function, and no data in this column yet. if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) { continue; @@ -86,7 +93,7 @@ void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) { } else { bool isNull = colDataIsNull_f(pDst->nullbitmap, 0); - memset(pDst->nullbitmap, 0, bmLen); + TAOS_MEMSET(pDst->nullbitmap, 0, bmLen); if (isNull) { colDataSetNull_f(pDst->nullbitmap, 0); } @@ -94,11 +101,14 @@ void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) { } pBlock->info.rows = 1; + + return TSDB_CODE_SUCCESS; } -void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) { +int32_t mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) { // int32_t totalRows = pBlock->info.rows; + int32_t code = TSDB_CODE_SUCCESS; int32_t bmLen = BitmapLen(totalRows); char* pBitmap = NULL; int32_t maxRows = 0; @@ -106,6 +116,10 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); + if (NULL == pDst) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + // it is a reserved column for scalar function, and no data in this column yet. if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) { continue; @@ -135,8 +149,12 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo len = varDataTLen(p1); } char* p2 = taosMemoryMalloc(len); - memcpy(p2, p1, len); - colDataSetVal(pDst, numOfRows, p2, false); + TAOS_MEMCPY(p2, p1, len); + code = colDataSetVal(pDst, numOfRows, p2, false); + if (code) { + taosMemoryFreeClear(p2); + MJ_ERR_RET(terrno); + } taosMemoryFree(p2); } numOfRows += 1; @@ -150,10 +168,13 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo } else { if (pBitmap == NULL) { pBitmap = taosMemoryCalloc(1, bmLen); + if (NULL == pBitmap) { + MJ_ERR_RET(terrno); + } } - memcpy(pBitmap, pDst->nullbitmap, bmLen); - memset(pDst->nullbitmap, 0, bmLen); + TAOS_MEMCPY(pBitmap, pDst->nullbitmap, bmLen); + TAOS_MEMSET(pDst->nullbitmap, 0, bmLen); int32_t j = 0; @@ -243,6 +264,8 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo if (pBitmap != NULL) { taosMemoryFree(pBitmap); } + + return TSDB_CODE_SUCCESS; } @@ -290,8 +313,10 @@ int32_t mJoinFilterAndMarkHashRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo code = TSDB_CODE_SUCCESS; _err: + colDataDestroy(p); taosMemoryFree(p); + return code; } @@ -300,18 +325,19 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM return TSDB_CODE_SUCCESS; } + int32_t code = TSDB_CODE_SUCCESS; SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; SColumnInfoData* p = NULL; - int32_t code = filterSetDataFromSlotId(pFilterInfo, ¶m1); + code = filterSetDataFromSlotId(pFilterInfo, ¶m1); if (code != TSDB_CODE_SUCCESS) { - goto _err; + goto _return; } int32_t status = 0; code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status); if (code != TSDB_CODE_SUCCESS) { - goto _err; + goto _return; } int32_t rowNum = 0; @@ -320,6 +346,9 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM if (status == FILTER_RESULT_ALL_QUALIFIED || status == FILTER_RESULT_PARTIAL_QUALIFIED) { for (int32_t i = startGrpIdx; i < grpNum && rowNum < pBlock->info.rows; startRowIdx = 0, ++i) { SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, i); + if (NULL == buildGrp) { + MJ_ERR_JRET(terrno); + } if (buildGrp->allRowsMatch) { rowNum += buildGrp->endIdx - startRowIdx + 1; continue; @@ -350,9 +379,11 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM code = TSDB_CODE_SUCCESS; -_err: +_return: + colDataDestroy(p); taosMemoryFree(p); + return code; } @@ -366,29 +397,31 @@ int32_t mJoinFilterAndKeepSingleRow(SSDataBlock* pBlock, SFilterInfo* pFilterInf int32_t code = filterSetDataFromSlotId(pFilterInfo, ¶m1); if (code != TSDB_CODE_SUCCESS) { - goto _err; + goto _return; } int32_t status = 0; code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status); if (code != TSDB_CODE_SUCCESS) { - goto _err; + goto _return; } if (status == FILTER_RESULT_ALL_QUALIFIED) { pBlock->info.rows = 1; - mJoinTrimKeepFirstRow(pBlock); + MJ_ERR_JRET(mJoinTrimKeepFirstRow(pBlock)); } else if (status == FILTER_RESULT_NONE_QUALIFIED) { pBlock->info.rows = 0; } else if (status == FILTER_RESULT_PARTIAL_QUALIFIED) { - mJoinTrimKeepOneRow(pBlock, pBlock->info.rows, (bool*)p->pData); + MJ_ERR_JRET(mJoinTrimKeepOneRow(pBlock, pBlock->info.rows, (bool*)p->pData)); } code = TSDB_CODE_SUCCESS; -_err: +_return: + colDataDestroy(p); taosMemoryFree(p); + return code; } @@ -418,8 +451,10 @@ int32_t mJoinFilterAndNoKeepRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo) code = TSDB_CODE_SUCCESS; _err: + colDataDestroy(p); taosMemoryFree(p); + return code; } @@ -480,12 +515,20 @@ int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app SMJoinColMap* pFirstCol = probe->finCols + c; SColumnInfoData* pInCol = taosArrayGet(pGrp->blk->pDataBlock, pFirstCol->srcSlot); SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot); - colDataAssignNRows(pOutCol, currRows, pInCol, pGrp->readIdx, firstRows); + if (NULL == pInCol || NULL == pOutCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + + MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows, pInCol, pGrp->readIdx, firstRows)); } for (int32_t c = 0; c < build->finNum; ++c) { SMJoinColMap* pSecondCol = build->finCols + c; SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot); + if (NULL == pOutCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + colDataSetNItemsNull(pOutCol, currRows, firstRows); } @@ -536,6 +579,10 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app SMJoinColMap* pFirstCol = probe->finCols + c; SColumnInfoData* pInCol = taosArrayGet(pFirst->blk->pDataBlock, pFirstCol->srcSlot); SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot); + if (NULL == pInCol || NULL == pOutCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + for (int32_t r = 0; r < firstRows; ++r) { if (colDataIsNull_s(pInCol, pFirst->readIdx + r)) { colDataSetNItemsNull(pOutCol, currRows + r * secondRows, secondRows); @@ -543,7 +590,7 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app ASSERT(pRes->info.capacity >= (pRes->info.rows + firstRows * secondRows)); uint32_t startOffset = (IS_VAR_DATA_TYPE(pOutCol->info.type)) ? pOutCol->varmeta.length : ((currRows + r * secondRows) * pOutCol->info.bytes); ASSERT((startOffset + 1 * pOutCol->info.bytes) <= pRes->info.capacity * pOutCol->info.bytes); - colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true); + MJ_ERR_RET(colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true)); } } } @@ -552,28 +599,40 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app SMJoinColMap* pSecondCol = build->finCols + c; SColumnInfoData* pInCol = taosArrayGet(pSecond->blk->pDataBlock, pSecondCol->srcSlot); SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot); + if (NULL == pInCol || NULL == pOutCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + for (int32_t r = 0; r < firstRows; ++r) { - colDataAssignNRows(pOutCol, currRows + r * secondRows, pInCol, pSecond->readIdx, secondRows); + MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows + r * secondRows, pInCol, pSecond->readIdx, secondRows)); } } pRes->info.rows = append ? (pRes->info.rows + firstRows * secondRows) : firstRows * secondRows; + return TSDB_CODE_SUCCESS; } -bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build) { +int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont) { + if (NULL != cont) { + *cont = false; + } + int32_t rowsLeft = append ? (pBlk->info.capacity - pBlk->info.rows) : pBlk->info.capacity; if (rowsLeft <= 0) { - return false; + return TSDB_CODE_SUCCESS; } int32_t buildGrpRows = taosArrayGetSize(build->pHashCurGrp); int32_t grpRows = buildGrpRows - build->grpRowIdx; if (grpRows <= 0 || build->grpRowIdx < 0) { build->grpRowIdx = -1; - return true; + if (NULL != cont) { + *cont = true; + } + return TSDB_CODE_SUCCESS; } int32_t actRows = TMIN(grpRows, rowsLeft); @@ -583,10 +642,14 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S SMJoinColMap* pFirstCol = probe->finCols + c; SColumnInfoData* pInCol = taosArrayGet(probeGrp->blk->pDataBlock, pFirstCol->srcSlot); SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pFirstCol->dstSlot); + if (NULL == pInCol || NULL == pOutCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (colDataIsNull_s(pInCol, probeGrp->readIdx)) { colDataSetNItemsNull(pOutCol, currRows, actRows); } else { - colDataSetNItems(pOutCol, currRows, colDataGetData(pInCol, probeGrp->readIdx), actRows, true); + MJ_ERR_RET(colDataSetNItems(pOutCol, currRows, colDataGetData(pInCol, probeGrp->readIdx), actRows, true)); } } @@ -595,8 +658,16 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pSecondCol->dstSlot); for (int32_t r = 0; r < actRows; ++r) { SMJoinRowPos* pRow = taosArrayGet(build->pHashCurGrp, build->grpRowIdx + r); + if (NULL == pRow) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SColumnInfoData* pInCol = taosArrayGet(pRow->pBlk->pDataBlock, pSecondCol->srcSlot); - colDataAssignNRows(pOutCol, currRows + r, pInCol, pRow->pos, 1); + if (NULL == pInCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + + MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows + r, pInCol, pRow->pos, 1)); } } @@ -609,16 +680,24 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S } if (actRows == rowsLeft) { - return false; + return TSDB_CODE_SUCCESS; } - return true; + if (NULL != cont) { + *cont = true; + } + + return TSDB_CODE_SUCCESS; } int32_t mJoinAllocGrpRowBitmap(SMJoinTableCtx* pTb) { int32_t grpNum = taosArrayGetSize(pTb->eqGrps); for (int32_t i = 0; i < grpNum; ++i) { SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGet(pTb->eqGrps, i); + if (NULL == pGrp) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + MJ_ERR_RET(mJoinGetRowBitmapOffset(pTb, pGrp->endIdx - pGrp->beginIdx + 1, &pGrp->rowBitmapOffset)); pGrp->rowMatchNum = 0; } @@ -632,9 +711,9 @@ int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastB pCtx->lastEqGrp = true; - mJoinBuildEqGroups(pJoin->probe, timestamp, NULL, true); + MJ_ERR_RET(mJoinBuildEqGroups(pJoin->probe, timestamp, NULL, true)); if (!lastBuildGrp) { - mJoinRetrieveEqGrpRows(pJoin, pJoin->build, timestamp); + MJ_ERR_RET(mJoinRetrieveEqGrpRows(pJoin, pJoin->build, timestamp)); } else { pJoin->build->grpIdx = 0; } @@ -661,7 +740,7 @@ int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastB pCtx->hashJoin = false; if (!lastBuildGrp && pJoin->build->rowBitmapSize > 0) { - mJoinAllocGrpRowBitmap(pJoin->build); + MJ_ERR_RET(mJoinAllocGrpRowBitmap(pJoin->build)); } return (*pCtx->mergeCartFp)(pCtx); @@ -721,7 +800,7 @@ int32_t mJoinInitDownstreamInfo(SMJoinOperatorInfo* pInfo, SOperatorInfo*** pDow *newDownstreams = true; *pDownstream = mJoinBuildDownstreams(pInfo, *pDownstream); if (NULL == *pDownstream) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *numOfDownstream = 2; } @@ -732,7 +811,7 @@ int32_t mJoinInitDownstreamInfo(SMJoinOperatorInfo* pInfo, SOperatorInfo*** pDow static int32_t mJoinInitPrimKeyInfo(SMJoinTableCtx* pTable, int32_t slotId) { pTable->primCol = taosMemoryMalloc(sizeof(SMJoinColInfo)); if (NULL == pTable->primCol) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pTable->primCol->srcSlot = slotId; @@ -745,7 +824,7 @@ static int32_t mJoinInitColsInfo(int32_t* colNum, int64_t* rowSize, SMJoinColInf *pCols = taosMemoryMalloc((*colNum) * sizeof(SMJoinColInfo)); if (NULL == *pCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *rowSize = 0; @@ -779,7 +858,7 @@ static int32_t mJoinInitKeyColsInfo(SMJoinTableCtx* pTable, SNodeList* pList, bo pTable->keyBuf = taosMemoryMalloc(TMAX(rowSize, pTable->keyNullSize)); if (NULL == pTable->keyBuf) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -790,7 +869,7 @@ static int32_t mJoinInitKeyColsInfo(SMJoinTableCtx* pTable, SNodeList* pList, bo static int32_t mJoinInitFinColsInfo(SMJoinTableCtx* pTable, SNodeList* pList) { pTable->finCols = taosMemoryMalloc(LIST_LENGTH(pList) * sizeof(SMJoinColMap)); if (NULL == pTable->finCols) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t i = 0; @@ -837,8 +916,20 @@ static int32_t mJoinInitPrimExprCtx(SNode* pNode, SMJoinPrimExprCtx* pCtx, SMJoi } SValueNode* pUnit = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); - SValueNode* pCurrTz = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2) : NULL; + if (NULL == pUnit) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + SValueNode* pCurrTz = NULL; + if (5 == pFunc->pParameterList->length){ + pCurrTz = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2); + if (NULL == pCurrTz) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + } SValueNode* pTimeZone = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 4) : (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3); + if (NULL == pTimeZone) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } pCtx->truncateUnit = pUnit->typeData; if ((NULL == pCurrTz || 1 == pCurrTz->typeData) && pCtx->truncateUnit >= (86400 * TSDB_TICK_PER_SECOND(pFunc->node.resType.precision))) { @@ -859,24 +950,32 @@ static int32_t mJoinInitTableInfo(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysi MJ_ERR_RET(mJoinInitKeyColsInfo(pTable, (0 == idx) ? pJoinNode->pEqLeft : pJoinNode->pEqRight, JOIN_TYPE_FULL == pJoin->joinType)); MJ_ERR_RET(mJoinInitFinColsInfo(pTable, pJoinNode->pTargets)); - memcpy(&pTable->inputStat, pStat, sizeof(*pStat)); + TAOS_MEMCPY(&pTable->inputStat, pStat, sizeof(*pStat)); pTable->eqGrps = taosArrayInit(8, sizeof(SMJoinGrpRows)); - //taosArrayReserve(pTable->eqGrps, 1); + if (NULL == pTable->eqGrps) { + return terrno; + } if (E_JOIN_TB_BUILD == pTable->type) { pTable->createdBlks = taosArrayInit(8, POINTER_BYTES); + if (NULL == pTable->createdBlks) { + return terrno; + } pTable->pGrpArrays = taosArrayInit(32, POINTER_BYTES); + if (NULL == pTable->pGrpArrays) { + return terrno; + } pTable->pGrpHash = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); - if (NULL == pTable->createdBlks || NULL == pTable->pGrpArrays || NULL == pTable->pGrpHash) { - return TSDB_CODE_OUT_OF_MEMORY; + if (NULL == pTable->pGrpHash) { + return terrno; } if (pJoin->pFPreFilter && IS_FULL_OUTER_JOIN(pJoin->joinType, pJoin->subType)) { pTable->rowBitmapSize = MJOIN_ROW_BITMAP_SIZE; pTable->pRowBitmap = taosMemoryMalloc(pTable->rowBitmapSize); if (NULL == pTable->pRowBitmap) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -945,7 +1044,15 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) { SMJoinPrimExprCtx* pCtx = &pTable->primCtx; SColumnInfoData* pPrimIn = taosArrayGet(pBlock->pDataBlock, pTable->primCol->srcSlot); + if (NULL == pPrimIn) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId); + if (NULL == pPrimOut) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + if (0 != pCtx->timezoneUnit) { for (int32_t i = 0; i < pBlock->info.rows; ++i) { ((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] + pCtx->timezoneUnit) % pCtx->truncateUnit; @@ -961,6 +1068,7 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) { SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable) { SSDataBlock* pTmp = NULL; + int32_t code = TSDB_CODE_SUCCESS; int32_t dsIdx = pTable->downStreamIdx; if (E_JOIN_TB_PROBE == pTable->type) { if (pTable->remainInBlk) { @@ -1028,7 +1136,12 @@ SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTa _return: - mJoinLaunchPrimExpr(pTmp, pTable); + code = mJoinLaunchPrimExpr(pTmp, pTable); + if (code) { + pJoin->errCode = code; + T_LONG_JMP(pJoin->pOperator->pTaskInfo->env, pJoin->errCode); + } + return pTmp; } @@ -1041,7 +1154,11 @@ static FORCE_INLINE SSDataBlock* mJoinRetrieveImpl(SMJoinOperatorInfo* pJoin, SM if (NULL == pTmp) { pTable->dsFetchDone = true; } else { - mJoinLaunchPrimExpr(pTmp, pTable); + int32_t code = mJoinLaunchPrimExpr(pTmp, pTable); + if (code) { + pJoin->errCode = code; + T_LONG_JMP(pJoin->pOperator->pTaskInfo->env, pJoin->errCode); + } } return pTmp; @@ -1107,7 +1224,7 @@ bool mJoinRetrieveBlk(SMJoinOperatorInfo* pJoin, int32_t* pIdx, SSDataBlock** pp static void mJoinDestroyCreatedBlks(SArray* pCreatedBlks) { int32_t blkNum = taosArrayGetSize(pCreatedBlks); for (int32_t i = 0; i < blkNum; ++i) { - blockDataDestroy(*(SSDataBlock**)TARRAY_GET_ELEM(pCreatedBlks, i)); + (void)blockDataDestroy(*(SSDataBlock**)TARRAY_GET_ELEM(pCreatedBlks, i)); } taosArrayClear(pCreatedBlks); } @@ -1119,12 +1236,12 @@ int32_t mJoinGetRowBitmapOffset(SMJoinTableCtx* pTable, int32_t rowNum, int32_t int64_t newSize = reqSize * 1.1; pTable->pRowBitmap = taosMemoryRealloc(pTable->pRowBitmap, newSize); if (NULL == pTable->pRowBitmap) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pTable->rowBitmapSize = newSize; } - memset(pTable->pRowBitmap + pTable->rowBitmapOffset, 0xFFFFFFFF, bitmapLen); + TAOS_MEMSET(pTable->pRowBitmap + pTable->rowBitmapOffset, 0xFFFFFFFF, bitmapLen); *rowBitmapOffset = pTable->rowBitmapOffset; pTable->rowBitmapOffset += bitmapLen; @@ -1140,12 +1257,16 @@ void mJoinResetForBuildTable(SMJoinTableCtx* pTable) { taosArrayClear(pTable->eqGrps); if (pTable->rowBitmapSize > 0) { pTable->rowBitmapOffset = 1; - memset(&pTable->nMatchCtx, 0, sizeof(pTable->nMatchCtx)); + TAOS_MEMSET(&pTable->nMatchCtx, 0, sizeof(pTable->nMatchCtx)); } } int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, bool restart) { SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + SMJoinGrpRows* pGrp = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -1159,7 +1280,10 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol bool keepGrp = true; pGrp = taosArrayReserve(pTable->eqGrps, 1); - + if (NULL == pGrp) { + MJ_ERR_RET(terrno); + } + pGrp->beginIdx = pTable->blkRowIdx++; pGrp->readIdx = pGrp->beginIdx; pGrp->endIdx = pGrp->beginIdx; @@ -1209,10 +1333,10 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol if (0 == pGrp->beginIdx && pTable->multiEqGrpRows && 0 == pTable->eqRowLimit) { pGrp->blk = createOneDataBlock(pTable->blk, true); if (NULL == pGrp->blk) { - MJ_ERR_JRET(terrno); + MJ_ERR_RET(terrno); } if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) { - MJ_ERR_JRET(terrno); + MJ_ERR_RET(terrno); } } else { if (!pTable->multiEqGrpRows) { @@ -1236,10 +1360,16 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol pTable->eqRowNum += rowNum; pGrp->blk = blockDataExtractBlock(pTable->blk, pGrp->beginIdx, rowNum); + if (NULL == pGrp->blk) { + MJ_ERR_RET(terrno); + } + pGrp->endIdx -= pGrp->beginIdx; pGrp->beginIdx = 0; pGrp->readIdx = 0; - taosArrayPush(pTable->createdBlks, &pGrp->blk); + if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) { + MJ_ERR_RET(terrno); + } } } @@ -1248,19 +1378,19 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol _return: if (pTable->noKeepEqGrpRows || !keepGrp || (!pTable->multiEqGrpRows && !restart)) { - taosArrayPop(pTable->eqGrps); + (void)taosArrayPop(pTable->eqGrps); } else { pTable->grpTotalRows += pGrp->endIdx - pGrp->beginIdx + 1; } - return TSDB_CODE_SUCCESS; + return code; } int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable, int64_t timestamp) { bool wholeBlk = false; - mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, true); + MJ_ERR_RET(mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, true)); while (wholeBlk && !pTable->dsFetchDone) { pTable->blk = (*pJoin->retrieveFp)(pJoin, pTable); @@ -1273,7 +1403,7 @@ int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable } wholeBlk = false; - mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, false); + MJ_ERR_RET(mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, false)); } return TSDB_CODE_SUCCESS; @@ -1282,6 +1412,10 @@ int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable) { for (int32_t i = 0; i < pTable->keyNum; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot); + if (NULL == pCol) { + MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } + if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) { qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata); return TSDB_CODE_INVALID_PARA; @@ -1326,15 +1460,15 @@ bool mJoinCopyKeyColsDataToBuf(SMJoinTableCtx* pTable, int32_t rowIdx, size_t *p } if (pTable->keyCols[0].jsonData) { pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx]; - memcpy(pTable->keyBuf + bufLen, pData, getJsonValueLen(pData)); + TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, getJsonValueLen(pData)); bufLen += getJsonValueLen(pData); } else if (pTable->keyCols[i].vardata) { pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx]; - memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData)); + TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, varDataTLen(pData)); bufLen += varDataTLen(pData); } else { pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx; - memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes); + TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes); bufLen += pTable->keyCols[i].bytes; } } @@ -1352,15 +1486,20 @@ static int32_t mJoinGetAvailableGrpArray(SMJoinTableCtx* pTable, SArray** ppRes) do { if (pTable->grpArrayIdx < taosArrayGetSize(pTable->pGrpArrays)) { *ppRes = taosArrayGetP(pTable->pGrpArrays, pTable->grpArrayIdx++); + if (NULL == *ppRes) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } taosArrayClear(*ppRes); return TSDB_CODE_SUCCESS; } SArray* pNew = taosArrayInit(4, sizeof(SMJoinRowPos)); if (NULL == pNew) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + if (NULL == taosArrayPush(pTable->pGrpArrays, &pNew)) { + return terrno; } - taosArrayPush(pTable->pGrpArrays, &pNew); } while (true); return TSDB_CODE_SUCCESS; @@ -1374,10 +1513,14 @@ static int32_t mJoinAddRowToHash(SMJoinOperatorInfo* pJoin, size_t keyLen, SSDat SArray* pNewGrp = NULL; MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp)); - taosArrayPush(pNewGrp, &pos); - tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, POINTER_BYTES); + if (NULL == taosArrayPush(pNewGrp, &pos)) { + return terrno; + } + MJ_ERR_RET(tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, POINTER_BYTES)); } else if (pBuild->multiRowsGrp) { - taosArrayPush(*pGrpRows, &pos); + if (NULL == taosArrayPush(*pGrpRows, &pos)) { + return terrno; + } } return TSDB_CODE_SUCCESS; @@ -1392,10 +1535,14 @@ static int32_t mJoinAddRowToFullHash(SMJoinOperatorInfo* pJoin, size_t keyLen, S SMJoinHashGrpRows pNewGrp = {0}; MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp.pRows)); - taosArrayPush(pNewGrp.pRows, &pos); - tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, sizeof(pNewGrp)); + if (NULL == taosArrayPush(pNewGrp.pRows, &pos)) { + return terrno; + } + MJ_ERR_RET(tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, sizeof(pNewGrp))); } else { - taosArrayPush(pGrpRows->pRows, &pos); + if (NULL == taosArrayPush(pGrpRows->pRows, &pos)) { + return terrno; + } } return TSDB_CODE_SUCCESS; @@ -1412,6 +1559,9 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT int32_t grpNum = taosArrayGetSize(pTable->eqGrps); for (int32_t g = 0; g < grpNum; ++g) { SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g); + if (NULL == pGrp) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable)); int32_t grpRows = GRP_REMAIN_ROWS(pGrp); @@ -1440,6 +1590,10 @@ int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable int32_t grpNum = taosArrayGetSize(pTable->eqGrps); for (int32_t g = 0; g < grpNum; ++g) { SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g); + if (NULL == pGrp) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable)); int32_t grpRows = GRP_REMAIN_ROWS(pGrp); @@ -1496,8 +1650,11 @@ void mWinJoinResetWindowCache(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache) { for (int32_t i = 0; i < grpNum; ++i) { SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i); + if (NULL == pGrp) { + continue; + } if (pGrp->blk != pCtx->cache.outBlk && pGrp->clonedBlk) { - blockDataDestroy(pGrp->blk); + (void)blockDataDestroy(pGrp->blk); } } @@ -1539,6 +1696,7 @@ void mJoinResetOperator(struct SOperatorInfo* pOperator) { SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) { SMJoinOperatorInfo* pJoin = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; if (pOperator->status == OP_EXEC_DONE) { if (NULL == pOperator->pDownstreamGetParams || NULL == pOperator->pDownstreamGetParams[0] || NULL == pOperator->pDownstreamGetParams[1]) { qDebug("%s merge join done", GET_TASKID(pOperator->pTaskInfo)); @@ -1567,7 +1725,11 @@ SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) { pBlock->info.id.blockId = pJoin->outBlkId; if (pJoin->pFinFilter != NULL) { - doFilter(pBlock, pJoin->pFinFilter, NULL); + code = doFilter(pBlock, pJoin->pFinFilter, NULL); + if (code) { + pJoin->errCode = code; + T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode); + } } if (pBlock->info.rows > 0 || pOperator->status == OP_EXEC_DONE) { @@ -1703,13 +1865,17 @@ int32_t mJoinSetImplFp(SMJoinOperatorInfo* pJoin) { SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { - SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); bool newDownstreams = false; - int32_t code = TSDB_CODE_SUCCESS; - if (pOperator == NULL || pInfo == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo)); + if (pInfo == NULL) { + code = terrno; + goto _return; + } + + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pOperator == NULL) { + code = terrno; goto _return; } @@ -1722,8 +1888,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t MJ_ERR_JRET(mJoinHandleConds(pInfo, pJoinNode)); - mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0], newDownstreams); - mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1], newDownstreams); + MJ_ERR_JRET(mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0], newDownstreams)); + MJ_ERR_JRET(mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1], newDownstreams)); MJ_ERR_JRET(mJoinInitCtx(pInfo, pJoinNode)); MJ_ERR_JRET(mJoinSetImplFp(pInfo)); @@ -1742,6 +1908,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t return pOperator; _return: + if (pInfo != NULL) { destroyMergeJoinOperator(pInfo); } @@ -1751,6 +1918,7 @@ _return: taosMemoryFree(pOperator); pTaskInfo->code = code; + return NULL; } From 2a675ccf6b63cce7a8221a00857b087206b90de6 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 22 Jul 2024 11:19:11 +0800 Subject: [PATCH 13/43] group const value func --- include/libs/function/functionMgt.h | 4 ++-- source/libs/executor/src/executorInt.c | 2 +- source/libs/executor/src/timesliceoperator.c | 8 ++++---- source/libs/function/inc/builtinsimpl.h | 2 ++ source/libs/function/src/builtins.c | 12 ++++++------ source/libs/function/src/builtinsimpl.c | 18 +++++++++++++----- source/libs/function/src/functionMgt.c | 4 ++-- source/libs/parser/src/parTranslater.c | 4 ++-- source/libs/planner/src/planLogicCreater.c | 2 +- 9 files changed, 33 insertions(+), 23 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 9fb4c80bb4..d3fb953dc3 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -138,7 +138,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_CACHE_LAST_ROW, FUNCTION_TYPE_CACHE_LAST, FUNCTION_TYPE_TABLE_COUNT, - FUNCTION_TYPE_SELECT_TAG, + FUNCTION_TYPE_GROUP_CONST_VALUE, // distributed splitting functions FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, @@ -257,7 +257,7 @@ bool fmIsConstantResFunc(SFunctionNode* pFunc); bool fmIsSkipScanCheckFunc(int32_t funcId); bool fmIsPrimaryKeyFunc(int32_t funcId); bool fmIsProcessByRowFunc(int32_t funcId); -bool fmIsSelectTagFunc(int32_t funcId); +bool fmisSelectGroupConstValueFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType, int32_t pkBytes); SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 3b4e0792a6..df70dfafe1 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -604,7 +604,7 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_key") == 0 || - strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_tag") == 0) { + strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_const_value") == 0) { // for groupkey along with functions that output multiple lines(e.g. Histogram) // need to match groupkey result for each output row of that function. if (pCtx[j].resultInfo->numOfRes != 0) { diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 5a448520f1..e5a1bd0701 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -233,9 +233,9 @@ static bool isGroupKeyFunc(SExprInfo* pExprInfo) { return (functionType == FUNCTION_TYPE_GROUP_KEY); } -static bool isSelectTagFunc(SExprInfo* pExprInfo) { +static bool isSelectGroupConstValueFunc(SExprInfo* pExprInfo) { int32_t functionType = pExprInfo->pExpr->_function.functionType; - return (functionType == FUNCTION_TYPE_SELECT_TAG); + return (functionType == FUNCTION_TYPE_GROUP_CONST_VALUE); } static bool getIgoreNullRes(SExprSupp* pExprSup) { @@ -301,7 +301,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); continue; } else if (!isInterpFunc(pExprInfo)) { - if (isGroupKeyFunc(pExprInfo) || isSelectTagFunc(pExprInfo)) { + if (isGroupKeyFunc(pExprInfo) || isSelectGroupConstValueFunc(pExprInfo)) { if (pSrcBlock != NULL) { int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); @@ -313,7 +313,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp char* v = colDataGetData(pSrc, index); colDataSetVal(pDst, pResBlock->info.rows, v, false); - } else if(!isSelectTagFunc(pExprInfo)){ + } else if(!isSelectGroupConstValueFunc(pExprInfo)){ // use stored group key SGroupKeys* pkey = pSliceInfo->pPrevGroupKey; if (pkey->isNull == false) { diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 7615584f8c..b48a617b9c 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -254,6 +254,8 @@ bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t groupKeyFunction(SqlFunctionCtx* pCtx); int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t groupConstValueFunction(SqlFunctionCtx* pCtx); +int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 75d6047c07..f5c98933fd 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -4109,14 +4109,14 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = NULL }, { - .name = "_select_tag", - .type = FUNCTION_TYPE_SELECT_TAG, + .name = "_group_const_value", + .type = FUNCTION_TYPE_GROUP_CONST_VALUE, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, - .translateFunc = translateGroupKey, - .getEnvFunc = getGroupKeyFuncEnv, + .translateFunc = translateSelectValue, + .getEnvFunc = getSelectivityFuncEnv, .initFunc = functionSetup, - .processFunc = groupKeyFunction, - .finalizeFunc = groupKeyFinalize, + .processFunc = groupConstValueFunction, + .finalizeFunc = groupConstValueFinalize, }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index aab5a52776..5f6565be1f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6615,7 +6615,7 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } -int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { +int32_t groupConstValueFunction(SqlFunctionCtx* pCtx) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -6626,13 +6626,13 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { // escape rest of data blocks to avoid first entry to be overwritten. if (pInfo->hasResult) { - goto _group_key_over; + goto _group_value_over; } if (pInputCol->pData == NULL || colDataIsNull_s(pInputCol, startIndex)) { pInfo->isNull = true; pInfo->hasResult = true; - goto _group_key_over; + goto _group_value_over; } char* data = colDataGetData(pInputCol, startIndex); @@ -6644,13 +6644,17 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { } pInfo->hasResult = true; -_group_key_over: +_group_value_over: SET_VAL(pResInfo, 1, 1); return TSDB_CODE_SUCCESS; } -int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { +int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { + return groupConstValueFunction(pCtx); +} + +int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -6670,6 +6674,10 @@ int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock){ + return groupConstValueFinalize(pCtx, pBlock); +} + int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SGroupKeyInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 9cb7d59a2c..8255997836 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -268,11 +268,11 @@ bool fmIsGroupKeyFunc(int32_t funcId) { return FUNCTION_TYPE_GROUP_KEY == funcMgtBuiltins[funcId].type; } -bool fmIsSelectTagFunc(int32_t funcId) { +bool fmisSelectGroupConstValueFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; } - return FUNCTION_TYPE_SELECT_TAG == funcMgtBuiltins[funcId].type; + return FUNCTION_TYPE_GROUP_CONST_VALUE == funcMgtBuiltins[funcId].type; } bool fmIsBlockDistFunc(int32_t funcId) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f4b318422a..fef58df3dc 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3169,7 +3169,7 @@ static EDealRes rewriteExprToSelectTagFunc(STranslateContext* pCxt, SNode** pNod return DEAL_RES_ERROR; } - strcpy(pFunc->functionName, "_select_tag"); + strcpy(pFunc->functionName, "_group_const_value"); strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); @@ -4974,7 +4974,7 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { static EDealRes needFillImpl(SNode* pNode, void* pContext) { if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType - && FUNCTION_TYPE_SELECT_TAG != ((SFunctionNode*)pNode)->funcType) { + && FUNCTION_TYPE_GROUP_CONST_VALUE != ((SFunctionNode*)pNode)->funcType) { *(bool*)pContext = true; return DEAL_RES_END; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 89a1da3585..cee765ff94 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -891,7 +891,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt } static bool isInterpFunc(int32_t funcId) { - return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId) || fmIsSelectTagFunc(funcId); + return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId) || fmisSelectGroupConstValueFunc(funcId); } static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { From d8e4f2d743f4d79bccf30fd971188aa97ce17b8a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 22 Jul 2024 11:34:25 +0800 Subject: [PATCH 14/43] operator --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 583dd7b41b..7a6d5cb730 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3556,7 +3556,7 @@ static int32_t tagScanCreateResultData(SDataType* pType, int32_t numOfRows, SSca } static EDealRes tagScanRewriteTagColumn(SNode** pNode, void* pContext) { - SColumnNode* pSColumnNode = NULL; + SColumnNode* pSColumnNode = NULL;; if (QUERY_NODE_COLUMN == nodeType((*pNode))) { pSColumnNode = *(SColumnNode**)pNode; } else if (QUERY_NODE_FUNCTION == nodeType((*pNode))) { From c01984941334c2ac8866411a320a363a98902763 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 22 Jul 2024 11:34:52 +0800 Subject: [PATCH 15/43] add test case --- tests/system-test/2-query/interp.py | 146 ++++++++++++++++++---------- 1 file changed, 93 insertions(+), 53 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index bb96f6342c..bcfc389d7b 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -43,60 +43,59 @@ class TDTestCase: tdSql.execute("create database db1 keep 36500") tdSql.execute("use db1") - tdSql.execute("CREATE STABLE db1.`stb1` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `v1` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium') TAGS (`t1` INT)") - - tdSql.execute("insert into db1.ttt_10000 using db1.stb1 tags( 44400 ) values('2024-02-19 16:05:17.649', 22300 ); ") - tdSql.execute("insert into db1.ttt_10000 using db1.stb1 tags( 44400 ) values('2024-02-19 16:05:48.818', 22300 ); ") - tdSql.execute("insert into db1.ttt_10 using db1.stb1 tags( 40 ) values('2024-02-19 16:25:36.013', 20 ); ") - tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-19 16:39:50.385' , 20 ); ") - tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-19 16:43:51.742' , 20 ); ") - tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-20 08:35:13.518' , 20 ); ") - tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-20 08:58:42.255' , 20 ); ") - tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 ) values('2024-02-21 09:57:49.477' , 20 ); ") - tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 ) values('2024-02-21 09:58:21.882' , 20 ); ") - tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 ) values('2024-02-26 16:08:31.675' , 20 ); ") - tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 ) values('2024-02-26 16:11:43.445' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:12:30.276' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:07.188' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:07.653' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:07.879' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.083' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.273' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.429' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.599' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.775' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:08.940' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:09.110' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:09.254' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:09.409' , NULL ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:34.750' , 12 ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:49.820' , 12 ); ") - tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 ) values('2024-02-26 16:25:59.551' , NULL ); ") - tdSql.execute("insert into db1.ttt_2 using db1.stb1 tags( 2 ) values('2024-02-19 15:26:39.644' , 2 ); ") - tdSql.execute("insert into db1.ttt_2 using db1.stb1 tags( 2 ) values('2024-02-19 15:26:40.433' , 2 ); ") - tdSql.execute("insert into db1.ttt_3 using db1.stb1 tags( 3 ) values('2024-02-19 15:27:22.613' , 1 ); ") - tdSql.execute("insert into db1.ttt_13 using db1.stb1 tags( 3 ) values('2024-02-19 15:27:39.719' , 1 ); ") - tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 ) values('2024-02-19 15:28:36.235' , 222 ); ") - tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 ) values('2024-02-19 15:28:59.310' , 222 ); ") - tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 ) values('2024-02-19 15:29:18.897' , 222 ); ") - tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 ) values('2024-02-19 15:50:24.682' , 223 ); ") - tdSql.execute("insert into db1.ttt_4 using db1.stb1 tags( 3 ) values('2024-02-19 15:31:19.945' , 222 ); ") - tdSql.execute("insert into db1.ttt_a using db1.stb1 tags( 3 ) values('2024-02-19 15:31:37.915' , 4 ); ") - tdSql.execute("insert into db1.ttt_axxxx using db1.stb1 tags( NULL ) values('2024-02-19 15:31:58.953' , 4 ); ") - tdSql.execute("insert into db1.ttt_axxx using db1.stb1 tags( 56 ) values('2024-02-19 15:32:22.323' , NULL ); ") - tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-19 15:36:44.625' , 5444 ); ") - tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-19 15:38:41.479' , 5444 ); ") - tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-19 15:57:23.249' , 5444 ); ") - tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-19 16:04:20.465' , 5444 ); ") - tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-26 16:11:29.364' , 5444 ); ") - tdSql.execute("insert into db1.ttt_123 using db1.stb1 tags( 0 ) values('2024-02-19 15:44:52.136' , 223 ); ") - tdSql.execute("insert into db1.ttt_145 using db1.stb1 tags( 0 ) values('2024-02-19 15:50:28.580' , 223 ); ") - tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 ) values('2024-02-19 15:50:32.493' , 223 ); ") - tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 ) values('2024-02-19 15:57:36.866' , 223 ); ") - tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 ) values('2024-02-19 16:04:52.794' , 221113 ); ") - tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-27 08:47:11.366' , 5444 ); ") - tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633 ) values('2024-02-28 09:35:46.474' , 5444 ); ") + tdSql.execute("CREATE STABLE db1.`stb1` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `v1` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium') TAGS (`t1` INT, t2 nchar(20))") + tdSql.execute("insert into db1.ttt_10000 using db1.stb1 tags(44400, '_ttt_10000') values('2024-02-19 16:05:17.649', 22300 ); ") + tdSql.execute("insert into db1.ttt_10000 using db1.stb1 tags(44400, '_ttt_10000') values('2024-02-19 16:05:48.818', 22300 ); ") + tdSql.execute("insert into db1.ttt_10 using db1.stb1 tags( 40 , '_ttt_10') values('2024-02-19 16:25:36.013', 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 , '_ttt_11') values('2024-02-19 16:39:50.385' , 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 , '_ttt_11') values('2024-02-19 16:43:51.742' , 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 , '_ttt_11') values('2024-02-20 08:35:13.518' , 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 , '_ttt_11') values('2024-02-20 08:58:42.255' , 20 ); ") + tdSql.execute("insert into db1.ttt_11 using db1.stb1 tags( 11 , '_ttt_11') values('2024-02-21 09:57:49.477' , 20 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 , '_ttt_2024-2-21') values('2024-02-21 09:58:21.882' , 20 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 , '_ttt_2024-2-21') values('2024-02-26 16:08:31.675' , 20 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-21` using db1.stb1 tags( 11 , '_ttt_2024-2-21') values('2024-02-26 16:11:43.445' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:12:30.276' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:07.188' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:07.653' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:07.879' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:08.083' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:08.273' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:08.429' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:08.599' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:08.775' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:08.940' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:09.110' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:09.254' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:09.409' , NULL ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:34.750' , 12 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:49.820' , 12 ); ") + tdSql.execute("insert into db1.`ttt_2024-2-33` using db1.stb1 tags( 11 , '_ttt_2024-2-33') values('2024-02-26 16:25:59.551' , NULL ); ") + tdSql.execute("insert into db1.ttt_2 using db1.stb1 tags( 2 , '_ttt_2') values('2024-02-19 15:26:39.644' , 2 ); ") + tdSql.execute("insert into db1.ttt_2 using db1.stb1 tags( 2 , '_ttt_2') values('2024-02-19 15:26:40.433' , 2 ); ") + tdSql.execute("insert into db1.ttt_3 using db1.stb1 tags( 3 , '_ttt_3') values('2024-02-19 15:27:22.613' , 1 ); ") + tdSql.execute("insert into db1.ttt_13 using db1.stb1 tags( 3 , '_ttt_13') values('2024-02-19 15:27:39.719' , 1 ); ") + tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 , '_ttt_14') values('2024-02-19 15:28:36.235' , 222 ); ") + tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 , '_ttt_14') values('2024-02-19 15:28:59.310' , 222 ); ") + tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 , '_ttt_14') values('2024-02-19 15:29:18.897' , 222 ); ") + tdSql.execute("insert into db1.ttt_14 using db1.stb1 tags( 3 , '_ttt_14') values('2024-02-19 15:50:24.682' , 223 ); ") + tdSql.execute("insert into db1.ttt_4 using db1.stb1 tags( 3 , '_ttt_4') values('2024-02-19 15:31:19.945' , 222 ); ") + tdSql.execute("insert into db1.ttt_a using db1.stb1 tags( 3 , '_ttt_a') values('2024-02-19 15:31:37.915' , 4 ); ") + tdSql.execute("insert into db1.ttt_axxxx using db1.stb1 tags( NULL, '_ttt_axxxx') values('2024-02-19 15:31:58.953' , 4 ); ") + tdSql.execute("insert into db1.ttt_axxx using db1.stb1 tags( 56 , '_ttt_axxx') values('2024-02-19 15:32:22.323' , NULL ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633, '_ttt_444') values('2024-02-19 15:36:44.625' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633, '_ttt_444') values('2024-02-19 15:38:41.479' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633, '_ttt_444') values('2024-02-19 15:57:23.249' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633, '_ttt_444') values('2024-02-19 16:04:20.465' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633, '_ttt_444') values('2024-02-26 16:11:29.364' , 5444 ); ") + tdSql.execute("insert into db1.ttt_123 using db1.stb1 tags( 0 , '_ttt_123') values('2024-02-19 15:44:52.136' , 223 ); ") + tdSql.execute("insert into db1.ttt_145 using db1.stb1 tags( 0 , '_ttt_145') values('2024-02-19 15:50:28.580' , 223 ); ") + tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 , '_ttt_1465') values('2024-02-19 15:50:32.493' , 223 ); ") + tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 , '_ttt_1465') values('2024-02-19 15:57:36.866' , 223 ); ") + tdSql.execute("insert into db1.ttt_1465 using db1.stb1 tags( 0 , '_ttt_1465') values('2024-02-19 16:04:52.794' , 221113 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633, '_ttt_444') values('2024-02-27 08:47:11.366' , 5444 ); ") + tdSql.execute("insert into db1.ttt_444 using db1.stb1 tags( 5633, '_ttt_444') values('2024-02-28 09:35:46.474' , 5444 ); ") tdSql.query("select *,tbname from db1.stb1 ;") tdSql.checkRows(51) @@ -232,6 +231,47 @@ class TDTestCase: tdSql.checkData(0, 1, 5) tdSql.checkData(0, 2, 0) tdSql.checkData(0, 3, "ttt_123") + + tdSql.query("select _irowts as ts,interp(v1),t1, t2, tbname from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(VALUE_F, 5) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 1, 5) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, "_ttt_123") + tdSql.checkData(0, 4, "ttt_123") + + tdSql.query("select _irowts as ts,interp(v1),t1,tbname, t2 from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) fill(prev)") + tdSql.checkRows(4) + + tdSql.query("select _irowts as ts,interp(v1),t1,tbname, t2 from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) fill(prev) order by tbname") + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 2, 2) + tdSql.checkData(3, 2, 3) + tdSql.checkData(0, 3, "ttt_13") + tdSql.checkData(1, 3, "ttt_14") + tdSql.checkData(2, 3, "ttt_2") + tdSql.checkData(3, 3, "ttt_3") + tdSql.checkData(0, 4, "_ttt_13") + tdSql.checkData(1, 4, "_ttt_14") + tdSql.checkData(2, 4, "_ttt_2") + tdSql.checkData(3, 4, "_ttt_3") + + tdSql.query("select _irowts as ts,interp(v1),t1,t2 from db1.stb1 \ + where ts>'2024-02-19T15:25:00+08:00' and ts<'2024-02-19T16:05:00+08:00' \ + partition by tbname range('2024-02-19T15:30:00+08:00','2024-02-19T15:30:00+08:00') every(1m) \ + fill(value, 0) order by tbname") + tdSql.checkRows(12) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, "_ttt_123") + tdSql.checkData(1, 2, 3) + tdSql.checkData(1, 3, "_ttt_13") def run(self): dbname = "db" From be162fb62fd9629abb0f39aba500d0cd78e099b0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Jul 2024 13:31:57 +0800 Subject: [PATCH 16/43] fix(stream): check return value. --- source/dnode/mnode/impl/inc/mndStream.h | 53 +- source/dnode/mnode/impl/src/mndStream.c | 304 +++++++---- source/dnode/mnode/impl/src/mndStreamHb.c | 86 +-- source/dnode/mnode/impl/src/mndStreamTrans.c | 32 +- source/dnode/mnode/impl/src/mndStreamUtil.c | 519 +++++++++++++------ source/libs/stream/src/streamMeta.c | 2 +- source/util/src/tcompression.c | 4 +- 7 files changed, 655 insertions(+), 345 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 0a93848843..bd0d97e34d 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -113,38 +113,39 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList); void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t retryCode, int32_t acceptCode); -STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg); +int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, + const char *pMsg, STrans **pTrans1); int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo); int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); -SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); -int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); -int32_t mndProcessStreamHb(SRpcMsg *pReq); -void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); -int32_t extractStreamNodeList(SMnode *pMnode); -int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated); -int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); -int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); -int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList); -int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq); -int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId, - int64_t ts); -void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); +int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream); +int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); +int32_t mndProcessStreamHb(SRpcMsg *pReq); +void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); +int32_t extractStreamNodeList(SMnode *pMnode); +int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated); +int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); +int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); +int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList); +int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq); +int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId, + int64_t ts); +void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); -SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); -void destroyStreamTaskIter(SStreamTaskIter *pIter); -bool streamTaskIterNextTask(SStreamTaskIter *pIter); -SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); -void mndInitExecInfo(); -void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo); -int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); -void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); +int32_t createStreamTaskIter(SStreamObj *pStream, SStreamTaskIter **pIter); +void destroyStreamTaskIter(SStreamTaskIter *pIter); +bool streamTaskIterNextTask(SStreamTaskIter *pIter); +int32_t streamTaskIterGetCurrent(SStreamTaskIter *pIter, SStreamTask **pTask); +int32_t mndInitExecInfo(); +void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo); +int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); +void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); int32_t mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo); void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index fa4b9b64f4..6362fbaa79 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -134,17 +134,18 @@ int32_t mndInitStream(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask); - mndInitExecInfo(); - - if (sdbSetTable(pMnode->pSdb, table) != 0) { - return -1; + int32_t code = mndInitExecInfo(); + if (code) { + return code; } - if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) { - return -1; + code = sdbSetTable(pMnode->pSdb, table); + if (code) { + return terrno; } - return 0; + code = sdbSetTable(pMnode->pSdb, tableSeq); + return code; } void mndCleanupStream(SMnode *pMnode) { @@ -252,6 +253,8 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream } int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream) { + terrno = 0; + SSdb *pSdb = pMnode->pSdb; (*pStream) = sdbAcquire(pSdb, SDB_STREAM, streamName); if ((*pStream) == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -530,9 +533,21 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { } int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) { - SStreamTaskIter *pIter = createStreamTaskIter(pStream); + SStreamTaskIter *pIter = NULL; + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + mError("failed to create task iter for stream:%s", pStream->name); + return code; + } + while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { + destroyStreamTaskIter(pIter); + return code; + } + if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { destroyStreamTaskIter(pIter); return -1; @@ -727,7 +742,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } code = mndAcquireStream(pMnode, createReq.name, &pStream); - if (pStream != NULL || code != 0) { + if (pStream != NULL || code == 0) { if (createReq.igExists) { mInfo("stream:%s, already exist, ignore exist is set", createReq.name); goto _OVER; @@ -760,8 +775,9 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg); - if (pTrans == NULL) { + STrans *pTrans = NULL; + code = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg, &pTrans); + if (pTrans == NULL || code) { goto _OVER; } @@ -802,11 +818,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { // add into buffer firstly // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name); saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo); -// mndRegisterConsensusChkptId(execInfo.pStreamConsensus, streamObj.uid); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); // execute creation if (mndTransPrepare(pMnode, pTrans) != 0) { @@ -867,7 +882,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) { { // check the max checkpoint id from all vnodes. int64_t maxCheckpointId = -1; if (lock) { - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); } for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { @@ -888,7 +903,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) { } if (lock) { - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); } if (maxCheckpointId > maxChkptId) { @@ -989,11 +1004,13 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre return -1; } - STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME, - "gen checkpoint for stream"); - if (pTrans == NULL) { + STrans *pTrans = NULL; + code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME, + "gen checkpoint for stream", &pTrans); + if (pTrans == NULL || code) { + code = TSDB_CODE_MND_TRANS_CONFLICT; mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId, - tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); + tstrerror(code)); goto _ERR; } @@ -1033,7 +1050,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre taosWUnLockLatch(&pStream->lock); if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY)) != TSDB_CODE_SUCCESS) { - return code; + goto _ERR; } if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) { @@ -1057,13 +1074,13 @@ int32_t extractStreamNodeList(SMnode *pMnode) { static bool taskNodeIsUpdated(SMnode *pMnode) { // check if the node update happens or not - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); int32_t numOfNodes = extractStreamNodeList(pMnode); if (numOfNodes == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); execInfo.ts = taosGetTimestampSec(); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); return false; } @@ -1071,7 +1088,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i); if (pNodeEntry->stageUpdated) { mDebug("stream task not ready due to node update detected, checkpoint not issued"); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); return true; } } @@ -1086,7 +1103,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { if (!allReady) { mWarn("not all vnodes ready, quit from vnodes status check"); taosArrayDestroy(pNodeSnapshot); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); return true; } @@ -1102,7 +1119,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { mDebug("stream tasks not ready due to node update"); } - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); return nodeUpdated; } @@ -1112,7 +1129,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { return -1; } - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); if (taosArrayGetSize(execInfo.pNodeList) == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); @@ -1157,7 +1174,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { removeTasksInBuf(pInvalidList, &execInfo); taosArrayDestroy(pInvalidList); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); return ready ? 0 : -1; } @@ -1220,14 +1237,14 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { continue; } - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); int64_t startTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid); if (startTs != -1 && (now - startTs) < tsStreamCheckpointInterval * 1000) { - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); sdbRelease(pSdb, pStream); continue; } - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration}; taosArrayPush(pList, &in); @@ -1270,8 +1287,9 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { for (int32_t i = 0; i < numOfQual; ++i) { SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i); - SStreamObj *p = mndGetStreamObj(pMnode, pCheckpointInfo->streamId); - if (p != NULL) { + SStreamObj *p = NULL; + code = mndGetStreamObj(pMnode, pCheckpointInfo->streamId, &p); + if (p != NULL || code != 0) { code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true); sdbRelease(pSdb, p); @@ -1362,8 +1380,9 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); - if (pTrans == NULL) { + STrans *pTrans = NULL; + code = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans); + if (pTrans == NULL || code) { mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr()); sdbRelease(pMnode->pSdb, pStream); tFreeMDropStreamReq(&dropReq); @@ -1863,9 +1882,9 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t numOfRows = 0; SStreamObj *pStream = NULL; - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); mndInitStreamExecInfo(pMnode, &execInfo); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); @@ -1882,11 +1901,24 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } // add row for each task - SStreamTaskIter *pIter = createStreamTaskIter(pStream); - while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); + SStreamTaskIter *pIter = NULL; + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + taosRUnLockLatch(&pStream->lock); + sdbRelease(pSdb, pStream); + mError("failed to create task iter for stream:%s", pStream->name); + continue; + } - int32_t code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows); + while (streamTaskIterNextTask(pIter)) { + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { + destroyStreamTaskIter(pIter); + break; + } + + code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows); if (code == TSDB_CODE_SUCCESS) { numOfRows++; } @@ -1961,7 +1993,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { { // check for tasks, if tasks are not ready, not allowed to pause bool found = false; bool readyToPause = true; - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { STaskId *p = taosArrayGet(execInfo.pTaskList, i); @@ -1984,7 +2016,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { found = true; } - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); if (!found) { mError("stream:%s task not report status yet, not ready for pause", pauseReq.name); sdbRelease(pMnode->pSdb, pStream); @@ -1998,42 +2030,49 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { } } - STrans *pTrans = - doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream"); - if (pTrans == NULL) { + STrans *pTrans = NULL; + code = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream", &pTrans); + if (pTrans == NULL || code) { mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); - return -1; + return code; } code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid); + if (code) { + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return code; + } // if nodeUpdate happened, not send pause trans - if (mndStreamSetPauseAction(pMnode, pTrans, pStream) < 0) { + code = mndStreamSetPauseAction(pMnode, pTrans, pStream); + if (code) { mError("stream:%s, failed to pause task since %s", pauseReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } // pause stream taosWLockLatch(&pStream->lock); pStream->status = STREAM_STATUS__PAUSE; - if (mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY) < 0) { + code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); + if (code) { taosWUnLockLatch(&pStream->lock); - sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } taosWUnLockLatch(&pStream->lock); - if (mndTransPrepare(pMnode, pTrans) != 0) { + code = mndTransPrepare(pMnode, pTrans); + if (code) { mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } sdbRelease(pMnode->pSdb, pStream); @@ -2087,22 +2126,28 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = - doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream"); - if (pTrans == NULL) { + STrans *pTrans = NULL; + code = + doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream", &pTrans); + if (pTrans == NULL || code) { mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); - return -1; + return code; } code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid); + if (code) { + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return code; + } // set the resume action if (mndStreamSetResumeAction(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) { mError("stream:%s, failed to drop task since %s", resumeReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } // resume stream @@ -2113,7 +2158,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } taosWUnLockLatch(&pStream->lock); @@ -2121,7 +2166,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } sdbRelease(pMnode->pSdb, pStream); @@ -2195,6 +2240,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange SStreamObj *pStream = NULL; void *pIter = NULL; STrans *pTrans = NULL; + int32_t code = 0; // conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool while (1) { @@ -2221,12 +2267,11 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange // here create only one trans if (pTrans == NULL) { - pTrans = - doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets"); - if (pTrans == NULL) { + code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets", &pTrans); + if (pTrans == NULL || code) { sdbRelease(pSdb, pStream); sdbCancelFetch(pSdb, pIter); - return terrno; + return terrno = code; } mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); @@ -2243,7 +2288,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, pStream->name, pTrans->id); - int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); + code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); // todo: not continue, drop all and retry again if (code != TSDB_CODE_SUCCESS) { @@ -2258,7 +2303,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange if (code != TSDB_CODE_SUCCESS) { sdbCancelFetch(pSdb, pIter); - return -1; + return code; } } @@ -2267,16 +2312,17 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange return 0; } - if (mndTransPrepare(pMnode, pTrans) != 0) { + code = mndTransPrepare(pMnode, pTrans); + if (code) { mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return 0; + return code; } static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { @@ -2293,9 +2339,21 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { taosWLockLatch(&pStream->lock); - SStreamTaskIter *pTaskIter = createStreamTaskIter(pStream); + SStreamTaskIter *pTaskIter = NULL; + int32_t code = createStreamTaskIter(pStream, &pTaskIter); + if (code) { + taosWUnLockLatch(&pStream->lock); + sdbRelease(pSdb, pStream); + mError("failed to create task iter for stream:%s", pStream->name); + continue; + } + while (streamTaskIterNextTask(pTaskIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pTaskIter); + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pTaskIter, &pTask); + if (code) { + break; + } SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId}; epsetAssign(&entry.epset, &pTask->info.epSet); @@ -2342,9 +2400,9 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); int32_t numOfNodes = extractStreamNodeList(pMnode); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); if (numOfNodes == 0) { mDebug("end to do stream task(s) node change checking, no stream tasks exist, do nothing"); @@ -2368,7 +2426,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { return 0; } - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot); @@ -2392,7 +2450,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { } taosArrayDestroy(pNodeSnapshot); - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); taosArrayDestroy(changeInfo.pUpdateNodeList); taosHashCleanup(changeInfo.pDBMap); @@ -2418,9 +2476,19 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) { } void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { - SStreamTaskIter *pIter = createStreamTaskIter(pStream); + SStreamTaskIter *pIter = NULL; + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + mError("failed to create task iter for stream:%s", pStream->name); + return; + } + while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { + break; + } STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); @@ -2490,10 +2558,11 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { mDebug("receive stream task checkpoint req msg, vgId:%d, s-task:0x%x", req.nodeId, req.taskId); // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); - SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); - if (pStream == NULL) { + SStreamObj *pStream = NULL; + int32_t code = mndGetStreamObj(pMnode, req.streamId, &pStream); + if (pStream == NULL || code != 0) { mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf", req.streamId); @@ -2504,7 +2573,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { if (p == NULL) { mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId); terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); return -1; } else { mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", @@ -2549,7 +2618,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { mndReleaseStream(pMnode, pStream); } - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); { SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)}; @@ -2609,10 +2678,11 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId); // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); - SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); - if (pStream == NULL) { + SStreamObj *pStream = NULL; + int32_t code = mndGetStreamObj(pMnode, req.streamId, &pStream); + if (pStream == NULL || code != 0) { mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId); // not in meta-store yet, try to acquire the task in exec buffer @@ -2622,7 +2692,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { if (p == NULL) { mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId); terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); return -1; } else { mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", @@ -2654,7 +2724,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { mndReleaseStream(pMnode, pStream); } - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS); return 0; @@ -2719,7 +2789,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, // req.nodeId, req.streamId, req.taskId, req.checkpointId); // // // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. -// taosThreadMutexLock(&execInfo.lock); +// streamMutexLock(&execInfo.lock); // // // mnode handle the create stream transaction too slow may cause this problem // SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); @@ -2733,7 +2803,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, // if (p == NULL) { // mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId); // terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; -// taosThreadMutexUnlock(&execInfo.lock); +// streamMutexUnlock(&execInfo.lock); // // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // return -1; @@ -2749,7 +2819,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, // // int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream); // if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly -// taosThreadMutexUnlock(&execInfo.lock); +// streamMutexUnlock(&execInfo.lock); // mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs); // // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); @@ -2766,7 +2836,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, // SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks); // mndAddConsensusTasks(pInfo, &req); // -// taosThreadMutexUnlock(&execInfo.lock); +// streamMutexUnlock(&execInfo.lock); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // return 0; // } @@ -2776,7 +2846,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, // req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId); // mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs); // -// taosThreadMutexUnlock(&execInfo.lock); +// streamMutexUnlock(&execInfo.lock); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // return 0; // } @@ -2789,7 +2859,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId, // mndReleaseStream(pMnode, pStream); // } // -// taosThreadMutexUnlock(&execInfo.lock); +// streamMutexUnlock(&execInfo.lock); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // return 0; //} @@ -2816,7 +2886,7 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) { return 0; } - taosThreadMutexLock(&execInfo.lock); + streamMutexLock(&execInfo.lock); void *pIter = NULL; while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) { @@ -2826,8 +2896,9 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) { int32_t num = taosArrayGetSize(pInfo->pTaskList); SArray *pList = taosArrayInit(4, sizeof(int32_t)); - SStreamObj *pStream = mndGetStreamObj(pMnode, pInfo->streamId); - if (pStream == NULL) { // stream has been dropped already + SStreamObj *pStream = NULL; + code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream); + if (pStream == NULL || code != 0) { // stream has been dropped already mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId); taosArrayDestroy(pList); continue; @@ -2886,14 +2957,14 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) { for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) { int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i); - mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId); + code = mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId); } - taosThreadMutexUnlock(&execInfo.lock); + streamMutexUnlock(&execInfo.lock); taosArrayDestroy(pStreamList); mDebug("end to process consensus-checkpointId in tmr"); - return TSDB_CODE_SUCCESS; + return code; } static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) { @@ -2944,32 +3015,41 @@ void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) { } int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) { - STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, - "update checkpoint-info"); - if (pTrans == NULL) { - return terrno; + STrans *pTrans = NULL; + int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, + "update checkpoint-info", &pTrans); + if (pTrans == NULL || code) { + sdbRelease(pMnode->pSdb, pStream); + return code; } - /*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); - int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream); - if (code != 0) { + code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); + if (code){ + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return code; + } + + code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream); + if (code) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); return code; } code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); - if (code != TSDB_CODE_SUCCESS) { + if (code) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } - if (mndTransPrepare(pMnode, pTrans) != 0) { + code = mndTransPrepare(pMnode, pTrans); + if (code) { mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } sdbRelease(pMnode->pSdb, pStream); diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 5f0434e3d0..c5297b5ba8 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -61,15 +61,23 @@ void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) { } int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { - STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, - " reset from failed checkpoint"); - if (pTrans == NULL) { + STrans *pTrans = NULL; + int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, + " reset from failed checkpoint", &pTrans); + if (pTrans == NULL || code) { + sdbRelease(pMnode->pSdb, pStream); return terrno; } - /*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); - int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); - if (code != 0) { + code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); + if (code) { + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return code; + } + + code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); + if (code) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); return code; @@ -79,14 +87,15 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { if (code != TSDB_CODE_SUCCESS) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } - if (mndTransPrepare(pMnode, pTrans) != 0) { + code = mndTransPrepare(pMnode, pTrans); + if (code != 0) { mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } sdbRelease(pMnode->pSdb, pStream); @@ -99,8 +108,9 @@ int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, int32_t t int32_t code = TSDB_CODE_SUCCESS; mndKillTransImpl(pMnode, transId, ""); - SStreamObj *pStream = mndGetStreamObj(pMnode, streamId); - if (pStream == NULL) { + SStreamObj *pStream = NULL; + code = mndGetStreamObj(pMnode, streamId, &pStream); + if (pStream == NULL || code != 0) { code = TSDB_CODE_STREAM_TASK_NOT_EXIST; mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid); } else { @@ -159,34 +169,39 @@ int32_t mndDropOrphanTasks(SMnode *pMnode, SArray *pList) { } SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""}; - STrans *pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); - if (pTrans == NULL) { + STrans *pTrans = NULL; + int32_t code = + doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans); + if (pTrans == NULL || code != 0) { mError("failed to create trans to drop orphan tasks since %s", terrstr()); - return -1; + return code; } - int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId); - + code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId); + if (code) { + return code; + } // drop all tasks - if (mndStreamSetDropActionFromList(pMnode, pTrans, pList) < 0) { + if ((code = mndStreamSetDropActionFromList(pMnode, pTrans, pList)) < 0) { mError("failed to create trans to drop orphan tasks since %s", terrstr()); mndTransDrop(pTrans); - return -1; + return code; } // drop stream - if (mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED) < 0) { + if ((code = mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED)) < 0) { mndTransDrop(pTrans); - return -1; + return code; } - if (mndTransPrepare(pMnode, pTrans) != 0) { + if ((code = mndTransPrepare(pMnode, pTrans)) != 0) { mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); - return -1; + return code; } + mndTransDrop(pTrans); - return 0; + return code; } int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) { @@ -230,9 +245,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray *pOrphanTasks = NULL; int32_t code = 0; - if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { + if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) { - return -1; + return code; } } @@ -242,8 +257,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (tDecodeStreamHbMsg(&decoder, &req) < 0) { tCleanupStreamHbMsg(&req); tDecoderClear(&decoder); - terrno = TSDB_CODE_INVALID_MSG; - return -1; + code = terrno = TSDB_CODE_INVALID_MSG; + return code; } tDecoderClear(&decoder); @@ -258,12 +273,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (!validateHbMsg(execInfo.pNodeList, req.vgId)) { mError("vgId:%d not exists in nodeList buf, discarded", req.vgId); - terrno = TSDB_CODE_INVALID_MSG; + code = terrno = TSDB_CODE_INVALID_MSG; doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); taosThreadMutexUnlock(&execInfo.lock); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); - return -1; + return code; } int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); @@ -294,9 +309,14 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { .startTs = pChkInfo->consensusTs, }; - SStreamObj *pStream = mndGetStreamObj(pMnode, p->id.streamId); - int32_t numOfTasks = mndGetNumOfStreamTasks(pStream); + SStreamObj *pStream = NULL; + code = mndGetStreamObj(pMnode, p->id.streamId, &pStream); + if (code) { + code = TSDB_CODE_STREAM_TASK_NOT_EXIST; + continue; + } + int32_t numOfTasks = mndGetNumOfStreamTasks(pStream); SCheckpointConsensusInfo *pInfo = NULL; code = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks, &pInfo); @@ -350,7 +370,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (pMnode != NULL) { SArray *p = NULL; - int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &p); + code = mndTakeVgroupSnapshot(pMnode, &allReady, &p); taosArrayDestroy(p); if (code) { mError("failed to get the vgroup snapshot, ignore it and continue"); @@ -388,7 +408,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); - return TSDB_CODE_SUCCESS; + return terrno; } void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index f252791618..b189ddb3cb 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -153,27 +153,30 @@ int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) { return 0; } -STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, - const char *pMsg) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); - if (pTrans == NULL) { +int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, + const char *pMsg, STrans ** pTrans1) { + *pTrans1 = NULL; + terrno = 0; + + STrans *p = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); + if (p == NULL) { mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return terrno; } - mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id); + mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, p->id); - mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName); - if (mndTransCheckConflict(pMnode, pTrans) != 0) { + mndTransSetDbName(p, pStream->sourceDb, pStream->targetSTbName); + if (mndTransCheckConflict(pMnode, p) != 0) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno)); - mndTransDrop(pTrans); - return NULL; + mndTransDrop(p); + return terrno; } - terrno = 0; - return pTrans; + *pTrans1 = p; + return 0; } SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { @@ -272,8 +275,9 @@ int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { continue; } - SStreamObj *pStream = mndGetStreamObj(pMnode, pTransInfo->streamId); - if (pStream != NULL) { + SStreamObj *pStream = NULL; + int32_t code = mndGetStreamObj(pMnode, pTransInfo->streamId, &pStream); + if (pStream != NULL || code != 0) { if (identicalName(pStream->sourceDb, pDBName, len)) { mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb); } else if (identicalName(pStream->targetDb, pDBName, len)) { diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 3a84856ae0..0b96626536 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -28,20 +28,20 @@ struct SStreamTaskIter { int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId); -SStreamTaskIter* createStreamTaskIter(SStreamObj* pStream) { - SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter)); - if (pIter == NULL) { +int32_t createStreamTaskIter(SStreamObj* pStream, SStreamTaskIter** pIter) { + *pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter)); + if (*pIter == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return terrno; } - pIter->level = -1; - pIter->ordinalIndex = 0; - pIter->pStream = pStream; - pIter->totalLevel = taosArrayGetSize(pStream->tasks); - pIter->pTask = NULL; + (*pIter)->level = -1; + (*pIter)->ordinalIndex = 0; + (*pIter)->pStream = pStream; + (*pIter)->totalLevel = taosArrayGetSize(pStream->tasks); + (*pIter)->pTask = NULL; - return pIter; + return 0; } bool streamTaskIterNextTask(SStreamTaskIter* pIter) { @@ -72,8 +72,15 @@ bool streamTaskIterNextTask(SStreamTaskIter* pIter) { return false; } -SStreamTask* streamTaskIterGetCurrent(SStreamTaskIter* pIter) { - return pIter->pTask; +int32_t streamTaskIterGetCurrent(SStreamTaskIter* pIter, SStreamTask** pTask) { + if (pTask) { + *pTask = pIter->pTask; + if (*pTask != NULL) { + return TSDB_CODE_SUCCESS; + } + } + + return TSDB_CODE_INVALID_PARA; } void destroyStreamTaskIter(SStreamTaskIter* pIter) { @@ -132,10 +139,15 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList) { } char buf[256] = {0}; - epsetToStr(&entry.epset, buf, tListLen(buf)); + (void) epsetToStr(&entry.epset, buf, tListLen(buf)); + + void* p = taosArrayPush(pVgroupList, &entry); + if (p == NULL) { + mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId); + } else { + mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf); + } - mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf); - taosArrayPush(pVgroupList, &entry); sdbRelease(pSdb, pVgroup); } @@ -146,15 +158,23 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList) { break; } - SNodeEntry entry = {0}; - addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); - entry.nodeId = SNODE_HANDLE; + SNodeEntry entry = {.nodeId = SNODE_HANDLE}; + code = addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); + if (code) { + sdbRelease(pSdb, pObj); + continue; + } char buf[256] = {0}; - epsetToStr(&entry.epset, buf, tListLen(buf)); - mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf); + (void) epsetToStr(&entry.epset, buf, tListLen(buf)); + + void* p = taosArrayPush(pVgroupList, &entry); + if (p == NULL) { + mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId); + } else { + mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf); + } - taosArrayPush(pVgroupList, &entry); sdbRelease(pSdb, pObj); } @@ -162,28 +182,33 @@ int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList) { return code; } -SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) { - void *pIter = NULL; - SSdb *pSdb = pMnode->pSdb; - SStreamObj *pStream = NULL; +int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj **pStream) { + void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + *pStream = NULL; - while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { - if (pStream->uid == streamId) { + SStreamObj *p = NULL; + while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&p)) != NULL) { + if (p->uid == streamId) { sdbCancelFetch(pSdb, pIter); - return pStream; + *pStream = p; + return TSDB_CODE_SUCCESS; } - sdbRelease(pSdb, pStream); + sdbRelease(pSdb, p); } - return NULL; + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName) { STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { mInfo("kill active transId:%d in Db:%s", transId, pDbName); - mndKillTrans(pMnode, pTrans); + int32_t code = mndKillTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); + if (code) { + mError("failed to kill trans:%d", pTrans->id); + } } else { mError("failed to acquire trans in Db:%s, transId:%d", pDbName, transId); } @@ -199,11 +224,16 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); if (pIter != NULL) { - addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); + int32_t code = addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); sdbRelease(pMnode->pSdb, pObj); sdbCancelFetch(pMnode->pSdb, pIter); - *hasEpset = true; - return TSDB_CODE_SUCCESS; + if (code) { + *hasEpset = false; + mError("failed to set epset"); + } else { + *hasEpset = true; + } + return code; } else { mError("failed to acquire snode epset"); return TSDB_CODE_INVALID_PARA; @@ -225,12 +255,14 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t } static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) { + terrno = 0; + SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); if (pReq == NULL) { mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq), tstrerror(TSDB_CODE_OUT_OF_MEMORY)); terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno; } pReq->head.vgId = htonl(pTask->info.nodeId); @@ -244,31 +276,45 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { terrno = code; taosMemoryFree(pReq); - return -1; + return terrno; } code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0); if (code != 0) { taosMemoryFree(pReq); - return -1; + return terrno; } mDebug("set the resume action for trans:%d", pTrans->id); return 0; } -SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { - SStreamTaskIter *pIter = createStreamTaskIter(pStream); +int32_t mndGetStreamTask(STaskId *pId, SStreamObj *pStream, SStreamTask **pTask) { + *pTask = NULL; + + SStreamTask *p = NULL; + SStreamTaskIter *pIter = NULL; + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + mError("failed to create stream task iter:%s", pStream->name); + return code; + } + while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - if (pTask->id.taskId == pId->taskId) { + code = streamTaskIterGetCurrent(pIter, &p); + if (code) { + continue; + } + + if (p->id.taskId == pId->taskId) { destroyStreamTaskIter(pIter); - return pTask; + *pTask = p; + return 0; } } destroyStreamTaskIter(pIter); - return NULL; + return TSDB_CODE_FAILED; } int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) { @@ -282,13 +328,25 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) { } int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) { - SStreamTaskIter *pIter = createStreamTaskIter(pStream); + SStreamTaskIter *pIter = NULL; + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + mError("failed to create stream task iter:%s", pStream->name); + return code; + } while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - if (doSetResumeAction(pTrans, pMnode, pTask, igUntreated) < 0) { + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code || pTask == NULL) { destroyStreamTaskIter(pIter); - return -1; + return code; + } + + code = doSetResumeAction(pTrans, pMnode, pTask, igUntreated); + if (code) { + destroyStreamTaskIter(pIter); + return code; } if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) { @@ -305,7 +363,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq), tstrerror(TSDB_CODE_OUT_OF_MEMORY)); terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno; } pReq->head.vgId = htonl(pTask->info.nodeId); @@ -322,25 +380,38 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa } char buf[256] = {0}; - epsetToStr(&epset, buf, tListLen(buf)); + (void) epsetToStr(&epset, buf, tListLen(buf)); mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0); if (code != 0) { taosMemoryFree(pReq); - return -1; + return code; } return 0; } int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { - SStreamTaskIter *pIter = createStreamTaskIter(pStream); + SStreamTaskIter *pIter = NULL; + + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + mError("failed to create stream task iter:%s", pStream->name); + return code; + } while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - if (doSetPauseAction(pMnode, pTrans, pTask) < 0) { + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { destroyStreamTaskIter(pIter); - return -1; + return code; + } + + code = doSetPauseAction(pMnode, pTrans, pTask); + if (code) { + destroyStreamTaskIter(pIter); + return code; } if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) { @@ -350,14 +421,14 @@ int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr } destroyStreamTaskIter(pIter); - return 0; + return code; } static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno; } pReq->head.vgId = htonl(pTask->info.nodeId); @@ -368,28 +439,40 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas bool hasEpset = false; int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); if (code != TSDB_CODE_SUCCESS || !hasEpset) { // no valid epset, return directly without redoAction - terrno = code; - return -1; + return code; } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); if (code != 0) { taosMemoryFree(pReq); - return -1; + return code; } return 0; } int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { - SStreamTaskIter *pIter = createStreamTaskIter(pStream); + SStreamTaskIter *pIter = NULL; + + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + mError("failed to create stream task iter:%s", pStream->name); + return code; + } while(streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - if (doSetDropAction(pMnode, pTrans, pTask) < 0) { + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { destroyStreamTaskIter(pIter); - return -1; + return code; + } + + code = doSetDropAction(pMnode, pTrans, pTask); + if (code) { + destroyStreamTaskIter(pIter); + return code; } } destroyStreamTaskIter(pIter); @@ -400,7 +483,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno; } pReq->head.vgId = htonl(pTask->nodeId); @@ -411,16 +494,15 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask bool hasEpset = false; int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId); if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { // no valid epset, return directly without redoAction - terrno = code; taosMemoryFree(pReq); - return -1; + return code; } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); if (code != 0) { taosMemoryFree(pReq); - return -1; + return code; } return 0; @@ -429,19 +511,35 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) { for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { SOrphanTask* pTask = taosArrayGet(pList, i); - mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId); - doSetDropActionFromId(pMnode, pTrans, pTask); + int32_t code = doSetDropActionFromId(pMnode, pTrans, pTask); + if (code != 0) { + return code; + } else { + mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId); + } } return 0; } static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId, int32_t transId) { + int32_t code = 0; + pMsg->streamId = pId->streamId; pMsg->taskId = pId->taskId; pMsg->transId = transId; pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo)); - taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList); + if (pMsg->pNodeList == NULL) { + mError("failed to prepare node list, code:out of memory"); + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (code == 0) { + void *p = taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList); + if (p == NULL) { + mError("failed to add update node list into nodeList"); + } + } } static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId, @@ -456,7 +554,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha if (code < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(req.pNodeList); - return -1; + return terrno; } int32_t tlen = sizeof(SMsgHead) + blen; @@ -465,13 +563,18 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(req.pNodeList); - return -1; + return terrno; } void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - tEncodeStreamTaskUpdateMsg(&encoder, &req); + code = tEncodeStreamTaskUpdateMsg(&encoder, &req); + if (code == -1) { + tEncoderClear(&encoder); + taosArrayDestroy(req.pNodeList); + return code; + } SMsgHead *pMsgHead = (SMsgHead *)buf; pMsgHead->contLen = htonl(tlen); @@ -489,15 +592,20 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { void *pBuf = NULL; int32_t len = 0; - streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); + int32_t code = streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); + if (code) { + return code; + } - doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); + code = doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); + if (code) { + return code; + } SEpSet epset = {0}; bool hasEpset = false; - int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); if (code != TSDB_CODE_SUCCESS || !hasEpset) { - terrno = code; return code; } @@ -512,16 +620,30 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask // build trans to update the epset int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid); - taosWLockLatch(&pStream->lock); + SStreamTaskIter *pIter = NULL; + + taosWLockLatch(&pStream->lock); + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + taosWUnLockLatch(&pStream->lock); + mError("failed to create stream task iter:%s", pStream->name); + return code; + } - SStreamTaskIter *pIter = createStreamTaskIter(pStream); while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo); + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { + destroyStreamTaskIter(pIter); + taosWUnLockLatch(&pStream->lock); + return code; + } + + code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo); if (code != TSDB_CODE_SUCCESS) { destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); - return -1; + return code; } } @@ -560,16 +682,30 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa } int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { - taosWLockLatch(&pStream->lock); + SStreamTaskIter *pIter = NULL; + + taosWLockLatch(&pStream->lock); + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + taosWUnLockLatch(&pStream->lock); + mError("failed to create stream task iter:%s", pStream->name); + return code; + } - SStreamTaskIter *pIter = createStreamTaskIter(pStream); while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - int32_t code = doSetResetAction(pMnode, pTrans, pTask); + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { + destroyStreamTaskIter(pIter); + taosWUnLockLatch(&pStream->lock); + return code; + } + + code = doSetResetAction(pMnode, pTrans, pTask); if (code != TSDB_CODE_SUCCESS) { destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); - return -1; + return code; } } @@ -583,8 +719,12 @@ static void freeTaskList(void* param) { taosArrayDestroy(*pList); } -void mndInitExecInfo() { - taosThreadMutexInit(&execInfo.lock, NULL); +int32_t mndInitExecInfo() { + int32_t code = taosThreadMutexInit(&execInfo.lock, NULL); + if (code) { + return code; + } + _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); @@ -598,6 +738,7 @@ void mndInitExecInfo() { taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList); taosHashSetFreeFp(execInfo.pStreamConsensus, freeTaskList); + return 0; } void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { @@ -610,7 +751,10 @@ void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { for (int32_t j = 0; j < size; ++j) { SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); if (pEntry->nodeId == p->nodeId) { - taosArrayPush(pValidList, p); + void* px = taosArrayPush(pValidList, p); + if (px == NULL) { + mError("failed to put node into list, nodeId:%d", p->nodeId); + } break; } } @@ -628,7 +772,10 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) { return TSDB_CODE_SUCCESS; } - taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); + int32_t code = taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); + if (code) { + return code; + } for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); @@ -647,28 +794,45 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) { void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) { for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) { STaskId *pId = taosArrayGet(pTaskIds, i); - doRemoveTasks(pExecInfo, pId); + int32_t code = doRemoveTasks(pExecInfo, pId); + if (code) { + mError("failed to remove task in buffer list, 0x%"PRIx64, pId->taskId); + } } } void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { - taosThreadMutexLock(&pExecNode->lock); + SStreamTaskIter *pIter = NULL; + streamMutexLock(&pExecNode->lock); // 1. remove task entries - SStreamTaskIter *pIter = createStreamTaskIter(pStream); + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + streamMutexUnlock(&pExecNode->lock); + mError("failed to create stream task iter:%s", pStream->name); + return; + } + while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { + continue; + } STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - doRemoveTasks(pExecNode, &id); + code = doRemoveTasks(pExecNode, &id); + if (code) { + mError("failed to remove task in buffer list, 0x%"PRIx64, id.taskId); + } } ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); // 2. remove stream entry in consensus hash table - mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); + (void) mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); - taosThreadMutexUnlock(&pExecNode->lock); + streamMutexUnlock(&pExecNode->lock); destroyStreamTaskIter(pIter); } @@ -699,7 +863,10 @@ int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) { bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); if (!existed) { - taosArrayPush(pRemovedTasks, pId); + void* p = taosArrayPush(pRemovedTasks, pId); + if (p == NULL) { + mError("failed to put task entry into remove list, taskId:0x%" PRIx64, pId->taskId); + } } } @@ -761,45 +928,64 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas } int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + SStreamTaskIter *pIter = NULL; + taosWLockLatch(&pStream->lock); + int32_t code = createStreamTaskIter(pStream, &pIter); + if (code) { + taosWUnLockLatch(&pStream->lock); + mError("failed to create stream task iter:%s", pStream->name); + return code; + } - SStreamTaskIter *pIter = createStreamTaskIter(pStream); while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + if (code) { + destroyStreamTaskIter(pIter); + taosWUnLockLatch(&pStream->lock); + return code; + } - int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask); + code = doSetUpdateChkptAction(pMnode, pTrans, pTask); if (code != TSDB_CODE_SUCCESS) { destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); - return -1; + return code; } } destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); - return 0; + return code; } int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; void *pIter = NULL; SArray *pDropped = taosArrayInit(4, sizeof(int64_t)); + int32_t code = 0; mDebug("start to scan checkpoint report info"); while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) { SArray *pList = *(SArray **)pIter; - STaskChkptInfo* pInfo = taosArrayGet(pList, 0); - SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId); - if (pStream == NULL) { + STaskChkptInfo *pInfo = taosArrayGet(pList, 0); + SStreamObj *pStream = NULL; + code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream); + if (pStream == NULL || code != 0) { mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId); - taosArrayPush(pDropped, &pInfo->streamId); + void* p = taosArrayPush(pDropped, &pInfo->streamId); + if (p == NULL) { + mError("failed to put stream into drop list:0x%" PRIx64, pInfo->streamId); + } + continue; } int32_t total = mndGetNumOfStreamTasks(pStream); - int32_t existed = (int32_t) taosArrayGetSize(pList); + int32_t existed = (int32_t)taosArrayGetSize(pList); if (total == existed) { mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info", @@ -807,17 +993,21 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); if (!conflict) { - int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList); - if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry - taosArrayPush(pDropped, &pInfo->streamId); - mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId); + code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList); + if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry + void* p = taosArrayPush(pDropped, &pInfo->streamId); + if (p == NULL) { + mError("failed to remove stream:0x%" PRIx64, pInfo->streamId); + } else { + mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId); + } } else { mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet", pInfo->streamId); } break; } else { - mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId); + mDebug("stream:0x%" PRIx64 " active checkpoint trans not finished yet, wait", pInfo->streamId); } } else { mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name, @@ -831,7 +1021,10 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { if (size > 0) { for (int32_t i = 0; i < size; ++i) { int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i); - taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId)); + code = taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId)); + if (code) { + mError("failed to remove stream in buf:0x%"PRIx64, streamId); + } } int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); @@ -856,29 +1049,30 @@ static int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStream int32_t blen; tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code); if (code < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } int32_t tlen = sizeof(SMsgHead) + blen; void *pBuf = taosMemoryMalloc(tlen); if (pBuf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - tEncodeRestoreCheckpointInfo(&encoder, &req); + code = tEncodeRestoreCheckpointInfo(&encoder, &req); + tEncoderClear(&encoder); + if (code == -1) { + taosMemoryFree(pBuf); + return code; + } SMsgHead *pMsgHead = (SMsgHead *)pBuf; pMsgHead->contLen = htonl(tlen); pMsgHead->vgId = htonl(pTask->info.nodeId); - tEncoderClear(&encoder); - SEpSet epset = {0}; bool hasEpset = false; code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); @@ -900,17 +1094,28 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i char msg[128] = {0}; snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId); - STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg); - if (pTrans == NULL) { + STrans *pTrans = NULL; + int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg, &pTrans); + if (pTrans == NULL || code != 0) { return terrno; } STaskId id = {.streamId = pStream->uid, .taskId = taskId}; - SStreamTask *pTask = mndGetStreamTask(&id, pStream); - ASSERT(pTask); + SStreamTask *pTask = NULL; + code = mndGetStreamTask(&id, pStream, &pTask); + if (code) { + mError("failed to get task:0x%x in stream:%s, failed to create consensus-checkpointId", taskId, pStream->name); + sdbRelease(pMnode->pSdb, pStream); + return code; + } - /*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid); - int32_t code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts); + code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid); + if (code) { + sdbRelease(pMnode->pSdb, pStream); + return code; + } + + code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts); if (code != 0) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); @@ -918,17 +1123,18 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i } code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); - if (code != TSDB_CODE_SUCCESS) { + if (code) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } - if (mndTransPrepare(pMnode, pTrans) != 0) { + code = mndTransPrepare(pMnode, pTrans); + if (code) { mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); - return -1; + return code; } sdbRelease(pMnode->pSdb, pStream); @@ -938,8 +1144,11 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i } int32_t mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo) { - *pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); - if (pInfo != NULL) { + *pInfo = NULL; + + void* px = taosHashGet(pHash, &streamId, sizeof(streamId)); + if (px != NULL) { + *pInfo = px; return 0; } @@ -977,11 +1186,15 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo } } - taosArrayPush(pInfo->pTaskList, &info); - int32_t num = taosArrayGetSize(pInfo->pTaskList); - mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64 - " waiting tasks:%d", - pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num); + void *p = taosArrayPush(pInfo->pTaskList, &info); + if (p == NULL) { + mError("s-task:0x%x failed to put task into consensus-checkpointId list, code: out of memory", info.req.taskId); + } else { + int32_t num = taosArrayGetSize(pInfo->pTaskList); + mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64 + " waiting tasks:%d", + pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num); + } } void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { @@ -990,22 +1203,14 @@ void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { } int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) { - taosHashRemove(pHash, &streamId, sizeof(streamId)); - int32_t numOfStreams = taosHashGetSize(pHash); - mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d", streamId, - numOfStreams); - return TSDB_CODE_SUCCESS; -} + int32_t code = taosHashRemove(pHash, &streamId, sizeof(streamId)); + if (code == 0) { + int32_t numOfStreams = taosHashGetSize(pHash); + mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d", + streamId, numOfStreams); + } else { + mError("failed to remove stream:0x%"PRIx64" in consensus-checkpointId list", streamId); + } -//int32_t mndRegisterConsensusChkptId(SHashObj* pHash, int64_t streamId) { -// void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); -// ASSERT(pInfo == NULL); -// -// SCheckpointConsensusInfo p = {.genTs = taosGetTimestampMs(), .checkpointId = 0, .pTaskList = NULL}; -// taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p)); -// -// SCheckpointConsensusInfo* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId)); -// ASSERT(pChkptInfo->genTs > 0 && pChkptInfo->checkpointId == 0); -// mDebug("s-task:0x%" PRIx64 " set the initial consensus-checkpointId:0", streamId); -// return TSDB_CODE_SUCCESS; -//} \ No newline at end of file + return code; +} \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 30ffcd71ac..1283f8e20b 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -639,7 +639,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { code = tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); - if (code) { + if (code == -1) { stError("s-task:%s vgId:%d task meta encode failed, code:%s", pTask->id.idStr, vgId, tstrerror(code)); return TSDB_CODE_INVALID_MSG; } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 884d7ea1b6..d8656c0f60 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -252,7 +252,7 @@ int32_t l2ComressInitImpl_xz(char *lossyColumns, float fPrecision, double dPreci } int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char *const output, int32_t outputSize, const char type, int8_t lvl) { - size_t len = FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); + size_t len = 0;//FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); if (len > inputSize) { output[0] = 0; memcpy(output + 1, input, inputSize); @@ -264,7 +264,7 @@ int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char int32_t l2DecompressImpl_xz(const char *const input, const int32_t compressedSize, char *const output, int32_t outputSize, const char type) { if (input[0] == 1) { - return FL2_decompress(output, outputSize, input + 1, compressedSize - 1); + return 0;//FL2_decompress(output, outputSize, input + 1, compressedSize - 1); } else if (input[0] == 0) { memcpy(output, input + 1, compressedSize - 1); return compressedSize - 1; From 116fc722b509b64a5c981806a3b0e0ce7714e732 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 05:46:01 +0000 Subject: [PATCH 17/43] error str --- source/dnode/mnode/impl/src/mndShow.c | 14 +++++++------- source/dnode/mnode/impl/src/mndSma.c | 18 +++++++++--------- source/dnode/mnode/impl/src/mndSnode.c | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 10b54e2afb..d4a28d75ca 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -37,7 +37,7 @@ int32_t mndInitShow(SMnode *pMnode) { pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show"); if (pMgmt->cache == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - mError("failed to alloc show cache since %s", terrstr()); + mError("failed to alloc show cache since %s", tstrerror(code)); TAOS_RETURN(code); } @@ -165,7 +165,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:0x%" PRIx64 ", failed to put into cache since %s", showId, terrstr()); + mError("show:0x%" PRIx64 ", failed to put into cache since %s", showId, tstrerror(code)); return NULL; } @@ -232,7 +232,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); if (pMeta == NULL) { code = TSDB_CODE_PAR_TABLE_NOT_EXIST; - mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); + mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code)); TAOS_RETURN(code); } } @@ -240,7 +240,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pShow = mndCreateShowObj(pMnode, &retrieveReq); if (pShow == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - mError("failed to process show-meta req since %s", terrstr()); + mError("failed to process show-meta req since %s", tstrerror(code)); TAOS_RETURN(code); } @@ -250,7 +250,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); if (pShow == NULL) { code = TSDB_CODE_MND_INVALID_SHOWOBJ; - mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); + mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code)); TAOS_RETURN(code); } } @@ -264,7 +264,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { if (retrieveFp == NULL) { mndReleaseShowObj(pShow, false); code = TSDB_CODE_MSG_NOT_PROCESSED; - mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); + mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code)); TAOS_RETURN(code); } @@ -327,7 +327,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { if (pRsp == NULL) { mndReleaseShowObj(pShow, false); code = TSDB_CODE_OUT_OF_MEMORY; - mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); + mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code)); blockDataDestroy(pBlock); TAOS_RETURN(code); } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 1a59b35cf1..0949055aa0 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -664,7 +664,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea } if ((code = mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg)) != 0) { - mError("sma:%s, failed to create since %s", smaObj.name, terrstr()); + mError("sma:%s, failed to create since %s", smaObj.name, tstrerror(code)); TAOS_RETURN(code); } smaObj.dstVgId = streamObj.fixedSinkVg.vgId; @@ -846,7 +846,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("sma:%s, failed to create since %s", createReq.name, terrstr()); + mError("sma:%s, failed to create since %s", createReq.name, tstrerror(code)); } mndReleaseStb(pMnode, pStb); @@ -991,14 +991,14 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p goto _OVER; } else { if ((code = mndStreamSetDropAction(pMnode, pTrans, pStream)) < 0) { - mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); + mError("stream:%s, failed to drop task since %s", pStream->name, tstrerror(code)); sdbRelease(pMnode->pSdb, pStream); goto _OVER; } // drop stream if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_DROPPED)) < 0) { - mError("stream:%s, failed to drop log since %s", pStream->name, terrstr()); + mError("stream:%s, failed to drop log since %s", pStream->name, tstrerror(code)); sdbRelease(pMnode->pSdb, pStream); goto _OVER; } @@ -1047,7 +1047,7 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p SStreamObj *pStream = mndAcquireStream(pMnode, streamName); if (pStream != NULL && pStream->smaId == pSma->uid) { if ((code = mndStreamSetDropAction(pMnode, pTrans, pStream)) < 0) { - mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); + mError("stream:%s, failed to drop task since %s", pStream->name, tstrerror(code)); mndReleaseStream(pMnode, pStream); goto _OVER; } @@ -1144,7 +1144,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("sma:%s, failed to drop since %s", dropReq.name, terrstr()); + mError("sma:%s, failed to drop since %s", dropReq.name, tstrerror(code)); } mndReleaseSma(pMnode, pSma); @@ -1300,7 +1300,7 @@ static int32_t mndProcessGetSmaReq(SRpcMsg *pReq) { _OVER: if (code != 0) { - mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); + mError("failed to get index %s since %s", indexReq.indexFName, tstrerror(code)); } TAOS_RETURN(code); @@ -1346,7 +1346,7 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { _OVER: if (code != 0) { - mError("failed to get table index %s since %s", indexReq.tbFName, terrstr()); + mError("failed to get table index %s since %s", indexReq.tbFName, tstrerror(code)); } tFreeSerializeSTableIndexRsp(&rsp); @@ -1852,7 +1852,7 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("tsma:%s, failed to create since %s", createReq.name, terrstr()); + mError("tsma:%s, failed to create since %s", createReq.name, tstrerror(code)); } if (pStb) mndReleaseStb(pMnode, pStb); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 4bb8ab92fb..e10fb394fc 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -332,7 +332,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); + mError("snode:%d, failed to create since %s", createReq.dnodeId, tstrerror(code)); TAOS_RETURN(code); } @@ -457,7 +457,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); + mError("snode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code)); } mndReleaseSnode(pMnode, pObj); From e8fef13d39ddb6513ceec09cb2a31f9795bb495b Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 05:48:47 +0000 Subject: [PATCH 18/43] error str --- source/dnode/mnode/impl/src/mndStb.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index efbe906d79..4e34132e06 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1344,7 +1344,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("stb:%s, failed to create since %s", createReq.name, terrstr()); + mError("stb:%s, failed to create since %s", createReq.name, tstrerror(code)); } mndReleaseStb(pMnode, pStb); @@ -2607,7 +2607,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("stb:%s, failed to alter since %s", alterReq.name, terrstr()); + mError("stb:%s, failed to alter since %s", alterReq.name, tstrerror(code)); } mndReleaseStb(pMnode, pStb); @@ -2884,7 +2884,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); + mError("stb:%s, failed to drop since %s", dropReq.name, tstrerror(code)); } mndReleaseDb(pMnode, pDb); @@ -2941,7 +2941,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { _OVER: if (code != 0) { - mError("stb:%s.%s, failed to retrieve meta since %s", infoReq.dbFName, infoReq.tbName, terrstr()); + mError("stb:%s.%s, failed to retrieve meta since %s", infoReq.dbFName, infoReq.tbName, tstrerror(code)); } mndReleaseUser(pMnode, pUser); @@ -2991,7 +2991,7 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { _OVER: if (code != 0) { - mError("stb:%s.%s, failed to retrieve cfg since %s", cfgReq.dbFName, cfgReq.tbName, terrstr()); + mError("stb:%s.%s, failed to retrieve cfg since %s", cfgReq.dbFName, cfgReq.tbName, tstrerror(code)); } tFreeSTableCfgRsp(&cfgRsp); From feba37b5ea3265ceb15d691bf2ec391c4a054901 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 22 Jul 2024 14:10:17 +0800 Subject: [PATCH 19/43] group by --- source/libs/parser/src/parTranslater.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fef58df3dc..c1995ab784 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3407,13 +3407,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (nodesEqualNode(pActualNode, *pNode)) { return DEAL_RES_IGNORE_CHILD; } - if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) && - ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { - return rewriteExprToGroupKeyFunc(pCxt, pNode); - } if (IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } + if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) && + ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { + return rewriteExprToSelectTagFunc(pCxt, pNode); + } } SNode* pPartKey = NULL; bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); From 64ad24077329b50c28cd26358c483f845d6cfd2a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Jul 2024 14:35:12 +0800 Subject: [PATCH 20/43] fix: crash issue --- .../libs/executor/src/dynqueryctrloperator.c | 59 +-- source/libs/executor/src/mergejoin.c | 5 +- source/libs/executor/test/joinTests.cpp | 348 +++++++++++------- 3 files changed, 258 insertions(+), 154 deletions(-) diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 6a4840b188..c3b3126f68 100755 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -105,7 +105,7 @@ static FORCE_INLINE bool tableNeedCache(int64_t uid, SStbJoinPrevJoinCtx* pPrev, return (NULL == num) ? false : true; } -static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) { +static int32_t updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) { SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev; SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post; SStbJoinTableList* pNode = pPrev->pListHead; @@ -141,32 +141,38 @@ static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) pPost->leftNeedCache = tableNeedCache(*leftUid, pPrev, pPost, false, pStbJoin->basic.batchFetch); pPost->rightNeedCache = tableNeedCache(*rightUid, pPrev, pPost, true, pStbJoin->basic.batchFetch); - if (pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) { - tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0); + if (!pStbJoin->basic.batchFetch && pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) { + QRY_ERR_RET(tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0)); pStbJoin->execInfo.rightCacheNum++; } + + return TSDB_CODE_SUCCESS; } static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid, bool needCache, SOperatorParam* pChild) { + int32_t code = TSDB_CODE_SUCCESS; *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { + code = terrno; freeOperatorParam(pChild, OP_GET_PARAM); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } if (pChild) { (*ppRes)->pChildren = taosArrayInit(1, POINTER_BYTES); if (NULL == (*ppRes)->pChildren) { + code = terrno; freeOperatorParam(pChild, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild)) { + code = terrno; freeOperatorParam(pChild, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } } else { (*ppRes)->pChildren = NULL; @@ -174,9 +180,10 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down SGcOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcOperatorParam)); if (NULL == pGc) { + code = terrno; freeOperatorParam(*ppRes, OP_GET_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } pGc->sessionId = atomic_add_fetch_64(&gSessionId, 1); @@ -194,16 +201,18 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid) { + int32_t code = TSDB_CODE_SUCCESS; *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*ppRes)->pChildren = NULL; SGcNotifyOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcNotifyOperatorParam)); if (NULL == pGc) { + code = terrno; freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } pGc->downstreamIdx = downstreamIdx; @@ -221,13 +230,13 @@ static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) { *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*ppRes)->pChildren = NULL; SExchangeOperatorParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorParam)); if (NULL == pExc) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pExc->multiParams = false; @@ -237,7 +246,7 @@ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downst pExc->basic.uidList = taosArrayInit(1, sizeof(int64_t)); if (NULL == pExc->basic.uidList) { taosMemoryFree(pExc); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } taosArrayPush(pExc->basic.uidList, pUid); @@ -252,14 +261,14 @@ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downst static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, SSHashObj* pVg) { *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*ppRes)->pChildren = NULL; SExchangeOperatorBatchParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorBatchParam)); if (NULL == pExc) { taosMemoryFreeClear(*ppRes); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pExc->multiParams = true; @@ -267,7 +276,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d if (NULL == pExc->pBatchs) { taosMemoryFree(pExc); taosMemoryFreeClear(*ppRes); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } tSimpleHashSetFreeFp(pExc->pBatchs, freeExchangeGetBasicOperatorParam); @@ -283,7 +292,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d basic.uidList = pUidList; basic.tableSeq = false; - tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic)); + QRY_ERR_RET(tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic))); qTrace("build downstreamIdx %d batch scan, vgId:%d, uidNum:%" PRId64, downstreamIdx, *pVgId, (int64_t)taosArrayGetSize(pUidList)); *(SArray**)p = NULL; @@ -298,39 +307,45 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initParam, SOperatorParam* pChild0, SOperatorParam* pChild1) { + int32_t code = TSDB_CODE_SUCCESS; *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { + code = terrno; freeOperatorParam(pChild0, OP_GET_PARAM); freeOperatorParam(pChild1, OP_GET_PARAM); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } (*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES); if (NULL == *ppRes) { + code = terrno; freeOperatorParam(pChild0, OP_GET_PARAM); freeOperatorParam(pChild1, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) { + code = terrno; freeOperatorParam(pChild0, OP_GET_PARAM); freeOperatorParam(pChild1, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) { + code = terrno; freeOperatorParam(pChild1, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } SSortMergeJoinOperatorParam* pJoin = taosMemoryMalloc(sizeof(SSortMergeJoinOperatorParam)); if (NULL == pJoin) { + code = terrno; freeOperatorParam(*ppRes, OP_GET_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } pJoin->initDownstream = initParam; @@ -435,7 +450,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS qDebug("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64, rowIdx, pPrev->tableNum, *leftVg, *leftUid, *rightVg, *rightUid); - updatePostJoinCurrTableInfo(&pInfo->stbJoin); + QRY_ERR_RET(updatePostJoinCurrTableInfo(&pInfo->stbJoin)); if (pInfo->stbJoin.basic.batchFetch) { if (pPrev->leftHash) { diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index 2c5121855a..cb0cb5cbde 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -2655,7 +2655,7 @@ static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo mJoinSetDone(pOperator); } - return false; + return TSDB_CODE_SUCCESS; } if (buildGot) { @@ -2690,8 +2690,9 @@ static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo } pCtx->probeGrp.blk = pJoin->probe->blk; + *newBlock = true; - return true; + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/test/joinTests.cpp b/source/libs/executor/test/joinTests.cpp index fd8c1d80b3..e3daf6ff27 100755 --- a/source/libs/executor/test/joinTests.cpp +++ b/source/libs/executor/test/joinTests.cpp @@ -70,6 +70,8 @@ enum { TEST_FULL_COND }; +#define JT_PRINTF (void)printf + #define COL_DISPLAY_WIDTH 18 #define JT_MAX_LOOP 1000 @@ -222,33 +224,33 @@ void printResRow(char* value, int32_t type) { return; } - printf(" "); + JT_PRINTF(" "); for (int32_t i = 0; i < jtCtx.resColNum; ++i) { int32_t slot = jtCtx.resColInSlot[i]; if (0 == type && ((jtCtx.leftColOnly && slot >= MAX_SLOT_NUM) || (jtCtx.rightColOnly && slot < MAX_SLOT_NUM))) { - printf("%18s", " "); + ("%18s", " "); continue; } if (*(bool*)(value + slot)) { - printf("%18s", " NULL"); + JT_PRINTF("%18s", " NULL"); continue; } switch (jtInputColType[slot % MAX_SLOT_NUM]) { case TSDB_DATA_TYPE_TIMESTAMP: - printf("%18" PRId64 , *(int64_t*)(value + jtCtx.resColOffset[slot])); + JT_PRINTF("%18" PRId64 , *(int64_t*)(value + jtCtx.resColOffset[slot])); break; case TSDB_DATA_TYPE_INT: - printf("%18d", *(int32_t*)(value + jtCtx.resColOffset[slot])); + JT_PRINTF("%18d", *(int32_t*)(value + jtCtx.resColOffset[slot])); break; case TSDB_DATA_TYPE_BIGINT: - printf("%18" PRId64, *(int64_t*)(value + jtCtx.resColOffset[slot])); + JT_PRINTF("%18" PRId64, *(int64_t*)(value + jtCtx.resColOffset[slot])); break; } } - printf("\t %s\n", 0 == type ? "-" : (1 == type ? "+" : "")); + JT_PRINTF("\t %s\n", 0 == type ? "-" : (1 == type ? "+" : "")); } void pushResRow(char* buf, int32_t size) { @@ -260,7 +262,7 @@ void pushResRow(char* buf, int32_t size) { (*rows)++; } else { int32_t n = 1; - tSimpleHashPut(jtCtx.jtResRows, buf, size, &n, sizeof(n)); + assert(0 == tSimpleHashPut(jtCtx.jtResRows, buf, size, &n, sizeof(n))); } } } @@ -270,10 +272,10 @@ void rmResRow() { if (rows) { (*rows)--; if ((*rows) == 0) { - tSimpleHashRemove(jtCtx.jtResRows, jtCtx.resColBuf, jtCtx.resColSize); + (void)tSimpleHashRemove(jtCtx.jtResRows, jtCtx.resColBuf, jtCtx.resColSize); } } else { - ASSERT(0); + assert(0); } } @@ -292,10 +294,10 @@ static int32_t jtMergeEqCond(SNode** ppDst, SNode** ppSrc) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppDst)) { SLogicConditionNode* pLogic = (SLogicConditionNode*)*ppDst; if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc)) { - nodesListStrictAppendList(pLogic->pParameterList, ((SLogicConditionNode*)(*ppSrc))->pParameterList); + assert(0 == nodesListStrictAppendList(pLogic->pParameterList, ((SLogicConditionNode*)(*ppSrc))->pParameterList)); ((SLogicConditionNode*)(*ppSrc))->pParameterList = NULL; } else { - nodesListStrictAppend(pLogic->pParameterList, *ppSrc); + assert(0 == nodesListStrictAppend(pLogic->pParameterList, *ppSrc)); *ppSrc = NULL; } nodesDestroyNode(*ppSrc); @@ -311,8 +313,8 @@ static int32_t jtMergeEqCond(SNode** ppDst, SNode** ppSrc) { pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogicCond->condType = LOGIC_COND_TYPE_AND; pLogicCond->pParameterList = nodesMakeList(); - nodesListStrictAppend(pLogicCond->pParameterList, *ppSrc); - nodesListStrictAppend(pLogicCond->pParameterList, *ppDst); + assert(0 == nodesListStrictAppend(pLogicCond->pParameterList, *ppSrc)); + assert(0 == nodesListStrictAppend(pLogicCond->pParameterList, *ppDst)); *ppDst = (SNode*)pLogicCond; *ppSrc = NULL; @@ -324,6 +326,7 @@ static int32_t jtMergeEqCond(SNode** ppDst, SNode** ppSrc) { void createDummyDownstreamOperators(int32_t num, SOperatorInfo** ppRes) { for (int32_t i = 0; i < num; ++i) { SOperatorInfo* p = (SOperatorInfo*)taosMemoryCalloc(1, sizeof(SOperatorInfo)); + assert(NULL != p); p->resultDataBlockId = i; ppRes[i] = p; } @@ -331,7 +334,7 @@ void createDummyDownstreamOperators(int32_t num, SOperatorInfo** ppRes) { void createTargetSlotList(SSortMergeJoinPhysiNode* p) { jtCtx.resColNum = 0; - memset(jtCtx.resColList, 0, sizeof(jtCtx.resColList)); + TAOS_MEMSET(jtCtx.resColList, 0, sizeof(jtCtx.resColList)); jtCtx.resColSize = MAX_SLOT_NUM * 2 * sizeof(bool); jtCtx.keyInSlotIdx = -1; @@ -386,6 +389,7 @@ void createTargetSlotList(SSortMergeJoinPhysiNode* p) { STargetNode* pTarget = (STargetNode*)nodesMakeNode(QUERY_NODE_TARGET); SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(NULL != pTarget && NULL != pCol); pCol->dataBlockId = LEFT_BLK_ID; pCol->slotId = i; pTarget->dataBlockId = RES_BLK_ID; @@ -394,7 +398,7 @@ void createTargetSlotList(SSortMergeJoinPhysiNode* p) { dstOffset += tDataTypes[jtInputColType[i]].bytes; jtCtx.resColSize += tDataTypes[jtInputColType[i]].bytes; - nodesListMakeStrictAppend(&p->pTargets, (SNode*)pTarget); + assert(0 == nodesListMakeStrictAppend(&p->pTargets, (SNode*)pTarget)); jtCtx.resColNum++; } @@ -410,6 +414,7 @@ void createTargetSlotList(SSortMergeJoinPhysiNode* p) { STargetNode* pTarget = (STargetNode*)nodesMakeNode(QUERY_NODE_TARGET); SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(NULL != pTarget && NULL != pCol); pCol->dataBlockId = RIGHT_BLK_ID; pCol->slotId = i; pTarget->dataBlockId = RES_BLK_ID; @@ -418,12 +423,13 @@ void createTargetSlotList(SSortMergeJoinPhysiNode* p) { dstOffset += tDataTypes[jtInputColType[i]].bytes; jtCtx.resColSize += tDataTypes[jtInputColType[i]].bytes; - nodesListMakeStrictAppend(&p->pTargets, (SNode*)pTarget); + assert(0 == nodesListMakeStrictAppend(&p->pTargets, (SNode*)pTarget)); jtCtx.resColNum++; } } jtCtx.resColBuf = (char*)taosMemoryRealloc(jtCtx.resColBuf, jtCtx.resColSize); + assert(NULL != jtCtx.resColBuf); } void createColEqCondStart(SSortMergeJoinPhysiNode* p) { @@ -433,7 +439,7 @@ void createColEqCondStart(SSortMergeJoinPhysiNode* p) { } while (0 == jtCtx.colEqNum); int32_t idx = 0; - memset(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); + TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); for (int32_t i = 0; i < jtCtx.colEqNum; ) { idx = taosRand() % MAX_SLOT_NUM; if (jtCtx.colEqList[idx]) { @@ -449,12 +455,14 @@ void createColEqCondStart(SSortMergeJoinPhysiNode* p) { for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { if (jtCtx.colEqList[i]) { SColumnNode* pCol1 = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(pCol1); + pCol1->dataBlockId = LEFT_BLK_ID; pCol1->slotId = i; pCol1->node.resType.type = jtInputColType[i]; pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - nodesListMakeStrictAppend(&p->pEqLeft, (SNode*)pCol1); + assert(0 == nodesListMakeStrictAppend(&p->pEqLeft, (SNode*)pCol1)); SColumnNode* pCol2 = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); pCol2->dataBlockId = RIGHT_BLK_ID; @@ -462,7 +470,7 @@ void createColEqCondStart(SSortMergeJoinPhysiNode* p) { pCol2->node.resType.type = jtInputColType[i]; pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - nodesListMakeStrictAppend(&p->pEqRight, (SNode*)pCol2); + assert(0 == nodesListMakeStrictAppend(&p->pEqRight, (SNode*)pCol2)); } } } @@ -474,7 +482,7 @@ void createColOnCondStart(SSortMergeJoinPhysiNode* p) { } while (0 == jtCtx.colOnNum || (jtCtx.colOnNum + jtCtx.colEqNum) > MAX_SLOT_NUM); int32_t idx = 0; - memset(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); + TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); for (int32_t i = 0; i < jtCtx.colOnNum; ) { idx = taosRand() % MAX_SLOT_NUM; if (jtCtx.colOnList[idx] || jtCtx.colEqList[idx]) { @@ -504,6 +512,7 @@ void createColEqCondEnd(SSortMergeJoinPhysiNode* p) { SLogicConditionNode* pLogic = NULL; if (jtCtx.colEqNum > 1) { pLogic = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + assert(pLogic); pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogic->condType = LOGIC_COND_TYPE_AND; @@ -512,18 +521,21 @@ void createColEqCondEnd(SSortMergeJoinPhysiNode* p) { for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { if (jtCtx.colEqList[i]) { SColumnNode* pCol1 = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(pCol1); pCol1->dataBlockId = RES_BLK_ID; pCol1->slotId = getDstSlotId(i); pCol1->node.resType.type = jtInputColType[i]; pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; SColumnNode* pCol2 = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(pCol2); pCol2->dataBlockId = RES_BLK_ID; pCol2->slotId = getDstSlotId(MAX_SLOT_NUM + i); pCol2->node.resType.type = jtInputColType[i]; pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; SOperatorNode* pOp = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + assert(pOp); pOp->opType = OP_TYPE_EQUAL; pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; @@ -531,7 +543,7 @@ void createColEqCondEnd(SSortMergeJoinPhysiNode* p) { pOp->pRight = (SNode*)pCol2; if (jtCtx.colEqNum > 1) { - nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp); + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); } else { p->pFullOnCond = (SNode*)pOp; break; @@ -552,6 +564,7 @@ void createColOnCondEnd(SSortMergeJoinPhysiNode* p) { SLogicConditionNode* pLogic = NULL; if (jtCtx.colOnNum > 1) { pLogic = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + assert(pLogic); pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogic->condType = LOGIC_COND_TYPE_AND; @@ -560,18 +573,21 @@ void createColOnCondEnd(SSortMergeJoinPhysiNode* p) { for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { if (jtCtx.colOnList[i]) { SColumnNode* pCol1 = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(pCol1); pCol1->dataBlockId = RES_BLK_ID; pCol1->slotId = getDstSlotId(i); pCol1->node.resType.type = jtInputColType[i]; pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; SColumnNode* pCol2 = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(pCol2); pCol2->dataBlockId = RES_BLK_ID; pCol2->slotId = getDstSlotId(MAX_SLOT_NUM + i); pCol2->node.resType.type = jtInputColType[i]; pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; SOperatorNode* pOp = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + assert(pOp); pOp->opType = OP_TYPE_GREATER_THAN; pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; @@ -579,7 +595,7 @@ void createColOnCondEnd(SSortMergeJoinPhysiNode* p) { pOp->pRight = (SNode*)pCol2; if (jtCtx.colOnNum > 1) { - nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp); + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); } else { p->pColOnCond = (SNode*)pOp; break; @@ -592,7 +608,8 @@ void createColOnCondEnd(SSortMergeJoinPhysiNode* p) { } SNode* pTmp = nodesCloneNode(p->pColOnCond); - jtMergeEqCond(&p->pFullOnCond, &pTmp); + assert(pTmp); + assert(0 == jtMergeEqCond(&p->pFullOnCond, &pTmp)); } @@ -603,18 +620,18 @@ void createColCond(SSortMergeJoinPhysiNode* p, int32_t cond) { case TEST_NO_COND: jtCtx.colEqNum = 0; jtCtx.colOnNum = 0; - memset(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); - memset(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); + TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); + TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); break; case TEST_EQ_COND: createColEqCondStart(p); jtCtx.colOnNum = 0; - memset(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); + TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); break; case TEST_ON_COND: createColOnCondStart(p); jtCtx.colEqNum = 0; - memset(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); + TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); break; case TEST_FULL_COND: createColEqCondStart(p); @@ -643,8 +660,8 @@ void createFilterStart(SSortMergeJoinPhysiNode* p, bool filter) { if (!filter) { jtCtx.leftFilterNum = 0; jtCtx.rightFilterNum = 0; - memset(jtCtx.leftFilterColList, 0, sizeof(jtCtx.leftFilterColList)); - memset(jtCtx.rightFilterColList, 0, sizeof(jtCtx.rightFilterColList)); + TAOS_MEMSET(jtCtx.leftFilterColList, 0, sizeof(jtCtx.leftFilterColList)); + TAOS_MEMSET(jtCtx.rightFilterColList, 0, sizeof(jtCtx.rightFilterColList)); return; } @@ -676,8 +693,8 @@ void createFilterStart(SSortMergeJoinPhysiNode* p, bool filter) { } int32_t idx = 0; - memset(jtCtx.leftFilterColList, 0, sizeof(jtCtx.leftFilterColList)); - memset(jtCtx.rightFilterColList, 0, sizeof(jtCtx.rightFilterColList)); + TAOS_MEMSET(jtCtx.leftFilterColList, 0, sizeof(jtCtx.leftFilterColList)); + TAOS_MEMSET(jtCtx.rightFilterColList, 0, sizeof(jtCtx.rightFilterColList)); for (int32_t i = 0; i < jtCtx.leftFilterNum; ) { idx = taosRand() % MAX_SLOT_NUM; if (jtCtx.leftFilterColList[idx]) { @@ -705,6 +722,7 @@ void createFilterEnd(SSortMergeJoinPhysiNode* p, bool filter) { SLogicConditionNode* pLogic = NULL; if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { pLogic = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + assert(pLogic); pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogic->condType = LOGIC_COND_TYPE_AND; @@ -713,18 +731,21 @@ void createFilterEnd(SSortMergeJoinPhysiNode* p, bool filter) { for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { if (jtCtx.leftFilterColList[i]) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(pCol); pCol->dataBlockId = RES_BLK_ID; pCol->slotId = getDstSlotId(i); pCol->node.resType.type = jtInputColType[i]; pCol->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - sprintf(pCol->colName, "l%d", i); + (void)sprintf(pCol->colName, "l%d", i); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + assert(pVal); pVal->node.resType.type = jtInputColType[i]; pVal->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - nodesSetValueNodeValue(pVal, getFilterValue(jtInputColType[i])); + assert(0 == nodesSetValueNodeValue(pVal, getFilterValue(jtInputColType[i]))); SOperatorNode* pOp = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + assert(pOp); pOp->opType = OP_TYPE_GREATER_THAN; pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; @@ -732,7 +753,7 @@ void createFilterEnd(SSortMergeJoinPhysiNode* p, bool filter) { pOp->pRight = (SNode*)pVal; if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { - nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp); + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); } else { p->node.pConditions = (SNode*)pOp; break; @@ -743,18 +764,21 @@ void createFilterEnd(SSortMergeJoinPhysiNode* p, bool filter) { for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { if (jtCtx.rightFilterColList[i]) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + assert(pCol); pCol->dataBlockId = RES_BLK_ID; pCol->slotId = getDstSlotId(MAX_SLOT_NUM + i); pCol->node.resType.type = jtInputColType[i]; pCol->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - sprintf(pCol->colName, "r%d", i); + (void)sprintf(pCol->colName, "r%d", i); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + assert(pVal); pVal->node.resType.type = jtInputColType[i]; pVal->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - nodesSetValueNodeValue(pVal, getFilterValue(jtInputColType[i])); + assert(0 == nodesSetValueNodeValue(pVal, getFilterValue(jtInputColType[i]))); SOperatorNode* pOp = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + assert(pOp); pOp->opType = OP_TYPE_GREATER_THAN; pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; @@ -762,7 +786,7 @@ void createFilterEnd(SSortMergeJoinPhysiNode* p, bool filter) { pOp->pRight = (SNode*)pVal; if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { - nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp); + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); } else { p->node.pConditions = (SNode*)pOp; break; @@ -787,17 +811,19 @@ void updateColRowInfo() { void createBlockDescNode(SDataBlockDescNode** ppNode) { SDataBlockDescNode* pDesc = (SDataBlockDescNode*)nodesMakeNode(QUERY_NODE_DATABLOCK_DESC); + assert(pDesc); pDesc->dataBlockId = RES_BLK_ID; pDesc->totalRowSize = jtCtx.resColSize - MAX_SLOT_NUM * 2 * sizeof(bool); pDesc->outputRowSize = pDesc->totalRowSize; for (int32_t i = 0; i < jtCtx.resColNum; ++i) { SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); + assert(pSlot); pSlot->slotId = i; int32_t slotIdx = jtCtx.resColInSlot[i] >= MAX_SLOT_NUM ? jtCtx.resColInSlot[i] - MAX_SLOT_NUM : jtCtx.resColInSlot[i]; pSlot->dataType.type = jtInputColType[slotIdx]; pSlot->dataType.bytes = tDataTypes[pSlot->dataType.type].bytes; - nodesListMakeStrictAppend(&pDesc->pSlots, (SNode *)pSlot); + assert(0 == nodesListMakeStrictAppend(&pDesc->pSlots, (SNode *)pSlot)); } *ppNode = pDesc; @@ -805,12 +831,14 @@ void createBlockDescNode(SDataBlockDescNode** ppNode) { SSortMergeJoinPhysiNode* createDummySortMergeJoinPhysiNode(SJoinTestParam* param) { SSortMergeJoinPhysiNode* p = (SSortMergeJoinPhysiNode*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN); + assert(p); p->joinType = param->joinType; p->subType = param->subType; p->asofOpType = param->asofOp; p->grpJoin = param->grpJoin; if (p->subType == JOIN_STYPE_WIN || param->jLimit > 1 || taosRand() % 2) { SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + assert(limitNode); limitNode->limit = param->jLimit; p->pJLimit = (SNode*)limitNode; } @@ -820,8 +848,11 @@ SSortMergeJoinPhysiNode* createDummySortMergeJoinPhysiNode(SJoinTestParam* param p->node.inputTsOrder = param->asc ? ORDER_ASC : ORDER_DESC; if (JOIN_STYPE_WIN == p->subType) { SWindowOffsetNode* pOffset = (SWindowOffsetNode*)nodesMakeNode(QUERY_NODE_WINDOW_OFFSET); + assert(pOffset); SValueNode* pStart = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + assert(pStart); SValueNode* pEnd = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + assert(pEnd); pStart->node.resType.type = TSDB_DATA_TYPE_BIGINT; pStart->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; pStart->datum.i = (taosRand() % 2) ? (((int32_t)-1) * (int64_t)(taosRand() % JT_MAX_WINDOW_OFFSET)) : (taosRand() % JT_MAX_WINDOW_OFFSET); @@ -863,6 +894,7 @@ SSortMergeJoinPhysiNode* createDummySortMergeJoinPhysiNode(SJoinTestParam* param SExecTaskInfo* createDummyTaskInfo(char* taskId) { SExecTaskInfo* p = (SExecTaskInfo*)taosMemoryCalloc(1, sizeof(SExecTaskInfo)); + assert(p); p->id.str = taskId; return p; @@ -870,6 +902,7 @@ SExecTaskInfo* createDummyTaskInfo(char* taskId) { SSDataBlock* createDummyBlock(int32_t blkId) { SSDataBlock* p = createDataBlock(); + assert(p); p->info.id.blockId = blkId; p->info.type = STREAM_INVALID; @@ -881,14 +914,14 @@ SSDataBlock* createDummyBlock(int32_t blkId) { SColumnInfoData idata = createColumnInfoData(jtInputColType[i], tDataTypes[jtInputColType[i]].bytes, i); - blockDataAppendColInfo(p, &idata); + assert(0 == blockDataAppendColInfo(p, &idata)); } return p; } void appendAsofLeftEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rightRows) { - memset(jtCtx.resColBuf, 0, jtCtx.resColSize); + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { if (!jtCtx.resColList[c]) { continue; @@ -897,7 +930,7 @@ void appendAsofLeftEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rig if (*((bool*)leftInRow + c)) { *(char*)(jtCtx.resColBuf + c) = true; } else { - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); } } @@ -913,10 +946,10 @@ void appendAsofLeftEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rig if (jtCtx.resColList[MAX_SLOT_NUM + c]) { if (*(bool*)(rightResRows + c)) { *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; - memset(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], 0, tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMSET(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], 0, tDataTypes[jtInputColType[c]].bytes); } else { *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = false; - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rightResRows + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rightResRows + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); } } } @@ -927,7 +960,7 @@ void appendAsofLeftEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rig void appendLeftNonMatchGrp(char* leftInRow) { if (!jtCtrl.noKeepResRows) { - memset(jtCtx.resColBuf, 0, jtCtx.resColSize); + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { if (!jtCtx.resColList[c]) { continue; @@ -936,7 +969,7 @@ void appendLeftNonMatchGrp(char* leftInRow) { if (*((bool*)leftInRow + c)) { *(char*)(jtCtx.resColBuf + c) = true; } else { - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); } } @@ -957,13 +990,15 @@ void appendAllAsofResRows() { if (0 == jtCtx.rightFilterNum) { for (int32_t i = 0; i < leftRows; ++i) { char* leftInRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); + assert(leftInRow); appendLeftNonMatchGrp(leftInRow); } } } else { - ASSERT(rightRows <= jtCtx.jLimit); + assert(rightRows <= jtCtx.jLimit); for (int32_t i = 0; i < leftRows; ++i) { char* leftInRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); + assert(leftInRow); appendAsofLeftEachResGrps(leftInRow, 0, rightRows); } } @@ -982,10 +1017,12 @@ void chkAppendAsofForwardGrpResRows(bool forceOut) { int32_t i = 0; for (; i < leftRows; ++i) { char* leftRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); + assert(leftRow); int64_t* leftTs = (int64_t*)(leftRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); bool append = false; for (int32_t r = rightOffset; r < rightRows; ++r) { char* rightRow = (char*)taosArrayGet(jtCtx.rightRowsList, r); + assert(rightRow); int64_t* rightTs = (int64_t*)(rightRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); if (((jtCtx.asc && *leftTs > *rightTs) || (!jtCtx.asc && *leftTs < *rightTs)) || (*leftTs == *rightTs && (OP_TYPE_LOWER_THAN == jtCtx.asofOpType || OP_TYPE_GREATER_THAN == jtCtx.asofOpType))) { rightOffset++; @@ -1030,7 +1067,7 @@ void appendWinEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rightRow return; } - memset(jtCtx.resColBuf, 0, jtCtx.resColSize); + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { if (!jtCtx.resColList[c]) { continue; @@ -1039,7 +1076,7 @@ void appendWinEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rightRow if (*((bool*)leftInRow + c)) { *(char*)(jtCtx.resColBuf + c) = true; } else { - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); } } @@ -1052,14 +1089,15 @@ void appendWinEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rightRow } char* rightResRows = (char*)taosArrayGet(jtCtx.rightRowsList, r); + assert(rightResRows); for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { if (jtCtx.resColList[MAX_SLOT_NUM + c]) { if (*(bool*)(rightResRows + c)) { *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; - memset(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], 0, tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMSET(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], 0, tDataTypes[jtInputColType[c]].bytes); } else { *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = false; - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rightResRows + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rightResRows + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); } } } @@ -1080,6 +1118,7 @@ void chkAppendWinResRows(bool forceOut) { int32_t i = 0; for (; i < leftRows; ++i) { char* leftRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); + assert(leftRow); int64_t* leftTs = (int64_t*)(leftRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); int64_t winStart = *leftTs + jtCtx.winStartOffset; int64_t winEnd = *leftTs + jtCtx.winEndOffset; @@ -1088,6 +1127,7 @@ void chkAppendWinResRows(bool forceOut) { bool winClosed = false; for (int32_t r = rightOffset; r < rightRows; ++r) { char* rightRow = (char*)taosArrayGet(jtCtx.rightRowsList, r); + assert(rightRow); int64_t* rightTs = (int64_t*)(rightRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); if ((jtCtx.asc && *rightTs < winStart) || (!jtCtx.asc && *rightTs > winEnd)) { rightOffset++; @@ -1154,8 +1194,9 @@ void createGrpRows(SSDataBlock** ppBlk, int32_t blkId, int32_t grpRows) { if (NULL == *ppBlk) { *ppBlk = createDummyBlock((blkId == LEFT_BLK_ID) ? LEFT_BLK_ID : RIGHT_BLK_ID); - blockDataEnsureCapacity(*ppBlk, jtCtx.blkRows); - taosArrayPush((blkId == LEFT_BLK_ID) ? jtCtx.leftBlkList : jtCtx.rightBlkList, ppBlk); + assert(*ppBlk); + assert(0 == blockDataEnsureCapacity(*ppBlk, jtCtx.blkRows)); + assert(NULL != taosArrayPush((blkId == LEFT_BLK_ID) ? jtCtx.leftBlkList : jtCtx.rightBlkList, ppBlk)); } if (jtCtx.grpJoin) { @@ -1207,8 +1248,9 @@ void createGrpRows(SSDataBlock** ppBlk, int32_t blkId, int32_t grpRows) { for (int32_t i = 0; i < grpRows; ++i) { if ((*ppBlk)->info.rows >= (*ppBlk)->info.capacity) { *ppBlk = createDummyBlock((blkId == LEFT_BLK_ID) ? LEFT_BLK_ID : RIGHT_BLK_ID); - blockDataEnsureCapacity(*ppBlk, jtCtx.blkRows); - taosArrayPush((blkId == LEFT_BLK_ID) ? jtCtx.leftBlkList : jtCtx.rightBlkList, ppBlk); + assert(*ppBlk); + assert(0 == blockDataEnsureCapacity(*ppBlk, jtCtx.blkRows)); + assert(NULL != taosArrayPush((blkId == LEFT_BLK_ID) ? jtCtx.leftBlkList : jtCtx.rightBlkList, ppBlk)); if (jtCtx.grpJoin) { (*ppBlk)->info.id.groupId = jtCtx.inGrpId; } @@ -1216,9 +1258,9 @@ void createGrpRows(SSDataBlock** ppBlk, int32_t blkId, int32_t grpRows) { filterOut = (peerFilterNum > 0 && (jtCtx.subType != JOIN_STYPE_ASOF && jtCtx.subType != JOIN_STYPE_WIN)) ? true : false; if (!filterOut) { - memset(jtCtx.resColBuf, 0, jtCtx.resColSize); + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); if (keepInput) { - memset(jtCtx.inColBuf, 0, jtCtx.inColSize); + TAOS_MEMSET(jtCtx.inColBuf, 0, jtCtx.inColSize); } } @@ -1262,14 +1304,15 @@ void createGrpRows(SSDataBlock** ppBlk, int32_t blkId, int32_t grpRows) { } SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet((*ppBlk)->pDataBlock, c); - colDataSetVal(pCol, (*ppBlk)->info.rows, pData, isNull); + assert(pCol); + assert(0 == colDataSetVal(pCol, (*ppBlk)->info.rows, pData, isNull)); if (keepInput) { if (!filterOut || (blkId != LEFT_BLK_ID)) { if (isNull) { *(char*)(jtCtx.inColBuf + c) = true; } else { - memcpy(jtCtx.inColBuf + jtCtx.inColOffset[c], pData, tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.inColBuf + jtCtx.inColOffset[c], pData, tDataTypes[jtInputColType[c]].bytes); } } else { addToRowList = false; @@ -1278,16 +1321,16 @@ void createGrpRows(SSDataBlock** ppBlk, int32_t blkId, int32_t grpRows) { if (isNull) { *(char*)(jtCtx.resColBuf + tableOffset + c) = true; } else { - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[tableOffset + c], pData, tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[tableOffset + c], pData, tDataTypes[jtInputColType[c]].bytes); } } } if (keepInput && addToRowList) { - taosArrayPush(pTableRows, jtCtx.inColBuf); + assert(NULL != taosArrayPush(pTableRows, jtCtx.inColBuf)); if (blkId == RIGHT_BLK_ID) { bool fout = filterOut ? true : false; - taosArrayPush(jtCtx.rightFilterOut, &fout); + assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &fout)); } } @@ -1328,27 +1371,28 @@ void createRowData(SSDataBlock* pBlk, int64_t tbOffset, int32_t rowIdx, int32_t for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlk->pDataBlock, c); + assert(pCol); int32_t rv = taosRand() % 10; switch (jtInputColType[c]) { case TSDB_DATA_TYPE_TIMESTAMP: *(int64_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = jtCtx.curTs; - colDataSetVal(pCol, pBlk->info.rows, (char*)&jtCtx.curTs, false); + assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&jtCtx.curTs, false)); break; case TSDB_DATA_TYPE_INT: if (rv) { tmpInt = (taosRand() % 2) ? INT_FILTER_VALUE + jtCtx.grpOffset[c] + taosRand() % vRange : INT_FILTER_VALUE - taosRand() % vRange; *(int32_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = tmpInt; - colDataSetVal(pCol, pBlk->info.rows, (char*)&tmpInt, false); + assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&tmpInt, false)); } else { *(bool*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + c) = true; - colDataSetVal(pCol, pBlk->info.rows, NULL, true); + assert(0 == colDataSetVal(pCol, pBlk->info.rows, NULL, true)); } break; case TSDB_DATA_TYPE_BIGINT: tmpBig = (taosRand() % 2) ? BIGINT_FILTER_VALUE + jtCtx.curKeyOffset++ : BIGINT_FILTER_VALUE - jtCtx.curKeyOffset++; *(int64_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = tmpBig; - colDataSetVal(pCol, pBlk->info.rows, (char*)&tmpBig, false); + assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&tmpBig, false)); break; default: break; @@ -1364,9 +1408,10 @@ void makeAppendBlkData(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t left if (jtCtx.colRowDataBufSize < totalSize) { jtCtx.colRowDataBuf = (char*)taosMemoryRealloc(jtCtx.colRowDataBuf, totalSize); + assert(jtCtx.colRowDataBuf); } - memset(jtCtx.colRowDataBuf, 0, totalSize); + TAOS_MEMSET(jtCtx.colRowDataBuf, 0, totalSize); for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { jtCtx.grpOffset[c] = c * TMAX(leftGrpRows, rightGrpRows); @@ -1376,8 +1421,9 @@ void makeAppendBlkData(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t left for (int32_t i = 0; i < leftGrpRows; ++i) { if ((*ppLeft)->info.rows >= (*ppLeft)->info.capacity) { *ppLeft = createDummyBlock(LEFT_BLK_ID); - blockDataEnsureCapacity(*ppLeft, jtCtx.blkRows); - taosArrayPush(jtCtx.leftBlkList, ppLeft); + assert(*ppLeft); + assert(0 == blockDataEnsureCapacity(*ppLeft, jtCtx.blkRows)); + assert(NULL != taosArrayPush(jtCtx.leftBlkList, ppLeft)); if (jtCtx.grpJoin) { (*ppLeft)->info.id.groupId = jtCtx.inGrpId; } @@ -1390,8 +1436,9 @@ void makeAppendBlkData(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t left for (int32_t i = 0; i < rightGrpRows; ++i) { if ((*ppRight)->info.rows >= (*ppRight)->info.capacity) { *ppRight = createDummyBlock(RIGHT_BLK_ID); - blockDataEnsureCapacity(*ppRight, jtCtx.blkRows); - taosArrayPush(jtCtx.rightBlkList, ppRight); + assert(*ppRight); + assert(0 == blockDataEnsureCapacity(*ppRight, jtCtx.blkRows)); + assert(NULL != taosArrayPush(jtCtx.rightBlkList, ppRight)); if (jtCtx.grpJoin) { (*ppRight)->info.id.groupId = jtCtx.inGrpId; } @@ -1404,14 +1451,14 @@ void makeAppendBlkData(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t left void putNMatchRowToRes(char* lrow, int32_t tableOffset, int32_t peerOffset) { if (!jtCtrl.noKeepResRows) { - memset(jtCtx.resColBuf, 0, jtCtx.resColSize); + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { if (jtCtx.resColList[tableOffset + c]) { if (*(bool*)(lrow + c)) { *(bool*)(jtCtx.resColBuf + tableOffset + c) = true; } else { - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[tableOffset + c], lrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[tableOffset + c], lrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); } } } @@ -1428,7 +1475,7 @@ void putNMatchRowToRes(char* lrow, int32_t tableOffset, int32_t peerOffset) { void putMatchRowToRes(char* lrow, char* rrow, int32_t cols) { if (!jtCtrl.noKeepResRows) { - memset(jtCtx.resColBuf, 0, jtCtx.resColSize); + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); if (cols & LEFT_TABLE_COLS) { for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { @@ -1436,7 +1483,7 @@ void putMatchRowToRes(char* lrow, char* rrow, int32_t cols) { if (*(bool*)(lrow + c)) { *(bool*)(jtCtx.resColBuf + c) = true; } else { - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[c], lrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], lrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); } } } @@ -1448,7 +1495,7 @@ void putMatchRowToRes(char* lrow, char* rrow, int32_t cols) { if (*(bool*)(rrow + c)) { *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; } else { - memcpy(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); } } } @@ -1824,7 +1871,7 @@ void antiJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { int64_t lBig = 0, rBig = 0, fbig = 0; int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - ASSERT(0 == jtCtx.rightFilterNum); + assert(0 == jtCtx.rightFilterNum); for (int32_t l = 0; l < leftGrpRows; ++l) { char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; @@ -1977,9 +2024,9 @@ void addAsofEqInRows(int32_t rowsNum, int64_t tbOffset, bool leftTable) { continue; } - taosArrayPush(rowList, row); + assert(NULL != taosArrayPush(rowList, row)); if (!leftTable) { - taosArrayPush(jtCtx.rightFilterOut, &filterOut); + assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &filterOut)); } } @@ -2103,9 +2150,9 @@ void addWinEqInRows(int32_t rowsNum, int64_t tbOffset, bool leftTable) { continue; } - taosArrayPush(rowList, row); + assert(NULL != taosArrayPush(rowList, row)); if (!leftTable) { - taosArrayPush(jtCtx.rightFilterOut, &filterOut); + assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &filterOut)); } } } @@ -2127,7 +2174,7 @@ void fullJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { int64_t lBig = 0, rBig = 0, fbig = 0; int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - memset(jtCtx.rightFinMatch, 0, rightGrpRows * sizeof(bool)); + TAOS_MEMSET(jtCtx.rightFinMatch, 0, rightGrpRows * sizeof(bool)); for (int32_t l = 0; l < leftGrpRows; ++l) { char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; @@ -2285,8 +2332,9 @@ void createTsEqGrpRows(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t left if (NULL == *ppLeft && leftGrpRows > 0) { *ppLeft = createDummyBlock(LEFT_BLK_ID); - blockDataEnsureCapacity(*ppLeft, jtCtx.blkRows); - taosArrayPush(jtCtx.leftBlkList, ppLeft); + assert(*ppLeft); + assert(0 == blockDataEnsureCapacity(*ppLeft, jtCtx.blkRows)); + assert(NULL != taosArrayPush(jtCtx.leftBlkList, ppLeft)); } if (jtCtx.grpJoin) { @@ -2295,8 +2343,9 @@ void createTsEqGrpRows(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t left if (NULL == *ppRight && rightGrpRows > 0) { *ppRight = createDummyBlock(RIGHT_BLK_ID); - blockDataEnsureCapacity(*ppRight, jtCtx.blkRows); - taosArrayPush(jtCtx.rightBlkList, ppRight); + assert(*ppRight); + assert(0 == blockDataEnsureCapacity(*ppRight, jtCtx.blkRows)); + assert(NULL != taosArrayPush(jtCtx.rightBlkList, ppRight)); } if (jtCtx.grpJoin) { @@ -2311,7 +2360,7 @@ void createTsEqGrpRows(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t left void forceFlushResRows() { if (JOIN_STYPE_ASOF == jtCtx.subType && taosArrayGetSize(jtCtx.leftRowsList) > 0) { - ASSERT((jtCtx.asc && (OP_TYPE_LOWER_EQUAL == jtCtx.asofOpType || OP_TYPE_LOWER_THAN == jtCtx.asofOpType)) + assert((jtCtx.asc && (OP_TYPE_LOWER_EQUAL == jtCtx.asofOpType || OP_TYPE_LOWER_THAN == jtCtx.asofOpType)) || (!jtCtx.asc && (OP_TYPE_GREATER_EQUAL == jtCtx.asofOpType || OP_TYPE_GREATER_THAN == jtCtx.asofOpType))); chkAppendAsofForwardGrpResRows(true); } else if (JOIN_STYPE_WIN == jtCtx.subType && taosArrayGetSize(jtCtx.leftRowsList) > 0) { @@ -2404,6 +2453,7 @@ void createDummyBlkList(int32_t leftMaxRows, int32_t leftMaxGrpRows, int32_t rig if (maxGrpRows > jtCtx.rightFinMatchNum) { jtCtx.rightFinMatchNum = maxGrpRows; jtCtx.rightFinMatch = (bool*)taosMemoryRealloc(jtCtx.rightFinMatch, maxGrpRows * sizeof(bool)); + assert(jtCtx.rightFinMatch); } taosArrayClear(jtCtx.leftRowsList); @@ -2466,43 +2516,44 @@ void joinTestReplaceRetrieveFp() { void printColList(char* title, bool left, int32_t* colList, bool filter, char* opStr) { bool first = true; - printf("\t %s:", title); + JT_PRINTF("\t %s:", title); for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { if (colList[i]) { if (!first) { - printf(" AND "); + JT_PRINTF(" AND "); } first = false; if (filter) { - printf("%sc%d%s%" PRId64 , left ? "l" : "r", i, opStr, jtFilterValue[i]); + JT_PRINTF("%sc%d%s%" PRId64 , left ? "l" : "r", i, opStr, jtFilterValue[i]); } else { - printf("lc%d%src%d", i, opStr, i); + JT_PRINTF("lc%d%src%d", i, opStr, i); } } } - printf("\n"); + JT_PRINTF("\n"); } void printInputRowData(SSDataBlock* pBlk, int32_t* rowIdx) { if (jtCtx.grpJoin) { - printf("%5" PRIu64, pBlk->info.id.groupId); + JT_PRINTF("%5" PRIu64, pBlk->info.id.groupId); } for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlk->pDataBlock, c); - ASSERT(pCol->info.type == jtInputColType[c]); + assert(pCol); + assert(pCol->info.type == jtInputColType[c]); if (colDataIsNull_s(pCol, *rowIdx)) { - printf("%18s", " NULL"); + JT_PRINTF("%18s", " NULL"); } else { switch (jtInputColType[c]) { case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: - printf("%18" PRId64, *(int64_t*)colDataGetData(pCol, *rowIdx)); + JT_PRINTF("%18" PRId64, *(int64_t*)colDataGetData(pCol, *rowIdx)); break; case TSDB_DATA_TYPE_INT: - printf("%18d", *(int32_t*)colDataGetData(pCol, *rowIdx)); + JT_PRINTF("%18d", *(int32_t*)colDataGetData(pCol, *rowIdx)); break; default: - ASSERT(0); + assert(0); } } } @@ -2513,41 +2564,43 @@ void printInputRowData(SSDataBlock* pBlk, int32_t* rowIdx) { void printInputData() { int32_t leftRowIdx = 0, rightRowIdx = 0; - printf("\nInput Data:\n"); + JT_PRINTF("\nInput Data:\n"); while (jtCtx.leftBlkReadIdx < taosArrayGetSize(jtCtx.leftBlkList) || jtCtx.rightBlkReadIdx < taosArrayGetSize(jtCtx.rightBlkList)) { if (jtCtx.leftBlkReadIdx < taosArrayGetSize(jtCtx.leftBlkList)) { while (true) { SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.leftBlkList, jtCtx.leftBlkReadIdx); + assert(pBlk); if (leftRowIdx < pBlk->info.rows) { printInputRowData(pBlk, &leftRowIdx); break; } - printf("\t%*s-------------------------blk end-------------------------------", jtCtx.grpJoin ? 6 : 0, " "); + JT_PRINTF("\t%*s-------------------------blk end-------------------------------", jtCtx.grpJoin ? 6 : 0, " "); jtCtx.leftBlkReadIdx++; leftRowIdx = 0; break; } } else { - printf("%*s", jtCtx.grpJoin ? 77 : 72, " "); + JT_PRINTF("%*s", jtCtx.grpJoin ? 77 : 72, " "); } if (jtCtx.rightBlkReadIdx < taosArrayGetSize(jtCtx.rightBlkList)) { while (true) { SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.rightBlkList, jtCtx.rightBlkReadIdx); + assert(pBlk); if (rightRowIdx < pBlk->info.rows) { printInputRowData(pBlk, &rightRowIdx); break; } - printf("\t%*s--------------------------blk end----------------------------\t", jtCtx.grpJoin ? 6 : 0, " "); + JT_PRINTF("\t%*s--------------------------blk end----------------------------\t", jtCtx.grpJoin ? 6 : 0, " "); jtCtx.rightBlkReadIdx++; rightRowIdx = 0; break; } } - printf("\n"); + JT_PRINTF("\n"); } jtCtx.leftBlkReadIdx = jtCtx.rightBlkReadIdx = 0; @@ -2555,13 +2608,13 @@ void printInputData() { char* getInputStatStr(char* inputStat) { if (jtCtx.inputStat & (1 << LEFT_BLK_ID)) { - strcat(inputStat, "L"); + TAOS_STRCAT(inputStat, "L"); } if (jtCtx.inputStat & (1 << RIGHT_BLK_ID)) { - strcat(inputStat, "R"); + TAOS_STRCAT(inputStat, "R"); } if (jtCtx.inputStat & (1 << 2)) { - strcat(inputStat, "E"); + TAOS_STRCAT(inputStat, "E"); } return inputStat; } @@ -2589,19 +2642,19 @@ void printBasicInfo(char* caseName) { } char inputStat[4] = {0}; - printf("\n%dth TEST [%s] START\nBasic Info:\n\t asc:%d\n\t filter:%d\n\t maxRows:left-%d right-%d\n\t " + JT_PRINTF("\n%dth TEST [%s] START\nBasic Info:\n\t asc:%d\n\t filter:%d\n\t maxRows:left-%d right-%d\n\t " "maxGrpRows:left-%d right-%d\n\t blkRows:%d\n\t colCond:%s\n\t joinType:%s\n\t " "subType:%s\n\t inputStat:%s\n\t groupJoin:%s\n", jtCtx.loopIdx, caseName, jtCtx.asc, jtCtx.filter, jtCtx.leftMaxRows, jtCtx.rightMaxRows, jtCtx.leftMaxGrpRows, jtCtx.rightMaxGrpRows, jtCtx.blkRows, jtColCondStr[jtCtx.colCond], jtJoinTypeStr[jtCtx.joinType], jtSubTypeStr[jtCtx.subType], getInputStatStr(inputStat), jtCtx.grpJoin ? "true" : "false"); if (JOIN_STYPE_ASOF == jtCtx.subType) { - printf("\t asofOp:%s\n\t JLimit:%" PRId64 "\n", getAsofOpStr(), jtCtx.jLimit); + JT_PRINTF("\t asofOp:%s\n\t JLimit:%" PRId64 "\n", getAsofOpStr(), jtCtx.jLimit); } else if (JOIN_STYPE_WIN == jtCtx.subType) { - printf("\t windowOffset:[%" PRId64 ", %" PRId64 "]\n\t JLimit:%" PRId64 "\n", jtCtx.winStartOffset, jtCtx.winEndOffset, jtCtx.jLimit); + JT_PRINTF("\t windowOffset:[%" PRId64 ", %" PRId64 "]\n\t JLimit:%" PRId64 "\n", jtCtx.winStartOffset, jtCtx.winEndOffset, jtCtx.jLimit); } - printf("Input Info:\n\t totalBlk:left-%d right-%d\n\t totalRows:left-%d right-%d\n\t " + JT_PRINTF("Input Info:\n\t totalBlk:left-%d right-%d\n\t totalRows:left-%d right-%d\n\t " "blkRowSize:%d\n\t inputCols:left-%s %s %s %s right-%s %s %s %s\n", (int32_t)taosArrayGetSize(jtCtx.leftBlkList), (int32_t)taosArrayGetSize(jtCtx.rightBlkList), jtCtx.leftTotalRows, jtCtx.rightTotalRows, @@ -2610,30 +2663,30 @@ void printBasicInfo(char* caseName) { tDataTypes[jtInputColType[1]].name, tDataTypes[jtInputColType[2]].name, tDataTypes[jtInputColType[3]].name); if (jtCtx.colEqNum) { - printf("\t colEqNum:%d\n", jtCtx.colEqNum); + JT_PRINTF("\t colEqNum:%d\n", jtCtx.colEqNum); printColList("colEqList", false, jtCtx.colEqList, false, "="); } if (jtCtx.colOnNum) { - printf("\t colOnNum:%d\n", jtCtx.colOnNum); + JT_PRINTF("\t colOnNum:%d\n", jtCtx.colOnNum); printColList("colOnList", false, jtCtx.colOnList, false, ">"); } if (jtCtx.leftFilterNum) { - printf("\t leftFilterNum:%d\n", jtCtx.leftFilterNum); + JT_PRINTF("\t leftFilterNum:%d\n", jtCtx.leftFilterNum); printColList("leftFilterList", true, jtCtx.leftFilterColList, true, ">"); } if (jtCtx.rightFilterNum) { - printf("\t rightFilterNum:%d\n", jtCtx.rightFilterNum); + JT_PRINTF("\t rightFilterNum:%d\n", jtCtx.rightFilterNum); printColList("rightFilterList", false, jtCtx.rightFilterColList, true, ">"); } - printf("\t resColSize:%d\n\t resColNum:%d\n\t resColList:", jtCtx.resColSize, jtCtx.resColNum); + JT_PRINTF("\t resColSize:%d\n\t resColNum:%d\n\t resColList:", jtCtx.resColSize, jtCtx.resColNum); for (int32_t i = 0; i < jtCtx.resColNum; ++i) { int32_t s = jtCtx.resColInSlot[i]; int32_t idx = s >= MAX_SLOT_NUM ? s - MAX_SLOT_NUM : s; - printf("%sc%d[%s]\t", s >= MAX_SLOT_NUM ? "r" : "l", s, tDataTypes[jtInputColType[idx]].name); + JT_PRINTF("%sc%d[%s]\t", s >= MAX_SLOT_NUM ? "r" : "l", s, tDataTypes[jtInputColType[idx]].name); } if (jtCtrl.printInputRow) { @@ -2646,8 +2699,8 @@ void printOutputInfo() { return; } - printf("\nOutput Info:\n\t expectedRows:%d\n\t ", jtCtx.resRows); - printf("Actual Result:\n"); + JT_PRINTF("\nOutput Info:\n\t expectedRows:%d\n\t ", jtCtx.resRows); + JT_PRINTF("Actual Result:\n"); } void printActualResInfo() { @@ -2655,7 +2708,7 @@ void printActualResInfo() { return; } - printf("Actual Result Summary:\n\t blkNum:%d\n\t rowNum:%d%s\n\t leftBlkRead:%d\n\t rightBlkRead:%d\n\t +rows:%d%s\n\t " + JT_PRINTF("Actual Result Summary:\n\t blkNum:%d\n\t rowNum:%d%s\n\t leftBlkRead:%d\n\t rightBlkRead:%d\n\t +rows:%d%s\n\t " "-rows:%d%s\n\t matchRows:%d%s\n\t executionTime:%" PRId64 "us\n", jtRes.blkNum, jtRes.rowNum, jtRes.rowNum == jtCtx.resRows ? "" : "*", @@ -2667,7 +2720,7 @@ void printActualResInfo() { } void printStatInfo(char* caseName) { - printf("\n TEST [%s] Stat:\n\t maxResRows:%d\n\t maxResBlkRows:%d\n\t totalResRows:%" PRId64 "\n\t useMSecs:%" PRId64 "\n", + JT_PRINTF("\n TEST [%s] Stat:\n\t maxResRows:%d\n\t maxResBlkRows:%d\n\t totalResRows:%" PRId64 "\n\t useMSecs:%" PRId64 "\n", caseName, jtStat.maxResRows, jtStat.maxResBlkRows, jtStat.totalResRows, jtStat.useMSecs); } @@ -2689,7 +2742,7 @@ void checkJoinDone(char* caseName) { printActualResInfo(); - printf("\n%dth TEST [%s] Final Result: %s\n", jtCtx.loopIdx, caseName, jtRes.succeed ? "SUCCEED" : "FAILED"); + JT_PRINTF("\n%dth TEST [%s] Final Result: %s\n", jtCtx.loopIdx, caseName, jtRes.succeed ? "SUCCEED" : "FAILED"); } void putRowToResColBuf(SSDataBlock* pBlock, int32_t r, bool ignoreTbCols) { @@ -2701,6 +2754,7 @@ void putRowToResColBuf(SSDataBlock* pBlock, int32_t r, bool ignoreTbCols) { } SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, c); + assert(pCol); switch (jtInputColType[slot % MAX_SLOT_NUM]) { case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: @@ -2739,7 +2793,7 @@ void checkJoinRes(SSDataBlock* pBlock) { jtRes.matchNum += pBlock->info.rows; } else { for (int32_t r = 0; r < pBlock->info.rows; ++r) { - memset(jtCtx.resColBuf, 0, jtCtx.resColSize); + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); putRowToResColBuf(pBlock, r, true); @@ -2766,7 +2820,7 @@ void resetForJoinRerun(int32_t dsNum, SSortMergeJoinPhysiNode* pNode, SExecTaskI jtCtx.rightBlkReadIdx = 0; jtCtx.curKeyOffset = 0; - memset(&jtRes, 0, sizeof(jtRes)); + TAOS_MEMSET(&jtRes, 0, sizeof(jtRes)); jtRes.succeed = true; SOperatorInfo* pDownstreams[2]; @@ -2800,10 +2854,10 @@ void jtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, TD_LOG_DIR_PATH); + TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); + JT_PRINTF("failed to open log file in directory:%s\n", tsLogDir); } } @@ -2812,12 +2866,15 @@ void jtInitLogFile() { void initJoinTest() { jtCtx.leftBlkList = taosArrayInit(10, POINTER_BYTES); jtCtx.rightBlkList = taosArrayInit(10, POINTER_BYTES); + assert(jtCtx.leftBlkList && jtCtx.rightBlkList); jtCtx.jtResRows = tSimpleHashInit(10000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); + assert(jtCtx.jtResRows); joinTestReplaceRetrieveFp(); if (jtCtrl.logHistory) { jtStat.pHistory = taosArrayInit(100000, sizeof(SJoinTestHistory)); + assert(jtStat.pHistory); } int32_t offset = MAX_SLOT_NUM * sizeof(bool); @@ -2827,10 +2884,12 @@ void initJoinTest() { } jtCtx.inColSize = offset; jtCtx.inColBuf = (char*)taosMemoryMalloc(jtCtx.inColSize); + assert(jtCtx.inColBuf); jtCtx.leftRowsList = taosArrayInit(1024, jtCtx.inColSize); jtCtx.rightRowsList = taosArrayInit(1024, jtCtx.inColSize); jtCtx.rightFilterOut = taosArrayInit(1024, sizeof(bool)); + assert(jtCtx.leftRowsList && jtCtx.rightRowsList && jtCtx.rightFilterOut); jtInitLogFile(); } @@ -2838,22 +2897,24 @@ void initJoinTest() { void handleTestDone() { if (jtCtrl.logHistory) { SJoinTestHistory h; - memcpy(&h.ctx, &jtCtx, sizeof(h.ctx)); - memcpy(&h.res, &jtRes, sizeof(h.res)); - taosArrayPush(jtStat.pHistory, &h); + TAOS_MEMCPY(&h.ctx, &jtCtx, sizeof(h.ctx)); + TAOS_MEMCPY(&h.res, &jtRes, sizeof(h.res)); + assert(NULL != taosArrayPush(jtStat.pHistory, &h)); } int32_t blkNum = taosArrayGetSize(jtCtx.leftBlkList); for (int32_t i = 0; i < blkNum; ++i) { SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.leftBlkList, i); - blockDataDestroy(pBlk); + assert(pBlk); + (void)blockDataDestroy(pBlk); } taosArrayClear(jtCtx.leftBlkList); blkNum = taosArrayGetSize(jtCtx.rightBlkList); for (int32_t i = 0; i < blkNum; ++i) { SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.rightBlkList, i); - blockDataDestroy(pBlk); + assert(pBlk); + (void)blockDataDestroy(pBlk); } taosArrayClear(jtCtx.rightBlkList); @@ -2867,6 +2928,7 @@ void runSingleTest(char* caseName, SJoinTestParam* param) { bool contLoop = true; SSortMergeJoinPhysiNode* pNode = createDummySortMergeJoinPhysiNode(param); + assert(pNode); createDummyBlkList(1000, 1000, 1000, 1000, 100); while (contLoop) { @@ -2906,6 +2968,7 @@ TEST(innerJoin, noCondTest) { SJoinTestParam param; char* caseName = "innerJoin:noCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_INNER; @@ -2933,6 +2996,7 @@ TEST(innerJoin, eqCondTest) { SJoinTestParam param; char* caseName = "innerJoin:eqCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_INNER; @@ -2960,6 +3024,7 @@ TEST(innerJoin, onCondTest) { SJoinTestParam param; char* caseName = "innerJoin:onCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_INNER; @@ -2987,6 +3052,7 @@ TEST(innerJoin, fullCondTest) { SJoinTestParam param; char* caseName = "innerJoin:fullCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_INNER; @@ -3017,6 +3083,7 @@ TEST(leftOuterJoin, noCondTest) { SJoinTestParam param; char* caseName = "leftOuterJoin:noCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3045,6 +3112,7 @@ TEST(leftOuterJoin, eqCondTest) { SJoinTestParam param; char* caseName = "leftOuterJoin:eqCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3072,6 +3140,7 @@ TEST(leftOuterJoin, onCondTest) { SJoinTestParam param; char* caseName = "leftOuterJoin:onCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3099,6 +3168,7 @@ TEST(leftOuterJoin, fullCondTest) { SJoinTestParam param; char* caseName = "leftOuterJoin:fullCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3128,6 +3198,7 @@ TEST(fullOuterJoin, noCondTest) { SJoinTestParam param; char* caseName = "fullOuterJoin:noCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_FULL; @@ -3155,6 +3226,7 @@ TEST(fullOuterJoin, eqCondTest) { SJoinTestParam param; char* caseName = "fullOuterJoin:eqCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_FULL; @@ -3183,6 +3255,7 @@ TEST(fullOuterJoin, onCondTest) { SJoinTestParam param; char* caseName = "fullOuterJoin:onCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_FULL; @@ -3210,6 +3283,7 @@ TEST(fullOuterJoin, fullCondTest) { SJoinTestParam param; char* caseName = "fullOuterJoin:fullCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_FULL; @@ -3239,6 +3313,7 @@ TEST(leftSemiJoin, noCondTest) { SJoinTestParam param; char* caseName = "leftSemiJoin:noCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3266,6 +3341,7 @@ TEST(leftSemiJoin, eqCondTest) { SJoinTestParam param; char* caseName = "leftSemiJoin:eqCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3294,6 +3370,7 @@ TEST(leftSemiJoin, onCondTest) { SJoinTestParam param; char* caseName = "leftSemiJoin:onCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3321,6 +3398,7 @@ TEST(leftSemiJoin, fullCondTest) { SJoinTestParam param; char* caseName = "leftSemiJoin:fullCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3350,6 +3428,7 @@ TEST(leftAntiJoin, noCondTest) { SJoinTestParam param; char* caseName = "leftAntiJoin:noCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3377,6 +3456,7 @@ TEST(leftAntiJoin, eqCondTest) { SJoinTestParam param; char* caseName = "leftAntiJoin:eqCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3405,6 +3485,7 @@ TEST(leftAntiJoin, onCondTest) { SJoinTestParam param; char* caseName = "leftAntiJoin:onCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3432,6 +3513,7 @@ TEST(leftAntiJoin, fullCondTest) { SJoinTestParam param; char* caseName = "leftAntiJoin:fullCondTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3461,6 +3543,7 @@ TEST(leftAsofJoin, noCondGreaterThanTest) { SJoinTestParam param; char* caseName = "leftAsofJoin:noCondGreaterThanTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3492,6 +3575,7 @@ TEST(leftAsofJoin, noCondGreaterEqTest) { SJoinTestParam param; char* caseName = "leftAsofJoin:noCondGreaterEqTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3523,6 +3607,7 @@ TEST(leftAsofJoin, noCondEqTest) { SJoinTestParam param; char* caseName = "leftAsofJoin:noCondEqTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3554,6 +3639,7 @@ TEST(leftAsofJoin, noCondLowerThanTest) { SJoinTestParam param; char* caseName = "leftAsofJoin:noCondLowerThanTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3586,6 +3672,7 @@ TEST(leftAsofJoin, noCondLowerEqTest) { SJoinTestParam param; char* caseName = "leftAsofJoin:noCondLowerEqTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; @@ -3621,6 +3708,7 @@ TEST(leftWinJoin, noCondProjectionTest) { SJoinTestParam param; char* caseName = "leftWinJoin:noCondProjectionTest"; SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); param.pTask = pTask; param.joinType = JOIN_TYPE_LEFT; From 46400b2dd7802dd769edd955b17743f78700d582 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Jul 2024 14:46:11 +0800 Subject: [PATCH 21/43] fix(stream): check return value. --- source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 6 ++++-- source/util/src/tcompression.c | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index e4ac503d3b..272897eadf 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1738,7 +1738,7 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { } code = mndAcquireStream(pMnode, streamName, &pStream); - if (pStream != NULL || code != 0) { + if (pStream != NULL || code != TSDB_CODE_MND_STREAM_NOT_EXIST) { mError("tsma:%s, failed to create since stream:%s already exist", createReq.name, streamName); terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; goto _OVER; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index cd6e87b9fc..45212df1dd 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -284,8 +284,10 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S memcpy(&tagVal.i64, pData, pTagData->info.bytes); } void* p = taosArrayPush(tagArray, &tagVal); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _end; + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } } } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index d8656c0f60..884d7ea1b6 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -252,7 +252,7 @@ int32_t l2ComressInitImpl_xz(char *lossyColumns, float fPrecision, double dPreci } int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char *const output, int32_t outputSize, const char type, int8_t lvl) { - size_t len = 0;//FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); + size_t len = FL2_compress(output + 1, outputSize - 1, input, inputSize, lvl); if (len > inputSize) { output[0] = 0; memcpy(output + 1, input, inputSize); @@ -264,7 +264,7 @@ int32_t l2CompressImpl_xz(const char *const input, const int32_t inputSize, char int32_t l2DecompressImpl_xz(const char *const input, const int32_t compressedSize, char *const output, int32_t outputSize, const char type) { if (input[0] == 1) { - return 0;//FL2_decompress(output, outputSize, input + 1, compressedSize - 1); + return FL2_decompress(output, outputSize, input + 1, compressedSize - 1); } else if (input[0] == 0) { memcpy(output, input + 1, compressedSize - 1); return compressedSize - 1; From ef96d37c1f2efd4a7fff5dac96c6b3d40b5e2f95 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Jul 2024 15:07:01 +0800 Subject: [PATCH 22/43] fix: return code validation --- .../libs/executor/src/dynqueryctrloperator.c | 163 ++++++++++++------ source/libs/executor/src/mergejoinoperator.c | 1 - 2 files changed, 109 insertions(+), 55 deletions(-) diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index c3b3126f68..e4ca946f4f 100755 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -358,31 +358,36 @@ static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initPara static int32_t buildMergeJoinNotifyOperatorParam(SOperatorParam** ppRes, SOperatorParam* pChild0, SOperatorParam* pChild1) { + int32_t code = TSDB_CODE_SUCCESS; *ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); if (NULL == *ppRes) { + code = terrno; freeOperatorParam(pChild0, OP_NOTIFY_PARAM); freeOperatorParam(pChild1, OP_NOTIFY_PARAM); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } (*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES); if (NULL == *ppRes) { + code = terrno; taosMemoryFreeClear(*ppRes); freeOperatorParam(pChild0, OP_NOTIFY_PARAM); freeOperatorParam(pChild1, OP_NOTIFY_PARAM); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } if (pChild0 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) { + code = terrno; freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); freeOperatorParam(pChild0, OP_NOTIFY_PARAM); freeOperatorParam(pChild1, OP_NOTIFY_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } if (pChild1 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) { + code = terrno; freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); freeOperatorParam(pChild1, OP_NOTIFY_PARAM); *ppRes = NULL; - return TSDB_CODE_OUT_OF_MEMORY; + return code; } (*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN; @@ -422,9 +427,11 @@ static int32_t buildBatchTableScanOperatorParam(SOperatorParam** ppRes, int32_t static int32_t buildSingleTableScanOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) { SArray* pUidList = taosArrayInit(1, sizeof(int64_t)); if (NULL == pUidList) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + if (NULL == taosArrayPush(pUidList, pUid)) { + return terrno; } - taosArrayPush(pUidList, pUid); int32_t code = buildTableScanOperatorParam(ppRes, pUidList, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true); taosArrayDestroy(pUidList); @@ -553,53 +560,56 @@ static int32_t notifySeqJoinTableCacheEnd(SOperatorInfo* pOperator, SStbJoinPost return optrDefaultNotifyFn(pOperator->pDownstream[1], pMergeJoinParam); } -static void handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) { +static int32_t handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) { SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post; pPost->isStarted = false; if (pStbJoin->basic.batchFetch) { - return; + return TSDB_CODE_SUCCESS; } if (pPost->leftNeedCache) { uint32_t* num = tSimpleHashGet(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); if (num && --(*num) <= 0) { - tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); - notifySeqJoinTableCacheEnd(pOperator, pPost, true); + (void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); + QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, true)); } } if (!pPost->rightNeedCache) { void* v = tSimpleHashGet(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid)); if (NULL != v) { - tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid)); - notifySeqJoinTableCacheEnd(pOperator, pPost, false); + (void)tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid)); + QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, false)); } } + + return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) { +static FORCE_INLINE int32_t seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) { SDynQueryCtrlOperatorInfo* pInfo = pOperator->info; SStbJoinPostJoinCtx* pPost = &pInfo->stbJoin.ctx.post; SStbJoinPrevJoinCtx* pPrev = &pInfo->stbJoin.ctx.prev; if (!pPost->isStarted) { - return; + return TSDB_CODE_SUCCESS; } qDebug("%s dynQueryCtrl continue to retrieve block from post op", GET_TASKID(pOperator->pTaskInfo)); *ppRes = getNextBlockFromDownstream(pOperator, 1); if (NULL == *ppRes) { - handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin); + QRY_ERR_RET(handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin)); pPrev->pListHead->readIdx++; } else { pInfo->stbJoin.execInfo.postBlkNum++; pInfo->stbJoin.execInfo.postBlkRows += (*ppRes)->info.rows; - return; } + + return TSDB_CODE_SUCCESS; } static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, int32_t keySize, void* pVal, int32_t valSize) { @@ -607,35 +617,38 @@ static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, in if (NULL == ppArray) { SArray* pArray = taosArrayInit(10, valSize); if (NULL == pArray) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (NULL == taosArrayPush(pArray, pVal)) { taosArrayDestroy(pArray); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tSimpleHashPut(pHash, pKey, keySize, &pArray, POINTER_BYTES)) { taosArrayDestroy(pArray); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return TSDB_CODE_SUCCESS; } if (NULL == taosArrayPush(*ppArray, pVal)) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } return TSDB_CODE_SUCCESS; } static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnceHash, void* pKey, int32_t keySize) { + int32_t code = TSDB_CODE_SUCCESS; uint32_t* pNum = tSimpleHashGet(pHash, pKey, keySize); if (NULL == pNum) { uint32_t n = 1; - if (tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n))) { - return TSDB_CODE_OUT_OF_MEMORY; + code = tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n)); + if (code) { + return code; } - if (tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0); + if (code) { + return code; } return TSDB_CODE_SUCCESS; } @@ -648,7 +661,7 @@ static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnc break; default: if (1 == (*pNum)) { - tSimpleHashRemove(pOnceHash, pKey, keySize); + (void)tSimpleHashRemove(pOnceHash, pKey, keySize); } (*pNum)++; break; @@ -670,23 +683,40 @@ static void freeStbJoinTableList(SStbJoinTableList* pList) { } static int32_t appendStbJoinTableList(SStbJoinPrevJoinCtx* pCtx, int64_t rows, int32_t* pLeftVg, int64_t* pLeftUid, int32_t* pRightVg, int64_t* pRightUid) { + int32_t code = TSDB_CODE_SUCCESS; SStbJoinTableList* pNew = taosMemoryMalloc(sizeof(SStbJoinTableList)); if (NULL == pNew) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pNew->pLeftVg = taosMemoryMalloc(rows * sizeof(*pLeftVg)); - pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid)); - pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg)); - pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid)); - if (NULL == pNew->pLeftVg || NULL == pNew->pLeftUid || NULL == pNew->pRightVg || NULL == pNew->pRightUid) { + if (NULL == pNew->pLeftVg) { + code = terrno; freeStbJoinTableList(pNew); - return TSDB_CODE_OUT_OF_MEMORY; + return code; + } + pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid)); + if (NULL == pNew->pLeftUid) { + code = terrno; + freeStbJoinTableList(pNew); + return code; + } + pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg)); + if (NULL == pNew->pRightVg) { + code = terrno; + freeStbJoinTableList(pNew); + return code; + } + pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid)); + if (NULL == pNew->pRightUid) { + code = terrno; + freeStbJoinTableList(pNew); + return code; } - memcpy(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg)); - memcpy(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid)); - memcpy(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg)); - memcpy(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid)); + TAOS_MEMCPY(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg)); + TAOS_MEMCPY(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid)); + TAOS_MEMCPY(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg)); + TAOS_MEMCPY(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid)); pNew->readIdx = 0; pNew->uidNum = rows; @@ -708,9 +738,21 @@ static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBloc SDynQueryCtrlOperatorInfo* pInfo = pOperator->info; SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin; SColumnInfoData* pVg0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[0]); + if (NULL == pVg0) { + QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } SColumnInfoData* pVg1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[1]); + if (NULL == pVg1) { + QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } SColumnInfoData* pUid0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[0]); + if (NULL == pUid0) { + QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } SColumnInfoData* pUid1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[1]); + if (NULL == pUid1) { + QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } if (pStbJoin->basic.batchFetch) { for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -746,6 +788,8 @@ static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBloc } } +_return: + if (TSDB_CODE_SUCCESS != code) { pOperator->pTaskInfo->code = code; T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code); @@ -769,7 +813,7 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) { uint64_t* pUid = NULL; int32_t iter = 0; while (NULL != (pUid = tSimpleHashIterate(pStbJoin->ctx.prev.onceTable, pUid, &iter))) { - tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid)); + (void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid)); } pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache); @@ -804,7 +848,7 @@ static void buildStbJoinTableList(SOperatorInfo* pOperator) { pStbJoin->ctx.prev.joinBuild = true; } -static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) { +static int32_t seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) { SDynQueryCtrlOperatorInfo* pInfo = pOperator->info; SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin; SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev; @@ -820,17 +864,17 @@ static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppR seqJoinLaunchNewRetrieveImpl(pOperator, ppRes); if (*ppRes) { - return; + return TSDB_CODE_SUCCESS; } - handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin); + QRY_ERR_RET(handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin)); pPrev->pListHead->readIdx++; } *ppRes = NULL; setOperatorCompleted(pOperator); - return; + return TSDB_CODE_SUCCESS; } static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo* pStbJoin, SSDataBlock* pBlock) { @@ -840,6 +884,7 @@ static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo* SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; SDynQueryCtrlOperatorInfo* pInfo = pOperator->info; SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin; SSDataBlock* pRes = NULL; @@ -861,18 +906,23 @@ SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) { } } - seqJoinContinueCurrRetrieve(pOperator, &pRes); + QRY_ERR_JRET(seqJoinContinueCurrRetrieve(pOperator, &pRes)); if (pRes) { goto _return; } - seqJoinLaunchNewRetrieve(pOperator, &pRes); + QRY_ERR_JRET(seqJoinLaunchNewRetrieve(pOperator, &pRes)); _return: if (pOperator->cost.openCost == 0) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } + + if (code) { + pOperator->pTaskInfo->code = code; + T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code); + } return pRes ? seqStableJoinComposeRes(pStbJoin, pRes) : NULL; } @@ -881,24 +931,24 @@ int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) { if (batchFetch) { pPrev->leftHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); if (NULL == pPrev->leftHash) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pPrev->rightHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); if (NULL == pPrev->rightHash) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } else { pPrev->leftCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); if (NULL == pPrev->leftCache) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pPrev->rightCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); if (NULL == pPrev->rightCache) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pPrev->onceTable = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); if (NULL == pPrev->onceTable) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -908,13 +958,16 @@ int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) { SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo) { - SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - __optr_fn_t nextFp = NULL; - int32_t code = TSDB_CODE_SUCCESS; - if (pOperator == NULL || pInfo == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + __optr_fn_t nextFp = NULL; + SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo)); + if (pInfo == NULL) { + code = terrno; + goto _error; + } + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pOperator == NULL) { + code = terrno; goto _error; } @@ -928,7 +981,7 @@ SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32 pInfo->qType = pPhyciNode->qType; switch (pInfo->qType) { case DYN_QTYPE_STB_HASH: - memcpy(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin)); + TAOS_MEMCPY(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin)); pInfo->stbJoin.outputBlkId = pPhyciNode->node.pOutputDataBlockDesc->dataBlockId; code = initSeqStbJoinTableHash(&pInfo->stbJoin.ctx.prev, pInfo->stbJoin.basic.batchFetch); if (TSDB_CODE_SUCCESS != code) { @@ -949,12 +1002,14 @@ SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32 return pOperator; _error: + if (pInfo != NULL) { destroyDynQueryCtrlOperator(pInfo); } taosMemoryFree(pOperator); pTaskInfo->code = code; + return NULL; } diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 8d94841847..395b04f45b 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -1717,7 +1717,6 @@ SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) { pBlock = (*pJoin->joinFp)(pOperator); if (NULL == pBlock) { if (pJoin->errCode) { - ASSERT(0); T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode); } break; From 869c08132c8122136ae82ae9c0a47ea962f9e816 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Jul 2024 15:53:10 +0800 Subject: [PATCH 23/43] fix(stream): fix exist check. --- source/dnode/mnode/impl/src/mndSma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 272897eadf..0523417f25 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -769,7 +769,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { mndGetStreamNameFromSmaName(streamName, createReq.name); code = mndAcquireStream(pMnode, streamName, &pStream); - if (pStream != NULL || code != 0) { + if (pStream != NULL || code == 0) { mError("sma:%s, failed to create since stream:%s already exist", createReq.name, streamName); terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; goto _OVER; From 6f77049c1f6de04970fb76b10a8ce21766ccbe7c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 22 Jul 2024 15:57:10 +0800 Subject: [PATCH 24/43] fix issue --- .../executor/src/streamcountwindowoperator.c | 3 ++- .../executor/src/streameventwindowoperator.c | 6 +++++- .../executor/src/streamtimewindowoperator.c | 18 +++++++++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 04310276a9..b79e6cf800 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -480,7 +480,8 @@ void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _end); } - len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true); + void* pTmpBuf = pBuf; + len = doStreamCountEncodeOpState(&pTmpBuf, len, pOperator, true); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), pBuf, len); saveStreamOperatorStateComplete(&pInfo->basic); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 2d6f9b1fc5..a3f0fb9839 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -540,7 +540,11 @@ void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) { if (needSaveStreamOperatorInfo(&pInfo->basic)) { int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator); void* buf = taosMemoryCalloc(1, len); - void* pBuf = buf; + if (!buf) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } + void* pBuf = buf; len = doStreamEventEncodeOpState(&pBuf, len, pOperator); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME, strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 50b20344d6..5724f2b3ba 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1408,7 +1408,11 @@ void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) { if (needSaveStreamOperatorInfo(&pInfo->basic)) { int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator); void* buf = taosMemoryCalloc(1, len); - void* pBuf = buf; + if (!buf) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } + void* pBuf = buf; len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME, strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len); @@ -3188,7 +3192,11 @@ void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) { if (needSaveStreamOperatorInfo(&pInfo->basic)) { int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true); void* buf = taosMemoryCalloc(1, len); - void* pBuf = buf; + if (!buf) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } + void* pBuf = buf; len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME, strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len); @@ -4390,7 +4398,11 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) { if (needSaveStreamOperatorInfo(&pInfo->basic)) { int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true); void* buf = taosMemoryCalloc(1, len); - void* pBuf = buf; + if (!buf) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return; + } + void* pBuf = buf; len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME, strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len); From f589cc68944123d466ce8ae15910a18efccde90a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 22 Jul 2024 15:58:46 +0800 Subject: [PATCH 25/43] fix issue --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7a6d5cb730..583dd7b41b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3556,7 +3556,7 @@ static int32_t tagScanCreateResultData(SDataType* pType, int32_t numOfRows, SSca } static EDealRes tagScanRewriteTagColumn(SNode** pNode, void* pContext) { - SColumnNode* pSColumnNode = NULL;; + SColumnNode* pSColumnNode = NULL; if (QUERY_NODE_COLUMN == nodeType((*pNode))) { pSColumnNode = *(SColumnNode**)pNode; } else if (QUERY_NODE_FUNCTION == nodeType((*pNode))) { From f5edf0c679801d57731d7c9bd7a4dd1170437e58 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 08:01:01 +0000 Subject: [PATCH 26/43] error str --- source/dnode/mnode/impl/src/mndShow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index d4a28d75ca..894d888e2d 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -165,7 +165,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:0x%" PRIx64 ", failed to put into cache since %s", showId, tstrerror(code)); + mError("show:0x%" PRIx64 ", failed to put into cache since %s", showId, terrstr()); return NULL; } From 005b51efebc8000a7f09f5fb3c52239992acc2f9 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 08:05:53 +0000 Subject: [PATCH 27/43] fix case --- source/dnode/mnode/impl/src/mndSma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0949055aa0..cfd48c0f44 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -769,6 +769,7 @@ static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { if (tNameFromString(&smaName, pCreate->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE) < 0) return -1; if (*(char *)tNameGetTableName(&smaName) == 0) return -1; + code = 0; TAOS_RETURN(code); } From 44fa1636f2fd91d8bcd107dbf2b3d57ea0eb51ab Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 22 Jul 2024 16:58:13 +0800 Subject: [PATCH 28/43] fix: (last cache) compare pk when merging ctb data --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 35 ++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 46c3ba4785..4eec3b87a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -294,6 +294,14 @@ void* tsdbCacherowsReaderClose(void* pReader) { taosMemoryFree(p->pCurrSchema); + if (p->rowKey.numOfPKs > 0) { + for (int32_t i = 0; i < p->rowKey.numOfPKs; i++) { + if (IS_VAR_DATA_TYPE(p->rowKey.pks[i].type)) { + taosMemoryFree(p->rowKey.pks[i].pData); + } + } + } + if (p->pLDataIterArray) { destroySttBlockReader(p->pLDataIterArray, NULL); } @@ -327,6 +335,12 @@ void* tsdbCacherowsReaderClose(void* pReader) { static void freeItem(void* pItem) { SLastCol* pCol = (SLastCol*)pItem; + for (int i = 0; i < pCol->rowKey.numOfPKs; i++) { + if (IS_VAR_DATA_TYPE(pCol->rowKey.pks[i].type)) { + taosMemoryFree(pCol->rowKey.pks[i].pData); + } + } + if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) { taosMemoryFree(pCol->colVal.value.pData); } @@ -407,6 +421,16 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 struct STColumn* pCol = &pr->pSchema->columns[slotId]; SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; + if (pr->rowKey.numOfPKs > 0) { + p.rowKey.numOfPKs = pr->rowKey.numOfPKs; + for (int32_t j = 0; j < pr->rowKey.numOfPKs; j++) { + p.rowKey.pks[j].type = pr->pkColumn.type; + if (IS_VAR_DATA_TYPE(pr->pkColumn.type)) { + p.rowKey.pks[j].pData = taosMemoryCalloc(1, pr->pkColumn.bytes); + } + } + } + if (IS_VAR_DATA_TYPE(pCol->type)) { p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); } @@ -432,7 +456,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 SLastCol* p = taosArrayGet(pLastCols, k); SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k); - if (pColVal->rowKey.ts > p->rowKey.ts) { + if (tRowKeyCompare(&pColVal->rowKey, &p->rowKey) > 0) { if (!COL_VAL_IS_VALUE(&pColVal->colVal) && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { if (!COL_VAL_IS_VALUE(&p->colVal)) { hasNotNullRow = false; @@ -445,6 +469,15 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 hasRes = true; p->rowKey.ts = pColVal->rowKey.ts; + for (int32_t j = 0; j < p->rowKey.numOfPKs; j++) { + if (IS_VAR_DATA_TYPE(p->rowKey.pks[j].type)) { + memcpy(p->rowKey.pks[j].pData, pColVal->rowKey.pks[j].pData, pColVal->rowKey.pks[j].nData); + p->rowKey.pks[j].nData = pColVal->rowKey.pks[j].nData; + } else { + p->rowKey.pks[j].val = pColVal->rowKey.pks[j].val; + } + } + if (k == 0) { if (TARRAY_SIZE(pTableUidList) == 0) { taosArrayPush(pTableUidList, &uid); From 14383a32a3080d61d2d779615248f6dff75a241d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Jul 2024 18:42:43 +0800 Subject: [PATCH 29/43] fix: sink function return code processing --- source/libs/executor/src/dataDeleter.c | 92 ++++++++++++++-------- source/libs/executor/src/dataDispatcher.c | 89 ++++++++++++--------- source/libs/executor/src/dataInserter.c | 95 ++++++++++++++++------- source/libs/executor/src/dataSinkMgt.c | 9 ++- 4 files changed, 184 insertions(+), 101 deletions(-) diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index aa377cd922..9f0ea0a87f 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -53,7 +53,7 @@ typedef struct SDataDeleterHandle { TdThreadMutex mutex; } SDataDeleterHandle; -static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) { +static int32_t toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) { int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots); SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; @@ -65,14 +65,23 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pBuf->useSize = sizeof(SDataCacheEntry); SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0); + if (NULL == pColRes) { + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } SColumnInfoData* pColSKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 1); + if (NULL == pColSKey) { + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } SColumnInfoData* pColEKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 2); + if (NULL == pColEKey) { + QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } SDeleterRes* pRes = (SDeleterRes*)pEntry->data; pRes->suid = pHandle->pParam->suid; pRes->uidList = pHandle->pParam->pUidList; - strcpy(pRes->tableName, pHandle->pDeleter->tableFName); - strcpy(pRes->tsColName, pHandle->pDeleter->tsColName); + TAOS_STRCPY(pRes->tableName, pHandle->pDeleter->tableFName); + TAOS_STRCPY(pRes->tsColName, pHandle->pDeleter->tsColName); pRes->affectedRows = *(int64_t*)pColRes->pData; if (pRes->affectedRows) { @@ -88,16 +97,18 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pBuf->useSize += pEntry->dataLen; - atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); - atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + (void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); + (void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + + return TSDB_CODE_SUCCESS; } -static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) { +static int32_t allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) { uint32_t capacity = pDeleter->pManager->cfg.maxDataBlockNumPerQuery; if (taosQueueItemSize(pDeleter->pDataBlocks) > capacity) { qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity, taosQueueItemSize(pDeleter->pDataBlocks)); - return false; + return TSDB_CODE_OUT_OF_MEMORY; } pBuf->allocSize = sizeof(SDataCacheEntry) + sizeof(SDeleterRes); @@ -105,55 +116,66 @@ static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDa pBuf->pData = taosMemoryMalloc(pBuf->allocSize); if (pBuf->pData == NULL) { qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); + return terrno; } - return NULL != pBuf->pData; + return TSDB_CODE_SUCCESS; } static int32_t updateStatus(SDataDeleterHandle* pDeleter) { - taosThreadMutexLock(&pDeleter->mutex); + (void)taosThreadMutexLock(&pDeleter->mutex); int32_t blockNums = taosQueueItemSize(pDeleter->pDataBlocks); int32_t status = (0 == blockNums ? DS_BUF_EMPTY : (blockNums < pDeleter->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); pDeleter->status = status; - taosThreadMutexUnlock(&pDeleter->mutex); + (void)taosThreadMutexUnlock(&pDeleter->mutex); + return status; } static int32_t getStatus(SDataDeleterHandle* pDeleter) { - taosThreadMutexLock(&pDeleter->mutex); + (void)taosThreadMutexLock(&pDeleter->mutex); int32_t status = pDeleter->status; - taosThreadMutexUnlock(&pDeleter->mutex); + (void)taosThreadMutexUnlock(&pDeleter->mutex); + return status; } static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; - SDataDeleterBuf* pBuf; + SDataDeleterBuf* pBuf = NULL; int32_t code = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0, (void**)&pBuf); if (code) { return code; } - if (!allocBuf(pDeleter, pInput, pBuf)) { + code = allocBuf(pDeleter, pInput, pBuf); + if (code) { taosFreeQitem(pBuf); - return TSDB_CODE_OUT_OF_MEMORY; + return code; } - toDataCacheEntry(pDeleter, pInput, pBuf); - taosWriteQitem(pDeleter->pDataBlocks, pBuf); + QRY_ERR_JRET(toDataCacheEntry(pDeleter, pInput, pBuf)); + QRY_ERR_JRET(taosWriteQitem(pDeleter->pDataBlocks, pBuf)); *pContinue = (DS_BUF_LOW == updateStatus(pDeleter) ? true : false); + return TSDB_CODE_SUCCESS; + +_return: + + taosFreeQitem(pBuf); + + return code; } static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; - taosThreadMutexLock(&pDeleter->mutex); + (void)taosThreadMutexLock(&pDeleter->mutex); pDeleter->queryEnd = true; pDeleter->useconds = useconds; - taosThreadMutexUnlock(&pDeleter->mutex); + (void)taosThreadMutexUnlock(&pDeleter->mutex); } static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) { @@ -165,9 +187,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw } SDataDeleterBuf* pBuf = NULL; - taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); + (void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); if (pBuf != NULL) { - memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); + TAOS_MEMCPY(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); taosFreeQitem(pBuf); } @@ -192,35 +214,35 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData); - memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); + TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen); pDeleter->pParam->pUidList = NULL; pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen); - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + (void)atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen); + (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDeleter); - taosThreadMutexLock(&pDeleter->mutex); + (void)taosThreadMutexLock(&pDeleter->mutex); pOutput->queryEnd = pDeleter->queryEnd; pOutput->useconds = pDeleter->useconds; pOutput->precision = pDeleter->pSchema->precision; - taosThreadMutexUnlock(&pDeleter->mutex); + (void)taosThreadMutexUnlock(&pDeleter->mutex); return TSDB_CODE_SUCCESS; } static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize); + (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize); taosMemoryFreeClear(pDeleter->nextOutput.pData); taosArrayDestroy(pDeleter->pParam->pUidList); taosMemoryFree(pDeleter->pParam); while (!taosQueueEmpty(pDeleter->pDataBlocks)) { SDataDeleterBuf* pBuf = NULL; - taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); + (void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); if (pBuf != NULL) { taosMemoryFreeClear(pBuf->pData); @@ -228,9 +250,10 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { } } taosCloseQueue(pDeleter->pDataBlocks); - taosThreadMutexDestroy(&pDeleter->mutex); + (void)taosThreadMutexDestroy(&pDeleter->mutex); taosMemoryFree(pDeleter->pManager); + return TSDB_CODE_SUCCESS; } @@ -247,8 +270,8 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); if (NULL == deleter) { + code = terrno; taosMemoryFree(pParam); - code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } @@ -276,17 +299,22 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData if (code) { goto _end; } - taosThreadMutexInit(&deleter->mutex, NULL); + code = taosThreadMutexInit(&deleter->mutex, NULL); + if (code) { + goto _end; + } *pHandle = deleter; return code; _end: + if (deleter != NULL) { - destroyDataSinker((SDataSinkHandle*)deleter); + (void)destroyDataSinker((SDataSinkHandle*)deleter); taosMemoryFree(deleter); } else { taosMemoryFree(pManager); } + return code; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3981cedd3f..9316ba960e 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -63,7 +63,7 @@ typedef struct SDataDispatchHandle { // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header // clang-format on -static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { +static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { int32_t numOfCols = 0; SNode* pNode; @@ -88,6 +88,9 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn if (pHandle->pCompressBuf == NULL) { // allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size pHandle->pCompressBuf = taosMemoryMalloc(pBuf->allocSize + 8); + if (NULL == pHandle->pCompressBuf) { + QRY_RET(terrno); + } pHandle->bufSize = pBuf->allocSize + 8; } else { if (pHandle->bufSize < pBuf->allocSize + 8) { @@ -96,9 +99,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn if (p != NULL) { pHandle->pCompressBuf = p; } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - qError("failed to prepare compress buf:%d, code: out of memory", pHandle->bufSize); - return; + qError("failed to prepare compress buf:%d, code: %x", pHandle->bufSize, terrno); + return terrno; } } } @@ -114,7 +116,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->compressed = 0; pEntry->dataLen = dataLen; pEntry->rawLen = dataLen; - memcpy(pEntry->data, pHandle->pCompressBuf, dataLen); + TAOS_MEMCPY(pEntry->data, pHandle->pCompressBuf, dataLen); } } else { pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); @@ -124,11 +126,13 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pBuf->useSize += pEntry->dataLen; - atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); - atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + (void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); + (void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + + return TSDB_CODE_SUCCESS; } -static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { +static int32_t allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { /* uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery; if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) { @@ -142,69 +146,73 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, pBuf->pData = taosMemoryMalloc(pBuf->allocSize); if (pBuf->pData == NULL) { - qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); + qError("SinkNode failed to malloc memory, size:%d, code:%x", pBuf->allocSize, terrno); + return terrno; } - return NULL != pBuf->pData; + return TSDB_CODE_SUCCESS; } static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { - taosThreadMutexLock(&pDispatcher->mutex); + (void)taosThreadMutexLock(&pDispatcher->mutex); int32_t blockNums = taosQueueItemSize(pDispatcher->pDataBlocks); int32_t status = (0 == blockNums ? DS_BUF_EMPTY : (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); pDispatcher->status = status; - taosThreadMutexUnlock(&pDispatcher->mutex); + (void)taosThreadMutexUnlock(&pDispatcher->mutex); return status; } static int32_t getStatus(SDataDispatchHandle* pDispatcher) { - taosThreadMutexLock(&pDispatcher->mutex); + (void)taosThreadMutexLock(&pDispatcher->mutex); int32_t status = pDispatcher->status; - taosThreadMutexUnlock(&pDispatcher->mutex); + (void)taosThreadMutexUnlock(&pDispatcher->mutex); return status; } static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { int32_t code = 0; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - SDataDispatchBuf* pBuf; + SDataDispatchBuf* pBuf = NULL; code = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0, (void**)&pBuf); if (code) { return code; } - if (!allocBuf(pDispatcher, pInput, pBuf)) { + code = allocBuf(pDispatcher, pInput, pBuf); + if (code) { taosFreeQitem(pBuf); - return TSDB_CODE_OUT_OF_MEMORY; - } - - toDataCacheEntry(pDispatcher, pInput, pBuf); - code = taosWriteQitem(pDispatcher->pDataBlocks, pBuf); - if (code != 0) { return code; } + QRY_ERR_JRET(toDataCacheEntry(pDispatcher, pInput, pBuf)); + QRY_ERR_JRET(taosWriteQitem(pDispatcher->pDataBlocks, pBuf)); + int32_t status = updateStatus(pDispatcher); *pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY); return TSDB_CODE_SUCCESS; + +_return: + + taosFreeQitem(pBuf); + return code; } static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - taosThreadMutexLock(&pDispatcher->mutex); + (void)taosThreadMutexLock(&pDispatcher->mutex); pDispatcher->queryEnd = true; pDispatcher->useconds = useconds; - taosThreadMutexUnlock(&pDispatcher->mutex); + (void)taosThreadMutexUnlock(&pDispatcher->mutex); } static void resetDispatcher(struct SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - taosThreadMutexLock(&pDispatcher->mutex); + (void)taosThreadMutexLock(&pDispatcher->mutex); pDispatcher->queryEnd = false; - taosThreadMutexUnlock(&pDispatcher->mutex); + (void)taosThreadMutexUnlock(&pDispatcher->mutex); } static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd) { @@ -216,9 +224,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow } SDataDispatchBuf* pBuf = NULL; - taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); + (void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); if (pBuf != NULL) { - memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); + TAOS_MEMCPY(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); } @@ -243,33 +251,34 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); - memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); + TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + (void)atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); + (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); - taosThreadMutexLock(&pDispatcher->mutex); + + (void)taosThreadMutexLock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->pSchema->precision; - taosThreadMutexUnlock(&pDispatcher->mutex); + (void)taosThreadMutexUnlock(&pDispatcher->mutex); return TSDB_CODE_SUCCESS; } static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize); + (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize); taosMemoryFreeClear(pDispatcher->nextOutput.pData); while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { SDataDispatchBuf* pBuf = NULL; - taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); + (void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); if (pBuf != NULL) { taosMemoryFreeClear(pBuf->pData); taosFreeQitem(pBuf); @@ -280,7 +289,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { taosMemoryFreeClear(pDispatcher->pCompressBuf); pDispatcher->bufSize = 0; - taosThreadMutexDestroy(&pDispatcher->mutex); + (void)taosThreadMutexDestroy(&pDispatcher->mutex); taosMemoryFree(pDispatcher->pManager); return TSDB_CODE_SUCCESS; } @@ -297,7 +306,6 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { - terrno = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -318,7 +326,11 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD terrno = code; goto _return; } - taosThreadMutexInit(&dispatcher->mutex, NULL); + code = taosThreadMutexInit(&dispatcher->mutex, NULL); + if (code) { + terrno = code; + goto _return; + } if (NULL == dispatcher->pDataBlocks) { taosMemoryFree(dispatcher); @@ -330,6 +342,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD return TSDB_CODE_SUCCESS; _return: + taosMemoryFree(pManager); return terrno; } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 5ba2f8bf42..6f226ecb21 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -58,10 +58,17 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { SSubmitRspParam* pParam = (SSubmitRspParam*)param; SDataInserterHandle* pInserter = pParam->pInserter; - pInserter->submitRes.code = code; - + if (code) { + pInserter->submitRes.code = code; + } + if (code == TSDB_CODE_SUCCESS) { pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2)); + if (NULL == pInserter->submitRes.pRsp) { + pInserter->submitRes.code = terrno; + goto _return; + } + SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp); @@ -77,6 +84,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { for (int32_t i = 0; i < numOfTables; ++i) { SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i); + if (NULL == pRsp) { + pInserter->submitRes.code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + goto _return; + } if (TSDB_CODE_SUCCESS != pRsp->code) { code = pRsp->code; taosMemoryFree(pInserter->submitRes.pRsp); @@ -94,8 +105,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } _return: - tsem_post(&pInserter->ready); + + (void)tsem_post(&pInserter->ready); taosMemoryFree(pMsg->pData); + return TSDB_CODE_SUCCESS; } @@ -105,11 +118,15 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { taosMemoryFreeClear(pMsg); - terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } SSubmitRspParam* pParam = taosMemoryCalloc(1, sizeof(SSubmitRspParam)); + if (NULL == pParam) { + taosMemoryFreeClear(pMsg); + taosMemoryFreeClear(pMsgSendInfo); + return terrno; + } pParam->pInserter = pInserter; pMsgSendInfo->param = pParam; @@ -133,7 +150,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int len += sizeof(SSubmitReq2Msg); pBuf = taosMemoryMalloc(len); if (NULL == pBuf) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } ((SSubmitReq2Msg*)pBuf)->header.vgId = htonl(vgId); ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); @@ -149,6 +166,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int } else { taosMemoryFree(pBuf); } + return code; } @@ -162,12 +180,10 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp if (NULL == pReq) { if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; goto _end; } if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; goto _end; } } @@ -208,6 +224,10 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx); + if (NULL == pColInfoData) { + terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + goto _end; + } void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); switch (pColInfoData->info.type) { @@ -217,13 +237,17 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp ASSERT(pColInfoData->info.type == pCol->type); if (colDataIsNull_s(pColInfoData, j)) { SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); - taosArrayPush(pVals, &cv); + if (NULL == taosArrayPush(pVals, &cv)) { + goto _end; + } } else { void* data = colDataGetVarData(pColInfoData, j); SValue sv = (SValue){ .type = pCol->type, .nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value SColVal cv = COL_VAL_VALUE(pCol->colId, sv); - taosArrayPush(pVals, &cv); + if (NULL == taosArrayPush(pVals, &cv)) { + goto _end; + } } break; } @@ -245,7 +269,9 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type - taosArrayPush(pVals, &cv); + if (NULL == taosArrayPush(pVals, &cv)) { + goto _end; + } } else { if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && !needSortMerge) { if (*(int64_t*)var <= lastTs) { @@ -256,9 +282,11 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } SValue sv = {.type = pCol->type}; - memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + TAOS_MEMCPY(&sv.val, var, tDataTypes[pCol->type].bytes); SColVal cv = COL_VAL_VALUE(pCol->colId, sv); - taosArrayPush(pVals, &cv); + if (NULL == taosArrayPush(pVals, &cv)) { + goto _end; + } } } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); @@ -274,7 +302,9 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } - taosArrayPush(tbData.aRowP, &pRow); + if (NULL == taosArrayPush(tbData.aRowP, &pRow)) { + goto _end; + } } if (needSortMerge) { @@ -284,9 +314,12 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } } - taosArrayPush(pReq->aSubmitTbData, &tbData); + if (NULL == taosArrayPush(pReq->aSubmitTbData, &tbData)) { + goto _end; + } _end: + taosArrayDestroy(pVals); if (terrno != 0) { *ppReq = NULL; @@ -294,9 +327,11 @@ _end: tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } + return terrno; } *ppReq = pReq; + return TSDB_CODE_SUCCESS; } @@ -312,7 +347,9 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32 for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); - + if (NULL == pDataBlock) { + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); if (code) { if (pReq) { @@ -334,7 +371,9 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32 static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; if (!pInserter->explain) { - taosArrayPush(pInserter->pDataBlocks, &pInput->pData); + if (NULL == taosArrayPush(pInserter->pDataBlocks, &pInput->pData)) { + return terrno; + } void* pMsg = NULL; int32_t msgLen = 0; int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen); @@ -350,7 +389,7 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, return code; } - tsem_wait(&pInserter->ready); + QRY_ERR_RET(tsem_wait(&pInserter->ready)); if (pInserter->submitRes.code) { return pInserter->submitRes.code; @@ -364,10 +403,10 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; - taosThreadMutexLock(&pInserter->mutex); + (void)taosThreadMutexLock(&pInserter->mutex); pInserter->queryEnd = true; pInserter->useconds = useconds; - taosThreadMutexUnlock(&pInserter->mutex); + (void)taosThreadMutexUnlock(&pInserter->mutex); } static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) { @@ -378,12 +417,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize); + (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize); taosArrayDestroy(pInserter->pDataBlocks); taosMemoryFree(pInserter->pSchema); taosMemoryFree(pInserter->pParam); taosHashCleanup(pInserter->pCols); - taosThreadMutexDestroy(&pInserter->mutex); + (void)taosThreadMutexDestroy(&pInserter->mutex); taosMemoryFree(pInserter->pManager); return TSDB_CODE_SUCCESS; @@ -401,7 +440,6 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); if (NULL == inserter) { taosMemoryFree(pParam); - terrno = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -432,28 +470,31 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat } inserter->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - taosThreadMutexInit(&inserter->mutex, NULL); if (NULL == inserter->pDataBlocks) { - terrno = TSDB_CODE_OUT_OF_MEMORY; goto _return; } + QRY_ERR_JRET(taosThreadMutexInit(&inserter->mutex, NULL)); inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols; inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + if (NULL == inserter->pCols) { + goto _return; + } + SNode* pNode = NULL; int32_t i = 0; FOREACH(pNode, pInserterNode->pCols) { SColumnNode* pCol = (SColumnNode*)pNode; - taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId)); + QRY_ERR_JRET(taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId))); if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) { inserter->fullOrderColList = false; } ++i; } - tsem_init(&inserter->ready, 0, 0); + QRY_ERR_JRET(tsem_init(&inserter->ready, 0, 0)); *pHandle = inserter; return TSDB_CODE_SUCCESS; @@ -461,7 +502,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat _return: if (inserter) { - destroyDataSinker((SDataSinkHandle*)inserter); + (void)destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); } else { taosMemoryFree(pManager); diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index e711ffdf5c..55fc520477 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -23,20 +23,20 @@ SDataSinkStat gDataSinkStat = {0}; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI, void** ppSinkManager) { SDataSinkManager* pSinkManager = taosMemoryMalloc(sizeof(SDataSinkManager)); if (NULL == pSinkManager) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pSinkManager->cfg = *cfg; pSinkManager->pAPI = pAPI; *ppSinkManager = pSinkManager; - return 0; // to avoid compiler eror + return TSDB_CODE_SUCCESS; // to avoid compiler eror } int32_t dsDataSinkGetCacheSize(SDataSinkStat* pStat) { pStat->cachedSize = atomic_load_64(&gDataSinkStat.cachedSize); - return 0; + return TSDB_CODE_SUCCESS; } int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam, const char* id) { @@ -56,6 +56,7 @@ int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, D taosMemoryFree(pSinkManager); qError("invalid input node type:%d, %s", nodeType(pDataSink), id); + return TSDB_CODE_QRY_INVALID_INPUT; } @@ -97,6 +98,6 @@ void dsScheduleProcess(void* ahandle, void* pItem) { void dsDestroyDataSinker(DataSinkHandle handle) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - pHandleImpl->fDestroy(pHandleImpl); + (void)pHandleImpl->fDestroy(pHandleImpl); taosMemoryFree(pHandleImpl); } From 5103b7dab7077c904a1829d81a76065894796be0 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 22 Jul 2024 11:50:19 +0000 Subject: [PATCH 30/43] fix case --- source/dnode/mnode/impl/src/mndStb.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 4e34132e06..7914d1dfcf 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2900,11 +2900,8 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { STableMetaRsp metaRsp = {0}; SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) { - code = TSDB_CODE_MND_RETURN_VALUE_NULL; - if (terrno != 0) code = terrno; - TAOS_RETURN(code); - } + //TODO why return 0 here + if (pUser == NULL) return 0; bool sysinfo = pUser->sysInfo; TAOS_CHECK_GOTO(tDeserializeSTableInfoReq(pReq->pCont, pReq->contLen, &infoReq), NULL, _OVER); @@ -2946,7 +2943,8 @@ _OVER: mndReleaseUser(pMnode, pUser); tFreeSTableMetaRsp(&metaRsp); - TAOS_RETURN(code); + //TODO change to TAOS_RETURN + return code; } static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { From bf03c2b3d8b013a5204b8af39b097961285b8bd5 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 22 Jul 2024 20:26:55 +0800 Subject: [PATCH 31/43] fix: tupleData --- source/libs/executor/src/executil.c | 3 +- tests/script/tsim/parser/select_with_tags.sim | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c623e94a12..1042e6b30d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1627,7 +1627,8 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; - if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) { + if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0) + || (strcmp(pName, "_group_const_value") == 0)) { pValCtx[num++] = &pCtx[i]; } else if (fmIsSelectFunc(pCtx[i].functionId)) { void* data = taosHashGet(pSelectFuncs, pName, strlen(pName)); diff --git a/tests/script/tsim/parser/select_with_tags.sim b/tests/script/tsim/parser/select_with_tags.sim index 0cc8a7db8a..a4c460a937 100644 --- a/tests/script/tsim/parser/select_with_tags.sim +++ b/tests/script/tsim/parser/select_with_tags.sim @@ -452,6 +452,43 @@ if $data04 != @abc0@ then return -1 endi +sql select ts, top(c1, 100), tbname, t1, t2 from select_tags_mt0 where tbname in ('select_tags_tb0', 'select_tags_tb1') partition by tbname order by ts; +if $row != 200 then + return -1 +endi + +if $data00 != @70-01-01 08:01:40.087@ then + return -1 +endi + +if $data10 != @70-01-01 08:01:40.088@ then + return -1 +endi + +if $data20 != @70-01-01 08:01:40.089@ then + return -1 +endi + +if $data90 != @70-01-01 08:01:40.096@ then + return -1 +endi + +if $data01 != 87 then + return -1 +endi + +if $data02 != @select_tags_tb0@ then + return -1 +endi + +if $data03 != 0 then + return -1 +endi + +if $data04 != @abc0@ then + return -1 +endi + sql select ts, top(c1, 2), t2, tbname, t2 from select_tags_mt0 where tbname in ('select_tags_tb0', 'select_tags_tb1') group by tbname,t2 order by ts; if $row != 4 then return -1 From e4bc6b33d550a068f6740fb13c8a66454e06153f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 01:31:53 +0800 Subject: [PATCH 32/43] fix(stream): check return value. --- source/dnode/mnode/impl/src/mndSma.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0523417f25..5826109d2e 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1249,8 +1249,7 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { rsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); if (NULL == rsp.pIndex) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -1260,14 +1259,12 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { } if (!exist) { - code = -1; - terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + code = terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; } else { int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &rsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; + code = terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } @@ -1724,14 +1721,15 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { code = 0; goto _OVER; } + if (pSma) { - terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + code = terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; goto _OVER; } SStbObj *pTargetStb = mndAcquireStb(pMnode, streamTargetStbFullName); if (pTargetStb) { - terrno = TSDB_CODE_TDB_STB_ALREADY_EXIST; + code = terrno = TSDB_CODE_TDB_STB_ALREADY_EXIST; mError("tsma: %s, failed to create since output stable already exists: %s", createReq.name, streamTargetStbFullName); goto _OVER; @@ -1740,13 +1738,13 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { code = mndAcquireStream(pMnode, streamName, &pStream); if (pStream != NULL || code != TSDB_CODE_MND_STREAM_NOT_EXIST) { mError("tsma:%s, failed to create since stream:%s already exist", createReq.name, streamName); - terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + code = terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; goto _OVER; } pDb = mndAcquireDbBySma(pMnode, createReq.name); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + code = terrno = TSDB_CODE_MND_DB_NOT_SELECTED; goto _OVER; } @@ -1758,7 +1756,7 @@ static int32_t mndProcessCreateTSMAReq(SRpcMsg* pReq) { pBaseTsma = sdbAcquire(pMnode->pSdb, SDB_SMA, createReq.baseTsmaName); if (!pBaseTsma) { mError("base tsma: %s not found when creating recursive tsma", createReq.baseTsmaName); - terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + code = terrno = TSDB_CODE_MND_SMA_NOT_EXIST; goto _OVER; } if (!pStb) { From f5f84c3cc697047a70d64efbd71f6c107819f3be Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 19 Jul 2024 10:59:21 +0800 Subject: [PATCH 33/43] feat: (errcode) ttime.c --- include/common/ttime.h | 4 +- source/common/src/ttime.c | 225 ++++++++------ source/dnode/mnode/impl/src/mndSma.c | 9 +- source/dnode/vnode/src/sma/smaOpen.c | 16 +- source/dnode/vnode/src/sma/smaRollup.c | 5 +- source/dnode/vnode/src/sma/smaTimeRange.c | 11 +- source/libs/command/inc/commandInt.h | 9 +- source/libs/command/src/explain.c | 103 ++++--- source/libs/parser/src/parTranslater.c | 10 +- tests/system-test/2-query/select_null.py | 340 +++++++++++----------- 10 files changed, 405 insertions(+), 327 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index f50b5ee9d7..2d40bd93a6 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -86,11 +86,11 @@ void deltaToUtcInitOnce(); char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision); -int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit); +int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes); int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal); int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision); -void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); +int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); struct STm { struct tm tm; diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index f7885caac0..4fca564804 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -35,7 +35,7 @@ void deltaToUtcInitOnce() { // printf("====delta:%lld\n\n", seconds); } -static int64_t parseFraction(char* str, char** end, int32_t timePrec); +static int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); @@ -95,7 +95,9 @@ char* forwardToTimeStringEnd(char* str) { return &str[i]; } -int64_t parseFraction(char* str, char** end, int32_t timePrec) { +int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t i = 0; int64_t fraction = 0; @@ -112,7 +114,7 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) { int32_t totalLen = i; if (totalLen <= 0) { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } /* parse the fraction */ @@ -134,21 +136,24 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) { } times = NANO_SEC_FRACTION_LEN - i; } else { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } fraction = strnatoi(str, i) * factor[times]; *end = str + totalLen; + *pFraction = fraction; - return fraction; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t parseTimezone(char* str, int64_t* tzOffset) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t hour = 0; int32_t i = 0; if (str[i] != '+' && str[i] != '-') { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } i++; @@ -160,7 +165,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { continue; } - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } char* sep = strchr(&str[i], ':'); @@ -175,18 +180,18 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { } if (hour > 12 || hour < 0) { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } // return error if there're illegal charaters after min(2 Digits) char* minStr = &str[i]; if (minStr[1] != '\0' && minStr[2] != '\0') { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } int64_t minute = strnatoi(&str[i], 2); if (minute > 59 || (hour == 12 && minute > 0)) { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } if (str[0] == '+') { @@ -195,13 +200,13 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { *tzOffset = hour * 3600 + minute * 60; } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t offsetOfTimezone(char* tzStr, int64_t* offset) { if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) { *offset = 0; - return 0; + return TSDB_CODE_SUCCESS; } return parseTimezone(tzStr, offset); } @@ -219,6 +224,8 @@ int32_t offsetOfTimezone(char* tzStr, int64_t* offset) { * 2013-04-12T15:52:01.123+0800 */ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t tzOffset = 0; @@ -234,7 +241,7 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch } if (str == NULL) { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } /* mktime will be affected by TZ, set by using taos_options */ @@ -253,22 +260,18 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch *time = seconds * factor; } else if (str[0] == '.') { str += 1; - if ((fraction = parseFraction(str, &str, timePrec)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(parseFraction(str, &str, timePrec, &fraction)); *time = seconds * factor + fraction; char seg = str[0]; if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } else if ((seg == 'Z' || seg == 'z') && str[1] != '\0') { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } else if (seg == '+' || seg == '-') { // parse the timezone - if (parseTimezone(str, &tzOffset) == -1) { - return -1; - } + TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset)); *time += tzOffset * factor; } @@ -277,16 +280,14 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch *time = seconds * factor + fraction; // parse the timezone - if (parseTimezone(str, &tzOffset) == -1) { - return -1; - } + TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset)); *time += tzOffset * factor; } else { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } static FORCE_INLINE bool validateTm(struct tm* pTm) { @@ -314,6 +315,8 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) { } int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { + int32_t code = TSDB_CODE_SUCCESS; + *utime = 0; struct tm tm = {0}; @@ -330,7 +333,7 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP // if parse failed, try "%Y-%m-%d" format str = taosStrpTime(timestr, "%Y-%m-%d", &tm); if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } } @@ -347,16 +350,16 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP if (*str == '.') { /* parse the second fraction part */ - if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction)); } *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { + int32_t code = TSDB_CODE_SUCCESS; + *utime = 0; struct tm tm = {0}; tm.tm_isdst = -1; @@ -374,7 +377,7 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti // if parse failed, try "%Y-%m-%d" format str = taosStrpTime(timestr, "%Y-%m-%d", &tm); if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } } @@ -384,13 +387,11 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti int64_t fraction = 0; if (*str == '.') { /* parse the second fraction part */ - if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction)); } *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } char getPrecisionUnit(int32_t precision) { @@ -482,10 +483,12 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre // !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = // 1626006833631000064 -int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { +int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO && fromPrecision != TSDB_TIME_PRECISION_NANO) { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; @@ -541,15 +544,23 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char time *= factors[fromPrecision]; break; default: { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } } - if (tmp >= (double)INT64_MAX) return INT64_MAX; - if (tmp <= (double)INT64_MIN) return INT64_MIN; - return time; + if (tmp >= (double)INT64_MAX) { + *pRes = INT64_MAX; + } else if (tmp <= (double)INT64_MIN) { + *pRes = INT64_MIN; + } else { + *pRes = time; + } + + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t charLen = varDataLen(inputData); char* newColData; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { @@ -558,7 +569,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); - return TSDB_CODE_INVALID_TIMESTAMP; + TAOS_RETURN(TSDB_CODE_INVALID_TIMESTAMP); } taosMemoryFree(newColData); } else if (type == TSDB_DATA_TYPE_NCHAR) { @@ -566,50 +577,52 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData); if (len < 0) { taosMemoryFree(newColData); - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_FAILED); } newColData[len] = 0; int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); - return ret; + TAOS_RETURN(ret); } taosMemoryFree(newColData); } else { - return TSDB_CODE_FAILED; + TAOS_RETURN(TSDB_CODE_FAILED); } - return TSDB_CODE_SUCCESS; + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { + int32_t code = TSDB_CODE_SUCCESS; + switch (unit) { case 's': if (val > INT64_MAX / MILLISECOND_PER_SECOND) { - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } (*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision); break; case 'm': if (val > INT64_MAX / MILLISECOND_PER_MINUTE) { - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } (*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision); break; case 'h': if (val > INT64_MAX / MILLISECOND_PER_MINUTE) { - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } (*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision); break; case 'd': if (val > INT64_MAX / MILLISECOND_PER_DAY) { - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } (*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision); break; case 'w': if (val > INT64_MAX / MILLISECOND_PER_WEEK) { - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } (*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision); break; @@ -623,10 +636,10 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi (*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_NANO, timePrecision); break; default: { - return -1; + TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE); } } - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } /* @@ -645,47 +658,50 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi */ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { + int32_t code = TSDB_CODE_SUCCESS; + errno = 0; char* endPtr = NULL; /* get the basic numeric value */ int64_t timestamp = taosStr2Int64(token, &endPtr, 10); if ((timestamp == 0 && token[0] != '0') || errno != 0) { - return -1; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } /* natual month/year are not allowed in absolute duration */ *unit = token[tokenlen - 1]; if (*unit == 'n' || *unit == 'y') { - return -1; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } return getDuration(timestamp, *unit, duration, timePrecision); } -int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) { +int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, + bool negativeAllow) { + int32_t code = TSDB_CODE_SUCCESS; + errno = 0; /* get the basic numeric value */ *duration = taosStr2Int64(token, NULL, 10); if ((*duration < 0 && !negativeAllow) || errno != 0) { - return -1; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } *unit = token[tokenLen - 1]; if (*unit == 'n' || *unit == 'y') { - return 0; + TAOS_RETURN(TSDB_CODE_SUCCESS); } - if(isdigit(*unit)) { + if (isdigit(*unit)) { *unit = getPrecisionUnit(timePrecision); } return getDuration(*duration, *unit, duration, timePrecision); } -static bool taosIsLeapYear(int32_t year) { - return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); -} +static bool taosIsLeapYear(int32_t year) { return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); } int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { @@ -900,7 +916,8 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { // used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) { if (pInterval->offset > 0) { - int64_t slideStart = taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); + int64_t slideStart = + taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision); return result; @@ -960,7 +977,9 @@ const char* fmtts(int64_t ts) { return buf; } -void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { +int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { + int32_t code = TSDB_CODE_SUCCESS; + char ts[40] = {0}; struct tm ptm; @@ -996,17 +1015,18 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) default: fractionLen = 0; - return; + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } if (taosLocalTime(", &ptm, buf) == NULL) { - return; + TAOS_RETURN(TAOS_SYSTEM_ERROR(errno)); } int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); length += snprintf(ts + length, fractionLen, format, mod); length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); tstrncpy(buf, ts, bufLen); + TAOS_RETURN(TSDB_CODE_SUCCESS); } int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) { @@ -1055,7 +1075,7 @@ typedef enum { TSFKW_Mon, TSFKW_MS, TSFKW_NS, - //TSFKW_OF, + // TSFKW_OF, TSFKW_PM, TSFKW_P_M, TSFKW_SS, @@ -1076,7 +1096,7 @@ typedef enum { TSFKW_day, TSFKW_ddd, TSFKW_dd, - TSFKW_dy, // mon, tue + TSFKW_dy, // mon, tue TSFKW_d, TSFKW_hh24, TSFKW_hh12, @@ -1268,7 +1288,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) { } if (*formatStr == '\\' && *(formatStr + 1)) { formatStr++; - last = NULL; // stop expanding last format, create new format + last = NULL; // stop expanding last format, create new format } if (last) { // expand @@ -1289,7 +1309,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) { // for other strings if (*formatStr == '\\' && *(formatStr + 1)) { formatStr++; - lastOtherFormat = NULL; // stop expanding + lastOtherFormat = NULL; // stop expanding } else { if (lastOtherFormat && !isSeperatorChar(*formatStr)) { // expanding @@ -1304,8 +1324,8 @@ static void parseTsFormat(const char* formatStr, SArray* formats) { formatStr++; } else { TSFormatNode format = { - .type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR, - .key = NULL}; + .type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR, + .key = NULL}; format.c = formatStr; format.len = 1; taosArrayPush(formats, &format); @@ -1318,7 +1338,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) { } static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { - int32_t size = taosArrayGetSize(formats); + int32_t size = taosArrayGetSize(formats); const char* start = s; for (int32_t i = 0; i < size; ++i) { TSFormatNode* format = taosArrayGet(formats, i); @@ -1923,8 +1943,8 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr return tm2char(*formats, &tm, out, outLen); } -int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, - int32_t errMsgLen) { +int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, + char* errMsg, int32_t errMsgLen) { const char* sErrPos; int32_t fErrIdx; if (!*formats) { @@ -1977,26 +1997,27 @@ static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1, #define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97] -static int64_t UNIT_MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/ - /*ns*/ { 1, 1000, 0}, - /*us*/ {1000, 1, 1000, 0}, - /*ms*/ { 0, 1000, 1, 1000, 0}, - /*s*/ { 0, 0, 1000, 1, 60, 0}, - /*min*/ { 0, 0, 0, 60, 1, 60, 0}, - /*h*/ { 0, 0, 0, 0, 60, 1, 1, 0}, - /*d*/ { 0, 0, 0, 0, 0, 24, 1, 7, 1, 0}, - /*w*/ { 0, 0, 0, 0, 0, 0, 7, 1, -1, 0}, - /*mon*/ { 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0}, - /*y*/ { 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}}; +static int64_t UNIT_MATRIX[10][11] = {/* ns, us, ms, s, min, h, d, w, month, y*/ + /*ns*/ {1, 1000, 0}, + /*us*/ {1000, 1, 1000, 0}, + /*ms*/ {0, 1000, 1, 1000, 0}, + /*s*/ {0, 0, 1000, 1, 60, 0}, + /*min*/ {0, 0, 0, 60, 1, 60, 0}, + /*h*/ {0, 0, 0, 0, 60, 1, 1, 0}, + /*d*/ {0, 0, 0, 0, 0, 24, 1, 7, 1, 0}, + /*w*/ {0, 0, 0, 0, 0, 0, 7, 1, -1, 0}, + /*mon*/ {0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0}, + /*y*/ {0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}}; -static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { +static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, + bool checkEq) { if (UNIT_MATRIX[baseIdx][idx] == -1) return false; if (baseIdx == idx) { if (interval < baseInterval) return false; if (checkEq && interval == baseInterval) return false; return interval % baseInterval == 0; } - int8_t next = baseIdx + 1; + int8_t next = baseIdx + 1; int64_t val = UNIT_MATRIX[baseIdx][next]; while (val != 0 && next <= idx) { if (val == -1) { @@ -2006,7 +2027,7 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in } if (val % baseInterval == 0 || baseInterval % val == 0) { int8_t extra = baseInterval >= val ? 0 : 1; - bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1); + bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1); if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, needCheckEq && checkEq)) { next++; val = UNIT_MATRIX[baseIdx][next]; @@ -2021,7 +2042,8 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in return false; } -static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { +static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, + bool checkEq) { if (UNIT_MATRIX[baseIdx][idx] == -1) return false; if (baseIdx == idx) { @@ -2030,7 +2052,7 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base return interval % baseInterval == 0; } - int8_t next = baseIdx - 1; + int8_t next = baseIdx - 1; int64_t val = UNIT_MATRIX[baseIdx][next]; while (val != 0 && next >= 0) { return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq); @@ -2041,18 +2063,27 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base /* * @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit * @param baseInterval, baseUnit, interval/unit of base tsma - * @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been - * translated to TICKS of [precision] + * @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already + * been translated to TICKS of [precision] * @param unit the tsma unit going to create * @param precision the precision of this db * @param checkEq pass true if same interval is not acceptable, false if acceptable. * @ret true the tsma can be created, else cannot * */ -bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, bool checkEq) { +bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, + bool checkEq) { bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit); - if (!baseIsCalendarDuration) baseInterval = convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit); + if (!baseIsCalendarDuration) { + if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit, &baseInterval)) { + return false; + } + } bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit); - if (!isCalendarDuration) interval = convertTimeFromPrecisionToUnit(interval, precision, unit); + if (!isCalendarDuration) { + if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(interval, precision, unit, &interval)) { + return false; + } + } bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 946df84a0f..ea53e82459 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -98,7 +98,7 @@ int32_t mndInitSma(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveIdx); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelRetrieveIdx); - + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TSMA, mndProcessCreateTSMAReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_TSMA, mndProcessDropTSMAReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_TABLE_TSMA, mndProcessGetTbTSMAReq); @@ -616,7 +616,12 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea // check the maxDelay if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { - int64_t msInterval = convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND); + int64_t msInterval = -1; + int32_t code = convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND, &msInterval); + if (TSDB_CODE_SUCCESS != code) { + mError("sma:%s, failed to create since convert time failed: %s", smaObj.name, tstrerror(code)); + return code; + } streamObj.conf.triggerParam = msInterval > TSDB_MIN_ROLLUP_MAX_DELAY ? msInterval : TSDB_MIN_ROLLUP_MAX_DELAY; } if (streamObj.conf.triggerParam > TSDB_MAX_ROLLUP_MAX_DELAY) { diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 633e096314..29854b4441 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -23,7 +23,9 @@ static int32_t rsmaRestore(SSma *pSma); #define SMA_SET_KEEP_CFG(v, l) \ do { \ SRetention *r = &pCfg->retentions[l]; \ - pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \ + int64_t keep = -1; \ + convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep); \ + pKeepCfg->keep2 = (int32_t)keep; \ pKeepCfg->keep0 = pKeepCfg->keep2; \ pKeepCfg->keep1 = pKeepCfg->keep2; \ pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \ @@ -60,8 +62,12 @@ static int32_t rsmaRestore(SSma *pSma); * @return int32_t */ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) { - int32_t freqDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE); - int32_t keepDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE); + int32_t code = TSDB_CODE_SUCCESS; + + int64_t freqDuration = -1; + int64_t keepDuration = -1; + code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration); + code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration); int32_t days = duration; // min if (days < freqDuration) { @@ -76,8 +82,8 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p goto _exit; } - freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); - keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); + code = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration); + code = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration); int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level days *= (nFreqTimes > 1 ? nFreqTimes : 1); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 5ab2a57954..823f65a9fd 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -335,8 +335,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat } if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { - int64_t msInterval = - convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); + int64_t msInterval = -1; + TAOS_CHECK_RETURN(convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, + TIME_UNIT_MILLISECOND, &msInterval)); pItem->maxDelay = (int32_t)msInterval; } else { pItem->maxDelay = (int32_t)param->maxdelay[idx]; diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index d0913081ac..fe3117de49 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -69,8 +69,9 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c } STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; - int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); - if (sInterval <= 0) { + int64_t sInterval = -1; + code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND, &sInterval); + if (TSDB_CODE_SUCCESS != code || 0 == sInterval) { *days = pTsdbCfg->days; goto _exit; } @@ -78,7 +79,11 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c if (records >= SMA_STORAGE_SPLIT_FACTOR) { *days = pTsdbCfg->days; } else { - int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); + int64_t mInterval = -1; + code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE, &mInterval); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index a583f21117..e433d61860 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -176,7 +176,14 @@ typedef struct SExplainCtx { #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") #define EXPLAIN_MERGE_MODE_STRING(_mode) ((_mode) == MERGE_TYPE_SORT ? "sort" : ((_mode) == MERGE_TYPE_NON_SORT ? "merge" : "column")) -#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) +#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p, _r) \ +do { \ + if ((_u) == 'n' || (_u) == 'y') { \ + _r = (_t); \ + } else { \ + code = convertTimeFromPrecisionToUnit(_t, _p, _u, &_r); \ + } \ +} while(0) #define EXPLAIN_ROW_NEW(level, ...) \ do { \ diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 843e3ee734..e0040ac0e1 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -55,7 +55,7 @@ char* qExplainGetAsofOpStr(int32_t opType) { case OP_TYPE_LOWER_EQUAL: return "<="; case OP_TYPE_EQUAL: - return "="; + return "="; default: return "UNKNOWN"; } @@ -192,13 +192,13 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S if (0 == group->physiPlanExecIdx) { group->nodeIdx = 0; } - + rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); return TSDB_CODE_APP_ERROR; } - + if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno; } else { for (int32_t i = 0; i < group->nodeNum; ++i) { @@ -338,7 +338,7 @@ static char* qExplainGetScanMode(STableScanPhysiNode* pScan) { isGroupByTag = (NULL != pScan->pGroupTags) && !isGroupByTbname; if ((((!isGroupByTag) || isGroupByTbname) && pScan->groupSort) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) { return "seq_grp_order"; - } + } if ((isGroupByTbname && (pScan->groupSort || pScan->scan.groupOrderScan)) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) { return "grp_order"; @@ -416,7 +416,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -581,7 +581,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -621,7 +621,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False"); EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); @@ -629,7 +629,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -689,7 +689,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_END(); - + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -705,7 +705,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) { @@ -733,7 +733,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET( nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } if (NULL != pJoinNode->pColOnCond) { @@ -741,7 +741,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET( nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; @@ -831,7 +831,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -844,7 +844,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId); QRY_ERR_RET(TSDB_CODE_APP_ERROR); } - + nodeNum += group->nodeNum; } @@ -876,7 +876,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } for (int32_t i = pExchNode->srcStartGroupId; i <= pExchNode->srcEndGroupId; ++i) { @@ -956,7 +956,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -991,10 +991,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); uint8_t precision = qExplainGetIntervalPrecision(pIntNode); + int64_t time1 = -1; + int64_t time2 = -1; + int32_t code = TSDB_CODE_SUCCESS; + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1); + QRY_ERR_RET(code); + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2); + QRY_ERR_RET(code); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), + time1, pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), + time2, pIntNode->slidingUnit); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -1043,10 +1050,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); uint8_t precision = qExplainGetIntervalPrecision(pIntNode); + int64_t time1 = -1; + int64_t time2 = -1; + int32_t code = TSDB_CODE_SUCCESS; + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1); + QRY_ERR_RET(code); + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2); + QRY_ERR_RET(code); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), + time1, pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), + time2, pIntNode->slidingUnit); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -1123,7 +1137,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1163,7 +1177,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1205,7 +1219,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1247,7 +1261,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1340,7 +1354,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1381,7 +1395,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1422,7 +1436,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1464,7 +1478,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1535,7 +1549,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1563,10 +1577,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); uint8_t precision = qExplainGetIntervalPrecision(pIntNode); + int64_t time1 = -1; + int64_t time2 = -1; + int32_t code = TSDB_CODE_SUCCESS; + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1); + QRY_ERR_RET(code); + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2); + QRY_ERR_RET(code); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), + time1, pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), + time2, pIntNode->slidingUnit); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -1577,7 +1598,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1644,7 +1665,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + } } break; } @@ -1669,7 +1690,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_END_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pEventNode->pEndCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); @@ -1722,7 +1743,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pJoinNode->pPrimKeyCond) { QRY_ERR_RET( nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - conditionsGot = true; + conditionsGot = true; } if (pJoinNode->pColEqCond) { if (conditionsGot) { @@ -1730,7 +1751,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } QRY_ERR_RET( nodesNodeToSQL(pJoinNode->pColEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - conditionsGot = true; + conditionsGot = true; } if (pJoinNode->pTagEqCond) { if (conditionsGot) { @@ -1738,7 +1759,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } QRY_ERR_RET( nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - conditionsGot = true; + conditionsGot = true; } if (pJoinNode->pFullOnCond) { if (conditionsGot) { @@ -1746,14 +1767,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - } + } EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pJoinNode->timeRangeTarget) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey); EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; @@ -2124,7 +2145,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_ERRNO(code); } - + groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); taosWUnLockLatch(&group->lock); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c2278649e1..26321a0215 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1523,7 +1523,7 @@ static int32_t biMakeTbnameProjectAstNode(char* funcName, char* tableAlias, SNod if (!multiResFunc) { code = TSDB_CODE_OUT_OF_MEMORY; } - + if (TSDB_CODE_SUCCESS == code) { tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN); code = nodesListMakeStrictAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc); @@ -5210,7 +5210,8 @@ static int32_t translateFill(STranslateContext* pCxt, SSelectStmt* pSelect, SInt } static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { - int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); + int64_t days = -1; + convertTimeFromPrecisionToUnit(val, fromPrecision, 'd', &days); switch (unit) { case 'b': case 'u': @@ -8209,7 +8210,8 @@ static SNode* makeIntervalVal(SRetention* pRetension, int8_t precision) { if (NULL == pVal) { return NULL; } - int64_t timeVal = convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit); + int64_t timeVal = -1; + convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit, &timeVal); char buf[20] = {0}; int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit); pVal->literal = strndup(buf, len); @@ -8758,7 +8760,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p if (TSDB_CODE_SUCCESS == code) { code = tNameExtractFullName(&name, usedbReq.db); } - if (TSDB_CODE_SUCCESS == code) + if (TSDB_CODE_SUCCESS == code) code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); diff --git a/tests/system-test/2-query/select_null.py b/tests/system-test/2-query/select_null.py index 682a98ad19..d6e48e4f99 100755 --- a/tests/system-test/2-query/select_null.py +++ b/tests/system-test/2-query/select_null.py @@ -33,7 +33,7 @@ class TDTestCase: self.testcasePath = os.path.split(__file__)[0] self.testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) - + self.db = "sel_null" def insert_data(self,database,vgroups): @@ -46,10 +46,10 @@ class TDTestCase: for i in range(5): tdSql.execute('''create table %s.stb0_%d using %s.stb0 tags(%d,'varchar%d',%d,%d, %d, %d,%d,'binary%d','nchar%d',%d,%d,%d ) ;'''%(database,i,database,i,i,i,i,i,i,i,i,i,i,i,i)) - + # insert data - for i in range(num_random): - for j in range(50): + for i in range(num_random): + for j in range(50): tdSql.execute('''insert into %s.stb0_0 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) tdSql.execute('''insert into %s.stb0_1 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) tdSql.execute('''insert into %s.stb0_2 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) @@ -60,378 +60,378 @@ class TDTestCase: tdSql.checkData(0,0,5*num_random*50) tdSql.query("select count(*) from %s.stb0_0;"%database) tdSql.checkData(0,0,num_random*50) - - def ts_3085(self,database): + + def ts_3085(self,database): sql = "select count(c0null) from(select * from %s.stb0 limit 20,4) "%(database) - tdSql.query(sql) + tdSql.query(sql) tdSql.checkData(0,0,0) - + offset = random.randint(10,100) for i in range(offset): sql = "select count(c0null) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) - tdSql.query(sql) + tdSql.query(sql) tdSql.checkData(0,0,0) sql = "select count(c1null) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) - tdSql.query(sql) + tdSql.query(sql) tdSql.checkData(0,0,0) sql = "select count(c0) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) - tdSql.query(sql) + tdSql.query(sql) tdSql.checkData(0,0,i) sql = "select count(c1) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) - tdSql.query(sql) + tdSql.query(sql) tdSql.checkData(0,0,i) sql = "select count(t0) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) - tdSql.query(sql) + tdSql.query(sql) tdSql.checkData(0,0,i) sql = "select count(t1) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) - tdSql.query(sql) + tdSql.query(sql) tdSql.checkData(0,0,i) - def ts_2974_max(self,database): + def ts_2974_max(self,database): sql = "select max(c0) from %s.stb0 where ts Date: Fri, 19 Jul 2024 11:31:46 +0800 Subject: [PATCH 34/43] feat: (errcode) geomFunc.c --- source/libs/geometry/src/geomFunc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 6e32aaf1a6..601588571e 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -180,7 +180,7 @@ int32_t executeGeomFromTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnI goto _exit; } - colDataSetVal(pOutputData, i, output, (output == NULL)); + code = colDataSetVal(pOutputData, i, output, (output == NULL)); _exit: if (output) { @@ -200,7 +200,7 @@ int32_t executeAsTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoDat goto _exit; } - colDataSetVal(pOutputData, i, output, (output == NULL)); + code = colDataSetVal(pOutputData, i, output, (output == NULL)); _exit: if (output) { @@ -228,7 +228,7 @@ int32_t executeRelationFunc(const GEOSGeometry *geom1, const GEOSPreparedGeometr } } - colDataSetVal(pOutputData, i, &res, (res==-1)); + code = colDataSetVal(pOutputData, i, &res, (res==-1)); return code; } From bebe2c5266a71565921519b8eee4ac4ec81e5664 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 19 Jul 2024 11:36:03 +0800 Subject: [PATCH 35/43] feat: (errcode) geosWrapper.c --- source/libs/geometry/src/geosWrapper.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index c7c83f4796..c5250c8481 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -86,9 +86,9 @@ _exit: return code; } -static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) { - int ret = 0; - char *wktPatternWithSpace = taosMemoryCalloc(4, 1024); +static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) { + int32_t code = 0; + char *wktPatternWithSpace = taosMemoryCalloc(4, 1024); sprintf( wktPatternWithSpace, "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " @@ -142,9 +142,9 @@ static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) { "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( " "*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$"); - ret = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace); + code = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace); taosMemoryFree(wktPatternWithSpace); - return ret; + return code; } int32_t initCtxGeomFromText() { From 1a992b65c33dd3809ec7f99a253f490688b4b764 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 19 Jul 2024 17:44:07 +0800 Subject: [PATCH 36/43] enh: rename tpcr2.c to tpcre2.c --- source/util/src/{tpcr2.c => tpcre2.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename source/util/src/{tpcr2.c => tpcre2.c} (100%) diff --git a/source/util/src/tpcr2.c b/source/util/src/tpcre2.c similarity index 100% rename from source/util/src/tpcr2.c rename to source/util/src/tpcre2.c From 5f9f6310a430cc588c68a71b696486c90ec06da4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 09:57:13 +0800 Subject: [PATCH 37/43] fix(stream): check the correct return value. --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 6362fbaa79..df8800aee4 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1289,7 +1289,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { SStreamObj *p = NULL; code = mndGetStreamObj(pMnode, pCheckpointInfo->streamId, &p); - if (p != NULL || code != 0) { + if (p != NULL && code == 0) { code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true); sdbRelease(pSdb, p); From 50e802d1d8ea528f60255d30218f0041e771187c Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 23 Jul 2024 10:13:24 +0800 Subject: [PATCH 38/43] fix: tsdbRetrieveCacheRows free memory issue --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 4eec3b87a6..27910d1746 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -333,7 +333,14 @@ void* tsdbCacherowsReaderClose(void* pReader) { return NULL; } -static void freeItem(void* pItem) { +static void freeItemOfRow(void* pItem) { + SLastCol* pCol = (SLastCol*)pItem; + if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) { + taosMemoryFree(pCol->colVal.value.pData); + } +} + +static void freeItemWithPk(void* pItem) { SLastCol* pCol = (SLastCol*)pItem; for (int i = 0; i < pCol->rowKey.numOfPKs; i++) { if (IS_VAR_DATA_TYPE(pCol->rowKey.pks[i].type)) { @@ -444,7 +451,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { - taosArrayClearEx(pRow, freeItem); + taosArrayClearEx(pRow, freeItemOfRow); continue; } @@ -516,26 +523,26 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } - taosArrayClearEx(pRow, freeItem); + taosArrayClearEx(pRow, freeItemOfRow); } if (hasRes) { saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); } - taosArrayDestroyEx(pLastCols, freeItem); + taosArrayDestroyEx(pLastCols, freeItemWithPk); } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) { for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) { tb_uid_t uid = pTableList[i].uid; tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { - taosArrayClearEx(pRow, freeItem); + taosArrayClearEx(pRow, freeItemOfRow); continue; } saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); - taosArrayClearEx(pRow, freeItem); + taosArrayClearEx(pRow, freeItemOfRow); taosArrayPush(pTableUidList, &uid); From 87abfd05bc4da2dbe8a79ea6b38c06e09bad2b44 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 23 Jul 2024 02:42:46 +0000 Subject: [PATCH 39/43] fix case --- source/dnode/mnode/impl/src/mndSma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index cfd48c0f44..1a0f7f1a8f 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1116,7 +1116,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { mInfo("sma:%s, start to drop", dropReq.name); SSIdx idx = {0}; - if (mndAcquireGlobalIdx(pMnode, dropReq.name, SDB_SMA, &idx) == 0) { + if ((code = mndAcquireGlobalIdx(pMnode, dropReq.name, SDB_SMA, &idx)) == 0) { pSma = idx.pIdx; } else { goto _OVER; From 687f7030fbfaf9369821c099c202fc60546d0f81 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 23 Jul 2024 03:06:31 +0000 Subject: [PATCH 40/43] fix case --- source/dnode/mnode/impl/src/mndStb.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7914d1dfcf..d4b086f32f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -997,10 +997,10 @@ _OVER: int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1; - if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; - if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; - if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; + TAOS_CHECK_RETURN (mndTransCheckConflict(pMnode, pTrans)); + TAOS_CHECK_RETURN (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb)); + TAOS_CHECK_RETURN (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb)); + TAOS_CHECK_RETURN (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb)); return 0; } From 0d131116e72697481de12273a4b1d39392c07a2e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 11:32:21 +0800 Subject: [PATCH 41/43] fix(stream): check return value. --- source/dnode/mnode/impl/src/mndStreamTrans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index b189ddb3cb..c0a869fb77 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -277,7 +277,7 @@ int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) { SStreamObj *pStream = NULL; int32_t code = mndGetStreamObj(pMnode, pTransInfo->streamId, &pStream); - if (pStream != NULL || code != 0) { + if (pStream != NULL && code == 0) { if (identicalName(pStream->sourceDb, pDBName, len)) { mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb); } else if (identicalName(pStream->targetDb, pDBName, len)) { From 93cc7aa9ad5a8e1858c5af7b59501271ab10b15d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 23 Jul 2024 12:25:57 +0800 Subject: [PATCH 42/43] fix(stream): set correct return code. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index f1abbeb77f..b56c474ed5 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -351,7 +351,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); - if (pTask) { + if (pTask && (code == 0)) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) { return -1; @@ -441,7 +441,7 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask); - if (pTask == NULL) { + if (pTask == NULL || code != 0) { tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, req.dstTaskId); tCleanupStreamRetrieveReq(&req); @@ -520,7 +520,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe SStreamTask* pTask = NULL; code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask); - if (pTask == NULL) { + if ((pTask == NULL) || (code != 0)) { return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId); } @@ -550,7 +550,7 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask); if (code != 0) { tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); - return TSDB_CODE_STREAM_TASK_NOT_EXIST; + return code; } tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId, @@ -642,7 +642,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve if (restored) { SStreamTask* p = NULL; code = streamMetaAcquireTask(pMeta, streamId, taskId, &p); - if ((p != NULL) && (p->info.fillHistory == 0)) { + if ((p != NULL) && (code == 0) && (p->info.fillHistory == 0)) { code = tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId); } @@ -831,7 +831,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); - if (pTask != NULL || (code != 0)) { + if (pTask != NULL && (code == 0)) { char* pStatus = NULL; if (streamTaskReadyToRun(pTask, &pStatus)) { int64_t execTs = pTask->status.lastExecTs; @@ -852,7 +852,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); - if (pTask != NULL || (code != 0)) { // even in halt status, the data in inputQ must be processed + if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, @@ -963,7 +963,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); - if (pTask == NULL) { + if (pTask == NULL || (code != 0)) { tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId); @@ -1031,11 +1031,11 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); - if (pTask == NULL) { + if (pTask == NULL || (code != 0)) { tqError( "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, pRsp->taskId); - return TSDB_CODE_STREAM_TASK_NOT_EXIST; + return code; } tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d", @@ -1051,7 +1051,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); - if (pTask == NULL) { + if (pTask == NULL || (code != 0)) { tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, pReq->taskId); // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active @@ -1065,7 +1065,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { pHistoryTask = NULL; code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask); - if (pHistoryTask == NULL) { + if (pHistoryTask == NULL || (code != 0)) { tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 ", it may have been dropped already", pMeta->vgId, pTask->hTaskInfo.id.taskId); @@ -1135,7 +1135,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); - if (pTask == NULL) { + if (pTask == NULL || (code != 0)) { tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } @@ -1153,7 +1153,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m STaskId* pHTaskId = &pTask->hTaskInfo.id; SStreamTask* pHTask = NULL; code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); - if (pHTask) { + if (pHTask && (code == 0)) { streamMutexLock(&pHTask->lock); SStreamTaskState p = streamTaskGetStatus(pHTask); tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name); @@ -1191,7 +1191,7 @@ int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { if (pTask == NULL || (code != 0)) { tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", pRsp->downstreamNodeId, pRsp->downstreamTaskId); - return TSDB_CODE_STREAM_TASK_NOT_EXIST; + return code; } code = streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId); From eaf499ad1ab0dc260a55bf1707c684af109904de Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 23 Jul 2024 06:17:46 +0000 Subject: [PATCH 43/43] fix/TD-30989 --- source/dnode/mnode/impl/src/mndSubscribe.c | 113 ++++++++++++--------- source/dnode/mnode/impl/src/mndSync.c | 51 ++++++---- 2 files changed, 94 insertions(+), 70 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e2bedc258a..c61949b316 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -44,11 +44,16 @@ static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash); static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - return 0; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); + TAOS_RETURN(code); } int32_t mndInitSubscribe(SMnode *pMnode) { @@ -75,6 +80,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { } static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) { + int32_t code = 0; SMqSubscribeObj *pSub = tNewSubscribeObj(subKey); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -86,7 +92,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj pSub->subType = pTopic->subType; pSub->withMeta = pTopic->withMeta; - if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { + if ((terrno = mndSchedInitSubEp(pMnode, pTopic, pSub)) < 0) { tDeleteSubscribeObj(pSub); taosMemoryFree(pSub); return NULL; @@ -97,6 +103,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + int32_t code = 0; SMqRebVgReq req = {0}; req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; @@ -106,8 +113,8 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj pPlan->execNode.nodeId = pRebVg->pVgEp->vgId; int32_t msgLen; if (qSubPlanToString(pPlan, &req.qmsg, &msgLen) < 0) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + code = TSDB_CODE_QRY_INVALID_INPUT; + TAOS_RETURN(code); } } else { req.qmsg = taosStrdup(""); @@ -122,7 +129,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret); if (ret < 0) { taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(ret); } tlen += sizeof(SMsgHead); @@ -130,7 +137,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(ret); } SMsgHead *pMsgHead = (SMsgHead *)buf; @@ -140,40 +147,41 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj SEncoder encoder = {0}; tEncoderInit(&encoder, POINTER_SHIFT(buf, sizeof(SMsgHead)), tlen); - if (tEncodeSMqRebVgReq(&encoder, &req) < 0) { + if ((code = tEncodeSMqRebVgReq(&encoder, &req)) < 0) { taosMemoryFreeClear(buf); tEncoderClear(&encoder); taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(code); } tEncoderClear(&encoder); *pBuf = buf; *pLen = tlen; taosMemoryFree(req.qmsg); - return 0; + TAOS_RETURN(code); } static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + int32_t code = 0; if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { if (pRebVg->oldConsumerId == -1) return 0; // drop stream, no consumer, while split vnode,all consumerId is -1 - terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_SUB_OPTION; + TAOS_RETURN(code); } void *buf; int32_t tlen; - if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan) < 0) { - return -1; + if ((code = mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan)) < 0) { + TAOS_RETURN(code); } int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); if (pVgObj == NULL) { taosMemoryFree(buf); - terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_VGROUP_NOT_EXIST; + TAOS_RETURN(code); } STransAction action = {0}; @@ -183,11 +191,11 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubsc action.msgType = TDMT_VND_TMQ_SUBSCRIBE; mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(buf); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, bool fullName) { @@ -209,6 +217,7 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, } static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + terrno = 0; SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); if (pRebInfo == NULL) { pRebInfo = tNewSMqRebSubscribe(key); @@ -612,7 +621,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu if (strcmp(pOutput->pSub->qmsg, "") != 0) { code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan); if (code != 0) { - terrno = code; goto END; } } @@ -623,7 +631,8 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb"); if (pTrans == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; goto END; } @@ -665,7 +674,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu END: nodesDestroyNode((SNode *)pPlan); mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static void freeRebalanceItem(void *param) { @@ -827,6 +836,7 @@ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { } static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { + int32_t code = 0; const char *key = rebInput->pRebInfo->key; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key); @@ -838,8 +848,10 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); if (pTopic == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; mError("[rebalance] mq rebalance %s ignored since topic %s doesn't exist", key, topic); - return -1; + TAOS_RETURN(code); } taosRLockLatch(&pTopic->lock); @@ -848,10 +860,12 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu rebOutput->pSub = mndCreateSubscription(pMnode, pTopic, key); if (rebOutput->pSub == NULL) { - mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, terrstr()); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, tstrerror(code)); taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); - return -1; + TAOS_RETURN(code); } memcpy(rebOutput->pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN); @@ -869,7 +883,7 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu mInfo("[rebalance] sub topic:%s has %d consumers sub till now", key, rebInput->oldConsumerNum); mndReleaseSubscribe(pMnode, pSub); } - return 0; + TAOS_RETURN(code); } static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { @@ -885,6 +899,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); if (rebSubHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + if (terrno != 0) code = terrno; goto END; } taosHashSetFreeFp(rebSubHash, freeRebalanceItem); @@ -915,8 +930,8 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mndDoRebalance(pMnode, &rebInput, &rebOutput); - if (mndPersistRebResult(pMnode, pMsg, &rebOutput) != 0) { - mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr()) + if ((code = mndPersistRebResult(pMnode, pMsg, &rebOutput)) != 0) { + mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", tstrerror(code)) } clearRebOutput(&rebOutput); @@ -931,7 +946,7 @@ END: taosHashCleanup(rebSubHash); mndRebCntDec(); - return code; + TAOS_RETURN(code); } static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) { @@ -950,8 +965,7 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran } SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - ret = -1; + ret = TSDB_CODE_OUT_OF_MEMORY; goto END; } pReq->head.vgId = htonl(pVgObj->vgId); @@ -968,15 +982,14 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST; sdbRelease(pMnode->pSdb, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - ret = -1; + if ((ret = mndTransAppendRedoAction(pTrans, &action)) != 0) { goto END; } } END: sdbRelease(pMnode->pSdb, pVgObj); sdbCancelFetch(pMnode->pSdb, pIter); - return ret; + TAOS_RETURN(ret); } static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { @@ -1012,7 +1025,7 @@ static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgro END: sdbRelease(pMnode->pSdb, pConsumer); sdbCancelFetch(pMnode->pSdb, pIter); - return ret; + TAOS_RETURN(ret); } static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { @@ -1022,8 +1035,8 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { int32_t code = TSDB_CODE_ACTION_IN_PROGRESS; if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; + code = TSDB_CODE_INVALID_MSG; + TAOS_RETURN(code); } SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, dropReq.cgroup, dropReq.topic); @@ -1032,24 +1045,24 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { mInfo("cgroup:%s on topic:%s, not exist, ignore not exist is set", dropReq.cgroup, dropReq.topic); return 0; } else { - terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; - mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, terrstr()); - return -1; + code = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; + mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, tstrerror(code)); + TAOS_RETURN(code); } } taosWLockLatch(&pSub->lock); if (taosHashGetSize(pSub->consumerHash) != 0) { - terrno = TSDB_CODE_MND_CGROUP_USED; - mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - code = -1; + code = TSDB_CODE_MND_CGROUP_USED; + mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code)); goto end; } pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup"); if (pTrans == NULL) { - mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - code = -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code)); goto end; } @@ -1088,9 +1101,9 @@ end: if (code != 0) { mError("cgroup %s on topic:%s, failed to drop", dropReq.cgroup, dropReq.topic); - return code; + TAOS_RETURN(code); } - return TSDB_CODE_ACTION_IN_PROGRESS; + TAOS_RETURN(TSDB_CODE_ACTION_IN_PROGRESS); } void mndCleanupSubscribe(SMnode *pMnode) {} @@ -1340,7 +1353,7 @@ END: sdbRelease(pSdb, pSub); sdbCancelFetch(pSdb, pIter); - return code; + TAOS_RETURN(code); } static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic, diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 573b75ff5a..70d0b858f6 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -123,15 +123,16 @@ _OUT: } static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) { + int32_t code = 0; if (pTrans->stage == TRN_STAGE_PREPARE) { - if (mndTransCheckConflict(pMnode, pTrans) < 0) { + if ((code = mndTransCheckConflict(pMnode, pTrans)) < 0) { mError("trans:%d, failed to validate trans conflicts.", pTrans->id); - return -1; + TAOS_RETURN(code); } return mndTransValidatePrepareStage(pMnode, pTrans); } - return 0; + TAOS_RETURN(code); } static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { @@ -139,10 +140,18 @@ static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { int32_t code = -1; SSdbRow *pRow = mndTransDecode(pRaw); - if (pRow == NULL) goto _OUT; + if (pRow == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OUT; + } pTrans = sdbGetRowObj(pRow); - if (pTrans == NULL) goto _OUT; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OUT; + } code = mndTransValidateImp(pMnode, pTrans); @@ -150,11 +159,10 @@ _OUT: if (pTrans) mndTransDropData(pTrans); if (pRow) taosMemoryFreeClear(pRow); if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT); - return code; + TAOS_RETURN(code); } int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { - terrno = TSDB_CODE_SUCCESS; SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; STrans *pTrans = NULL; @@ -163,7 +171,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (transId <= 0) { mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OUT; } @@ -176,7 +184,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (code != 0) { mError("trans:%d, failed to validate requested trans since %s", transId, terrstr()); code = 0; - pMeta->code = terrno; + pMeta->code = code; goto _OUT; } @@ -184,13 +192,15 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (code != 0) { mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); code = 0; - pMeta->code = terrno; + pMeta->code = code; goto _OUT; } pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { - mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("trans:%d, not found while execute in mnode since %s", transId, tstrerror(code)); goto _OUT; } @@ -207,7 +217,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { _OUT: if (pTrans) mndReleaseTrans(pMnode, pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { @@ -258,7 +268,7 @@ _OUT: mndPostMgmtCode(pMnode, code ? code : pMeta->code); rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; - return code; + TAOS_RETURN(code); } SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) { @@ -466,16 +476,18 @@ int32_t mndInitSync(SMnode *pMnode) { pNode->clusterId); } + int32_t code = 0; tsem_init(&pMgmt->syncSem, 0, 0); pMgmt->sync = syncOpen(&syncInfo, true); if (pMgmt->sync <= 0) { - mError("failed to open sync since %s", terrstr()); - return -1; + if (terrno != 0) code = terrno; + mError("failed to open sync since %s", tstrerror(code)); + TAOS_RETURN(code); } pMnode->pSdb->sync = pMgmt->sync; mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); - return 0; + TAOS_RETURN(code); } void mndCleanupSync(SMnode *pMnode) { @@ -518,10 +530,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; - if (req.contLen <= 0) return -1; + if (req.contLen <= 0) return TSDB_CODE_OUT_OF_MEMORY; req.pCont = rpcMallocCont(req.contLen); - if (req.pCont == NULL) return -1; + if (req.pCont == NULL) return TSDB_CODE_OUT_OF_MEMORY; memcpy(req.pCont, pRaw, req.contLen); taosThreadMutexLock(&pMgmt->lock); @@ -531,8 +543,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); taosThreadMutexUnlock(&pMgmt->lock); rpcFreeCont(req.pCont); - terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; - return terrno; + TAOS_RETURN(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED); } mInfo("trans:%d, will be proposed", transId);