From e78a75d1836eefd8e2d89fe923dd32774b570a1d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 15 Jul 2024 09:48:04 +0800 Subject: [PATCH 01/45] opti:[TS-4593] transform offset from file to tdb in tmq --- source/common/src/tmsg.c | 4 +- source/dnode/mnode/impl/src/mndTopic.c | 6 +- source/dnode/vnode/CMakeLists.txt | 5 +- source/dnode/vnode/src/inc/tq.h | 51 +- source/dnode/vnode/src/inc/vnodeInt.h | 29 +- source/dnode/vnode/src/tq/tq.c | 144 +++-- .../dnode/vnode/src/tq/tqCheckInfoSnapshot.c | 196 ------ source/dnode/vnode/src/tq/tqCommit.c | 28 - source/dnode/vnode/src/tq/tqMeta.c | 561 +++++++++--------- source/dnode/vnode/src/tq/tqOffset.c | 206 ++----- source/dnode/vnode/src/tq/tqOffsetSnapshot.c | 168 ------ .../tq/{tqHandleSnapshot.c => tqSnapshot.c} | 56 +- source/dnode/vnode/src/tq/tqUtil.c | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 5 - source/dnode/vnode/src/vnd/vnodeSnapshot.c | 58 +- source/libs/wal/src/walRef.c | 5 +- 16 files changed, 506 insertions(+), 1018 deletions(-) delete mode 100644 source/dnode/vnode/src/tq/tqCheckInfoSnapshot.c delete mode 100644 source/dnode/vnode/src/tq/tqCommit.c delete mode 100644 source/dnode/vnode/src/tq/tqOffsetSnapshot.c rename source/dnode/vnode/src/tq/{tqHandleSnapshot.c => tqSnapshot.c} (71%) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7e89753241..60d794c85c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9297,12 +9297,12 @@ int32_t tEncodeSTqCheckInfo(SEncoder *pEncoder, const STqCheckInfo *pInfo) { int32_t tDecodeSTqCheckInfo(SDecoder *pDecoder, STqCheckInfo *pInfo) { if (tDecodeCStrTo(pDecoder, pInfo->topic) < 0) return -1; if (tDecodeI64(pDecoder, &pInfo->ntbUid) < 0) return -1; - int32_t sz; + int32_t sz = 0; if (tDecodeI32(pDecoder, &sz) < 0) return -1; pInfo->colIdList = taosArrayInit(sz, sizeof(int16_t)); if (pInfo->colIdList == NULL) return -1; for (int32_t i = 0; i < sz; i++) { - int16_t colId; + int16_t colId = 0; if (tDecodeI16(pDecoder, &colId) < 0) return -1; taosArrayPush(pInfo->colIdList, &colId); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index bcb38a3902..b8cb1d00d0 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -725,19 +725,19 @@ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicO continue; } - buf = taosMemoryCalloc(1, sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN); + buf = taosMemoryCalloc(1, sizeof(SMsgHead) + sizeof(pTopic->ntbUid)); if (buf == NULL){ code = TSDB_CODE_OUT_OF_MEMORY; goto end; } void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); ((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId); - memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN); + *(int64_t*)abuf = pTopic->ntbUid; STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = buf; - action.contLen = sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN; + action.contLen = sizeof(SMsgHead) + sizeof(pTopic->ntbUid); action.msgType = TDMT_VND_TMQ_DEL_CHECKINFO; code = mndTransAppendRedoAction(pTrans, &action); if (code != 0) { diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index d776271131..400725b041 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -68,11 +68,8 @@ set( "src/tq/tqOffset.c" "src/tq/tqPush.c" "src/tq/tqSink.c" - "src/tq/tqCommit.c" "src/tq/tqStreamTask.c" - "src/tq/tqHandleSnapshot.c" - "src/tq/tqCheckInfoSnapshot.c" - "src/tq/tqOffsetSnapshot.c" + "src/tq/tqSnapshot.c" "src/tq/tqStreamStateSnap.c" "src/tq/tqStreamTaskSnap.c" diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 651fe2cda4..d97a953d94 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -41,8 +41,6 @@ extern "C" { #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -typedef struct STqOffsetStore STqOffsetStore; - #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) // tqExec @@ -101,10 +99,11 @@ struct STQ { SHashObj* pPushMgr; // subKey -> STqHandle SHashObj* pHandle; // subKey -> STqHandle SHashObj* pCheckInfo; // topic -> SAlterCheckInfo - STqOffsetStore* pOffsetStore; + SHashObj* pOffset; // subKey -> STqOffsetVal TDB* pMetaDB; TTB* pExecStore; TTB* pCheckStore; + TTB* pOffsetStore; SStreamMeta* pStreamMeta; }; @@ -128,29 +127,22 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); int32_t tqMetaOpen(STQ* pTq); int32_t tqMetaClose(STQ* pTq); int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); -int32_t tqMetaDeleteHandle(STQ* pTq, const char* key); -int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen); -int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); -int32_t tqMetaRestoreCheckInfo(STQ* pTq); -int32_t tqMetaGetHandle(STQ* pTq, const char* key); -int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); - -STqOffsetStore* tqOffsetOpen(STQ* pTq); -int32_t tqMetaTransform(STQ* pTq); -void tqOffsetClose(STqOffsetStore*); -STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey); -int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset); -int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey); -int32_t tqOffsetCommitFile(STqOffsetStore* pStore); - +int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen); +int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen); +int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); +int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen); +int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle); +void* tqMetaGetCheckInfo(STQ* pTq, int64_t tbUid); +void* tqMetaGetOffset(STQ* pTq, const char* subkey); +int32_t tqMetaTransform(STQ* pTq); // tqSink int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr, bool newSubTableRule); void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data); // tqOffset -char* tqOffsetBuildFName(const char* path, int32_t fVer); -int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); +int32_t tqBuildFName(char** data, const char* path, char* name); +int32_t tqOffsetRestoreFromFile(SHashObj* pOffset, char* name); // tq util int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); @@ -166,6 +158,25 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule); +#define TQ_ERR_GO_TO_END(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + goto END; \ + } \ + } while (0) + +#define TQ_ERR_RETURN(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + return code; \ + } \ + } while (0) + +#define TQ_SUBSCRIBE_NAME "subscribe" +#define TQ_OFFSET_NAME "offset-ver0" + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4a47e08730..a7f87300b5 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -71,10 +71,6 @@ typedef struct STsdbSnapRAWReader STsdbSnapRAWReader; typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter; typedef struct STqSnapReader STqSnapReader; typedef struct STqSnapWriter STqSnapWriter; -typedef struct STqOffsetReader STqOffsetReader; -typedef struct STqOffsetWriter STqOffsetWriter; -typedef struct STqCheckInfoReader STqCheckInfoReader; -typedef struct STqCheckInfoWriter STqCheckInfoWriter; typedef struct SStreamTaskReader SStreamTaskReader; typedef struct SStreamTaskWriter SStreamTaskWriter; typedef struct SStreamStateReader SStreamStateReader; @@ -268,7 +264,6 @@ int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver); int32_t tqScanWal(STQ* pTq); -int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); // tq-mq @@ -351,29 +346,17 @@ int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr); int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter); int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback); // STqSnapshotReader == -int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader); +int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader); int32_t tqSnapReaderClose(STqSnapReader** ppReader); int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData); // STqSnapshotWriter ====================================== int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter); int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback); -int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData); -// STqCheckInfoshotReader == -int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader); -int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader); -int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData); -// STqCheckInfoshotWriter ====================================== -int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter); -int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback); -int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData); -// STqOffsetReader ======================================== -int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader); -int32_t tqOffsetReaderClose(STqOffsetReader** ppReader); -int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData); -// STqOffsetWriter ======================================== -int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter** ppWriter); -int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback); -int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nData); +int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData); + +int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData); +int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData); + // SStreamTaskWriter ====================================== int32_t streamTaskSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskReader** ppReader); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ac57a003c5..bb64a0ccf6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -76,9 +76,11 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { taosInitRWLatch(&pTq->lock); pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); + pTq->pCheckInfo = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); + pTq->pOffset = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_ENTRY_LOCK); + int32_t code = tqInitialize(pTq); if (code != TSDB_CODE_SUCCESS) { tqClose(pTq); @@ -98,18 +100,10 @@ int32_t tqInitialize(STQ* pTq) { streamMetaLoadAllTasks(pTq->pStreamMeta); - if (tqMetaTransform(pTq) < 0) { + if (tqMetaOpen(pTq) < 0) { return -1; } - if (tqMetaRestoreCheckInfo(pTq) < 0) { - return -1; - } - - pTq->pOffsetStore = tqOffsetOpen(pTq); - if (pTq->pOffsetStore == NULL) { - return -1; - } return 0; } @@ -133,10 +127,10 @@ void tqClose(STQ* pTq) { pIter = taosHashIterate(pTq->pPushMgr, pIter); } - tqOffsetClose(pTq->pOffsetStore); taosHashCleanup(pTq->pHandle); taosHashCleanup(pTq->pPushMgr); taosHashCleanup(pTq->pCheckInfo); + taosHashCleanup(pTq->pOffset); taosMemoryFree(pTq->path); tqMetaClose(pTq); @@ -221,7 +215,7 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t goto end; } - STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); + STqOffset* pSavedOffset = (STqOffset*)tqMetaGetOffset(pTq, pOffset->subKey); if (pSavedOffset != NULL && tqOffsetEqual(pOffset, pSavedOffset)) { tqInfo("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version); @@ -229,10 +223,14 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } // save the new offset value - code = tqOffsetWrite(pTq->pOffsetStore, pOffset); - if(code != 0) { - code = TSDB_CODE_INVALID_MSG; - goto end; + if (taosHashPut(pTq->pOffset, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset))){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (tqMetaSaveInfo(pTq, pTq->pOffsetStore, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset)) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } return 0; @@ -284,25 +282,16 @@ end: } int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { - void* pIter = NULL; + STqCheckInfo* pCheck = tqMetaGetCheckInfo(pTq, tbUid); + if(pCheck == NULL) { + return 0; + } - while (1) { - pIter = taosHashIterate(pTq->pCheckInfo, pIter); - if (pIter == NULL) { - break; - } - - STqCheckInfo* pCheck = (STqCheckInfo*)pIter; - - if (pCheck->ntbUid == tbUid) { - int32_t sz = taosArrayGetSize(pCheck->colIdList); - for (int32_t i = 0; i < sz; i++) { - int16_t forbidColId = *(int16_t*)taosArrayGet(pCheck->colIdList, i); - if (forbidColId == colId) { - taosHashCancelIterate(pTq->pCheckInfo, pIter); - return -1; - } - } + int32_t sz = taosArrayGetSize(pCheck->colIdList); + for (int32_t i = 0; i < sz; i++) { + int16_t forbidColId = *(int16_t*)taosArrayGet(pCheck->colIdList, i); + if (forbidColId == colId) { + return -1; } } @@ -360,21 +349,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { while (1) { taosWLockLatch(&pTq->lock); // 1. find handle - pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle == NULL) { - do { - if (tqMetaGetHandle(pTq, req.subKey) == 0) { - pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle != NULL) { - break; - } - } - tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); - terrno = TSDB_CODE_INVALID_MSG; - taosWUnLockLatch(&pTq->lock); - code = -1; - goto END; - } while (0); + code = tqMetaGetHandle(pTq, req.subKey, &pHandle); + if (code != TDB_CODE_SUCCESS) { + tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + taosWUnLockLatch(&pTq->lock); + return -1; } // 2. check rebalance status @@ -443,7 +423,7 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) { tDecoderClear(&decoder); - STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, vgOffset.offset.subKey); + STqOffset* pSavedOffset = (STqOffset*)tqMetaGetOffset(pTq, vgOffset.offset.subKey); if (pSavedOffset == NULL) { terrno = TSDB_CODE_TMQ_NO_COMMITTED; return terrno; @@ -523,7 +503,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { if (reqOffset.type == TMQ_OFFSET__LOG) { dataRsp.common.rspOffset.version = reqOffset.version; } else if (reqOffset.type < 0) { - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); + STqOffset* pOffset = (STqOffset*)(STqOffset*)tqMetaGetOffset(pTq, req.subKey); if (pOffset != NULL) { if (pOffset->val.type != TMQ_OFFSET__LOG) { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s, no valid wal info", consumerId, vgId, req.subKey); @@ -590,12 +570,14 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } taosWLockLatch(&pTq->lock); - code = tqOffsetDelete(pTq->pOffsetStore, pReq->subKey); - if (code != 0) { - tqError("cannot process tq delete req %s, since no such offset in cache", pReq->subKey); + if (taosHashRemove(pTq->pOffset, pReq->subKey, strlen(pReq->subKey) != 0)) { + tqError("cannot process tq delete req %s, since no such offset in hash", pReq->subKey); + } + if (tqMetaDeleteInfo(pTq, pTq->pOffsetStore, pReq->subKey, strlen(pReq->subKey)) != 0) { + tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey); } - if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) { + if (tqMetaDeleteInfo(pTq, pTq->pExecStore, pReq->subKey, strlen(pReq->subKey)) < 0) { tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey); } taosWUnLockLatch(&pTq->lock); @@ -603,20 +585,39 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg return 0; } +void mergeTwoArray(SArray* a, SArray* b){ + size_t bLen = taosArrayGetSize(b); + for(int i = 0; i < bLen; i++){ + void* dataB = taosArrayGet(b, i); + size_t aLen = taosArrayGetSize(a); + int j = 0; + for(; j < aLen; j++){ + void* dataA = taosArrayGet(a, i); + if(*(int64_t*)dataA == *(int64_t*)dataB){ + break; + } + } + if(j == aLen){ + taosArrayPush(a, dataB); + } + } +} + int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { STqCheckInfo info = {0}; - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeSTqCheckInfo(&decoder, &info) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + if(tqMetaDecodeCheckInfo(&info, msg, msgLen) != 0){ return -1; } - tDecoderClear(&decoder); - if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) { + STqCheckInfo *old = tqMetaGetCheckInfo(pTq, info.ntbUid); + if (old != NULL){ + mergeTwoArray(info.colIdList, old->colIdList); + } + if (taosHashPut(pTq->pCheckInfo, &info.ntbUid, sizeof(info.ntbUid), &info, sizeof(STqCheckInfo)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tDeleteSTqCheckInfo(&info); return -1; } - if (tqMetaSaveCheckInfo(pTq, info.topic, msg, msgLen) < 0) { + if (tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.ntbUid, sizeof(info.ntbUid), msg, msgLen) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -628,7 +629,7 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (tqMetaDeleteCheckInfo(pTq, msg) < 0) { + if (tqMetaDeleteInfo(pTq, pTq->pCheckStore, msg, msgLen) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -652,19 +653,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); + taosRLockLatch(&pTq->lock); STqHandle* pHandle = NULL; - while (1) { - pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle) { - break; - } - taosRLockLatch(&pTq->lock); - ret = tqMetaGetHandle(pTq, req.subKey); - taosRUnLockLatch(&pTq->lock); - if (ret < 0) { - break; - } - } + ret = tqMetaGetHandle(pTq, req.subKey, &pHandle); + taosRUnLockLatch(&pTq->lock); if (pHandle == NULL) { if (req.oldConsumerId != -1) { tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, @@ -675,7 +667,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } STqHandle handle = {0}; - ret = tqCreateHandle(pTq, &req, &handle); + ret = tqMetaCreateHandle(pTq, &req, &handle); if (ret < 0) { tqDestroyTqHandle(&handle); goto end; diff --git a/source/dnode/vnode/src/tq/tqCheckInfoSnapshot.c b/source/dnode/vnode/src/tq/tqCheckInfoSnapshot.c deleted file mode 100644 index a3bd22eef0..0000000000 --- a/source/dnode/vnode/src/tq/tqCheckInfoSnapshot.c +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "meta.h" -#include "tdbInt.h" -#include "tq.h" - -// STqCheckInfoReader ======================================== -struct STqCheckInfoReader { - STQ* pTq; - int64_t sver; - int64_t ever; - TBC* pCur; -}; - -int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader) { - int32_t code = 0; - STqCheckInfoReader* pReader = NULL; - - // alloc - pReader = (STqCheckInfoReader*)taosMemoryCalloc(1, sizeof(STqCheckInfoReader)); - if (pReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pReader->pTq = pTq; - pReader->sver = sver; - pReader->ever = ever; - - // impl - code = tdbTbcOpen(pTq->pCheckStore, &pReader->pCur, NULL); - if (code) { - taosMemoryFree(pReader); - goto _err; - } - - code = tdbTbcMoveToFirst(pReader->pCur); - if (code) { - taosMemoryFree(pReader); - goto _err; - } - - tqInfo("vgId:%d, vnode checkinfo tq reader opened", TD_VID(pTq->pVnode)); - - *ppReader = pReader; - return code; - -_err: - tqError("vgId:%d, vnode checkinfo tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); - *ppReader = NULL; - return code; -} - -int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader) { - int32_t code = 0; - - tdbTbcClose((*ppReader)->pCur); - taosMemoryFree(*ppReader); - *ppReader = NULL; - - return code; -} - -int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData) { - int32_t code = 0; - void* pKey = NULL; - void* pVal = NULL; - int32_t kLen = 0; - int32_t vLen = 0; - - if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) { - goto _exit; - } - - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); - if (*ppData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); - pHdr->type = SNAP_DATA_TQ_CHECKINFO; - pHdr->size = vLen; - memcpy(pHdr->data, pVal, vLen); - -_exit: - tdbFree(pKey); - tdbFree(pVal); - - tqInfo("vgId:%d, vnode check info tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen); - return code; - -_err: - tdbFree(pKey); - tdbFree(pVal); - - tqError("vgId:%d, vnode check info tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code)); - return code; -} - -// STqCheckInfoWriter ======================================== -struct STqCheckInfoWriter { - STQ* pTq; - int64_t sver; - int64_t ever; - TXN* txn; -}; - -int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter) { - int32_t code = 0; - STqCheckInfoWriter* pWriter; - - // alloc - pWriter = (STqCheckInfoWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pWriter->pTq = pTq; - pWriter->sver = sver; - pWriter->ever = ever; - - if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - code = -1; - taosMemoryFree(pWriter); - goto _err; - } - - *ppWriter = pWriter; - return code; - -_err: - tqError("vgId:%d, tq check info writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); - *ppWriter = NULL; - return code; -} - -int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback) { - int32_t code = 0; - STqCheckInfoWriter* pWriter = *ppWriter; - STQ* pTq = pWriter->pTq; - - if (rollback) { - tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); - } else { - code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); - if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn); - if (code) goto _err; - } - - taosMemoryFree(pWriter); - *ppWriter = NULL; - - return code; - -_err: - tqError("vgId:%d, tq check info writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); - return code; -} - -int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - STQ* pTq = pWriter->pTq; - STqCheckInfo info = {0}; - SDecoder decoder; - SDecoder* pDecoder = &decoder; - - tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); - code = tDecodeSTqCheckInfo(pDecoder, &info); - if (code) goto _err; - code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)); - if (code) goto _err; - code = tqMetaSaveCheckInfo(pTq, info.topic, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); - if (code) goto _err; - tDecoderClear(pDecoder); - - return code; - -_err: - tDecoderClear(pDecoder); - tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); - return code; -} diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c deleted file mode 100644 index 0f5daa31ad..0000000000 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tq.h" - -int tqCommit(STQ* pTq) { -#if 0 - // stream meta commit does not be aligned to the vnode commit - if (streamMetaCommit(pTq->pStreamMeta) < 0) { - tqError("vgId:%d, failed to commit stream meta since %s", TD_VID(pTq->pVnode), terrstr()); - return -1; - } -#endif - - return tqOffsetCommitFile(pTq->pOffsetStore); -} diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 02184f1d50..4b30e79d44 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -75,125 +75,116 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { return 0; } -int32_t tqMetaOpen(STQ* pTq) { - if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL) < 0) { - return -1; +int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){ + SDecoder decoder = {0}; + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); + int32_t code = tDecodeSTqCheckInfo(&decoder, info); + if (code != 0) { + tDeleteSTqCheckInfo(info); + return TSDB_CODE_OUT_OF_MEMORY; } - - if (tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0) < 0) { - return -1; - } - - if (tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0) < 0) { - return -1; - } - - return 0; -} - -int32_t tqMetaClose(STQ* pTq) { - if (pTq->pExecStore) { - tdbTbClose(pTq->pExecStore); - } - if (pTq->pCheckStore) { - tdbTbClose(pTq->pCheckStore); - } - tdbClose(pTq->pMetaDB); - return 0; -} - -int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen) { - TXN* txn; - - if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - - if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, txn) < 0) { - return -1; - } - - if (tdbCommit(pTq->pMetaDB, txn) < 0) { - return -1; - } - - if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { - return -1; - } - - return 0; -} - -int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { - TXN* txn; - - if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - - if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) { - tqWarn("vgId:%d, tq try delete checkinfo failed %s", pTq->pVnode->config.vgId, key); - } - - if (tdbCommit(pTq->pMetaDB, txn) < 0) { - return -1; - } - - if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { - return -1; - } - - return 0; -} - -int32_t tqMetaRestoreCheckInfo(STQ* pTq) { - TBC* pCur = NULL; - if (tdbTbcOpen(pTq->pCheckStore, &pCur, NULL) < 0) { - return -1; - } - - void* pKey = NULL; - int kLen = 0; - void* pVal = NULL; - int vLen = 0; - SDecoder decoder; - int32_t code = 0; - - tdbTbcMoveToFirst(pCur); - - while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { - STqCheckInfo info; - tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - code = tDecodeSTqCheckInfo(&decoder, &info); - if (code != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - tDecoderClear(&decoder); - code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)); - if (code != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - } - -END: - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); + tDecoderClear(&decoder); return code; } +void* tqMetaGetCheckInfo(STQ* pTq, int64_t tbUid){ + void* data = taosHashGet(pTq->pCheckInfo, &tbUid, sizeof(tbUid)); + if (data == NULL) { + int vLen = 0; + if (tdbTbGet(pTq->pCheckStore, &tbUid, sizeof(tbUid), &data, &vLen) < 0) { + tdbFree(data); + return NULL; + } + STqCheckInfo info= {0}; + if(tqMetaDecodeCheckInfo(&info, data, vLen) != 0) { + tdbFree(data); + return NULL; + } + tdbFree(data); + + if(taosHashPut(pTq->pCheckInfo, &tbUid, sizeof(tbUid), &info, sizeof(STqCheckInfo)) != 0){ + tDeleteSTqCheckInfo(&info); + return NULL; + } + return taosHashGet(pTq->pCheckInfo, &tbUid, sizeof(tbUid)); + } else { + return data; + } +} + +int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) { + int32_t code = TDB_CODE_SUCCESS; + TXN* txn = NULL; + + TQ_ERR_RETURN(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + TQ_ERR_RETURN(tdbTbUpsert(ttb, key, kLen, value, vLen, txn)); + TQ_ERR_RETURN(tdbCommit(pTq->pMetaDB, txn)); + TQ_ERR_RETURN(tdbPostCommit(pTq->pMetaDB, txn)); + + return 0; +} + +int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) { + int32_t code = TDB_CODE_SUCCESS; + TXN* txn = NULL; + + TQ_ERR_RETURN(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + TQ_ERR_RETURN(tdbTbDelete(ttb, key, kLen, txn)); + TQ_ERR_RETURN(tdbCommit(pTq->pMetaDB, txn)); + TQ_ERR_RETURN(tdbPostCommit(pTq->pMetaDB, txn)); + + return 0; +} + +static int32_t tqMetaTransformOffsetInfo(STQ* pTq, char* path) { + int32_t code = TDB_CODE_SUCCESS; + void* pIter = NULL; + + TQ_ERR_RETURN(tqOffsetRestoreFromFile(pTq->pOffset, path)); + while (1) { + pIter = taosHashIterate(pTq->pOffset, pIter); + if (pIter == NULL) { + break; + } + + STqOffset* offset = (STqOffset*)pIter; + TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset->subKey, strlen(offset->subKey), offset, sizeof(STqOffset))); + } + +END: + taosHashCancelIterate(pTq->pOffset, pIter); + return code; +} + +void* tqMetaGetOffset(STQ* pTq, const char* subkey){ + void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey)); + if (data == NULL) { + int vLen = 0; + if (tdbTbGet(pTq->pOffsetStore, subkey, strlen(subkey), &data, &vLen) < 0) { + tdbFree(data); + return NULL; + } + + if(taosHashPut(pTq->pOffset, subkey, strlen(subkey), data, sizeof(STqOffset)) != 0){ + tdbFree(data); + return NULL; + } + tdbFree(data); + + return taosHashGet(pTq->pOffset, subkey, strlen(subkey)); + } else { + return data; + } +} + int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { - int32_t code; + int32_t code = TDB_CODE_SUCCESS; int32_t vlen; void* buf = NULL; SEncoder encoder; tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); if (code < 0) { - goto end; + goto END; } tqDebug("tq save %s(%d) handle consumer:0x%" PRIx64 " epoch:%d vgId:%d", pHandle->subKey, @@ -202,71 +193,33 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { buf = taosMemoryCalloc(1, vlen); if (buf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + goto END; } - tEncoderInit(&encoder, buf, vlen); - code = tEncodeSTqHandle(&encoder, pHandle); if (code < 0) { - goto end; + goto END; } - TXN* txn = NULL; - code = tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - if (code < 0) { - goto end; - } + TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pExecStore, key, (int)strlen(key), buf, vlen)); - code = tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn); - if (code < 0) { - goto end; - } - - code = tdbCommit(pTq->pMetaDB, txn); - if (code < 0) { - goto end; - } - - code = tdbPostCommit(pTq->pMetaDB, txn); -end: +END: tEncoderClear(&encoder); taosMemoryFree(buf); return code; } -int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { - TXN* txn; - - if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - - if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) { - } - - if (tdbCommit(pTq->pMetaDB, txn) < 0) { - return -1; - } - - if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { - return -1; - } - - return 0; -} - -static int buildHandle(STQ* pTq, STqHandle* handle){ +static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){ + int32_t code = TDB_CODE_SUCCESS; SVnode* pVnode = pTq->pVnode; int32_t vgId = TD_VID(pVnode); handle->pRef = walOpenRef(pVnode->pWal); if (handle->pRef == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } - walSetRefVer(handle->pRef, handle->snapshotVer); + TQ_ERR_RETURN(walSetRefVer(handle->pRef, handle->snapshotVer)); SReadHandle reader = { .vnode = pVnode, @@ -282,18 +235,18 @@ static int buildHandle(STQ* pTq, STqHandle* handle){ qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId); if (handle->execHandle.task == NULL) { tqError("cannot create exec task for %s", handle->subKey); - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } void* scanner = NULL; - qExtractStreamScanner(handle->execHandle.task, &scanner); + (void)qExtractStreamScanner(handle->execHandle.task, &scanner); if (scanner == NULL) { tqError("cannot extract stream scanner for %s", handle->subKey); - return -1; + return TSDB_CODE_SCH_INTERNAL_ERROR; } handle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); if (handle->execHandle.pTqReader == NULL) { tqError("cannot extract exec reader for %s", handle->subKey); - return -1; + return TSDB_CODE_SCH_INTERNAL_ERROR; } } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) { handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0); @@ -308,7 +261,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){ if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) { tqError("nodesStringToNode error in sub stable, since %s", terrstr()); - return -1; + return TSDB_CODE_SCH_INTERNAL_ERROR; } } buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType, @@ -320,7 +273,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){ if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId); taosArrayDestroy(tbUidList); - return -1; + return TSDB_CODE_SCH_INTERNAL_ERROR; } tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid); handle->execHandle.pTqReader = tqReaderOpen(pVnode); @@ -330,24 +283,23 @@ static int buildHandle(STQ* pTq, STqHandle* handle){ return 0; } -static int restoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){ +static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){ int32_t vgId = TD_VID(pTq->pVnode); - SDecoder decoder; - int32_t code = 0; + SDecoder decoder = {0}; + int32_t code = TDB_CODE_SUCCESS; + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - code = tDecodeSTqHandle(&decoder, handle); - if (code) goto end; - code = buildHandle(pTq, handle); - if (code) goto end; - tqInfo("restoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId); + TQ_ERR_GO_TO_END(tDecodeSTqHandle(&decoder, handle)); + TQ_ERR_GO_TO_END(tqMetaInitHandle(pTq, handle)); + tqInfo("tqMetaRestoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId); code = taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); -end: +END: tDecoderClear(&decoder); return code; } -int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ +int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ int32_t vgId = TD_VID(pTq->pVnode); memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); @@ -367,165 +319,186 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal); - if(buildHandle(pTq, handle) < 0){ + if(tqMetaInitHandle(pTq, handle) < 0){ return -1; } - tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer); + tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer); return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); } -static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pExecStoreOld, TTB* pExecStoreNew){ - TBC* pCur = NULL; - if (tdbTbcOpen(pExecStoreOld, &pCur, NULL) < 0) { - return -1; - } - - TXN* txn; - if (tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - return -1; - } - +static int32_t tqMetaTransformStoreInfo(TDB* pMetaDB, TTB* pExecStoreOld, TTB* pExecStoreNew){ + TBC* pCur = NULL; void* pKey = NULL; int kLen = 0; void* pVal = NULL; int vLen = 0; + TXN* txn = NULL; - tdbTbcMoveToFirst(pCur); + int32_t code = TDB_CODE_SUCCESS; + + TQ_ERR_GO_TO_END(tdbTbcOpen(pExecStoreOld, &pCur, NULL)); + TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + + TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur)); while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { - if (tdbTbUpsert(pExecStoreNew, pKey, kLen, pVal, vLen, txn) < 0) { - tqError("transform sub info error"); - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); - return -1; - } + TQ_ERR_GO_TO_END (tdbTbUpsert(pExecStoreNew, pKey, kLen, pVal, vLen, txn)); } + + TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn)); + TQ_ERR_GO_TO_END (tdbPostCommit(pMetaDB, txn)); + +END: tdbFree(pKey); tdbFree(pVal); - tdbTbcClose(pCur); + (void)tdbTbcClose(pCur); + return code; +} - if (tdbCommit(pMetaDB, txn) < 0) { - return -1; +static int32_t tqMetaTransformCheckInfo(TDB* pMetaDB, TTB* pCheckOld, TTB* pCheckNew){ + TBC* pCur = NULL; + void* pKey = NULL; + int kLen = 0; + void* pVal = NULL; + int vLen = 0; + TXN* txn = NULL; + + int32_t code = TDB_CODE_SUCCESS; + + TQ_ERR_GO_TO_END(tdbTbcOpen(pCheckOld, &pCur, NULL)); + TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + + TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur)); + while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { + STqCheckInfo info= {0}; + TQ_ERR_GO_TO_END(tqMetaDecodeCheckInfo(&info, pVal, vLen)); + int64_t uid = info.ntbUid; + tDeleteSTqCheckInfo(&info); + TQ_ERR_GO_TO_END (tdbTbUpsert(pCheckNew, &uid, sizeof(uid), pVal, vLen, txn)); } - if (tdbPostCommit(pMetaDB, txn) < 0) { - return -1; + TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn)); + TQ_ERR_GO_TO_END (tdbPostCommit(pMetaDB, txn)); +END: + tdbFree(pKey); + tdbFree(pVal); + (void)tdbTbcClose(pCur); + return code; +} + +int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { + void* data = taosHashGet(pTq->pHandle, key, strlen(key)); + if(data == NULL){ + int vLen = 0; + if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &data, &vLen) < 0) { + tdbFree(data); + return TSDB_CODE_OUT_OF_MEMORY; + } + STqHandle handle = {0}; + if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0){ + tdbFree(data); + tqDestroyTqHandle(&handle); + return TSDB_CODE_OUT_OF_MEMORY; + } + tdbFree(data); + if(taosHashPut(pTq->pHandle, key, strlen(key), &handle, sizeof(STqHandle)) != 0){ + tqDestroyTqHandle(&handle); + return TSDB_CODE_OUT_OF_MEMORY; + } + *pHandle = taosHashGet(pTq->pCheckInfo, key, strlen(key)); + if(*pHandle == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else{ + *pHandle = data; } - return 0; + return TDB_CODE_SUCCESS; +} + +int32_t tqMetaOpenTdb(STQ* pTq) { + int32_t code = TDB_CODE_SUCCESS; + TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL)); + TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0)); + TQ_ERR_GO_TO_END(tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0)); + TQ_ERR_GO_TO_END(tdbTbOpen("tq.offset.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pOffsetStore, 0)); + +END: + return code; +} + +static int32_t replaceTqPath(char** path){ + char* tpath = NULL; + int32_t code = TDB_CODE_SUCCESS; + TQ_ERR_RETURN(tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME)); + taosMemoryFree(*path); + *path = tpath; + return TDB_CODE_SUCCESS; +} + +int32_t tqMetaOpen(STQ* pTq) { + char* maindb = NULL; + int32_t code = TDB_CODE_SUCCESS; + TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME)); + if(!taosCheckExistFile(maindb)){ + TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path)); + TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq)); + }else{ + TQ_ERR_GO_TO_END(tqMetaTransform(pTq)); + } + +END: + taosMemoryFree(maindb); + return code; } int32_t tqMetaTransform(STQ* pTq) { - int32_t len = strlen(pTq->path) + 64; - char* maindb = taosMemoryCalloc(1, len); - sprintf(maindb, "%s%s%s", pTq->path, TD_DIRSEP, TDB_MAINDB_NAME); + int32_t code = TDB_CODE_SUCCESS; + TDB* pMetaDB = NULL; + TTB* pExecStore = NULL; + TTB* pCheckStore = NULL; + char* offsetNew = NULL; + char* offset = NULL; + TQ_ERR_GO_TO_END(tqBuildFName(&offset, pTq->path, TQ_OFFSET_NAME)); - if(!taosCheckExistFile(maindb)){ - taosMemoryFree(maindb); - char* tpath = taosMemoryCalloc(1, len); - if(tpath == NULL){ - return -1; - } - sprintf(tpath, "%s%s%s", pTq->path, TD_DIRSEP, "subscribe"); - taosMemoryFree(pTq->path); - pTq->path = tpath; - return tqMetaOpen(pTq); - } + TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL)); + TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pMetaDB, &pExecStore, 0)); + TQ_ERR_GO_TO_END(tdbTbOpen("tq.check.db", -1, -1, NULL, pMetaDB, &pCheckStore, 0)); - int32_t code = 0; - TDB* pMetaDB = NULL; - TTB* pExecStore = NULL; - TTB* pCheckStore = NULL; - char* offsetNew = NULL; - char* offset = tqOffsetBuildFName(pTq->path, 0); - if(offset == NULL){ - code = -1; - goto END; - } + TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path)); + TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq)); + TQ_ERR_GO_TO_END(tqMetaTransformStoreInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore)); + TQ_ERR_GO_TO_END(tqMetaTransformCheckInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore)); - if (tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL) < 0) { - code = -1; - goto END; - } - - if (tdbTbOpen("tq.db", -1, -1, NULL, pMetaDB, &pExecStore, 0) < 0) { - code = -1; - goto END; - } - - if (tdbTbOpen("tq.check.db", -1, -1, NULL, pMetaDB, &pCheckStore, 0) < 0) { - code = -1; - goto END; - } - - char* tpath = taosMemoryCalloc(1, len); - if(tpath == NULL){ - code = -1; - goto END; - } - sprintf(tpath, "%s%s%s", pTq->path, TD_DIRSEP, "subscribe"); - taosMemoryFree(pTq->path); - pTq->path = tpath; - if (tqMetaOpen(pTq) < 0) { - code = -1; - goto END; - } - - if( tqMetaTransformInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore) < 0){ - code = -1; - goto END; - } - - if(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore) < 0){ - code = -1; - goto END; - } - - tdbTbClose(pExecStore); - pExecStore = NULL; - tdbTbClose(pCheckStore); - pCheckStore = NULL; - tdbClose(pMetaDB); - pMetaDB = NULL; - - offsetNew = tqOffsetBuildFName(pTq->path, 0); - if(offsetNew == NULL){ - code = -1; - goto END; - } + TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME)); if(taosCheckExistFile(offset) && taosCopyFile(offset, offsetNew) < 0){ tqError("copy offset file error"); - code = -1; - goto END; } - taosRemoveFile(maindb); - taosRemoveFile(offset); + TQ_ERR_GO_TO_END(tqMetaTransformOffsetInfo(pTq, offsetNew)); + (void)taosRemoveFile(offset); - END: - taosMemoryFree(maindb); +END: taosMemoryFree(offset); taosMemoryFree(offsetNew); - tdbTbClose(pExecStore); - tdbTbClose(pCheckStore); - tdbClose(pMetaDB); + //return 0 always, so ignore + (void)tdbTbClose(pExecStore); + (void)tdbTbClose(pCheckStore); + (void)tdbClose(pMetaDB); return code; } -int32_t tqMetaGetHandle(STQ* pTq, const char* key) { - void* pVal = NULL; - int vLen = 0; - - if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &pVal, &vLen) < 0) { - return -1; +int32_t tqMetaClose(STQ* pTq) { + if (pTq->pExecStore) { + (void)tdbTbClose(pTq->pExecStore); } - STqHandle handle = {0}; - int code = restoreHandle(pTq, pVal, vLen, &handle); - if (code < 0){ - tqDestroyTqHandle(&handle); + if (pTq->pCheckStore) { + (void)tdbTbClose(pTq->pCheckStore); } - tdbFree(pVal); - return code; -} + if (pTq->pOffsetStore) { + (void)tdbTbClose(pTq->pOffsetStore); + } + (void)tdbClose(pTq->pMetaDB); + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 8b0e039ad5..8295ec4824 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -16,192 +16,76 @@ #include "tq.h" -struct STqOffsetStore { - STQ* pTq; - SHashObj* pHash; // SHashObj - int8_t needCommit; -}; - -char* tqOffsetBuildFName(const char* path, int32_t fVer) { - int32_t len = strlen(path); - char* fname = taosMemoryCalloc(1, len + 40); +int32_t tqBuildFName(char** data, const char* path, char* name) { + int32_t len = strlen(path) + strlen(name) + 2; + char* fname = taosMemoryCalloc(1, len); if(fname == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - snprintf(fname, len + 40, "%s/offset-ver%d", path, fVer); - return fname; + int32_t code = snprintf(fname, len, "%s%s%s", path, TD_DIRSEP, name); + if (code < 0){ + code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(fname); + return code; + } + *data = fname; + return TDB_CODE_SUCCESS; } -int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); +int32_t tqOffsetRestoreFromFile(SHashObj* pOffset, char* name) { + int32_t code = TDB_CODE_SUCCESS; + void* pMemBuf = NULL; + SDecoder decoder = {0}; + + TdFilePtr pFile = taosOpenFile(name, TD_FILE_READ); if (pFile == NULL) { - return TSDB_CODE_SUCCESS; + code = TAOS_SYSTEM_ERROR(errno); + goto END; } - int32_t vgId = TD_VID(pStore->pTq->pVnode); - int64_t code = 0; + int64_t ret = 0; int32_t size = 0; while (1) { - if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { - if (code == 0) { - break; - } else { - return -1; + if ((ret = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { + if (ret != 0) { + code = TSDB_CODE_INVALID_MSG; } + goto END; } size = htonl(size); - void* pMemBuf = taosMemoryCalloc(1, size); + pMemBuf = taosMemoryCalloc(1, size); if (pMemBuf == NULL) { - tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; } - if ((code = taosReadFile(pFile, pMemBuf, size)) != size) { - taosMemoryFree(pMemBuf); - return -1; + if (taosReadFile(pFile, pMemBuf, size) != size) { + terrno = TSDB_CODE_INVALID_MSG; + goto END; } - STqOffset offset = {0}; - SDecoder decoder; + STqOffset offset; tDecoderInit(&decoder, pMemBuf, size); if (tDecodeSTqOffset(&decoder, &offset) < 0) { - taosMemoryFree(pMemBuf); - tDecoderClear(&decoder); - return code; + code = TSDB_CODE_INVALID_MSG; + goto END; + } + + if (taosHashPut(pOffset, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; } tDecoderClear(&decoder); - if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { - return -1; - } - - // todo remove this - if (offset.val.type == TMQ_OFFSET__LOG) { - STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); - if (pHandle) { - if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) { -// tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, pHandle->subKey, -// offset.val.version); - } - } - } - taosMemoryFree(pMemBuf); + pMemBuf = NULL; } +END: taosCloseFile(&pFile); - return TSDB_CODE_SUCCESS; -} - -STqOffsetStore* tqOffsetOpen(STQ* pTq) { - STqOffsetStore* pStore = taosMemoryCalloc(1, sizeof(STqOffsetStore)); - if (pStore == NULL) { - return NULL; - } - - pStore->pTq = pTq; - pStore->needCommit = 0; - pTq->pOffsetStore = pStore; - - pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); - if (pStore->pHash == NULL) { - taosMemoryFree(pStore); - return NULL; - } - - taosHashSetFreeFp(pStore->pHash, tOffsetDestroy); - char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); - if (tqOffsetRestoreFromFile(pStore, fname) < 0) { - taosMemoryFree(fname); - taosMemoryFree(pStore); - return NULL; - } - - taosMemoryFree(fname); - return pStore; -} - -void tqOffsetClose(STqOffsetStore* pStore) { - if(pStore == NULL) return; - tqOffsetCommitFile(pStore); - taosHashCleanup(pStore->pHash); - taosMemoryFree(pStore); -} - -STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) { - return (STqOffset*)taosHashGet(pStore->pHash, subscribeKey, strlen(subscribeKey)); -} - -int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) { - pStore->needCommit = 1; - return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset)); -} - -int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) { - return taosHashRemove(pStore->pHash, subscribeKey, strlen(subscribeKey)); -} - -int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { - if (!pStore->needCommit) { - return 0; - } - - // TODO file name should be with a newer version - char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - const char* err = strerror(errno); - tqError("vgId:%d, failed to open offset file %s, since %s", TD_VID(pStore->pTq->pVnode), fname, err); - taosMemoryFree(fname); - return -1; - } - - taosMemoryFree(fname); - - void* pIter = NULL; - while (1) { - pIter = taosHashIterate(pStore->pHash, pIter); - if (pIter == NULL) { - break; - } - - STqOffset* pOffset = (STqOffset*)pIter; - int32_t bodyLen; - int32_t code; - tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); - - if (code < 0) { - taosHashCancelIterate(pStore->pHash, pIter); - return -1; - } - - int32_t totLen = INT_BYTES + bodyLen; - void* buf = taosMemoryCalloc(1, totLen); - void* abuf = POINTER_SHIFT(buf, INT_BYTES); - - *(int32_t*)buf = htonl(bodyLen); - SEncoder encoder; - tEncoderInit(&encoder, abuf, bodyLen); - tEncodeSTqOffset(&encoder, pOffset); - - // write file - int64_t writeLen; - if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { - tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen); - taosHashCancelIterate(pStore->pHash, pIter); - taosMemoryFree(buf); - return -1; - } - - taosMemoryFree(buf); - } - - // close and rename file - taosCloseFile(&pFile); - pStore->needCommit = 0; - return 0; + taosMemoryFree(pMemBuf); + tDecoderClear(&decoder); + + return code; } diff --git a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c deleted file mode 100644 index 8a7f672e5d..0000000000 --- a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "meta.h" -#include "tdbInt.h" -#include "tq.h" - -// STqOffsetReader ======================================== -struct STqOffsetReader { - STQ* pTq; - int64_t sver; - int64_t ever; - int8_t readEnd; -}; - -int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader) { - STqOffsetReader* pReader = NULL; - - pReader = taosMemoryCalloc(1, sizeof(STqOffsetReader)); - if (pReader == NULL) { - *ppReader = NULL; - return -1; - } - pReader->pTq = pTq; - pReader->sver = sver; - pReader->ever = ever; - - tqInfo("vgId:%d, vnode snapshot tq offset reader opened", TD_VID(pTq->pVnode)); - - *ppReader = pReader; - return 0; -} - -int32_t tqOffsetReaderClose(STqOffsetReader** ppReader) { - taosMemoryFree(*ppReader); - *ppReader = NULL; - return 0; -} - -int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) { - if (pReader->readEnd != 0) return 0; - - char* fname = tqOffsetBuildFName(pReader->pTq->path, 0); - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); - if (pFile == NULL) { - taosMemoryFree(fname); - return 0; - } - - int64_t sz = 0; - if (taosStatFile(fname, &sz, NULL, NULL) < 0) { - taosCloseFile(&pFile); - taosMemoryFree(fname); - return -1; - } - taosMemoryFree(fname); - - SSnapDataHdr* buf = taosMemoryCalloc(1, sz + sizeof(SSnapDataHdr)); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosCloseFile(&pFile); - return terrno; - } - void* abuf = POINTER_SHIFT(buf, sizeof(SSnapDataHdr)); - int64_t contLen = taosReadFile(pFile, abuf, sz); - if (contLen != sz) { - taosCloseFile(&pFile); - taosMemoryFree(buf); - return -1; - } - buf->size = sz; - buf->type = SNAP_DATA_TQ_OFFSET; - *ppData = (uint8_t*)buf; - - pReader->readEnd = 1; - taosCloseFile(&pFile); - return 0; -} - -// STqOffseWriter ======================================== -struct STqOffsetWriter { - STQ* pTq; - int64_t sver; - int64_t ever; - int32_t tmpFileVer; - char* fname; -}; - -int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter** ppWriter) { - int32_t code = 0; - STqOffsetWriter* pWriter; - - pWriter = (STqOffsetWriter*)taosMemoryCalloc(1, sizeof(STqOffsetWriter)); - if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pWriter->pTq = pTq; - pWriter->sver = sver; - pWriter->ever = ever; - - *ppWriter = pWriter; - return code; - -_err: - tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); - *ppWriter = NULL; - return code; -} - -int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback) { - STqOffsetWriter* pWriter = *ppWriter; - STQ* pTq = pWriter->pTq; - char* fname = tqOffsetBuildFName(pTq->path, 0); - - if (rollback) { - if (taosRemoveFile(pWriter->fname) < 0) { - taosMemoryFree(fname); - return -1; - } - } else { - if (taosRenameFile(pWriter->fname, fname) < 0) { - taosMemoryFree(fname); - return -1; - } - if (tqOffsetRestoreFromFile(pTq->pOffsetStore, fname) < 0) { - taosMemoryFree(fname); - return -1; - } - } - taosMemoryFree(fname); - taosMemoryFree(pWriter->fname); - taosMemoryFree(pWriter); - *ppWriter = NULL; - return 0; -} - -int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nData) { - STQ* pTq = pWriter->pTq; - pWriter->tmpFileVer = 1; - pWriter->fname = tqOffsetBuildFName(pTq->path, pWriter->tmpFileVer); - TdFilePtr pFile = taosOpenFile(pWriter->fname, TD_FILE_CREATE | TD_FILE_WRITE); - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - int64_t size = pHdr->size; - if (pFile) { - int64_t contLen = taosWriteFile(pFile, pHdr->data, size); - if (contLen != size) { - taosCloseFile(&pFile); - return -1; - } - taosCloseFile(&pFile); - } else { - return -1; - } - return 0; -} diff --git a/source/dnode/vnode/src/tq/tqHandleSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c similarity index 71% rename from source/dnode/vnode/src/tq/tqHandleSnapshot.c rename to source/dnode/vnode/src/tq/tqSnapshot.c index 28fd315eb6..755274ff5e 100644 --- a/source/dnode/vnode/src/tq/tqHandleSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -23,9 +23,10 @@ struct STqSnapReader { int64_t sver; int64_t ever; TBC* pCur; + int8_t type; }; -int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader) { +int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader) { int32_t code = 0; STqSnapReader* pReader = NULL; @@ -38,9 +39,21 @@ int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** p pReader->pTq = pTq; pReader->sver = sver; pReader->ever = ever; + pReader->type = type; // impl - code = tdbTbcOpen(pTq->pExecStore, &pReader->pCur, NULL); + TTB *pTb = NULL; + if (type == SNAP_DATA_TQ_CHECKINFO) { + pTb = pTq->pCheckStore; + } else if (type == SNAP_DATA_TQ_HANDLE) { + pTb = pTq->pExecStore; + } else if (type == SNAP_DATA_TQ_OFFSET) { + pTb = pTq->pOffsetStore; + } else { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } + code = tdbTbcOpen(pTb, &pReader->pCur, NULL); if (code) { taosMemoryFree(pReader); goto _err; @@ -91,7 +104,7 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { } SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); - pHdr->type = SNAP_DATA_TQ_HANDLE; + pHdr->type = pReader->type; pHdr->size = vLen; memcpy(pHdr->data, pVal, vLen); @@ -169,7 +182,7 @@ _err: return code; } -int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { +int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; STQ* pTq = pWriter->pTq; SDecoder decoder = {0}; @@ -180,7 +193,7 @@ int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { code = tDecodeSTqHandle(pDecoder, &handle); if (code) goto end; taosWLockLatch(&pTq->lock); - code = tqMetaSaveHandle(pTq, handle.subKey, &handle); + code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, (int)strlen(handle.subKey), pData, nData); taosWUnLockLatch(&pTq->lock); end: @@ -189,3 +202,36 @@ end: tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code); return code; } + +int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { + int32_t code = 0; + STQ* pTq = pWriter->pTq; + STqCheckInfo info = {0}; + if(tqMetaDecodeCheckInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)) != 0){ + goto _err; + } + + code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.ntbUid, sizeof(info.ntbUid), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + if (code) goto _err; + + return code; + + _err: + tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); + return code; +} + +int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { + int32_t code = 0; + STQ* pTq = pWriter->pTq; + STqOffset *info = (STqOffset*)(pData + sizeof(SSnapDataHdr)); + + code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info->subKey, strlen(info->subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + if (code) goto _err; + + return code; + + _err: + tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); + return code; +} diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5290c39d42..8c3417239b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -77,7 +77,7 @@ static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) { uint64_t consumerId = pRequest->consumerId; - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); + STqOffset* pOffset = (STqOffset*)tqMetaGetOffset(pTq, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); *pBlockReturned = false; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 5acaf2bce4..12b26a4b26 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -458,11 +458,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); } - if (tqCommit(pVnode->pTq) < 0) { - code = TSDB_CODE_FAILED; - TSDB_CHECK_CODE(code, lino, _exit); - } - // commit info if (vnodeCommitInfo(dir) < 0) { code = terrno; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 611a603c63..a050734cb9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -59,9 +59,9 @@ struct SVSnapReader { int8_t tqHandleDone; STqSnapReader *pTqSnapReader; int8_t tqOffsetDone; - STqOffsetReader *pTqOffsetReader; + STqSnapReader *pTqOffsetReader; int8_t tqCheckInfoDone; - STqCheckInfoReader *pTqCheckInfoReader; + STqSnapReader *pTqCheckInfoReader; // stream int8_t streamTaskDone; SStreamTaskReader *pStreamTaskReader; @@ -233,11 +233,11 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { } if (pReader->pTqOffsetReader) { - tqOffsetReaderClose(&pReader->pTqOffsetReader); + tqSnapReaderClose(&pReader->pTqOffsetReader); } if (pReader->pTqCheckInfoReader) { - tqCheckInfoReaderClose(&pReader->pTqCheckInfoReader); + tqSnapReaderClose(&pReader->pTqCheckInfoReader); } taosMemoryFree(pReader); @@ -365,7 +365,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) vInfo("vgId:%d tq transform start", vgId); if (!pReader->tqHandleDone) { if (pReader->pTqSnapReader == NULL) { - code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqSnapReader); + code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE, &pReader->pTqSnapReader); if (code < 0) goto _err; } @@ -384,11 +384,11 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } if (!pReader->tqCheckInfoDone) { if (pReader->pTqCheckInfoReader == NULL) { - code = tqCheckInfoReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqCheckInfoReader); + code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO, &pReader->pTqCheckInfoReader); if (code < 0) goto _err; } - code = tqCheckInfoRead(pReader->pTqCheckInfoReader, ppData); + code = tqSnapRead(pReader->pTqCheckInfoReader, ppData); if (code) { goto _err; } else { @@ -396,18 +396,18 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) goto _exit; } else { pReader->tqCheckInfoDone = 1; - code = tqCheckInfoReaderClose(&pReader->pTqCheckInfoReader); + code = tqSnapReaderClose(&pReader->pTqCheckInfoReader); if (code) goto _err; } } } if (!pReader->tqOffsetDone) { if (pReader->pTqOffsetReader == NULL) { - code = tqOffsetReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqOffsetReader); + code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET, &pReader->pTqOffsetReader); if (code < 0) goto _err; } - code = tqOffsetSnapRead(pReader->pTqOffsetReader, ppData); + code = tqSnapRead(pReader->pTqOffsetReader, ppData); if (code) { goto _err; } else { @@ -415,7 +415,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) goto _exit; } else { pReader->tqOffsetDone = 1; - code = tqOffsetReaderClose(&pReader->pTqOffsetReader); + code = tqSnapReaderClose(&pReader->pTqOffsetReader); if (code) goto _err; } } @@ -536,9 +536,9 @@ struct SVSnapWriter { // tsdb raw STsdbSnapRAWWriter *pTsdbSnapRAWWriter; // tq - STqSnapWriter *pTqSnapWriter; - STqOffsetWriter *pTqOffsetWriter; - STqCheckInfoWriter *pTqCheckInfoWriter; + STqSnapWriter *pTqSnapHandleWriter; + STqSnapWriter *pTqSnapOffsetWriter; + STqSnapWriter *pTqSnapCheckInfoWriter; // stream SStreamTaskWriter *pStreamTaskWriter; SStreamStateWriter *pStreamStateWriter; @@ -736,18 +736,18 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } - if (pWriter->pTqSnapWriter) { - code = tqSnapWriterClose(&pWriter->pTqSnapWriter, rollback); + if (pWriter->pTqSnapHandleWriter) { + code = tqSnapWriterClose(&pWriter->pTqSnapHandleWriter, rollback); if (code) goto _exit; } - if (pWriter->pTqCheckInfoWriter) { - code = tqCheckInfoWriterClose(&pWriter->pTqCheckInfoWriter, rollback); + if (pWriter->pTqSnapCheckInfoWriter) { + code = tqSnapWriterClose(&pWriter->pTqSnapCheckInfoWriter, rollback); if (code) goto _exit; } - if (pWriter->pTqOffsetWriter) { - code = tqOffsetWriterClose(&pWriter->pTqOffsetWriter, rollback); + if (pWriter->pTqSnapOffsetWriter) { + code = tqSnapWriterClose(&pWriter->pTqSnapOffsetWriter, rollback); if (code) goto _exit; } @@ -872,32 +872,32 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { } break; case SNAP_DATA_TQ_HANDLE: { // tq handle - if (pWriter->pTqSnapWriter == NULL) { - code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapWriter); + if (pWriter->pTqSnapHandleWriter == NULL) { + code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapHandleWriter); if (code) goto _err; } - code = tqSnapWrite(pWriter->pTqSnapWriter, pData, nData); + code = tqSnapHandleWrite(pWriter->pTqSnapHandleWriter, pData, nData); if (code) goto _err; } break; case SNAP_DATA_TQ_CHECKINFO: { // tq checkinfo - if (pWriter->pTqCheckInfoWriter == NULL) { - code = tqCheckInfoWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqCheckInfoWriter); + if (pWriter->pTqSnapCheckInfoWriter == NULL) { + code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapCheckInfoWriter); if (code) goto _err; } - code = tqCheckInfoWrite(pWriter->pTqCheckInfoWriter, pData, nData); + code = tqSnapCheckInfoWrite(pWriter->pTqSnapCheckInfoWriter, pData, nData); if (code) goto _err; } break; case SNAP_DATA_TQ_OFFSET: { // tq offset - if (pWriter->pTqOffsetWriter == NULL) { - code = tqOffsetWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqOffsetWriter); + if (pWriter->pTqSnapOffsetWriter == NULL) { + code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapOffsetWriter); if (code) goto _err; } - code = tqOffsetSnapWrite(pWriter->pTqOffsetWriter, pData, nData); + code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData); if (code) goto _err; } break; case SNAP_DATA_STREAM_TASK: diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index b7169dec53..7f47517257 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -52,15 +52,14 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { taosThreadMutexLock(&pWal->mutex); if (ver < pWal->vers.firstVer || ver > pWal->vers.lastVer) { taosThreadMutexUnlock(&pWal->mutex); - terrno = TSDB_CODE_WAL_INVALID_VER; - return -1; + return TSDB_CODE_WAL_INVALID_VER; } pRef->refVer = ver; taosThreadMutexUnlock(&pWal->mutex); } - return 0; + return TSDB_CODE_SUCCESS; } void walRefFirstVer(SWal *pWal, SWalRef *pRef) { From 1e1612a5fa0dba18b6ead946e14105e0c5f0cc06 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 15 Jul 2024 10:19:03 +0800 Subject: [PATCH 02/45] opti:[TS-4593] transform offset from file to tdb in tmq --- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqMeta.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index bb64a0ccf6..ca1527d5e7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -655,7 +655,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg taosRLockLatch(&pTq->lock); STqHandle* pHandle = NULL; - ret = tqMetaGetHandle(pTq, req.subKey, &pHandle); + (void)tqMetaGetHandle(pTq, req.subKey, &pHandle); //ignore return code taosRUnLockLatch(&pTq->lock); if (pHandle == NULL) { if (req.oldConsumerId != -1) { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 4b30e79d44..3ddd7d953f 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -391,7 +391,7 @@ int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { int vLen = 0; if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &data, &vLen) < 0) { tdbFree(data); - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; } STqHandle handle = {0}; if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0){ @@ -404,7 +404,7 @@ int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { tqDestroyTqHandle(&handle); return TSDB_CODE_OUT_OF_MEMORY; } - *pHandle = taosHashGet(pTq->pCheckInfo, key, strlen(key)); + *pHandle = taosHashGet(pTq->pHandle, key, strlen(key)); if(*pHandle == NULL){ return TSDB_CODE_OUT_OF_MEMORY; } From 0704dcf45d5d4efbd94856386f510c96e257fc57 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Jul 2024 18:56:20 +0800 Subject: [PATCH 03/45] fix:[TS-4593] check info error --- source/dnode/mnode/impl/src/mndTopic.c | 6 +- source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tq.c | 59 +++++++--------- source/dnode/vnode/src/tq/tqMeta.c | 93 +++++++++----------------- source/dnode/vnode/src/tq/tqSnapshot.c | 2 +- 5 files changed, 60 insertions(+), 101 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b8cb1d00d0..bcb38a3902 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -725,19 +725,19 @@ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicO continue; } - buf = taosMemoryCalloc(1, sizeof(SMsgHead) + sizeof(pTopic->ntbUid)); + buf = taosMemoryCalloc(1, sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN); if (buf == NULL){ code = TSDB_CODE_OUT_OF_MEMORY; goto end; } void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); ((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId); - *(int64_t*)abuf = pTopic->ntbUid; + memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN); STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = buf; - action.contLen = sizeof(SMsgHead) + sizeof(pTopic->ntbUid); + action.contLen = sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN; action.msgType = TDMT_VND_TMQ_DEL_CHECKINFO; code = mndTransAppendRedoAction(pTrans, &action); if (code != 0) { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index d97a953d94..f946dc76f8 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -132,7 +132,6 @@ int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen); int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen); int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle); -void* tqMetaGetCheckInfo(STQ* pTq, int64_t tbUid); void* tqMetaGetOffset(STQ* pTq, const char* subkey); int32_t tqMetaTransform(STQ* pTq); // tqSink diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ca1527d5e7..6611aa4caa 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -76,7 +76,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { taosInitRWLatch(&pTq->lock); pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - pTq->pCheckInfo = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + pTq->pCheckInfo = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); pTq->pOffset = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_ENTRY_LOCK); @@ -282,16 +282,25 @@ end: } int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { - STqCheckInfo* pCheck = tqMetaGetCheckInfo(pTq, tbUid); - if(pCheck == NULL) { - return 0; - } + void* pIter = NULL; - int32_t sz = taosArrayGetSize(pCheck->colIdList); - for (int32_t i = 0; i < sz; i++) { - int16_t forbidColId = *(int16_t*)taosArrayGet(pCheck->colIdList, i); - if (forbidColId == colId) { - return -1; + while (1) { + pIter = taosHashIterate(pTq->pCheckInfo, pIter); + if (pIter == NULL) { + break; + } + + STqCheckInfo* pCheck = (STqCheckInfo*)pIter; + + if (pCheck->ntbUid == tbUid) { + int32_t sz = taosArrayGetSize(pCheck->colIdList); + for (int32_t i = 0; i < sz; i++) { + int16_t forbidColId = *(int16_t*)taosArrayGet(pCheck->colIdList, i); + if (forbidColId == colId) { + taosHashCancelIterate(pTq->pCheckInfo, pIter); + return -1; + } + } } } @@ -585,39 +594,18 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg return 0; } -void mergeTwoArray(SArray* a, SArray* b){ - size_t bLen = taosArrayGetSize(b); - for(int i = 0; i < bLen; i++){ - void* dataB = taosArrayGet(b, i); - size_t aLen = taosArrayGetSize(a); - int j = 0; - for(; j < aLen; j++){ - void* dataA = taosArrayGet(a, i); - if(*(int64_t*)dataA == *(int64_t*)dataB){ - break; - } - } - if(j == aLen){ - taosArrayPush(a, dataB); - } - } -} - int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { STqCheckInfo info = {0}; if(tqMetaDecodeCheckInfo(&info, msg, msgLen) != 0){ return -1; } - STqCheckInfo *old = tqMetaGetCheckInfo(pTq, info.ntbUid); - if (old != NULL){ - mergeTwoArray(info.colIdList, old->colIdList); - } - if (taosHashPut(pTq->pCheckInfo, &info.ntbUid, sizeof(info.ntbUid), &info, sizeof(STqCheckInfo)) < 0) { + + if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSTqCheckInfo(&info); return -1; } - if (tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.ntbUid, sizeof(info.ntbUid), msg, msgLen) < 0) { + if (tqMetaSaveInfo(pTq, pTq->pCheckStore, info.topic, strlen(info.topic), msg, msgLen) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -629,7 +617,8 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (tqMetaDeleteInfo(pTq, pTq->pCheckStore, msg, msgLen) < 0) { + if (tqMetaDeleteInfo(pTq, pTq->pCheckStore, msg, strlen(msg)) < 0) { + tqError("cannot process tq delete check info req %s, since no such check info", msg); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 3ddd7d953f..622ba786ab 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -87,31 +87,6 @@ int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){ return code; } -void* tqMetaGetCheckInfo(STQ* pTq, int64_t tbUid){ - void* data = taosHashGet(pTq->pCheckInfo, &tbUid, sizeof(tbUid)); - if (data == NULL) { - int vLen = 0; - if (tdbTbGet(pTq->pCheckStore, &tbUid, sizeof(tbUid), &data, &vLen) < 0) { - tdbFree(data); - return NULL; - } - STqCheckInfo info= {0}; - if(tqMetaDecodeCheckInfo(&info, data, vLen) != 0) { - tdbFree(data); - return NULL; - } - tdbFree(data); - - if(taosHashPut(pTq->pCheckInfo, &tbUid, sizeof(tbUid), &info, sizeof(STqCheckInfo)) != 0){ - tDeleteSTqCheckInfo(&info); - return NULL; - } - return taosHashGet(pTq->pCheckInfo, &tbUid, sizeof(tbUid)); - } else { - return data; - } -} - int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) { int32_t code = TDB_CODE_SUCCESS; TXN* txn = NULL; @@ -326,7 +301,7 @@ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); } -static int32_t tqMetaTransformStoreInfo(TDB* pMetaDB, TTB* pExecStoreOld, TTB* pExecStoreNew){ +static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew){ TBC* pCur = NULL; void* pKey = NULL; int kLen = 0; @@ -336,12 +311,12 @@ static int32_t tqMetaTransformStoreInfo(TDB* pMetaDB, TTB* pExecStoreOld, TTB* p int32_t code = TDB_CODE_SUCCESS; - TQ_ERR_GO_TO_END(tdbTbcOpen(pExecStoreOld, &pCur, NULL)); + TQ_ERR_GO_TO_END(tdbTbcOpen(pOld, &pCur, NULL)); TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur)); while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { - TQ_ERR_GO_TO_END (tdbTbUpsert(pExecStoreNew, pKey, kLen, pVal, vLen, txn)); + TQ_ERR_GO_TO_END (tdbTbUpsert(pNew, pKey, kLen, pVal, vLen, txn)); } TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn)); @@ -354,37 +329,6 @@ END: return code; } -static int32_t tqMetaTransformCheckInfo(TDB* pMetaDB, TTB* pCheckOld, TTB* pCheckNew){ - TBC* pCur = NULL; - void* pKey = NULL; - int kLen = 0; - void* pVal = NULL; - int vLen = 0; - TXN* txn = NULL; - - int32_t code = TDB_CODE_SUCCESS; - - TQ_ERR_GO_TO_END(tdbTbcOpen(pCheckOld, &pCur, NULL)); - TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); - - TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur)); - while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { - STqCheckInfo info= {0}; - TQ_ERR_GO_TO_END(tqMetaDecodeCheckInfo(&info, pVal, vLen)); - int64_t uid = info.ntbUid; - tDeleteSTqCheckInfo(&info); - TQ_ERR_GO_TO_END (tdbTbUpsert(pCheckNew, &uid, sizeof(uid), pVal, vLen, txn)); - } - - TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn)); - TQ_ERR_GO_TO_END (tdbPostCommit(pMetaDB, txn)); -END: - tdbFree(pKey); - tdbFree(pVal); - (void)tdbTbcClose(pCur); - return code; -} - int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { void* data = taosHashGet(pTq->pHandle, key, strlen(key)); if(data == NULL){ @@ -434,6 +378,32 @@ static int32_t replaceTqPath(char** path){ return TDB_CODE_SUCCESS; } +static int32_t tqMetaRestoreCheckInfo(STQ* pTq) { + TBC* pCur = NULL; + void* pKey = NULL; + int kLen = 0; + void* pVal = NULL; + int vLen = 0; + int32_t code = 0; + STqCheckInfo info = {0}; + + TQ_ERR_GO_TO_END(tdbTbcOpen(pTq->pCheckStore, &pCur, NULL)); + TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur)); + + while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { + TQ_ERR_GO_TO_END(tqMetaDecodeCheckInfo(&info, pVal, vLen)); + TQ_ERR_GO_TO_END(taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo))); + } + info.colIdList = NULL; + +END: + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + tDeleteSTqCheckInfo(&info); + return code; +} + int32_t tqMetaOpen(STQ* pTq) { char* maindb = NULL; int32_t code = TDB_CODE_SUCCESS; @@ -444,6 +414,7 @@ int32_t tqMetaOpen(STQ* pTq) { }else{ TQ_ERR_GO_TO_END(tqMetaTransform(pTq)); } + TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq)); END: taosMemoryFree(maindb); @@ -466,8 +437,8 @@ int32_t tqMetaTransform(STQ* pTq) { TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path)); TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq)); - TQ_ERR_GO_TO_END(tqMetaTransformStoreInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore)); - TQ_ERR_GO_TO_END(tqMetaTransformCheckInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore)); + TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore)); + TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore)); TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME)); if(taosCheckExistFile(offset) && taosCopyFile(offset, offsetNew) < 0){ diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index 755274ff5e..6cb5734aa2 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -211,7 +211,7 @@ int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nD goto _err; } - code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.ntbUid, sizeof(info.ntbUid), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.topic, strlen(info.topic), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); if (code) goto _err; return code; From 1ed620e702fc70fe06cdeefcb12e37f382b3acc1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 11:17:27 +0800 Subject: [PATCH 04/45] fix(stream): return after get value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 58 +++++++---- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 112 ++++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 1 - 3 files changed, 124 insertions(+), 47 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index cf2d23cfc8..dcca320f7f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -48,7 +48,7 @@ typedef struct { static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo); static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); -static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); +static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRow); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader); static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, @@ -2025,9 +2025,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SArray* pDelList = pBlockScanInfo->delSkyline; int32_t pkSrcSlot = pReader->suppInfo.pkSrcSlot; + TSDBROW* pRow = NULL; + TSDBROW* piRow = NULL; - TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); - TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); + getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader, &pRow); + getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader, &piRow); SRowKey* pSttKey = NULL; if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { @@ -2201,9 +2203,10 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { SRowKey rowKey = {0}; + TSDBROW* pRow = NULL; while (1) { - TSDBROW* pRow = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader); + getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader, &pRow); if (!pIter->hasVal) { break; } @@ -2559,11 +2562,11 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } if (pBlockScanInfo->iter.hasVal) { - pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow); } if (pBlockScanInfo->iiter.hasVal) { - piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow); } // two levels of mem-table does contain the valid rows @@ -2810,13 +2813,16 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL}; bool hasKey = false, hasIKey = false; - TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); + TSDBROW* pRow = NULL; + TSDBROW* pIRow = NULL; + + getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader, &pRow); if (pRow != NULL) { hasKey = true; key = TSDBROW_KEY(pRow); } - TSDBROW* pIRow = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader); + getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader, &pIRow); if (pIRow != NULL) { hasIKey = true; ikey = TSDBROW_KEY(pIRow); @@ -3742,9 +3748,11 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t return false; } -FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) { +FORCE_INLINE int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes) { + *pRes = NULL; + if (!pIter->hasVal) { - return NULL; + return TSDB_CODE_SUCCESS; } int32_t order = pReader->info.order; @@ -3754,18 +3762,20 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S TSDBROW_INIT_KEY(pRow, key); if (outOfTimeWindow(key.ts, &pReader->info.window)) { pIter->hasVal = false; - return NULL; + return TSDB_CODE_SUCCESS; } // it is a valid data version if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { - return pRow; + *pRes = pRow; + return TSDB_CODE_SUCCESS; } else { bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, pReader->suppInfo.numOfPks > 0); if (!dropped) { - return pRow; + *pRes = pRow; + return TSDB_CODE_SUCCESS; } } } @@ -3773,7 +3783,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S while (1) { pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { - return NULL; + return TSDB_CODE_SUCCESS; } pRow = tsdbTbDataIterGet(pIter->iter); @@ -3781,17 +3791,19 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S TSDBROW_INIT_KEY(pRow, key); if (outOfTimeWindow(key.ts, &pReader->info.window)) { pIter->hasVal = false; - return NULL; + return TSDB_CODE_SUCCESS; } if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { - return pRow; + *pRes = pRow; + return TSDB_CODE_SUCCESS; } else { bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, pReader->suppInfo.numOfPks > 0); if (!dropped) { - return pRow; + *pRes = pRow; + return TSDB_CODE_SUCCESS; } } } @@ -3809,7 +3821,8 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArra } // data exists but not valid - TSDBROW* pRow = getValidMemRow(pIter, pDelList, pReader); + TSDBROW* pRow = NULL; + getValidMemRow(pIter, pDelList, pReader, &pRow); if (pRow == NULL) { break; } @@ -3974,7 +3987,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt *freeTSRow = false; return TSDB_CODE_SUCCESS; } else { // has next point in mem/imem - pNextRow = getValidMemRow(pIter, pDelList, pReader); + getValidMemRow(pIter, pDelList, pReader, &pNextRow); if (pNextRow == NULL) { *pResRow = current; *freeTSRow = false; @@ -4118,8 +4131,11 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRow static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, int64_t endKey, bool* freeTSRow) { - TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); - TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + TSDBROW* pRow = NULL; + TSDBROW* piRow = NULL; + + getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow); + getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow); SArray* pDelList = pBlockScanInfo->delSkyline; uint64_t uid = pBlockScanInfo->uid; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 84f235ab25..f0918ceb22 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -37,7 +37,10 @@ static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pBuf->pData, &p); + void* px = taosArrayPush(pBuf->pData, &p); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } if (remainder > 0) { @@ -45,7 +48,10 @@ static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pBuf->pData, &p); + void* px = taosArrayPush(pBuf->pData, &p); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } pBuf->numOfTables = numOfTables; @@ -86,7 +92,10 @@ int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pBuf->pData, &p); + void* px = taosArrayPush(pBuf->pData, &p); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } if (remainder > 0) { @@ -94,7 +103,10 @@ int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pBuf->pData, &p); + void* px = taosArrayPush(pBuf->pData, &p); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } pBuf->numOfTables = numOfTables; @@ -214,7 +226,8 @@ void clearRowKey(SRowKey* pKey) { taosMemoryFreeClear(pKey->pks[0].pData); } -static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) { +static int32_t initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) { + int32_t code = 0; int32_t numOfPks = pReader->suppInfo.numOfPks; bool asc = ASCENDING_TRAVERSE(pReader->info.order); int8_t type = pReader->suppInfo.pk.type; @@ -225,18 +238,37 @@ static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader int64_t skey = pReader->info.window.skey; int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey; - initRowKey(pRowKey, ts, numOfPks, type, bytes, asc); - initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc); + code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } else { int64_t ekey = pReader->info.window.ekey; int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey; - initRowKey(pRowKey, ts, numOfPks, type, bytes, asc); - initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc); + code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc); - initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc); + code = initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc); + return code; } int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, @@ -248,10 +280,13 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH pScanInfo->cleanSttBlocks = false; pScanInfo->sttBlockReturned = false; - initLastProcKey(pScanInfo, pReader); + int32_t code = initLastProcKey(pScanInfo, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; - int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -607,7 +642,10 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, 0); for (int32_t i = 0; i < numOfBlocks; ++i) { STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i}; - taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); + void* px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); @@ -640,11 +678,18 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 int32_t index = sup.indexPerTable[pos]++; SFileDataBlockInfo* pBlockInfo = taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); - taosArrayPush(pBlockIter->blockList, pBlockInfo); + void* px = taosArrayPush(pBlockIter->blockList, pBlockInfo); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; STableDataBlockIdx tableDataBlockIdx = {.globalIndex = numOfTotal}; - taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); + + px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } // set data block index overflow, in order to disable the offset comparator if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) { @@ -752,7 +797,10 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_ if (record.version <= pReader->info.verRange.maxVer) { SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; - taosArrayPush(pScanInfo->pFileDelData, &delData); + void* px = taosArrayPush(pScanInfo->pFileDelData, &delData); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } } @@ -858,7 +906,10 @@ int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piM p = pMemTbData->pHead; while (p) { if (p->version <= ver) { - taosArrayPush(pMemDelData, p); + void* px = taosArrayPush(pMemDelData, p); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } p = p->pNext; @@ -870,7 +921,10 @@ int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piM p = piMemTbData->pHead; while (p) { if (p->version <= ver) { - taosArrayPush(pMemDelData, p); + void* px = taosArrayPush(pMemDelData, p); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } p = p->pNext; } @@ -914,7 +968,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo } if (index >= pStatisBlock->numOfRecords) { - tStatisBlockDestroy(pStatisBlock); + (void) tStatisBlockDestroy(pStatisBlock); taosMemoryFreeClear(pStatisBlock); return num; } @@ -924,7 +978,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) { p = &pStatisBlkArray->data[i]; if (p->minTbid.suid > suid) { - tStatisBlockDestroy(pStatisBlock); + (void) tStatisBlockDestroy(pStatisBlock); taosMemoryFreeClear(pStatisBlock); return num; } @@ -944,7 +998,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo } } - tStatisBlockDestroy(pStatisBlock); + (void) tStatisBlockDestroy(pStatisBlock); taosMemoryFreeClear(pStatisBlock); return num; } @@ -961,14 +1015,17 @@ static void loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlo } } -void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { +int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { int32_t size = taosArrayGetSize(pLDIterList); if (size < numOfFileObj) { int32_t inc = numOfFileObj - size; for (int32_t k = 0; k < inc; ++k) { SLDataIter* pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); - taosArrayPush(pLDIterList, &pIter); + void* px = taosArrayPush(pLDIterList, &pIter); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } } else if (size > numOfFileObj) { // remove unused LDataIter int32_t inc = size - numOfFileObj; @@ -978,6 +1035,8 @@ void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { destroyLDataIter(pIter); } } + + return TSDB_CODE_SUCCESS; } int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) { @@ -986,7 +1045,10 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) // add the list/iter placeholder while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) { SArray* pList = taosArrayInit(4, POINTER_BYTES); - taosArrayPush(pSttFileBlockIterArray, &pList); + void* px = taosArrayPush(pSttFileBlockIterArray, &pList); + if (px == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } for (int32_t j = 0; j < numOfLevels; ++j) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 45a2384b65..865e8e2d41 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -247,7 +247,6 @@ typedef struct SDataBlockIter { typedef struct SFileBlockDumpInfo { int32_t totalRows; int32_t rowIndex; -// STsdbRowKey lastKey; // this key should be removed bool allDumped; } SFileBlockDumpInfo; From a61d5fa82760d6c72c543912e6acdcd3f14fb63d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Jul 2024 11:26:22 +0800 Subject: [PATCH 05/45] opti:[TS-4593] transform offset from file to tdb in tmq --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6611aa4caa..0d167ccfa6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -579,7 +579,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } taosWLockLatch(&pTq->lock); - if (taosHashRemove(pTq->pOffset, pReq->subKey, strlen(pReq->subKey) != 0)) { + if (taosHashRemove(pTq->pOffset, pReq->subKey, strlen(pReq->subKey)) != 0) { tqError("cannot process tq delete req %s, since no such offset in hash", pReq->subKey); } if (tqMetaDeleteInfo(pTq, pTq->pOffsetStore, pReq->subKey, strlen(pReq->subKey)) != 0) { From 14f149cdd3385a157409e8eb2a67cc56415c5598 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 11:47:05 +0800 Subject: [PATCH 06/45] refactor: check return value for each function. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 45 ++++++++++++++++------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index dcca320f7f..9ee5e1c7ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -708,7 +708,7 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S while (1) { int32_t code = getNextBrinRecord(&iter, &pRecord); if (code != TSDB_CODE_SUCCESS) { - break; + return code; } if (pRecord->suid > pReader->info.suid) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index f0918ceb22..f937dac606 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -27,8 +27,12 @@ static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { int32_t num = numOfTables / pBuf->numPerBucket; int32_t remainder = numOfTables % pBuf->numPerBucket; + if (pBuf->pData == NULL) { pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); + if (pBuf->pData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } for (int32_t i = 0; i < num; ++i) { @@ -310,7 +314,10 @@ int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, c } int64_t st = taosGetTimestampUs(); - initBlockScanInfoBuf(pBuf, numOfTables); + code = initBlockScanInfoBuf(pBuf, numOfTables); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { @@ -459,7 +466,7 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); - tBrinBlockClear(&pIter->block); + (void) tBrinBlockClear(&pIter->block); int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); @@ -470,13 +477,13 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { } pIter->recordIndex += 1; - tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); + int32_t code = tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); *pRecord = &pIter->record; - return TSDB_CODE_SUCCESS; + return code; } -void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } +void clearBrinBlockIter(SBrinRecordIter* pIter) { (void) tBrinBlockDestroy(&pIter->block); } // initialize the file block access order // sort the file blocks according to the offset of each data block in the files @@ -648,7 +655,11 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 } } - taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); + void* p = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); + if (p != NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArrayDestroy(pTableScanInfo->pBlockList); pTableScanInfo->pBlockList = NULL; @@ -854,7 +865,7 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs ETombBlkCheckEnum ret = 0; code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret); - tTombBlockDestroy(&block); + (void) tTombBlockDestroy(&block); if (code != TSDB_CODE_SUCCESS || ret == BLK_CHECK_QUIT) { return code; } @@ -953,10 +964,13 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo SStatisBlk* p = &pStatisBlkArray->data[i]; STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock)); - tStatisBlockInit(pStatisBlock); + (void) tStatisBlockInit(pStatisBlock); int64_t st = taosGetTimestampMs(); - tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock); + int32_t code = tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock); + if (code != TSDB_CODE_SUCCESS) { + return 0; + } double el = (taosGetTimestampMs() - st) / 1000.0; pBlockLoadInfo->cost.loadStatisBlocks += 1; @@ -1010,7 +1024,7 @@ static void loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlo (*i) += 1; (*j) = 0; if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) { - tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock); + (void) tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock); } } } @@ -1041,6 +1055,7 @@ int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) { int32_t numOfLevels = pFileSet->lvlArr->size; + int32_t code = 0; // add the list/iter placeholder while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) { @@ -1054,7 +1069,10 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) for (int32_t j = 0; j < numOfLevels; ++j) { SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); - doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr)); + code = doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TSDB_CODE_SUCCESS; @@ -1072,7 +1090,10 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra } // add the list/iter placeholder - adjustSttDataIters(pSttFileBlockIterArray, pFileSet); + code = adjustSttDataIters(pSttFileBlockIterArray, pFileSet); + if (code != TSDB_CODE_SUCCESS) { + return numOfRows; + } for (int32_t j = 0; j < numOfLevels; ++j) { SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; From 552e59f42b8255f257779dc8fef95fb2448cae28 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 15:40:00 +0800 Subject: [PATCH 07/45] refactor: check return value for each function. --- include/libs/stream/tstream.h | 98 +++++----- source/dnode/mnode/impl/src/mndScheduler.c | 63 +++---- source/dnode/snode/src/snode.c | 6 +- source/dnode/vnode/src/sma/smaRollup.c | 13 +- source/dnode/vnode/src/tq/tq.c | 21 ++- source/dnode/vnode/src/tq/tqRead.c | 6 +- source/dnode/vnode/src/tq/tqStreamTask.c | 5 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 62 ++++--- source/libs/stream/src/streamCheckStatus.c | 10 +- source/libs/stream/src/streamCheckpoint.c | 192 +++++++++----------- source/libs/stream/src/streamData.c | 9 +- source/libs/stream/src/streamDispatch.c | 18 +- source/libs/stream/src/streamExec.c | 9 +- source/libs/stream/src/streamMeta.c | 103 ++++++----- source/libs/stream/src/streamStartHistory.c | 15 +- source/libs/stream/src/streamTask.c | 80 +++++--- source/libs/stream/src/streamTaskSm.c | 18 +- 17 files changed, 403 insertions(+), 325 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5ba0ce454c..8f6b512741 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -59,8 +59,6 @@ extern "C" { #define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_RESUME_TASK (-6) #define STREAM_EXEC_T_ADD_FAILED_TASK (-7) -// the load and start stream task should be executed after snode has started successfully, since the load of stream -// tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources. typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; @@ -190,8 +188,8 @@ typedef struct { SSDataBlock* pBlock; } SStreamRefDataBlock; -SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); -void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); +int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit** pSubmit); +void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); typedef struct { char* qmsg; @@ -528,13 +526,13 @@ typedef struct STaskUpdateEntry { typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param); -SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, - SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5); -void tFreeStreamTask(SStreamTask* pTask); -int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); -int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); -int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver); -void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status); +int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, + SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5, SStreamTask** pTask); +void tFreeStreamTask(SStreamTask* pTask); +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); +int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver); +void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status); int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo); int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId); @@ -629,8 +627,8 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask); int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); -SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); -SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); +int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo); +SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); void streamTaskInputFail(SStreamTask* pTask); @@ -722,36 +720,36 @@ int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* p SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st); // stream task meta -void streamMetaInit(); -void streamMetaCleanup(); -SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn, - int32_t vgId, int64_t stage, startComplete_fn_t fn); -void streamMetaClose(SStreamMeta* streamMeta); -int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store -int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); -int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); -int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); -int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); -SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); -SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); -void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); -SStreamTask* streamMetaAcquireOneTask(SStreamTask* pTask); -void streamMetaClear(SStreamMeta* pMeta); -void streamMetaInitBackend(SStreamMeta* pMeta); -int32_t streamMetaCommit(SStreamMeta* pMeta); -int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); -void streamMetaNotifyClose(SStreamMeta* pMeta); -void streamMetaStartHb(SStreamMeta* pMeta); -bool streamMetaTaskInTimer(SStreamMeta* pMeta); -int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, - int64_t endTs, bool ready); -int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta); -int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); -void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs); -void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, - int64_t startTs); -void streamMetaClearUpdateTaskList(SStreamMeta* pMeta); -void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId); +void streamMetaInit(); +void streamMetaCleanup(); +int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn, int32_t vgId, + int64_t stage, startComplete_fn_t fn, SStreamMeta** pMeta); +void streamMetaClose(SStreamMeta* streamMeta); +int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store +int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); +int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); +int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); +int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); +int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask); +int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask); +void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); +void streamMetaAcquireOneTask(SStreamTask* pTask); +void streamMetaClear(SStreamMeta* pMeta); +void streamMetaInitBackend(SStreamMeta* pMeta); +int32_t streamMetaCommit(SStreamMeta* pMeta); +int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); +void streamMetaNotifyClose(SStreamMeta* pMeta); +void streamMetaStartHb(SStreamMeta* pMeta); +bool streamMetaTaskInTimer(SStreamMeta* pMeta); +int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, + int64_t endTs, bool ready); +int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta); +int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); +void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs); +void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, + int64_t startTs); +void streamMetaClearUpdateTaskList(SStreamMeta* pMeta); +void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId); void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta); @@ -785,14 +783,14 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp int32_t setCode); int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask); int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq); -SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo(); +int32_t streamTaskCreateActiveChkptInfo(SActiveCheckpointInfo** pRes); // stream task state machine, and event handling -SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); -void* streamDestroyStateMachine(SStreamTaskSM* pSM); -int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); -int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, - void* param); +int32_t streamCreateStateMachine(SStreamTask* pTask); +void streamDestroyStateMachine(SStreamTaskSM* pSM); +int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); +int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, + void* param); int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param); int32_t streamTaskRestoreStatus(SStreamTask* pTask); @@ -803,7 +801,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp); int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp); -int32_t streamTaskSendPreparedCheckpointsourceRsp(SStreamTask* pTask); +int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index fc2e3b4619..db8c08e2e3 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -242,11 +242,11 @@ static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgrou int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); - SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, - pStream->conf.fillHistory, pStream->subTableWithoutMd5); - if (pTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return terrno; + SStreamTask* pTask = NULL; + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, pStream->conf.fillHistory, + pStream->subTableWithoutMd5, &pTask); + if (code != 0) { + return code; } mDebug("doAddSinkTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); @@ -353,18 +353,13 @@ static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFi } } -static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) { +static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam, SStreamTask** pTask) { uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); - SStreamTask* pTask = - tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0, - *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5); - if (pTask == NULL) { - return NULL; - } - - return pTask; + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0, + *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5, pTask); + return code; } static void addNewTaskList(SStreamObj* pStream) { @@ -398,10 +393,10 @@ static void setHTasksId(SStreamObj* pStream) { static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey, SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) { - SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam); - if (pTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return terrno; + SStreamTask* pTask = NULL; + int32_t code = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask); + if (code != TSDB_CODE_SUCCESS) { + return code; } mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); @@ -412,10 +407,9 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId); - int32_t code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup); - if (code != 0) { - terrno = code; - return terrno; + code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup); + if (code != TSDB_CODE_SUCCESS) { + return code; } return TDB_CODE_SUCCESS; @@ -498,28 +492,29 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream return TSDB_CODE_SUCCESS; } -static SStreamTask* buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) { +static int32_t buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam, + SStreamTask** pAggTask) { + *pAggTask = NULL; + uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); - SStreamTask* pAggTask = + int32_t code = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0, - *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5); - if (pAggTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - return pAggTask; + *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5, pAggTask); + return code; } static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup, SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) { int32_t code = 0; - SStreamTask* pTask = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam); - if (pTask == NULL) { - return terrno; + SStreamTask* pTask = NULL; + + code = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask); + if (code != TSDB_CODE_SUCCESS) { + return code; } + if (pSnode != NULL) { code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode); mDebug("doAddAggTask taskId:%s, snode id:%d, isFillHistory:%d", pTask->id.idStr, pSnode->id, isFillhistory); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index cfa24b2430..db2ead2f37 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -71,9 +71,9 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { startRsync(); pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); - if (pSnode->pMeta == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE, + taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index a48188430f..a6002d2d65 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -297,8 +297,17 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG); pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); - pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask); - pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); + + int32_t code = streamCreateStateMachine(pStreamTask); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = streamTaskCreateActiveChkptInfo(&pStreamTask->chkInfo.pActiveInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ac57a003c5..1c3866080b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -90,10 +90,11 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { int32_t tqInitialize(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); - pTq->pStreamMeta = - streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback); - if (pTq->pStreamMeta == NULL) { - return -1; + + int32_t code = streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, + tqStartTaskCompleteCallback, &pTq->pStreamMeta); + if (code != TSDB_CODE_SUCCESS) { + return code; } streamMetaLoadAllTasks(pTq->pStreamMeta); @@ -856,7 +857,8 @@ int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) { SStreamMeta* pMeta = pStreamTask->pMeta; STaskId hId = pStreamTask->hTaskInfo.id; - SStreamTask* pTask = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId, &pTask); if (pTask == NULL) { tqWarn("s-task:0x%x failed to acquired it to exec step 2, scan wal quit", (int32_t) hId.taskId); return TSDB_CODE_SUCCESS; @@ -874,7 +876,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t code = TSDB_CODE_SUCCESS; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask == NULL) { tqError("vgId:%d failed to acquire stream task:0x%x during scan history data, task may have been destroyed", pMeta->vgId, pReq->taskId); @@ -961,7 +964,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { ASSERT(pTask->info.fillHistory == 1); // 1. get the related stream task - SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + SStreamTask* pStreamTask = NULL; + code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask); if (pStreamTask == NULL) { tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", pTask->streamTaskId.taskId, pTask->id.idStr); @@ -1121,7 +1125,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) return TSDB_CODE_SUCCESS; } - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask == NULL) { tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64 " transId:%d it may have been destroyed", diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index d3be8fa666..b57af518b4 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -341,10 +341,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con memcpy(data, pBody, len); SPackedData data1 = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; - *pItem = (SStreamQueueItem*)streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT); - if (*pItem == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - terrno = code; + code = streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT, (SStreamDataSubmit**)pItem); + if (code != 0) { tqError("%s failed to create data submit for stream since out of memory", id); return code; } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index bcf17bf1e1..22658cc650 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -324,7 +324,8 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { for (int32_t i = 0; i < numOfTasks; ++i) { STaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId, &pTask); if (pTask == NULL) { continue; } @@ -337,7 +338,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = false; // seek the stored version and extract data from WAL - int32_t code = setWalReaderStartOffset(pTask, vgId); + code = setWalReaderStartOffset(pTask, vgId); if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pStreamMeta, pTask); continue; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 1f3c049211..95ee1b70f8 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -140,17 +140,19 @@ int32_t tqStreamTaskRestoreCheckpoint(SStreamMeta* pMeta, int64_t streamId, int3 } tqDebug("vgId:%d restore task:0x%" PRIx64 "-0x%x checkpointId", vgId, streamId, taskId); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, streamId, taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask); if (pTask == NULL) { tqError("failed to acquire task:0x%x when trying to restore checkpointId", taskId); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } - int32_t code = streamTaskSendRestoreChkptMsg(pTask); + code = streamTaskSendRestoreChkptMsg(pTask); streamMetaReleaseTask(pMeta, pTask); return code; } +// this is to process request from transaction, always return true. int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { int32_t vgId = pMeta->vgId; char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); @@ -230,7 +232,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList); // send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode - streamTaskSendPreparedCheckpointsourceRsp(pTask); + (void) streamTaskSendCheckpointsourceRsp(pTask); streamTaskResetStatus(pTask); streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr); @@ -329,7 +331,8 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) { @@ -388,7 +391,8 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { tqDebug("s-task:0x%x vgId:%d recv dispatch-rsp from 0x%x vgId:%d", pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pMeta, pTask); @@ -411,7 +415,8 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { tDecodeStreamRetrieveReq(&decoder, &req); tDecoderClear(&decoder); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask); if (pTask == NULL) { tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, req.dstTaskId); @@ -419,7 +424,6 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { return -1; } - int32_t code = 0; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { code = streamProcessRetrieveReq(pTask, &req); } else { @@ -484,7 +488,8 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId); } - SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask); if (pTask == NULL) { return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId); } @@ -511,7 +516,8 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) } tDecoderClear(&decoder); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask); if (pTask == NULL) { tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); return TSDB_CODE_STREAM_TASK_NOT_EXIST; @@ -598,7 +604,8 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve tqDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); if (restored) { - SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); + SStreamTask* p = NULL; + code = streamMetaAcquireTask(pMeta, streamId, taskId, &p); if ((p != NULL) && (p->info.fillHistory == 0)) { tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId); } @@ -773,7 +780,8 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); return code; } else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask != NULL) { char* pStatus = NULL; @@ -794,7 +802,8 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead return 0; } - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask != NULL) { // even in halt status, the data in inputQ must be processed char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { @@ -862,7 +871,8 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask == NULL) { tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, pReq->taskId); @@ -903,7 +913,8 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); if (pTask == NULL) { tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", @@ -971,7 +982,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); if (pTask == NULL) { tqError( "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", @@ -990,7 +1002,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask == NULL) { tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, pReq->taskId); @@ -1003,7 +1016,8 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { SStreamTask* pHistoryTask = NULL; if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); + pHistoryTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask); if (pHistoryTask == NULL) { tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 ", it may have been dropped already", @@ -1070,7 +1084,8 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); if (pTask == NULL) { tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; @@ -1081,13 +1096,14 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState->name); taosThreadMutexUnlock(&pTask->lock); - int32_t code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); + code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); if (code != 0) { return code; } STaskId* pHTaskId = &pTask->hTaskInfo.id; - SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); + SStreamTask* pHTask = NULL; + code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); if (pHTask) { taosThreadMutexLock(&pHTask->lock); SStreamTaskState* p = streamTaskGetStatus(pHTask); @@ -1121,7 +1137,8 @@ int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { r int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask); if (pTask == NULL) { tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", pRsp->downstreamNodeId, pRsp->downstreamTaskId); @@ -1154,7 +1171,8 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { tDecoderClear(&decoder); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); if (pTask == NULL) { tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, req.taskId); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 3ea3e814ed..c73b245940 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -40,8 +40,11 @@ static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage) { - SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); - ASSERT(pInfo != NULL); + SStreamUpstreamEpInfo* pInfo = NULL; + int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } *oldStage = pInfo->stage; const char* id = pTask->id.idStr; @@ -166,7 +169,8 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS taskId, pReq->upstreamTaskId, pReq->upstreamNodeId, pMeta->vgId); pRsp->status = TASK_DOWNSTREAM_NOT_LEADER; } else { - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); + SStreamTask* pTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, taskId, &pTask); if (pTask != NULL) { pRsp->status = streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 990e7fb987..6906f1eccd 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -29,12 +29,11 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList); static void checkpointTriggerMonitorFn(void* param, void* tmrId); -SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, - int32_t transId, int32_t srcTaskId) { +int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId, + int32_t srcTaskId, SStreamDataBlock** pRes) { SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock)); if (pChkpoint == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pChkpoint->type = checkpointType; @@ -46,8 +45,7 @@ SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpoint SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pBlock == NULL) { taosFreeQitem(pChkpoint); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pBlock->info.type = STREAM_CHECKPOINT; @@ -60,28 +58,37 @@ SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpoint if (pChkpoint->blocks == NULL) { taosMemoryFree(pBlock); taosFreeQitem(pChkpoint); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pChkpoint->blocks, pBlock); + void* p = taosArrayPush(pChkpoint->blocks, pBlock); + if (p == NULL) { + taosArrayDestroy(pChkpoint->blocks); + taosMemoryFree(pBlock); + taosFreeQitem(pChkpoint); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pRes = pChkpoint; taosMemoryFree(pBlock); - terrno = 0; - - return pChkpoint; + return TSDB_CODE_SUCCESS; } +// this message must be put into inputq successfully, continue retrying until it succeeds int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId, int32_t srcTaskId) { - SStreamDataBlock* pCheckpoint = createChkptTriggerBlock(pTask, checkpointType, checkpointId, transId, srcTaskId); + SStreamDataBlock* pCheckpoint = NULL; + int32_t code = createChkptTriggerBlock(pTask, checkpointType, checkpointId, transId, srcTaskId, &pCheckpoint); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pCheckpoint) < 0) { return TSDB_CODE_OUT_OF_MEMORY; } - streamTrySchedExec(pTask); - return TSDB_CODE_SUCCESS; + return streamTrySchedExec(pTask); } int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) { @@ -176,22 +183,26 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); if (pTask->chkInfo.checkpointId > checkpointId) { stError("s-task:%s vgId:%d current checkpointId:%" PRId64 " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); - taosThreadMutexUnlock(&pTask->lock); + code = taosThreadMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); - return TSDB_CODE_SUCCESS; + return code; } if (pTask->chkInfo.checkpointId == checkpointId) { { // send checkpoint-ready msg to upstream SRpcMsg msg = {0}; + SStreamUpstreamEpInfo* pInfo = NULL; + code = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId); initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); tmsgSendReq(&pInfo->epSet, &msg); } @@ -202,10 +213,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock id, vgId, pBlock->srcTaskId); streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); - taosThreadMutexUnlock(&pTask->lock); + code = taosThreadMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); - return TSDB_CODE_SUCCESS; + return code; } if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) { @@ -213,19 +224,19 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 " discard", id, vgId, pActiveInfo->activeId, checkpointId); - taosThreadMutexUnlock(&pTask->lock); + code = taosThreadMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); - return TSDB_CODE_SUCCESS; + return code; } else { // checkpointId == pActiveInfo->activeId if (pActiveInfo->allUpstreamTriggerRecv == 1) { stDebug( "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "checkpointId:%" PRId64 " transId:%d", id, vgId, checkpointId, transId); - taosThreadMutexUnlock(&pTask->lock); + code = taosThreadMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); - return TSDB_CODE_SUCCESS; + return code; } if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { @@ -238,16 +249,16 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock ", prev recvTs:%" PRId64 " discard", pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); - taosThreadMutexUnlock(&pTask->lock); + code = taosThreadMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); - return TSDB_CODE_SUCCESS; + return code; } } } } } - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64 ", transId:%d current active checkpointId:%" PRId64, @@ -357,7 +368,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId return -1; } - taosThreadMutexLock(&pInfo->lock); + (void) taosThreadMutexLock(&pInfo->lock); // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList); @@ -384,7 +395,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList); int32_t transId = pInfo->transId; - taosThreadMutexUnlock(&pInfo->lock); + (void) taosThreadMutexUnlock(&pInfo->lock); if (notReady == 0) { stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id); @@ -399,7 +410,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream int64_t now = taosGetTimestampMs(); int32_t numOfConfirmed = 0; - taosThreadMutexLock(&pInfo->lock); + (void) taosThreadMutexLock(&pInfo->lock); for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { @@ -420,7 +431,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr, numOfConfirmed, checkpointId); - taosThreadMutexUnlock(&pInfo->lock); + (void) taosThreadMutexUnlock(&pInfo->lock); return TSDB_CODE_SUCCESS; } @@ -428,12 +439,12 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { pTask->chkInfo.startTs = 0; // clear the recorded start time streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks - taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); + (void) taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo); if (clearChkpReadyMsg) { streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo); } - taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); + (void) taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); } int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) { @@ -443,7 +454,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV const char* id = pTask->id.idStr; SCheckpointInfo* pInfo = &pTask->chkInfo; - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); if (pReq->checkpointId <= pInfo->checkpointId) { stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64 @@ -451,13 +462,13 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV " transId:%d ignored", id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer, pReq->transId); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); { // destroy the related fill-history tasks // drop task should not in the meta-lock, and drop the related fill-history task now streamMetaWUnLock(pMeta); if (pReq->dropRelHTask) { - streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); + (void) streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped in update checkpointInfo, remain tasks:%d", id, vgId, pReq->taskId, numOfTasks); @@ -519,12 +530,12 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV return code; } - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); streamMetaWUnLock(pMeta); // drop task should not in the meta-lock, and drop the related fill-history task now if (pReq->dropRelHTask) { - streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); + (void) streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId, (int32_t)pReq->hTaskId, numOfTasks); @@ -718,9 +729,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask); } } else { // clear the checkpoint info if failed - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, id, ckId); @@ -758,13 +769,13 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { pActiveInfo->checkCounter = 0; stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); SStreamTaskState* pState = streamTaskGetStatus(pTask); if (pState->state != TASK_STATUS__CK) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pTask->pMeta, pTask); return; } @@ -775,14 +786,14 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pTask->pMeta, pTask); return; } - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); - taosThreadMutexLock(&pActiveInfo->lock); + (void) taosThreadMutexLock(&pActiveInfo->lock); // send msg to retrieve checkpoint trigger msg SArray* pList = pTask->upstreamInfo.pList; @@ -815,7 +826,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { // do send retrieve checkpoint trigger msg to upstream int32_t size = taosArrayGetSize(pNotSendList); doSendRetrieveTriggerMsg(pTask, pNotSendList); - taosThreadMutexUnlock(&pActiveInfo->lock); + (void) taosThreadMutexUnlock(&pActiveInfo->lock); // check every 100ms if (size > 0) { @@ -885,9 +896,9 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) return false; } - taosThreadMutexLock(&pInfo->lock); + (void) taosThreadMutexLock(&pInfo->lock); if (!pInfo->dispatchTrigger) { - taosThreadMutexUnlock(&pInfo->lock); + (void) taosThreadMutexUnlock(&pInfo->lock); return false; } @@ -909,7 +920,7 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId); } - taosThreadMutexUnlock(&pInfo->lock); + (void) taosThreadMutexUnlock(&pInfo->lock); return true; } @@ -928,11 +939,12 @@ void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_ } // record the dispatch checkpoint trigger info in the list +// memory insufficient may cause the stream computing stopped void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int64_t now = taosGetTimestampMs(); - taosThreadMutexLock(&pInfo->lock); + (void) taosThreadMutexLock(&pInfo->lock); // outputQ should be empty here ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0); @@ -942,31 +954,37 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher; STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId}; - taosArrayPush(pInfo->pDispatchTriggerList, &p); + void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); + if (px == NULL) { + // pause the stream task, if memory not enough + } } else { for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) { SVgroupInfo* pVgInfo = taosArrayGet(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos, i); STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; - taosArrayPush(pInfo->pDispatchTriggerList, &p); + void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); + if (px == NULL) { + // pause the stream task, if memory not enough + } } } - taosThreadMutexUnlock(&pInfo->lock); + (void) taosThreadMutexUnlock(&pInfo->lock); } int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int32_t num = 0; - taosThreadMutexLock(&pInfo->lock); + (void) taosThreadMutexLock(&pInfo->lock); for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); if (p->recved) { num++; } } - taosThreadMutexUnlock(&pInfo->lock); + (void) taosThreadMutexUnlock(&pInfo->lock); return num; } @@ -974,7 +992,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int32_t taskId = 0; - taosThreadMutexLock(&pInfo->lock); + (void) taosThreadMutexLock(&pInfo->lock); for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); @@ -988,7 +1006,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) { } } - taosThreadMutexUnlock(&pInfo->lock); + (void) taosThreadMutexUnlock(&pInfo->lock); int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask); int32_t total = streamTaskGetNumOfDownstream(pTask); @@ -1045,7 +1063,8 @@ static int32_t uploadCheckpointToS3(const char* id, const char* path) { stDebug("[s3] upload checkpoint:%s", filename); } } - taosCloseDir(&pDir); + + (void) taosCloseDir(&pDir); return code; } @@ -1162,78 +1181,37 @@ int32_t deleteCheckpointFile(const char* id, const char* name) { } int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) { - int32_t code; - int32_t tlen = 0; - int32_t vgId = pTask->pMeta->vgId; - const char* id = pTask->id.idStr; - SCheckpointInfo* pInfo = &pTask->chkInfo; + const char* id = pTask->id.idStr; - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); if (pTask->status.sendConsensusChkptId == true) { stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_SUCCESS; } else { pTask->status.sendConsensusChkptId = true; } - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); ASSERT(pTask->pBackend == NULL); pTask->status.requireConsensusChkptId = true; -#if 0 - SRestoreCheckpointInfo req = { - .streamId = pTask->id.streamId, - .taskId = pTask->id.taskId, - .nodeId = vgId, - .checkpointId = pInfo->checkpointId, - .startTs = pTask->execInfo.created, - }; - tEncodeSize(tEncodeRestoreCheckpointInfo, &req, tlen, code); - if (code < 0) { - stError("s-task:%s vgId:%d encode stream task latest-checkpoint-id failed, code:%s", id, vgId, tstrerror(code)); - return TSDB_CODE_INVALID_MSG; - } - - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - stError("s-task:%s vgId:%d encode stream task latest-checkpoint-id msg failed, code:%s", id, vgId, - tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; - } - - SEncoder encoder; - tEncoderInit(&encoder, buf, tlen); - if ((code = tEncodeRestoreCheckpointInfo(&encoder, &req)) < 0) { - rpcFreeCont(buf); - stError("s-task:%s vgId:%d encode stream task latest-checkpoint-id msg failed, code:%s", id, vgId, tstrerror(code)); - return -1; - } - tEncoderClear(&encoder); - - SRpcMsg msg = {0}; - initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CONSEN_CHKPT, buf, tlen); - stDebug("s-task:%s vgId:%d send latest checkpointId:%" PRId64 " to mnode to get the consensus checkpointId", id, vgId, - pInfo->checkpointId); - - tmsgSendReq(&pTask->info.mnodeEpset, &msg); -#endif return 0; } -int32_t streamTaskSendPreparedCheckpointsourceRsp(SStreamTask* pTask) { +int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) { int32_t code = 0; if (pTask->info.taskLevel != TASK_LEVEL__SOURCE) { return code; } - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); SStreamTaskState* p = streamTaskGetStatus(pTask); if (p->state == TASK_STATUS__CK) { code = streamTaskSendCheckpointSourceRsp(pTask); } - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); return code; } \ No newline at end of file diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index fae90f4db8..e54794d75f 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -131,17 +131,20 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return TSDB_CODE_SUCCESS; } -SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type) { +int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit** pSubmit) { + *pSubmit = NULL; + SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, pData->msgLen); if (pDataSubmit == NULL) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pDataSubmit->ver = pData->ver; pDataSubmit->submit = *pData; pDataSubmit->type = type; - return pDataSubmit; + *pSubmit = pDataSubmit; + return TSDB_CODE_SUCCESS; } void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 617adaa016..f406c7ac48 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1118,7 +1118,11 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, return TSDB_CODE_SUCCESS; } - SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); + SStreamUpstreamEpInfo* pInfo = NULL; + int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } STaskCheckpointReadyInfo info = {0}; initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); @@ -1419,8 +1423,11 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S stDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64 ", msgId:%d", id, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId); - SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId); - ASSERT(pInfo != NULL); + SStreamUpstreamEpInfo* pInfo = NULL; + int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pMeta->role == NODE_ROLE_FOLLOWER) { stError("s-task:%s task on follower received dispatch msgs, dispatch msg rejected", id); @@ -1453,10 +1460,9 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S { // do send response with the input status - int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); + code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code)); - terrno = code; return code; } @@ -1465,5 +1471,5 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } streamTrySchedExec(pTask); - return 0; + return code; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b0915640cc..ffada66cca 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -314,7 +314,8 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { SStreamMeta* pMeta = pTask->pMeta; const char* id = pTask->id.idStr; - SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + SStreamTask* pStreamTask = NULL; + int32_t code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask); if (pStreamTask == NULL) { stError( "s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related " @@ -416,7 +417,8 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) { code = streamTransferStateDoPrepare(pTask); } else { // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task. - SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + SStreamTask* pStreamTask = NULL; + code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask); if (pStreamTask != NULL) { // halt the related stream sink task code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, haltCallback, NULL); @@ -590,7 +592,8 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask)); STaskId* pHTaskId = &pTask->hTaskInfo.id; - SStreamTask* pHTask = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId); + SStreamTask* pHTask = NULL; + int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); if (pHTask != NULL) { streamTaskReleaseState(pHTask); streamTaskReloadState(pTask); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d0b1f6ca93..9cf3909867 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -292,32 +292,39 @@ void streamMetaRemoveDB(void* arg, char* key) { taosThreadMutexUnlock(&pMeta->backendMutex); } -SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, - int32_t vgId, int64_t stage, startComplete_fn_t fn) { +int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId, + int64_t stage, startComplete_fn_t fn, SStreamMeta** p) { + *p = NULL; + int32_t code = 0; + SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; stError("vgId:%d failed to prepare stream meta, alloc size:%" PRIzu ", out of memory", vgId, sizeof(SStreamMeta)); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t len = strlen(path) + 64; char* tpath = taosMemoryCalloc(1, len); + if (tpath == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream"); pMeta->path = tpath; - if (streamMetaOpenTdb(pMeta) < 0) { + code = streamMetaOpenTdb(pMeta); + if (code != TSDB_CODE_SUCCESS) { goto _err; } - if (streamMetaMayCvtDbFormat(pMeta) < 0) { + if ((code = streamMetaMayCvtDbFormat(pMeta)) < 0) { stError("vgId:%d convert sub info format failed, open stream meta failed, reason: %s", pMeta->vgId, tstrerror(terrno)); goto _err; } - if (streamMetaBegin(pMeta) < 0) { + if ((code = streamMetaBegin(pMeta) < 0)) { stError("vgId:%d begin trans for stream meta failed", pMeta->vgId); goto _err; } @@ -325,32 +332,32 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); pMeta->pTasksMap = taosHashInit(64, fp, true, HASH_NO_LOCK); if (pMeta->pTasksMap == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } pMeta->updateInfo.pTasks = taosHashInit(64, fp, false, HASH_NO_LOCK); if (pMeta->updateInfo.pTasks == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } pMeta->startInfo.pReadyTaskSet = taosHashInit(64, fp, false, HASH_NO_LOCK); if (pMeta->startInfo.pReadyTaskSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } pMeta->startInfo.pFailedTaskSet = taosHashInit(4, fp, false, HASH_NO_LOCK); if (pMeta->startInfo.pFailedTaskSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } // task list pMeta->pTaskList = taosArrayInit(4, sizeof(SStreamTaskId)); if (pMeta->pTaskList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -376,10 +383,16 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas // set the attribute when running on Linux OS TdThreadRwlockAttr attr; - taosThreadRwlockAttrInit(&attr); + code = taosThreadRwlockAttrInit(&attr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } #ifdef LINUX - pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + code = pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } #endif taosThreadRwlockInit(&pMeta->lock, &attr); @@ -391,7 +404,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas pMeta->pHbInfo = createMetaHbInfo(pRid); if (pMeta->pHbInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -403,7 +416,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas } taosThreadMutexInit(&pMeta->backendMutex, NULL); - return pMeta; + *p = pMeta; + return code; _err: taosMemoryFree(pMeta->path); @@ -420,7 +434,7 @@ _err: taosMemoryFree(pMeta); stError("failed to open stream meta, reason:%s", tstrerror(terrno)); - return NULL; + return code; } // todo refactor: the lock shoud be restricted in one function @@ -630,29 +644,30 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) { return (int32_t)size; } -SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { +int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask) { STaskId id = {.streamId = streamId, .taskId = taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL || streamTaskShouldStop(*ppTask)) { - return NULL; + *pTask = NULL; + return TSDB_CODE_FAILED; } int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref); - return *ppTask; + *pTask = *ppTask; + return TSDB_CODE_SUCCESS; } -SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { +int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask) { streamMetaRLock(pMeta); - SStreamTask* p = streamMetaAcquireTaskNoLock(pMeta, streamId, taskId); + int32_t code = streamMetaAcquireTaskNoLock(pMeta, streamId, taskId, pTask); streamMetaRUnLock(pMeta); - return p; + return code; } -SStreamTask* streamMetaAcquireOneTask(SStreamTask* pTask) { +void streamMetaAcquireOneTask(SStreamTask* pTask) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); stTrace("s-task:%s acquire task, ref:%d", pTask->id.idStr, ref); - return pTask; } void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) { @@ -719,20 +734,19 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, pMeta->vgId); while (1) { - streamMetaRLock(pMeta); + int32_t timerActive = 0; + streamMetaRLock(pMeta); ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask) { - if ((*ppTask)->status.timerActive == 0) { - streamMetaRUnLock(pMeta); - break; - } + timerActive = (*ppTask)->status.timerActive; + } + streamMetaRUnLock(pMeta); - taosMsleep(10); - stDebug("s-task:%s wait for quit from timer", (*ppTask)->id.idStr); - streamMetaRUnLock(pMeta); + if (timerActive > 0) { + taosMsleep(100); + stDebug("s-task:0x%x wait for quit from timer", id.taskId); } else { - streamMetaRUnLock(pMeta); break; } } @@ -1105,8 +1119,10 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { int32_t numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { + SStreamTask* pTask = NULL; + + int32_t code = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); + if (code != TSDB_CODE_SUCCESS) { continue; } @@ -1201,7 +1217,8 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { // initialization, when the operation of check downstream tasks status is executed far quickly. for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); @@ -1223,7 +1240,8 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); @@ -1298,8 +1316,10 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { + SStreamTask* pTask = NULL; + + int32_t code = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); + if (code != TSDB_CODE_SUCCESS) { continue; } @@ -1339,7 +1359,8 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas int32_t vgId = pMeta->vgId; stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, streamId, taskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask); if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId); streamMetaAddFailedTask(pMeta, streamId, taskId); diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 30bccd6479..cf23b776e2 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -92,7 +92,8 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) } // add ref for task - SStreamTask* p = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId); + SStreamTask* p = NULL; + int32_t code = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, &p); if (p == NULL) { stError("s-task:0x%x failed to acquire task, status:%s, not exec scan-history data", pTask->id.taskId, streamTaskGetStatus(pTask)->name); @@ -223,7 +224,8 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { streamMetaRUnLock(pMeta); if (pHTask != NULL) { // it is already added into stream meta store. - SStreamTask* pHisTask = streamMetaAcquireTask(pMeta, hStreamId, hTaskId); + SStreamTask* pHisTask = NULL; + code = streamMetaAcquireTask(pMeta, hStreamId, hTaskId, &pHisTask); if (pHisTask == NULL) { stDebug("s-task:%s failed acquire and start fill-history task, it may have been dropped/stopped", idStr); streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); @@ -353,7 +355,11 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { return; } - SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pInfo->id.streamId, pInfo->id.taskId); + SStreamTask* pTask = NULL; + code = streamMetaAcquireTaskNoLock(pMeta, pInfo->id.streamId, pInfo->id.taskId, &pTask); + if (code != TSDB_CODE_SUCCESS) { + // todo + } streamMetaWUnLock(pMeta); if (pTask != NULL) { @@ -373,7 +379,8 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { ASSERT(pTask->status.timerActive >= 1); // abort the timer if intend to stop task - SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId); + SStreamTask* pHTask = NULL; + code = streamMetaAcquireTask(pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId, &pHTask); if (pHTask == NULL) { doRetryLaunchFillHistoryTask(pTask, pInfo, now); streamMetaReleaseTask(pMeta, pTask); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 4cbe0cb136..6264a262da 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -91,14 +91,15 @@ static SStreamUpstreamEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) { return pEpInfo; } -SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, - SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5) { +int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, + SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5, SStreamTask** p) { + *p = NULL; + SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; stError("s-task:0x%" PRIx64 " failed malloc new stream task, size:%d, code:%s", streamId, (int32_t)sizeof(SStreamTask), tstrerror(terrno)); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pTask->ver = SSTREAM_TASK_VER; @@ -110,10 +111,10 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, pTask->info.delaySchedParam = triggerParam; pTask->subtableWithoutMd5 = subtableWithoutMd5; - pTask->status.pSM = streamCreateStateMachine(pTask); - if (pTask->status.pSM == NULL) { + int32_t code = streamCreateStateMachine(pTask); + if (pTask->status.pSM == NULL || code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTask); - return NULL; + return code; } char buf[128] = {0}; @@ -135,7 +136,9 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, epsetAssign(&(pTask->info.mnodeEpset), pEpset); addToTaskset(pTaskList, pTask); - return pTask; + *p = pTask; + + return code; } int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo) { @@ -274,7 +277,9 @@ void tFreeStreamTask(SStreamTask* pTask) { tSimpleHashCleanup(pTask->pNameMap); } - pTask->status.pSM = streamDestroyStateMachine(pTask->status.pSM); + streamDestroyStateMachine(pTask->status.pSM); + pTask->status.pSM = NULL; + streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo); taosMemoryFree(pTask->outputInfo.pTokenBucket); @@ -369,8 +374,9 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->status.timerActive = 0; - pTask->status.pSM = streamCreateStateMachine(pTask); - if (pTask->status.pSM == NULL) { + + int32_t code = streamCreateStateMachine(pTask); + if (pTask->status.pSM == NULL || code != TSDB_CODE_SUCCESS) { stError("s-task:%s failed create state-machine for stream task, initialization failed, code:%s", pTask->id.idStr, tstrerror(terrno)); return terrno; @@ -391,7 +397,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i TdThreadMutexAttr attr = {0}; - int code = taosThreadMutexAttrInit(&attr); + code = taosThreadMutexAttrInit(&attr); if (code != 0) { stError("s-task:%s initElapsed mutex attr failed, code:%s", pTask->id.idStr, tstrerror(code)); return code; @@ -433,10 +439,10 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i } if (pTask->chkInfo.pActiveInfo == NULL) { - pTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); } - return TSDB_CODE_SUCCESS; + return code; } int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) { @@ -636,7 +642,13 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) { } void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { - SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId); + SStreamUpstreamEpInfo* pInfo = NULL; + + int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return; + } + if ((pInfo != NULL) && pInfo->dataAllowed) { pInfo->dataAllowed = false; int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); @@ -645,8 +657,14 @@ void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { } void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) { - SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId); - if ((pInfo != NULL) && (!pInfo->dataAllowed)) { + SStreamUpstreamEpInfo* pInfo = NULL; + + int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + return; + } + + if (pInfo != NULL && (!pInfo->dataAllowed)) { int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); ASSERT(t >= 0); pInfo->dataAllowed = true; @@ -941,17 +959,24 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { return 0; } -SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId) { +int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) { + *pEpInfo = NULL; + int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); for (int32_t i = 0; i < num; ++i) { SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + if (pInfo == NULL) { + return TSDB_CODE_FAILED; + } + if (pInfo->taskId == taskId) { - return pInfo; + *pEpInfo = pInfo; + return TSDB_CODE_SUCCESS; } } stError("s-task:%s failed to find upstream task:0x%x", pTask->id.idStr, taskId); - return NULL; + return TSDB_CODE_FAILED; } SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { @@ -1041,14 +1066,23 @@ int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_ return TSDB_CODE_SUCCESS; } -SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo() { +int32_t streamTaskCreateActiveChkptInfo(SActiveCheckpointInfo** pRes) { SActiveCheckpointInfo* pInfo = taosMemoryCalloc(1, sizeof(SActiveCheckpointInfo)); - taosThreadMutexInit(&pInfo->lock, NULL); + if (pInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = taosThreadMutexInit(&pInfo->lock, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->pDispatchTriggerList = taosArrayInit(4, sizeof(STaskTriggerSendInfo)); pInfo->pReadyMsgList = taosArrayInit(4, sizeof(STaskCheckpointReadyInfo)); pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskDownstreamReadyInfo)); - return pInfo; + + *pRes = pInfo; + return TSDB_CODE_SUCCESS; } void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index f2bd99cdaf..e2d988b876 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -257,43 +257,41 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask) { return code; } -SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) { +int32_t streamCreateStateMachine(SStreamTask* pTask) { initStateTransferTable(); const char* id = pTask->id.idStr; SStreamTaskSM* pSM = taosMemoryCalloc(1, sizeof(SStreamTaskSM)); if (pSM == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM), tstrerror(terrno)); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } pSM->pTask = pTask; pSM->pWaitingEventList = taosArrayInit(4, sizeof(SFutureHandleEventInfo)); if (pSM->pWaitingEventList == NULL) { taosMemoryFree(pSM); - - terrno = TSDB_CODE_OUT_OF_MEMORY; stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM), tstrerror(terrno)); - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } // set the initial state for the state-machine of stream task pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT]; pSM->startTs = taosGetTimestampMs(); - return pSM; + + pTask->status.pSM = pSM; + return TSDB_CODE_SUCCESS; } -void* streamDestroyStateMachine(SStreamTaskSM* pSM) { +void streamDestroyStateMachine(SStreamTaskSM* pSM) { if (pSM == NULL) { - return NULL; + return; } taosArrayDestroy(pSM->pWaitingEventList); taosMemoryFree(pSM); - return NULL; } static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans) { From 646731b5dd033b3b85e56b0d4248204b5b392bf5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Jul 2024 15:58:37 +0800 Subject: [PATCH 08/45] opti:[TS-4593] transform offset from file to tdb in tmq --- source/dnode/vnode/src/tq/tqMeta.c | 2 ++ source/dnode/vnode/src/tq/tqOffset.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 622ba786ab..94311c0fbb 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -413,6 +413,7 @@ int32_t tqMetaOpen(STQ* pTq) { TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq)); }else{ TQ_ERR_GO_TO_END(tqMetaTransform(pTq)); + taosRemoveFile(maindb); } TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq)); @@ -447,6 +448,7 @@ int32_t tqMetaTransform(STQ* pTq) { TQ_ERR_GO_TO_END(tqMetaTransformOffsetInfo(pTq, offsetNew)); (void)taosRemoveFile(offset); + (void)taosRemoveFile(offsetNew); END: taosMemoryFree(offset); diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 8295ec4824..e37cdfc697 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -39,7 +39,7 @@ int32_t tqOffsetRestoreFromFile(SHashObj* pOffset, char* name) { TdFilePtr pFile = taosOpenFile(name, TD_FILE_READ); if (pFile == NULL) { - code = TAOS_SYSTEM_ERROR(errno); + code = TDB_CODE_SUCCESS; goto END; } From f078be357d21d8ee5b9fe4acd4f5d829ae35937e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Jul 2024 16:25:51 +0800 Subject: [PATCH 09/45] opti:[TS-4593] transform offset from file to tdb in tmq --- source/dnode/vnode/src/tq/tqMeta.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 94311c0fbb..2ab459416d 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -344,10 +344,6 @@ int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { return TSDB_CODE_OUT_OF_MEMORY; } tdbFree(data); - if(taosHashPut(pTq->pHandle, key, strlen(key), &handle, sizeof(STqHandle)) != 0){ - tqDestroyTqHandle(&handle); - return TSDB_CODE_OUT_OF_MEMORY; - } *pHandle = taosHashGet(pTq->pHandle, key, strlen(key)); if(*pHandle == NULL){ return TSDB_CODE_OUT_OF_MEMORY; From 30186f466b30b2cba3713d4d34d821372eba0989 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 16:47:56 +0800 Subject: [PATCH 10/45] refactor: check return value for each function. --- include/libs/stream/tstream.h | 4 +- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tq/tq.c | 12 +++--- source/dnode/vnode/src/tq/tqStreamTask.c | 14 +++---- source/dnode/vnode/src/tqCommon/tqCommon.c | 28 ++++++------- source/libs/stream/src/streamCheckStatus.c | 26 ++++++------ source/libs/stream/src/streamCheckpoint.c | 36 ++++++++-------- source/libs/stream/src/streamDispatch.c | 8 ++-- source/libs/stream/src/streamExec.c | 42 +++++++++---------- source/libs/stream/src/streamMeta.c | 6 +-- source/libs/stream/src/streamSched.c | 8 ++-- source/libs/stream/src/streamStartHistory.c | 46 ++++++++++----------- source/libs/stream/src/streamTask.c | 14 +++---- source/libs/stream/src/streamTaskSm.c | 8 ++-- 14 files changed, 128 insertions(+), 126 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8f6b512741..609b977536 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -628,7 +628,9 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo); +#if 0 SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); +#endif void streamTaskInputFail(SStreamTask* pTask); @@ -638,7 +640,7 @@ bool streamTaskIsIdle(const SStreamTask* pTask); bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus); char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); -SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask); +SStreamTaskState streamTaskGetStatus(const SStreamTask* pTask); const char* streamTaskGetStatusStr(ETaskStatus status); void streamTaskResetStatus(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index db2ead2f37..da60d7d30e 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -41,7 +41,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce SCheckpointInfo *pChkInfo = &pTask->chkInfo; tqSetRestoreVersionInfo(pTask); - char *p = streamTaskGetStatus(pTask)->name; + char *p = streamTaskGetStatus(pTask).name; if (pTask->info.fillHistory) { sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1c3866080b..99d57bd5bd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -768,7 +768,7 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV SCheckpointInfo* pChkInfo = &pTask->chkInfo; tqSetRestoreVersionInfo(pTask); - char* p = streamTaskGetStatus(pTask)->name; + char* p = streamTaskGetStatus(pTask).name; const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus); if (pTask->info.fillHistory) { @@ -886,7 +886,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // do recovery step1 const char* id = pTask->id.idStr; - char* pStatus = streamTaskGetStatus(pTask)->name; + char* pStatus = streamTaskGetStatus(pTask).name; // avoid multi-thread exec while (1) { @@ -942,15 +942,15 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { if (retInfo.ret == TASK_SCANHISTORY_REXEC) { streamExecScanHistoryInFuture(pTask, retInfo.idleTime); } else { - SStreamTaskState* p = streamTaskGetStatus(pTask); - ETaskStatus s = p->state; + SStreamTaskState p = streamTaskGetStatus(pTask); + ETaskStatus s = p.state; if (s == TASK_STATUS__PAUSE) { tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr, el, pTask->execInfo.step1El, status); } else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) { tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr, - p->name, pTask->execInfo.step1El); + p.name, pTask->execInfo.step1El); } } @@ -1152,7 +1152,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // todo save the checkpoint failed info taosThreadMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask)->state; + ETaskStatus status = streamTaskGetStatus(pTask).state; if (req.mndTrigger == 1) { if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) { diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 22658cc650..b4829d1dd8 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -226,15 +226,15 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) { } // not in ready state, do not handle the data from wal - SStreamTaskState* pState = streamTaskGetStatus(pTask); - if (pState->state != TASK_STATUS__READY) { - tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState->name); + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state != TASK_STATUS__READY) { + tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState.name); return false; } // fill-history task has entered into the last phase, no need to anything if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) { - ASSERT(pState->state == TASK_STATUS__READY); + ASSERT(pState.state == TASK_STATUS__READY); // the maximum version of data in the WAL has reached already, the step2 is done tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr, pTask->dataRange.range.maxVer); @@ -349,9 +349,9 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { taosThreadMutexLock(&pTask->lock); - SStreamTaskState* pState = streamTaskGetStatus(pTask); - if (pState->state != TASK_STATUS__READY) { - tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState->name); + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state != TASK_STATUS__READY) { + tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pStreamMeta, pTask); continue; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 95ee1b70f8..3c6100a8f4 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -885,8 +885,8 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { streamTaskClearCheckInfo(pTask, true); // clear flag set during do checkpoint, and open inputQ for all upstream tasks - SStreamTaskState* pState = streamTaskGetStatus(pTask); - if (pState->state == TASK_STATUS__CK) { + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state == TASK_STATUS__CK) { int32_t tranId = 0; int64_t activeChkId = 0; streamTaskGetActiveCheckpointInfo(pTask, &tranId, &activeChkId); @@ -895,13 +895,13 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { pTask->id.idStr, activeChkId, tranId); streamTaskSetStatusReady(pTask); - } else if (pState->state == TASK_STATUS__UNINIT) { + } else if (pState.state == TASK_STATUS__UNINIT) { // tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); // ASSERT(pTask->status.downstreamReady == 0); // tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId); - tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState->name); + tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); } else { - tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState->name); + tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); } taosThreadMutexUnlock(&pTask->lock); @@ -936,8 +936,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) return TSDB_CODE_SUCCESS; } - SStreamTaskState* pState = streamTaskGetStatus(pTask); - if (pState->state == TASK_STATUS__CK) { // recv the checkpoint-source/trigger already + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state == TASK_STATUS__CK) { // recv the checkpoint-source/trigger already int32_t transId = 0; int64_t checkpointId = 0; @@ -966,7 +966,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) TSDB_CODE_ACTION_IN_PROGRESS); } } else { // upstream not recv the checkpoint-source/trigger till now - ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT); + ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT); tqWarn( "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "upstream sending checkpoint-source/trigger", @@ -1017,7 +1017,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { SStreamTask* pHistoryTask = NULL; if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { pHistoryTask = NULL; - int32_t code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask); + code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask); if (pHistoryTask == NULL) { tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 ", it may have been dropped already", @@ -1047,7 +1047,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t } streamTaskResume(pTask); - ETaskStatus status = streamTaskGetStatus(pTask)->state; + ETaskStatus status = streamTaskGetStatus(pTask).state; int32_t level = pTask->info.taskLevel; if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) { @@ -1092,8 +1092,8 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m } taosThreadMutexLock(&pTask->lock); - SStreamTaskState* pState = streamTaskGetStatus(pTask); - tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState->name); + SStreamTaskState pState = streamTaskGetStatus(pTask); + tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name); taosThreadMutexUnlock(&pTask->lock); code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); @@ -1106,8 +1106,8 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); if (pHTask) { taosThreadMutexLock(&pHTask->lock); - SStreamTaskState* p = streamTaskGetStatus(pHTask); - tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p->name); + SStreamTaskState p = streamTaskGetStatus(pHTask); + tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name); taosThreadMutexUnlock(&pHTask->lock); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index c73b245940..e588be0784 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -66,7 +66,7 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ id, upstreamTaskId, vgId, stage, pInfo->stage); // record the checkpoint failure id and sent to mnode taosThreadMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask)->state; + ETaskStatus status = streamTaskGetStatus(pTask).state; if (status == TASK_STATUS__CK) { streamTaskSetFailedCheckpointId(pTask); } @@ -75,7 +75,7 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ if (pInfo->stage != stage) { taosThreadMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask)->state; + ETaskStatus status = streamTaskGetStatus(pTask).state; if (status == TASK_STATUS__CK) { streamTaskSetFailedCheckpointId(pTask); } @@ -175,10 +175,10 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS pRsp->status = streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); - SStreamTaskState* pState = streamTaskGetStatus(pTask); + SStreamTaskState pState = streamTaskGetStatus(pTask); stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", - pTask->id.idStr, pState->name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, + pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); streamMetaReleaseTask(pMeta, pTask); } else { @@ -651,12 +651,12 @@ int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64 void rspMonitorFn(void* param, void* tmrId) { SStreamTask* pTask = param; SStreamMeta* pMeta = pTask->pMeta; - SStreamTaskState* pStat = streamTaskGetStatus(pTask); + SStreamTaskState pStat = streamTaskGetStatus(pTask); STaskCheckInfo* pInfo = &pTask->taskCheckInfo; int32_t vgId = pTask->pMeta->vgId; int64_t now = taosGetTimestampMs(); int64_t timeoutDuration = now - pInfo->timeoutStartTs; - ETaskStatus state = pStat->state; + ETaskStatus state = pStat.state; const char* id = pTask->id.idStr; int32_t numOfReady = 0; int32_t numOfFault = 0; @@ -669,7 +669,7 @@ void rspMonitorFn(void* param, void* tmrId) { if (state == TASK_STATUS__STOP) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref); + stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat.name, vgId, ref); streamTaskCompleteCheckRsp(pInfo, true, id); @@ -685,7 +685,7 @@ void rspMonitorFn(void* param, void* tmrId) { if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref); + stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat.name, vgId, ref); streamTaskCompleteCheckRsp(pInfo, true, id); streamMetaReleaseTask(pMeta, pTask); @@ -695,7 +695,7 @@ void rspMonitorFn(void* param, void* tmrId) { taosThreadMutexLock(&pInfo->checkInfoLock); if (pInfo->notReadyTasks == 0) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat->name, + stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat.name, vgId, ref); streamTaskCompleteCheckRsp(pInfo, false, id); @@ -707,7 +707,7 @@ void rspMonitorFn(void* param, void* tmrId) { SArray* pNotReadyList = taosArrayInit(4, sizeof(int64_t)); SArray* pTimeoutList = taosArrayInit(4, sizeof(int64_t)); - if (pStat->state == TASK_STATUS__UNINIT) { + if (pStat.state == TASK_STATUS__UNINIT) { getCheckRspStatus(pInfo, timeoutDuration, &numOfReady, &numOfFault, &numOfNotRsp, pTimeoutList, pNotReadyList, id); numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList); @@ -720,7 +720,7 @@ void rspMonitorFn(void* param, void* tmrId) { stDebug( "s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart " "detected, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d", - id, pStat->name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); + id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); streamTaskCompleteCheckRsp(pInfo, false, id); taosThreadMutexUnlock(&pInfo->checkInfoLock); @@ -731,7 +731,7 @@ void rspMonitorFn(void* param, void* tmrId) { return; } } else { // unexpected status - stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat->name); + stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat.name); } // checking of downstream tasks has been stopped by other threads @@ -740,7 +740,7 @@ void rspMonitorFn(void* param, void* tmrId) { stDebug( "s-task:%s status:%s vgId:%d stopped by other threads to check downstream process, total:%d, notRsp:%d, " "notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d", - id, pStat->name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); + id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); streamTaskCompleteCheckRsp(pInfo, false, id); taosThreadMutexUnlock(&pInfo->checkInfoLock); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 6906f1eccd..3b15e5122d 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -219,7 +219,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock return code; } - if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) { + if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) { if (pActiveInfo->activeId != checkpointId) { stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 " discard", @@ -265,7 +265,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock id, vgId, pTask->chkInfo.checkpointId, pTask->chkInfo.checkpointVer, transId, checkpointId); // set task status - if (streamTaskGetStatus(pTask)->state != TASK_STATUS__CK) { + if (streamTaskGetStatus(pTask).state != TASK_STATUS__CK) { pActiveInfo->activeId = checkpointId; pActiveInfo->transId = transId; @@ -354,9 +354,9 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId ASSERT(total > 0); // 1. not in checkpoint status now - SStreamTaskState* pStat = streamTaskGetStatus(pTask); - if (pStat->state != TASK_STATUS__CK) { - stError("s-task:%s status:%s discard checkpoint-ready msg from task:0x%x", id, pStat->name, downstreamTaskId); + SStreamTaskState pStat = streamTaskGetStatus(pTask); + if (pStat.state != TASK_STATUS__CK) { + stError("s-task:%s status:%s discard checkpoint-ready msg from task:0x%x", id, pStat.name, downstreamTaskId); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } @@ -364,7 +364,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId if (pTask->chkInfo.checkpointId > checkpointId || pInfo->activeId != checkpointId) { stError("s-task:%s status:%s checkpointId:%" PRId64 " new arrival checkpoint-ready msg (checkpointId:%" PRId64 ") from task:0x%x, expired and discard ", - id, pStat->name, pTask->chkInfo.checkpointId, checkpointId, downstreamTaskId); + id, pStat.name, pTask->chkInfo.checkpointId, checkpointId, downstreamTaskId); return -1; } @@ -482,17 +482,17 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV return TSDB_CODE_SUCCESS; } - SStreamTaskState* pStatus = streamTaskGetStatus(pTask); + SStreamTaskState pStatus = streamTaskGetStatus(pTask); if (!restored) { // during restore procedure, do update checkpoint-info stDebug("s-task:%s vgId:%d status:%s update the checkpoint-info during restore, checkpointId:%" PRId64 "->%" PRId64 " checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64, - id, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer, + id, vgId, pStatus.name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer, pInfo->checkpointTime, pReq->checkpointTs); } else { // not in restore status, must be in checkpoint status stDebug("s-task:%s vgId:%d status:%s start to update the checkpoint-info, checkpointId:%" PRId64 "->%" PRId64 " checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64, - id, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer, + id, vgId, pStatus.name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer, pInfo->checkpointTime, pReq->checkpointTs); } @@ -505,11 +505,11 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV streamTaskClearCheckInfo(pTask, true); - if (pStatus->state == TASK_STATUS__CK) { + if (pStatus.state == TASK_STATUS__CK) { // todo handle error code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); } else { - stDebug("s-task:0x%x vgId:%d not handle checkpoint-done event, status:%s", pReq->taskId, vgId, pStatus->name); + stDebug("s-task:0x%x vgId:%d not handle checkpoint-done event, status:%s", pReq->taskId, vgId, pStatus.name); } if (pReq->dropRelHTask) { @@ -519,7 +519,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV } stDebug("s-task:0x%x set the persistent status attr to be ready, prev:%s, status in sm:%s", pReq->taskId, - streamTaskGetStatusStr(pTask->status.taskStatus), streamTaskGetStatus(pTask)->name); + streamTaskGetStatusStr(pTask->status.taskStatus), streamTaskGetStatus(pTask).name); pTask->status.taskStatus = TASK_STATUS__READY; @@ -770,8 +770,8 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); (void) taosThreadMutexLock(&pTask->lock); - SStreamTaskState* pState = streamTaskGetStatus(pTask); - if (pState->state != TASK_STATUS__CK) { + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state != TASK_STATUS__CK) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); @@ -890,9 +890,9 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) int64_t now = taosGetTimestampMs(); const char* id = pTask->id.idStr; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; - SStreamTaskState* pStatus = streamTaskGetStatus(pTask); + SStreamTaskState pStatus = streamTaskGetStatus(pTask); - if (pStatus->state != TASK_STATUS__CK) { + if (pStatus.state != TASK_STATUS__CK) { return false; } @@ -1207,8 +1207,8 @@ int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) { } (void) taosThreadMutexLock(&pTask->lock); - SStreamTaskState* p = streamTaskGetStatus(pTask); - if (p->state == TASK_STATUS__CK) { + SStreamTaskState p = streamTaskGetStatus(pTask); + if (p.state == TASK_STATUS__CK) { code = streamTaskSendCheckpointSourceRsp(pTask); } (void) taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index f406c7ac48..d6929c5f00 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -799,11 +799,11 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id); taosThreadMutexLock(&pTask->lock); - SStreamTaskState* pState = streamTaskGetStatus(pTask); - if (pState->state != TASK_STATUS__CK) { + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state != TASK_STATUS__CK) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId, - pState->name, ref); + pState.name, ref); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pTask->pMeta, pTask); return; @@ -1303,7 +1303,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (delayDispatch) { taosThreadMutexLock(&pTask->lock); // we only set the dispatch msg info for current checkpoint trans - if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK && + if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK && pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) { ASSERT(pTask->chkInfo.pActiveInfo->transId == pMsgInfo->transId); stDebug("s-task:%s checkpoint-trigger msg to 0x%x rsp for checkpointId:%" PRId64 " transId:%d confirmed", diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index ffada66cca..b2c491de05 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -27,12 +27,12 @@ static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); bool streamTaskShouldStop(const SStreamTask* pTask) { - SStreamTaskState* pState = streamTaskGetStatus(pTask); - return (pState->state == TASK_STATUS__STOP) || (pState->state == TASK_STATUS__DROPPING); + SStreamTaskState pState = streamTaskGetStatus(pTask); + return (pState.state == TASK_STATUS__STOP) || (pState.state == TASK_STATUS__DROPPING); } bool streamTaskShouldPause(const SStreamTask* pTask) { - return (streamTaskGetStatus(pTask)->state == TASK_STATUS__PAUSE); + return (streamTaskGetStatus(pTask).state == TASK_STATUS__PAUSE); } static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBlock) { @@ -337,10 +337,10 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { stDebug( "s-task:%s fill-history task end, status:%s, scan wal elapsed time:%.2fSec, update related stream task:%s " "info, prepare transfer exec state", - id, streamTaskGetStatus(pTask)->name, el, pStreamTask->id.idStr); + id, streamTaskGetStatus(pTask).name, el, pStreamTask->id.idStr); } - ETaskStatus status = streamTaskGetStatus(pStreamTask)->state; + ETaskStatus status = streamTaskGetStatus(pStreamTask).state; STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; // It must be halted for a source stream task, since when the related scan-history-data task start scan the history @@ -350,7 +350,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { } else { ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP); - int32_t code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); + code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id, pStreamTask->id.idStr, tstrerror(code)); @@ -364,9 +364,9 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { // In case of sink tasks, no need to halt them. // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to // start the task state transfer procedure. - SStreamTaskState* pState = streamTaskGetStatus(pStreamTask); - status = pState->state; - char* p = pState->name; + SStreamTaskState pState = streamTaskGetStatus(pStreamTask); + status = pState.state; + char* p = pState.name; if (status == TASK_STATUS__STOP || status == TASK_STATUS__DROPPING) { stError("s-task:%s failed to transfer state from fill-history task:%s, status:%s", id, pStreamTask->id.idStr, p); streamMetaReleaseTask(pMeta, pStreamTask); @@ -391,7 +391,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { streamTaskSendCheckpointReq(pStreamTask); // 3. assign the status to the value that will be kept in disk - pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask)->state; + pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state; // 4. open the inputQ for all upstream tasks streamTaskOpenAllUpstreamInput(pStreamTask); @@ -598,7 +598,7 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB streamTaskReleaseState(pHTask); streamTaskReloadState(pTask); stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, - streamTaskGetStatus(pHTask)->name); + streamTaskGetStatus(pHTask).name); streamMetaReleaseTask(pTask->pMeta, pHTask); } else { @@ -628,7 +628,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { int32_t numOfBlocks = 0; SStreamQueueItem* pInput = NULL; - if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask)->state == TASK_STATUS__UNINIT)) { + if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask).state == TASK_STATUS__UNINIT)) { stDebug("s-task:%s stream task is stopped", id); return 0; } @@ -706,9 +706,9 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } else { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. taosThreadMutexLock(&pTask->lock); - SStreamTaskState* pState = streamTaskGetStatus(pTask); - if (pState->state == TASK_STATUS__CK) { - stDebug("s-task:%s checkpoint block received, set status:%s", id, pState->name); + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state == TASK_STATUS__CK) { + stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); streamTaskBuildCheckpoint(pTask); } else { // todo refactor int32_t code = 0; @@ -735,17 +735,17 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { // the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not // be updated by tryExec function, therefore, the schedStatus will always be the TASK_SCHED_STATUS__WAITING. bool streamTaskIsIdle(const SStreamTask* pTask) { - ETaskStatus status = streamTaskGetStatus(pTask)->state; + ETaskStatus status = streamTaskGetStatus(pTask).state; return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || status == TASK_STATUS__STOP || status == TASK_STATUS__DROPPING); } bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { - SStreamTaskState* pState = streamTaskGetStatus(pTask); + SStreamTaskState pState = streamTaskGetStatus(pTask); - ETaskStatus st = pState->state; + ETaskStatus st = pState.state; if (pStatus != NULL) { - *pStatus = pState->name; + *pStatus = pState.name; } // pause & halt will still run for sink tasks. @@ -776,7 +776,7 @@ int32_t streamResumeTask(SStreamTask* pTask) { setLastExecTs(pTask, taosGetTimestampMs()); - char* p = streamTaskGetStatus(pTask)->name; + char* p = streamTaskGetStatus(pTask).name; stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p, pTask->status.schedStatus, pTask->status.lastExecTs); @@ -804,7 +804,7 @@ int32_t streamExecTask(SStreamTask* pTask) { if (schedStatus == TASK_SCHED_STATUS__WAITING) { streamResumeTask(pTask); } else { - char* p = streamTaskGetStatus(pTask)->name; + char* p = streamTaskGetStatus(pTask).name; stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, p, pTask->status.schedStatus); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9cf3909867..a3f833294a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1128,11 +1128,11 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { taosThreadMutexLock(&pTask->lock); - SStreamTaskState* pState = streamTaskGetStatus(pTask); - if (pState->state == TASK_STATUS__CK) { + SStreamTaskState pState = streamTaskGetStatus(pTask); + if (pState.state == TASK_STATUS__CK) { streamTaskSetFailedCheckpointId(pTask); } else { - stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState->name); + stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState.name); } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 0a54dfa4c8..4cb87820c6 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -94,13 +94,13 @@ int32_t streamTaskResumeInFuture(SStreamTask* pTask) { void streamTaskResumeHelper(void* param, void* tmrId) { SStreamTask* pTask = (SStreamTask*)param; SStreamTaskId* pId = &pTask->id; - SStreamTaskState* p = streamTaskGetStatus(pTask); + SStreamTaskState p = streamTaskGetStatus(pTask); - if (p->state == TASK_STATUS__DROPPING || p->state == TASK_STATUS__STOP) { + if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) { streamTaskSetSchedStatusInactive(pTask); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p->name, ref); + stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref); streamMetaReleaseTask(pTask->pMeta, pTask); return; @@ -130,7 +130,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { return; } - if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) { + if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) { stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger); } else { if (status == TASK_TRIGGER_STATUS__ACTIVE) { diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index cf23b776e2..3c7ad2639a 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -44,13 +44,13 @@ static void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunc static void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now); static int32_t streamTaskSetReady(SStreamTask* pTask) { - int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); - SStreamTaskState* p = streamTaskGetStatus(pTask); + int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); + SStreamTaskState p = streamTaskGetStatus(pTask); - if ((p->state == TASK_STATUS__SCAN_HISTORY) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { + if ((p.state == TASK_STATUS__SCAN_HISTORY) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { int32_t numOfUps = taosArrayGetSize(pTask->upstreamInfo.pList); stDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", - pTask->id.idStr, pTask->info.taskLevel, numOfUps, p->name); + pTask->id.idStr, pTask->info.taskLevel, numOfUps, p.name); } ASSERT(pTask->status.downstreamReady == 0); @@ -59,7 +59,7 @@ static int32_t streamTaskSetReady(SStreamTask* pTask) { pTask->execInfo.readyTs = taosGetTimestampMs(); int64_t el = (pTask->execInfo.readyTs - pTask->execInfo.checkTs); stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", - pTask->id.idStr, numOfDowns, el, p->name); + pTask->id.idStr, numOfDowns, el, p.name); return TSDB_CODE_SUCCESS; } @@ -96,7 +96,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) int32_t code = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, &p); if (p == NULL) { stError("s-task:0x%x failed to acquire task, status:%s, not exec scan-history data", pTask->id.taskId, - streamTaskGetStatus(pTask)->name); + streamTaskGetStatus(pTask).name); return TSDB_CODE_SUCCESS; } @@ -118,7 +118,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) int32_t streamTaskStartScanHistory(SStreamTask* pTask) { int32_t level = pTask->info.taskLevel; - ETaskStatus status = streamTaskGetStatus(pTask)->state; + ETaskStatus status = streamTaskGetStatus(pTask).state; ASSERT((pTask->status.downstreamReady == 1) && (status == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1)); @@ -139,8 +139,8 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { streamTaskSetReady(pTask); streamTaskSetRangeStreamCalc(pTask); - SStreamTaskState* p = streamTaskGetStatus(pTask); - ASSERT(p->state == TASK_STATUS__READY); + SStreamTaskState p = streamTaskGetStatus(pTask); + ASSERT(p.state == TASK_STATUS__READY); int8_t schedStatus = pTask->status.schedStatus; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { @@ -149,10 +149,10 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { startVer = pTask->chkInfo.nextProcessVer; } - stDebug("s-task:%s status:%s, sched-status:%d, ready for data from wal ver:%" PRId64, id, p->name, schedStatus, + stDebug("s-task:%s status:%s, sched-status:%d, ready for data from wal ver:%" PRId64, id, p.name, schedStatus, startVer); } else { - stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p->name, schedStatus); + stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p.name, schedStatus); } return TSDB_CODE_SUCCESS; @@ -163,10 +163,10 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) { streamTaskSetReady(pTask); streamTaskSetRangeStreamCalc(pTask); - SStreamTaskState* p = streamTaskGetStatus(pTask); - ASSERT((p->state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1)); + SStreamTaskState p = streamTaskGetStatus(pTask); + ASSERT((p.state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1)); - stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p->name); + stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name); streamTaskStartScanHistory(pTask); // NOTE: there will be an deadlock if launch fill history here. @@ -206,11 +206,11 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { ASSERT(hTaskId != 0); // check stream task status in the first place. - SStreamTaskState* pStatus = streamTaskGetStatus(pTask); - if (pStatus->state != TASK_STATUS__READY && pStatus->state != TASK_STATUS__HALT && - pStatus->state != TASK_STATUS__PAUSE) { + SStreamTaskState pStatus = streamTaskGetStatus(pTask); + if (pStatus.state != TASK_STATUS__READY && pStatus.state != TASK_STATUS__HALT && + pStatus.state != TASK_STATUS__PAUSE) { stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, - pStatus->name); + pStatus.name); streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); return -1; // todo set the correct error code @@ -310,7 +310,7 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); taosMemoryFree(pInfo); } else { - char* p = streamTaskGetStatus(pTask)->name; + char* p = streamTaskGetStatus(pTask).name; int32_t hTaskId = pHTaskInfo->id.taskId; stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", @@ -342,7 +342,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { if (streamTaskShouldStop(*ppTask)) { ASSERT((*ppTask)->status.timerActive >= 1); - char* p = streamTaskGetStatus(*ppTask)->name; + char* p = streamTaskGetStatus(*ppTask).name; int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1); stDebug("s-task:%s status:%s should stop, quit launch fill-history task timer, retry:%d, ref:%d", (*ppTask)->id.idStr, p, (*ppTask)->hTaskInfo.retryTimes, ref); @@ -544,10 +544,10 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) { SStreamTask* pTask = param; pTask->schedHistoryInfo.numOfTicks -= 1; - SStreamTaskState* p = streamTaskGetStatus(pTask); - if (p->state == TASK_STATUS__DROPPING || p->state == TASK_STATUS__STOP) { + SStreamTaskState p = streamTaskGetStatus(pTask); + if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p->name, ref); + stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p.name, ref); streamMetaReleaseTask(pTask->pMeta, pTask); return; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 6264a262da..6876cf5beb 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -199,9 +199,9 @@ void tFreeStreamTask(SStreamTask* pTask) { ETaskStatus status1 = TASK_STATUS__UNINIT; taosThreadMutexLock(&pTask->lock); if (pTask->status.pSM != NULL) { - SStreamTaskState* pStatus = streamTaskGetStatus(pTask); - p = pStatus->name; - status1 = pStatus->state; + SStreamTaskState pStatus = streamTaskGetStatus(pTask); + p = pStatus.name; + status1 = pStatus.state; } taosThreadMutexUnlock(&pTask->lock); @@ -728,7 +728,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { if (resetRelHalt) { stDebug("s-task:0x%" PRIx64 " set the persistent status attr to be ready, prev:%s, status in sm:%s", sTaskId.taskId, streamTaskGetStatusStr((*ppStreamTask)->status.taskStatus), - streamTaskGetStatus(*ppStreamTask)->name); + streamTaskGetStatus(*ppStreamTask).name); (*ppStreamTask)->status.taskStatus = TASK_STATUS__READY; } @@ -861,7 +861,7 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) { STaskStatusEntry entry = { .id = streamTaskGetTaskId(pTask), - .status = streamTaskGetStatus(pTask)->state, + .status = streamTaskGetStatus(pTask).state, .nodeId = pMeta->vgId, .stage = pMeta->stage, @@ -906,12 +906,12 @@ void streamTaskPause(SStreamTask* pTask) { } void streamTaskResume(SStreamTask* pTask) { - SStreamTaskState prevState = *streamTaskGetStatus(pTask); + SStreamTaskState prevState = streamTaskGetStatus(pTask); SStreamMeta* pMeta = pTask->pMeta; int32_t code = streamTaskRestoreStatus(pTask); if (code == TSDB_CODE_SUCCESS) { - char* pNew = streamTaskGetStatus(pTask)->name; + char* pNew = streamTaskGetStatus(pTask).name; int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); } else { diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index e2d988b876..1af47cf43f 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -67,7 +67,7 @@ static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus nex static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; } static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) { - char* p = streamTaskGetStatus(pTask)->name; + char* p = streamTaskGetStatus(pTask).name; stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p, GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); @@ -305,7 +305,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt while (1) { // wait for the task to be here taosThreadMutexLock(&pTask->lock); - ETaskStatus s = streamTaskGetStatus(pTask)->state; + ETaskStatus s = streamTaskGetStatus(pTask).state; taosThreadMutexUnlock(&pTask->lock); if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already @@ -510,8 +510,8 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even return TSDB_CODE_SUCCESS; } -SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask) { - return &pTask->status.pSM->current; // copy one obj in case of multi-thread environment +SStreamTaskState streamTaskGetStatus(const SStreamTask* pTask) { + return pTask->status.pSM->current; // copy one obj in case of multi-thread environment } ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask) { From 2aa4d6028ba40fda38dccd02ebec045f7da9b193 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 16:56:36 +0800 Subject: [PATCH 11/45] refactor: check return value for each function. --- include/libs/stream/tstream.h | 4 ++-- source/libs/stream/inc/streamInt.h | 5 ----- source/libs/stream/src/streamMeta.c | 16 +++++++++++----- source/libs/stream/src/streamTask.c | 12 +++++++++--- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 609b977536..b6569ea771 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -639,7 +639,7 @@ bool streamTaskShouldPause(const SStreamTask* pStatus); bool streamTaskIsIdle(const SStreamTask* pTask); bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus); -char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); +int32_t createStreamTaskIdStr(int64_t streamId, int32_t taskId, const char** pId); SStreamTaskState streamTaskGetStatus(const SStreamTask* pTask); const char* streamTaskGetStatusStr(ETaskStatus status); void streamTaskResetStatus(SStreamTask* pTask); @@ -758,7 +758,7 @@ void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta); void streamMetaResetStartInfo(STaskStartInfo* pMeta, int32_t vgId); -SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); +int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pTaskList); void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); void streamMetaLoadAllTasks(SStreamMeta* pMeta); int32_t streamMetaStartAllTasks(SStreamMeta* pMeta); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 008d066717..3027abc4be 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -65,11 +65,6 @@ struct SActiveCheckpointInfo { tmr_h pSendReadyMsgTmr; }; -struct SConsensusCheckpoint { - int8_t inProcess; - -}; - typedef struct { int8_t type; SSDataBlock* pBlock; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a3f833294a..3940866488 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1104,13 +1104,14 @@ int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, voi return taosScheduleTask(pMeta->qHandle, &schedMsg); } -SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { +int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { + *pList = NULL; SArray* pTaskList = taosArrayDup(pMeta->pTaskList, NULL); bool sendMsg = pMeta->sendMsgBeforeClosing; if (!sendMsg) { stDebug("vgId:%d no need to send msg to mnode before closing tasks", pMeta->vgId); - return pTaskList; + return TSDB_CODE_SUCCESS; } stDebug("vgId:%d send msg to mnode before closing all tasks", pMeta->vgId); @@ -1141,7 +1142,7 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { streamMetaSendHbHelper(pMeta); pMeta->sendMsgBeforeClosing = false; - return pTaskList; + return TSDB_CODE_SUCCESS; } void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) { @@ -1311,14 +1312,19 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { int64_t st = taosGetTimestampMs(); // send hb msg to mnode before closing all tasks. - SArray* pTaskList = streamMetaSendMsgBeforeCloseTasks(pMeta); + SArray* pTaskList = NULL; + int32_t code = streamMetaSendMsgBeforeCloseTasks(pMeta, &pTaskList); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + int32_t numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); + code = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); if (code != TSDB_CODE_SUCCESS) { continue; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 6876cf5beb..892ed6f5bf 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -359,7 +359,7 @@ static void setInitialVersionInfo(SStreamTask* pTask, int64_t ver) { } int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver) { - pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId); + (void) createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId, &pTask->id.idStr); pTask->refCnt = 1; pTask->inputq.status = TASK_INPUT_STATUS__NORMAL; @@ -997,10 +997,16 @@ SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { return NULL; } -char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { +int32_t createStreamTaskIdStr(int64_t streamId, int32_t taskId, const char** pId) { char buf[128] = {0}; sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId); - return taosStrdup(buf); + *pId = taosStrdup(buf); + + if (*pId == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } else { + return TSDB_CODE_SUCCESS; + } } static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq) { From aee48971de58a42d8c82d037584cc2cef336ee7d Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 17 Jul 2024 17:52:29 +0800 Subject: [PATCH 12/45] fix: taoshash put --- source/client/src/clientHb.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 6 +++--- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqOffset.c | 2 +- source/libs/parser/src/parTranslater.c | 2 +- source/libs/stream/src/streamMeta.c | 2 +- source/util/src/thash.c | 29 +++++++++++++------------- 7 files changed, 22 insertions(+), 23 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index a7d459ff31..e45ca2b872 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -744,7 +744,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); } - if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { + if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) != 0) { taosMemoryFree(user); code = terrno ? terrno : TSDB_CODE_APP_ERROR; goto _return; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index a48188430f..37b7ed944f 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -407,7 +407,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con goto _err; } - if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { + if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != 0) { goto _err; } @@ -533,12 +533,12 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) taosArrayDestroy(pUidArray); return TSDB_CODE_FAILED; } - if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) < 0) { + if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) { return TSDB_CODE_FAILED; } } } else { - if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) < 0) { + if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) { return TSDB_CODE_FAILED; } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ac57a003c5..7daf807c27 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -612,7 +612,7 @@ int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } tDecoderClear(&decoder); - if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) { + if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 8b0e039ad5..4456359717 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -74,7 +74,7 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { } tDecoderClear(&decoder); - if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { + if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) != 0) { return -1; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 104ffcc945..4684e4a62c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -12753,7 +12753,7 @@ static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFil if (code) break; - if (taosHashPut(pIdxHash, &idx, sizeof(idx), NULL, 0) < 0) { + if (taosHashPut(pIdxHash, &idx, sizeof(idx), NULL, 0) != 0) { code = terrno; goto _OUT; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d0b1f6ca93..3ef5a4ac10 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -922,7 +922,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { continue; } - if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) < 0) { + if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) != 0) { stError("s-task:0x%x failed to put into hashTable, code:%s, continue", pTask->id.taskId, tstrerror(terrno)); taosArrayPop(pMeta->pTaskList); tFreeStreamTask(pTask); diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 8c0ca3e5a7..54fb9ade93 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -316,7 +316,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) { if (pHashObj == NULL || key == NULL || keyLen == 0) { terrno = TSDB_CODE_INVALID_PTR; - return -1; + return TSDB_CODE_INVALID_PTR; } uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); @@ -331,6 +331,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo // disable resize taosHashRLock(pHashObj); + int32_t code = TSDB_CODE_SUCCESS; uint32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; @@ -352,36 +353,34 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo // no data in hash table with the specified key, add it into hash table SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); if (pNewNode == NULL) { - return -1; + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _exit; } pushfrontNodeInEntryList(pe, pNewNode); - taosHashEntryWUnlock(pHashObj, pe); - - // enable resize - taosHashRUnlock(pHashObj); atomic_add_fetch_64(&pHashObj->size, 1); - - return 0; } else { // not support the update operation, return error if (pHashObj->enableUpdate) { SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); if (pNewNode == NULL) { - return -1; + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _exit; } doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); } else { terrno = TSDB_CODE_DUP_KEY; + code = terrno; + goto _exit; } - - taosHashEntryWUnlock(pHashObj, pe); - - // enable resize - taosHashRUnlock(pHashObj); - return pHashObj->enableUpdate ? 0 : -2; } +_exit: + taosHashEntryWUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); + return code; } static void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef); From 2b00d6549dfa7328fd0cf0e566b070a5fc38c7f4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 17:57:08 +0800 Subject: [PATCH 13/45] refactor: check return value for each function. --- source/libs/stream/inc/streamInt.h | 15 +++++---- source/libs/stream/src/streamCheckpoint.c | 7 ++-- source/libs/stream/src/streamData.c | 39 +++++++++++------------ source/libs/stream/src/streamDispatch.c | 8 +++-- source/libs/stream/src/streamExec.c | 15 ++++++--- source/libs/stream/src/streamHb.c | 12 +++---- source/libs/stream/src/streamMeta.c | 8 ++--- 7 files changed, 54 insertions(+), 50 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 3027abc4be..ae74ed204e 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -169,11 +169,12 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask); void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); void clearBufferedDispatchMsg(SStreamTask* pTask); -int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); -SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); -SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, - SArray* pRes); -void destroyStreamDataBlock(SStreamDataBlock* pBlock); +int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); +int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg, + SStreamDataBlock** pBlock); +int32_t createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes, + SStreamDataBlock** pBlock); +void destroyStreamDataBlock(SStreamDataBlock* pBlock); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData, const char* idstr); int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock); @@ -214,8 +215,8 @@ int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); void streamMetaRemoveDB(void* arg, char* key); void streamMetaHbToMnode(void* param, void* tmrId); -SMetaHbInfo* createMetaHbInfo(int64_t* pRid); -void* destroyMetaHbInfo(SMetaHbInfo* pInfo); +int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes); +void destroyMetaHbInfo(SMetaHbInfo* pInfo); void streamMetaWaitForHbTmrQuit(SStreamMeta* pMeta); void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSendCount); int32_t streamMetaSendHbHelper(SStreamMeta* pMeta); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index d0ad38ea79..8de923e900 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -29,10 +29,9 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList); static void checkpointTriggerMonitorFn(void* param, void* tmrId); -int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, - int32_t transId, int32_t srcTaskId) { - SStreamDataBlock* pChkpoint; - +int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId, + int32_t srcTaskId, SStreamDataBlock** pRes) { + SStreamDataBlock* pChkpoint = NULL; int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pChkpoint); if (code) { return code; diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index bf499293b4..4235b6c1bb 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -15,13 +15,11 @@ #include "streamInt.h" -SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg) { - SStreamDataBlock* pData; - +int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg, SStreamDataBlock** pRes) { + SStreamDataBlock* pData = NULL; int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen, (void**)&pData); if (code) { - terrno = code; - return NULL; + return terrno = code; } pData->type = blockType; @@ -32,7 +30,7 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe SArray* pArray = taosArrayInit_s(sizeof(SSDataBlock), blockNum); if (pArray == NULL) { taosFreeQitem(pData); - return NULL; + return code; } ASSERT((pReq->blockNum == taosArrayGetSize(pReq->data)) && (pReq->blockNum == taosArrayGetSize(pReq->dataLen))); @@ -69,37 +67,36 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe } pData->blocks = pArray; - return pData; + *pRes = pData; + + return code; } -SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, - SArray* pRes) { - SStreamDataBlock* pStreamBlocks; - - int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, resultSize, (void**)&pStreamBlocks); +int32_t createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes, + SStreamDataBlock** pBlock) { + int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, resultSize, (void**)pBlock); if (code) { taosArrayClearEx(pRes, (FDelete)blockDataFreeRes); - terrno = code; - return NULL; + return terrno = code; } - pStreamBlocks->srcTaskId = pTask->id.taskId; - pStreamBlocks->type = STREAM_INPUT__DATA_BLOCK; - pStreamBlocks->blocks = pRes; + (*pBlock)->srcTaskId = pTask->id.taskId; + (*pBlock)->type = STREAM_INPUT__DATA_BLOCK; + (*pBlock)->blocks = pRes; if (pItem == NULL) { - return pStreamBlocks; + return code; } if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)pItem; - pStreamBlocks->sourceVer = pSubmit->ver; + (*pBlock)->sourceVer = pSubmit->ver; } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)pItem; - pStreamBlocks->sourceVer = pMerged->ver; + (*pBlock)->sourceVer = pMerged->ver; } - return pStreamBlocks; + return code; } void destroyStreamDataBlock(SStreamDataBlock* pBlock) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 2023da43ad..9e4b6bc09d 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1408,8 +1408,10 @@ static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchR static int32_t streamTaskAppendInputBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq) { int8_t status = 0; - SStreamDataBlock* pBlock = createStreamBlockFromDispatchMsg(pReq, pReq->type, pReq->srcVgId); - if (pBlock == NULL) { + SStreamDataBlock* pBlock = NULL; + + int32_t code = createStreamBlockFromDispatchMsg(pReq, pReq->type, pReq->srcVgId, &pBlock); + if (code) { streamTaskInputFail(pTask); status = TASK_INPUT_STATUS__FAILED; stError("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, @@ -1419,7 +1421,7 @@ static int32_t streamTaskAppendInputBlocks(SStreamTask* pTask, const SStreamDisp pTask->status.appendTranstateBlock = true; } - int32_t code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pBlock); + code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pBlock); // input queue is full, upstream is blocked now status = (code == TSDB_CODE_SUCCESS) ? TASK_INPUT_STATUS__NORMAL : TASK_INPUT_STATUS__BLOCKED; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b2c491de05..238db27d60 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -67,8 +67,10 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return TSDB_CODE_SUCCESS; } - SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(pItem, pTask, size, pRes); - if (pStreamBlocks == NULL) { + SStreamDataBlock* pStreamBlocks = NULL; + + int32_t code = createStreamBlockFromResults(pItem, pTask, size, pRes, &pStreamBlocks); + if (code) { stError("s-task:%s failed to create result stream data block, code:%s", pTask->id.idStr, tstrerror(terrno)); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return TSDB_CODE_OUT_OF_MEMORY; @@ -77,7 +79,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* stDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, SIZE_IN_MiB(size)); - int32_t code = doOutputResultBlockImpl(pTask, pStreamBlocks); + code = doOutputResultBlockImpl(pTask, pStreamBlocks); if (code != TSDB_CODE_SUCCESS) { // back pressure and record position return code; } @@ -187,7 +189,12 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { int32_t code = TSDB_CODE_SUCCESS; if (taosArrayGetSize(pRes) > 0) { - SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes); + SStreamDataBlock* pStreamBlocks = NULL; + code = createStreamBlockFromResults(NULL, pTask, size, pRes, &pStreamBlocks); + if (code) { + return code; + } + code = doOutputResultBlockImpl(pTask, pStreamBlocks); if (code != TSDB_CODE_SUCCESS) { // should not have error code stError("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code)); diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 16cb23de10..d8ee15c909 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -275,11 +275,11 @@ void streamMetaHbToMnode(void* param, void* tmrId) { taosReleaseRef(streamMetaId, rid); } -SMetaHbInfo* createMetaHbInfo(int64_t* pRid) { +int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes) { + *pRes = NULL; SMetaHbInfo* pInfo = taosMemoryCalloc(1, sizeof(SMetaHbInfo)); if (pInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return pInfo; + return TSDB_CODE_OUT_OF_MEMORY; } pInfo->hbTmr = taosTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); @@ -287,10 +287,10 @@ SMetaHbInfo* createMetaHbInfo(int64_t* pRid) { pInfo->stopFlag = 0; pInfo->msgSendTs = -1; pInfo->hbCount = 0; - return pInfo; + return TSDB_CODE_SUCCESS; } -void* destroyMetaHbInfo(SMetaHbInfo* pInfo) { +void destroyMetaHbInfo(SMetaHbInfo* pInfo) { if (pInfo != NULL) { tCleanupStreamHbMsg(&pInfo->hbMsg); @@ -301,8 +301,6 @@ void* destroyMetaHbInfo(SMetaHbInfo* pInfo) { taosMemoryFree(pInfo); } - - return NULL; } void streamMetaWaitForHbTmrQuit(SStreamMeta* pMeta) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 3940866488..d9c937a14f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -402,9 +402,8 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); metaRefMgtAdd(pMeta->vgId, pRid); - pMeta->pHbInfo = createMetaHbInfo(pRid); - if (pMeta->pHbInfo == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = createMetaHbInfo(pRid, &pMeta->pHbInfo); + if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -538,7 +537,8 @@ void streamMetaCloseImpl(void* arg) { taosHashCleanup(pMeta->startInfo.pReadyTaskSet); taosHashCleanup(pMeta->startInfo.pFailedTaskSet); - pMeta->pHbInfo = destroyMetaHbInfo(pMeta->pHbInfo); + destroyMetaHbInfo(pMeta->pHbInfo); + pMeta->pHbInfo = NULL; taosMemoryFree(pMeta->path); taosThreadMutexDestroy(&pMeta->backendMutex); From b35d107bf9930cdaa808eb70c05bc0b49842c0f3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 18:55:04 +0800 Subject: [PATCH 14/45] refactor: check return value for each function. --- source/libs/stream/inc/streamInt.h | 33 ++--- source/libs/stream/src/streamData.c | 20 ++- source/libs/stream/src/streamQueue.c | 29 ++-- source/libs/stream/src/streamTask.c | 10 +- source/libs/stream/src/streamTaskSm.c | 202 +++++++++++++++----------- 5 files changed, 167 insertions(+), 127 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index ae74ed204e..fd248861e3 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -161,8 +161,7 @@ extern int32_t streamMetaId; int32_t streamTimerInit(); void streamTimerCleanUp(); - -void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); +void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); @@ -202,24 +201,24 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIt int32_t streamQueueItemGetSize(const SStreamQueueItem* pItem); void streamQueueItemIncSize(const SStreamQueueItem* pItem, int32_t size); const char* streamQueueItemGetTypeStr(int32_t type); -SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); +int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes); int32_t streamTransferStatePrepare(SStreamTask* pTask); -SStreamQueue* streamQueueOpen(int64_t cap); -void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); -void streamQueueProcessSuccess(SStreamQueue* queue); -void streamQueueProcessFail(SStreamQueue* queue); -void* streamQueueNextItem(SStreamQueue* pQueue); -void streamFreeQitem(SStreamQueueItem* data); -int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); +int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ); +void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); +void streamQueueProcessSuccess(SStreamQueue* queue); +void streamQueueProcessFail(SStreamQueue* queue); +void* streamQueueNextItem(SStreamQueue* pQueue); +void streamFreeQitem(SStreamQueueItem* data); +int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); -void streamMetaRemoveDB(void* arg, char* key); -void streamMetaHbToMnode(void* param, void* tmrId); -int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes); -void destroyMetaHbInfo(SMetaHbInfo* pInfo); -void streamMetaWaitForHbTmrQuit(SStreamMeta* pMeta); -void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSendCount); -int32_t streamMetaSendHbHelper(SStreamMeta* pMeta); +void streamMetaRemoveDB(void* arg, char* key); +void streamMetaHbToMnode(void* param, void* tmrId); +int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes); +void destroyMetaHbInfo(SMetaHbInfo* pInfo); +void streamMetaWaitForHbTmrQuit(SStreamMeta* pMeta); +void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSendCount); +int32_t streamMetaSendHbHelper(SStreamMeta* pMeta); ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType(); diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 4235b6c1bb..af4946cf81 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -185,8 +185,8 @@ int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubm } // todo handle memory error -SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) { - terrno = 0; +int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes) { + *pRes = NULL; if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; @@ -196,7 +196,8 @@ SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueI streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); taosFreeQitem(pElem); - return dst; + *pRes = dst; + return TSDB_CODE_SUCCESS; } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem; @@ -204,12 +205,13 @@ SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueI streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); taosFreeQitem(pElem); - return dst; + *pRes = dst; + *pRes = dst; + return TSDB_CODE_SUCCESS; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); if (pMerged == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } streamQueueItemIncSize((SStreamQueueItem*)pMerged, streamQueueItemGetSize(pElem)); @@ -219,11 +221,13 @@ SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueI taosFreeQitem(dst); taosFreeQitem(pElem); - return (SStreamQueueItem*)pMerged; + + *pRes = (SStreamQueueItem*)pMerged; + return TSDB_CODE_SUCCESS; } else { stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type), dst->type); - return NULL; + return TSDB_CODE_FAILED; } } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 7bc50417bd..b8cdcd4cf5 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -41,34 +41,34 @@ static void streamQueueCleanup(SStreamQueue* pQueue) { static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } -SStreamQueue* streamQueueOpen(int64_t cap) { - int32_t code; +int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ) { + *pQ = NULL; + int32_t code = 0; SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue)); if (pQueue == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } code = taosOpenQueue(&pQueue->pQueue); if (code) { taosMemoryFreeClear(pQueue); - terrno = code; - return NULL; + return code; } code = taosAllocateQall(&pQueue->qall); if (code) { taosCloseQueue(pQueue->pQueue); taosMemoryFree(pQueue); - terrno = code; - return NULL; + return code; } pQueue->status = STREAM_QUEUE__SUCESS; taosSetQueueCapacity(pQueue->pQueue, cap); taosSetQueueMemoryCapacity(pQueue->pQueue, cap * 1024); - return pQueue; + + *pQ = pQueue; + return code; } void streamQueueClose(SStreamQueue* pQueue, int32_t taskId) { @@ -227,12 +227,11 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte *pInput = qItem; } else { // merge current block failed, let's handle the already merged blocks. - void* newRet = streamQueueMergeQueueItem(*pInput, qItem); - if (newRet == NULL) { - if (terrno != 0) { - stError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks, - tstrerror(terrno)); - } + void* newRet = NULL; + int32_t code = streamQueueMergeQueueItem(*pInput, qItem, (SStreamQueueItem**)&newRet); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks, + tstrerror(terrno)); *blockSize = streamQueueItemGetSize(*pInput); if (taskLevel == TASK_LEVEL__SINK) { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 11eea98e98..39e12a9da7 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -366,9 +366,9 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->inputq.status = TASK_INPUT_STATUS__NORMAL; pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL; - pTask->inputq.queue = streamQueueOpen(512 << 10); - pTask->outputq.queue = streamQueueOpen(512 << 10); - if (pTask->inputq.queue == NULL || pTask->outputq.queue == NULL) { + int32_t code1 = streamQueueOpen(512 << 10, &pTask->inputq.queue); + int32_t code2 = streamQueueOpen(512 << 10, &pTask->outputq.queue); + if (code1 || code2) { stError("s-task:%s failed to prepare the input/output queue, initialize task failed", pTask->id.idStr); return TSDB_CODE_OUT_OF_MEMORY; } @@ -379,8 +379,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i int32_t code = streamCreateStateMachine(pTask); if (pTask->status.pSM == NULL || code != TSDB_CODE_SUCCESS) { stError("s-task:%s failed create state-machine for stream task, initialization failed, code:%s", pTask->id.idStr, - tstrerror(terrno)); - return terrno; + tstrerror(code)); + return code; } pTask->execInfo.created = taosGetTimestampMs(); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 1af47cf43f..7e47857a39 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -20,7 +20,17 @@ #include "ttimer.h" #include "wal.h" +static int32_t initRes = 0; + #define GET_EVT_NAME(_ev) (StreamTaskEventList[(_ev)].name) +#define CHECK_RET_VAL(_exec) \ + do { \ + void* p = (_exec); \ + if (p == NULL) { \ + initRes = TSDB_CODE_OUT_OF_MEMORY; \ + return; \ + } \ + } while (0); SStreamTaskState StreamTaskStatusList[9] = { {.state = TASK_STATUS__READY, .name = "ready"}, @@ -73,10 +83,17 @@ static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEv GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); SArray* pList = pTask->status.pSM->pWaitingEventList; - taosArrayPush(pList, pEvtInfo); - stDebug("s-task:%s add into waiting list, total waiting events:%d", pTask->id.idStr, (int32_t)taosArrayGetSize(pList)); - return 0; + void* px = taosArrayPush(pList, pEvtInfo); + if (px == NULL) { + stError("s-task:%s failed to add into waiting list, total waiting events:%d, code: out of memory", pTask->id.idStr, + (int32_t)taosArrayGetSize(pList)); + return TSDB_CODE_OUT_OF_MEMORY; + } else { + stDebug("s-task:%s add into waiting list, total waiting events:%d", pTask->id.idStr, + (int32_t)taosArrayGetSize(pList)); + return TSDB_CODE_SUCCESS; + } } static int32_t stopTaskSuccFn(SStreamTask* pTask) { @@ -177,14 +194,14 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, GET_EVT_NAME(pEvtInfo->event), pSM->current.name); // remove it - taosArrayPop(pSM->pWaitingEventList); + (void) taosArrayPop(pSM->pWaitingEventList); STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event); ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL); pSM->pActiveTrans = pNextTrans; pSM->startTs = taosGetTimestampMs(); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); code = pNextTrans->pAction(pSM->pTask); if (pNextTrans->autoInvokeEndFn) { @@ -193,7 +210,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, return code; } } else { - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, pSM->current.name, GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); @@ -228,9 +245,9 @@ static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent eve int32_t streamTaskRestoreStatus(SStreamTask* pTask) { SStreamTaskSM* pSM = pTask->status.pSM; - int32_t code = 0; + int32_t code = 0; - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) { SStreamTaskState state = pSM->current; @@ -244,21 +261,25 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask) { if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); - doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); + code = doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); } else { stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); } } else { - removeEventInWaitingList(pTask, TASK_EVENT_PAUSE); - code = -1; // failed to restore the status + (void)removeEventInWaitingList(pTask, TASK_EVENT_PAUSE); // ignore the return value, + code = TSDB_CODE_FAILED; // failed to restore the status, since it is not in pause status } - taosThreadMutexUnlock(&pTask->lock); + (void)taosThreadMutexUnlock(&pTask->lock); return code; } int32_t streamCreateStateMachine(SStreamTask* pTask) { - initStateTransferTable(); + int32_t code = initStateTransferTable(); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + const char* id = pTask->id.idStr; SStreamTaskSM* pSM = taosMemoryCalloc(1, sizeof(SStreamTaskSM)); @@ -297,16 +318,21 @@ void streamDestroyStateMachine(SStreamTaskSM* pSM) { static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans) { SStreamTask* pTask = pSM->pTask; const char* id = pTask->id.idStr; + int32_t code = 0; if (pTrans->attachEvent.event != 0) { - attachWaitedEvent(pTask, &pTrans->attachEvent); - taosThreadMutexUnlock(&pTask->lock); + code = attachWaitedEvent(pTask, &pTrans->attachEvent); + if (code) { + return code; + } + + (void) taosThreadMutexUnlock(&pTask->lock); while (1) { // wait for the task to be here - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); ETaskStatus s = streamTaskGetStatus(pTask).state; - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event)); @@ -323,42 +349,49 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt } else { // override current active trans pSM->pActiveTrans = pTrans; pSM->startTs = taosGetTimestampMs(); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); - int32_t code = pTrans->pAction(pTask); - // todo handle error code; + code = pTrans->pAction(pTask); if (pTrans->autoInvokeEndFn) { - streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL); + int32_t c = streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = c; + } } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans, __state_trans_user_fn callbackFn, void* param) { SStreamTask* pTask = pSM->pTask; + int32_t code = 0; + if (pTrans->attachEvent.event != 0) { SFutureHandleEventInfo info = pTrans->attachEvent; info.pParam = param; info.callBackFn = callbackFn; - attachWaitedEvent(pTask, &info); - taosThreadMutexUnlock(&pTask->lock); + code = attachWaitedEvent(pTask, &info); + (void) taosThreadMutexUnlock(&pTask->lock); } else { // override current active trans pSM->pActiveTrans = pTrans; pSM->startTs = taosGetTimestampMs(); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); - int32_t code = pTrans->pAction(pTask); + code = pTrans->pAction(pTask); // todo handle error code; if (pTrans->autoInvokeEndFn) { - streamTaskOnHandleEventSuccess(pSM, event, callbackFn, param); + int32_t c = streamTaskOnHandleEventSuccess(pSM, event, callbackFn, param); + if (code == TSDB_CODE_SUCCESS) { + code = c; + } } } - return TSDB_CODE_SUCCESS; + return code; } int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { @@ -367,11 +400,11 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { STaskStateTrans* pTrans = NULL; while (1) { - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { EStreamTaskEvent evt = pSM->pActiveTrans->event; - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); @@ -381,7 +414,7 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { pTrans = streamTaskFindTransform(pSM->current.state, event); if (pTrans == NULL) { stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event)); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -406,11 +439,11 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _ STaskStateTrans* pTrans = NULL; while (1) { - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { EStreamTaskEvent evt = pSM->pActiveTrans->event; - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); @@ -420,7 +453,7 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _ pTrans = streamTaskFindTransform(pSM->current.state, event); if (pTrans == NULL) { stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -449,9 +482,10 @@ static void keepPrevInfo(SStreamTaskSM* pSM) { int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { SStreamTask* pTask = pSM->pTask; const char* id = pTask->id.idStr; + int32_t code = 0; // do update the task status - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); STaskStateTrans* pTrans = pSM->pActiveTrans; if (pTrans == NULL) { @@ -463,14 +497,14 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } if (pTrans->event != event) { stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pTrans->event)); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -480,34 +514,38 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even pSM->pActiveTrans = NULL; // todo remove it + // todo: handle the error code // on success callback, add into lock if necessary, or maybe we should add an option for this? - pTrans->pSuccAction(pTask); + code = pTrans->pSuccAction(pTask); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); // todo: add parameter to control lock // after handling the callback function assigned by invoker, go on handling the waiting tasks if (callbackFn != NULL) { stDebug("s-task:%s start to handle user-specified callback fn for event:%s", id, GET_EVT_NAME(pTrans->event)); - callbackFn(pSM->pTask, param); + int32_t ret = callbackFn(pSM->pTask, param); + if (ret != TSDB_CODE_SUCCESS) { + // todo handle error + } stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event)); } - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); // tasks in waiting list if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { - doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); + code = doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); } else { - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); int64_t el = (taosGetTimestampMs() - pSM->startTs); stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id, GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name); } - return TSDB_CODE_SUCCESS; + return code; } SStreamTaskState streamTaskGetStatus(const SStreamTask* pTask) { @@ -525,14 +563,14 @@ const char* streamTaskGetStatusStr(ETaskStatus status) { void streamTaskResetStatus(SStreamTask* pTask) { SStreamTaskSM* pSM = pTask->status.pSM; - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr, pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name); pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT]; pSM->pActiveTrans = NULL; taosArrayClear(pSM->pWaitingEventList); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); // clear the downstream ready status pTask->status.downstreamReady = 0; @@ -575,8 +613,7 @@ STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStr } int32_t initStateTransferTable() { - taosThreadOnce(&streamTaskStateMachineInit, doInitStateTransferTable); - return TSDB_CODE_SUCCESS; + return taosThreadOnce(&streamTaskStateMachineInit, doInitStateTransferTable); } //clang-format off @@ -585,92 +622,93 @@ void doInitStateTransferTable(void) { // initialization event handle STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanHistoryTaskReady, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); // scan-history related event trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); // halt stream task, from other task status trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); SFutureHandleEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); // checkpoint related event trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); // pause & resume related event handle trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); info = (SFutureHandleEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); // resume is completed by restore status of state-machine // stop related event trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); // dropping related event trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); - taosArrayPush(streamTaskSMTrans, &trans); + CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans)); } //clang-format on From 9bdc65ceb1dceccbaaf860b5682e96290b1dba80 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 18:57:29 +0800 Subject: [PATCH 15/45] fix(stream):fix syntax error. --- source/libs/stream/src/streamMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d9c937a14f..a2b1d39e0c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -745,7 +745,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t if (timerActive > 0) { taosMsleep(100); - stDebug("s-task:0x%x wait for quit from timer", id.taskId); + stDebug("s-task:0x%" PRIx64 " wait for quit from timer", id.taskId); } else { break; } From 0b6a49ac7d4fd26f65ce6bdd7c2fbe875d3c919d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 19:17:53 +0800 Subject: [PATCH 16/45] fix(stream): fix error and do some internal refactor. --- source/libs/stream/src/streamMsg.c | 48 +++++++++++++++++++++---- source/libs/stream/test/backendTest.cpp | 4 ++- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c index 1cc48f02b6..b08280f9ed 100644 --- a/source/libs/stream/src/streamMsg.c +++ b/source/libs/stream/src/streamMsg.c @@ -99,6 +99,8 @@ int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpda } int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg) { + int32_t code = 0; + if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pMsg->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pMsg->taskId) < 0) return -1; @@ -111,13 +113,17 @@ int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* if (tDecodeI32(pDecoder, &info.nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &info.prevEp) < 0) return -1; if (tDecodeSEpSet(pDecoder, &info.newEp) < 0) return -1; - taosArrayPush(pMsg->pNodeList, &info); + + void* p = taosArrayPush(pMsg->pNodeList, &info); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } if (tDecodeI32(pDecoder, &pMsg->transId) < 0) return -1; tEndDecode(pDecoder); - return 0; + return code; } int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { @@ -257,8 +263,18 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { if (tDecodeI32(pDecoder, &len1) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &data, &len2) < 0) return -1; ASSERT(len1 == len2); - taosArrayPush(pReq->dataLen, &len1); - taosArrayPush(pReq->data, &data); + + void* p = taosArrayPush(pReq->dataLen, &len1); + if (p == NULL) { + tEndDecode(pDecoder); + return TSDB_CODE_OUT_OF_MEMORY; + } + + p = taosArrayPush(pReq->data, &data); + if (p == NULL) { + tEndDecode(pDecoder); + return TSDB_CODE_OUT_OF_MEMORY; + } } tEndDecode(pDecoder); @@ -371,6 +387,8 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { } int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { + int32_t code = 0; + if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->numOfTasks) < 0) return -1; @@ -413,7 +431,11 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { if (tDecodeI64(pDecoder, &entry.hTaskId) < 0) return -1; entry.id.taskId = taskId; - taosArrayPush(pReq->pTaskStatus, &entry); + void* p = taosArrayPush(pReq->pTaskStatus, &entry); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } int32_t numOfVgs = 0; @@ -424,12 +446,20 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { for (int j = 0; j < numOfVgs; ++j) { int32_t vgId = 0; if (tDecodeI32(pDecoder, &vgId) < 0) return -1; - taosArrayPush(pReq->pUpdateNodes, &vgId); + void* p = taosArrayPush(pReq->pUpdateNodes, &vgId); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } if (tDecodeI32(pDecoder, &pReq->msgId) < 0) return -1; tEndDecode(pDecoder); return 0; + + _err: + tEndDecode(pDecoder); + return code; } void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) { @@ -572,7 +602,11 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { taosMemoryFreeClear(pInfo); return -1; } - taosArrayPush(pTask->upstreamInfo.pList, &pInfo); + void* p = taosArrayPush(pTask->upstreamInfo.pList, &pInfo); + if (p == NULL) { + tEndDecode(pDecoder); + return -1; + } } if (pTask->info.taskLevel != TASK_LEVEL__SINK) { diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp index 38d48a2a32..104b1c27d8 100644 --- a/source/libs/stream/test/backendTest.cpp +++ b/source/libs/stream/test/backendTest.cpp @@ -43,7 +43,9 @@ SStreamState *stateCreate(const char *path) { pTask->ver = 1024; pTask->id.streamId = 1023; pTask->id.taskId = 1111111; - SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL); + SStreamMeta *pMeta = NULL; + + int32_t code = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL, &pMeta); pTask->pMeta = pMeta; SStreamState *p = streamStateOpen((char *)path, pTask, 0, 0); From 2eec17b20cbd3da2efb64b7da32312712e012962 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 Jul 2024 19:23:20 +0800 Subject: [PATCH 17/45] enh: add catalog return code processing --- include/os/osString.h | 3 +- source/libs/catalog/inc/catalogInt.h | 15 +- source/libs/catalog/src/catalog.c | 212 ++- source/libs/catalog/src/ctgAsync.c | 1652 ++++++++++++----- source/libs/catalog/src/ctgCache.c | 8 +- source/libs/catalog/src/ctgRemote.c | 4 +- source/libs/catalog/src/ctgUtil.c | 30 +- source/libs/scheduler/test/schedulerTests.cpp | 76 +- 8 files changed, 1408 insertions(+), 592 deletions(-) diff --git a/include/os/osString.h b/include/os/osString.h index 5fa5a25c6d..e53aceb83a 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -55,11 +55,12 @@ typedef enum { M2C = 0, C2M } ConvType; #define tstrncpy(dst, src, size) \ do { \ - strncpy((dst), (src), (size)); \ + (void)strncpy((dst), (src), (size)); \ (dst)[(size)-1] = 0; \ } while (0) #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) +#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 6aaa79bb31..4f5e1b069b 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -669,12 +669,12 @@ typedef struct SCtgCacheItemInfo { #define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_AUTH_WRITE(_t) ((_t) == AUTH_TYPE_WRITE || (_t) == AUTH_TYPE_READ_OR_WRITE) -#define CTG_QUEUE_INC() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_QUEUE_DEC() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_INC() (void)atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_DEC() (void)atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) -#define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) -#define CTG_STAT_GET(_item) atomic_load_64(&(_item)) +#define CTG_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n) +#define CTG_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n) +#define CTG_STAT_GET(_item) (void)atomic_load_64(&(_item)) #define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) #define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) @@ -971,7 +971,7 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx, SArray* pList); -void* ctgCloneDbCfgInfo(void* pSrc); +int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst); int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action); int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation); @@ -1112,7 +1112,7 @@ int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgRemoveCacheUser(SCatalog* pCtg, SCtgUserAuth* pUser, const char* user); int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); -SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); +int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName); int32_t ctgdGetOneHandle(SCatalog** pHandle); int ctgVgInfoComp(const void* lp, const void* rp); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); @@ -1165,6 +1165,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, void* bInput); int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag, CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName); +void ctgFreeTask(SCtgTask* pTask, bool freeRes); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 4048c8841b..07982afd5d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -86,7 +86,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* if (code) { if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) { ctgDebug("db no longer exist, dbFName:%s, dbId:0x%" PRIx64, input.db, input.dbId); - ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId); + CTG_ERR_RET(ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId)); } CTG_ERR_RET(code); @@ -116,8 +116,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName)); - CTG_ERR_JRET( - ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL)); } else if (CTG_FLAG_IS_STB(ctx->flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); @@ -128,8 +127,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), - ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL)); @@ -166,7 +164,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* if (CTG_IS_META_NULL(output->metaType)) { ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + CTG_ERR_JRET(ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false)); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -216,7 +214,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx } if (CTG_IS_META_BOTH(output->metaType)) { - memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + TAOS_MEMCPY(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); *pTableMeta = output->tbMeta; goto _return; @@ -233,7 +231,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx taosMemoryFreeClear(output->tbMeta); SName stbName = *ctx->pName; - strcpy(stbName.tname, output->tbName); + TAOS_STRCPY(stbName.tname, output->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = ctx->flag; stbCtx.pName = &stbName; @@ -244,7 +242,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx continue; } - memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + TAOS_MEMCPY(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); break; } @@ -254,15 +252,15 @@ _return: if (CTG_TABLE_NOT_EXIST(code) && ctx->tbInfo.inCache) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - strcpy(dbFName, ctx->pName->dbname); + TAOS_STRCPY(dbFName, ctx->pName->dbname); } else { - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); } if (TSDB_SUPER_TABLE == ctx->tbInfo.tbType) { - ctgDropStbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); + (void)ctgDropStbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); // already in error } else { - ctgDropTbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); + (void)ctgDropTbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); // already in error } } @@ -285,18 +283,18 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) { int32_t code = 0; - strcpy(output->dbFName, rspMsg->dbFName); + TAOS_STRCPY(output->dbFName, rspMsg->dbFName); output->dbId = rspMsg->dbId; if (TSDB_CHILD_TABLE == rspMsg->tableType && NULL == rspMsg->pSchemas) { - strcpy(output->ctbName, rspMsg->tbName); + TAOS_STRCPY(output->ctbName, rspMsg->tbName); SET_META_TYPE_CTABLE(output->metaType); CTG_ERR_JRET(queryCreateCTableMetaFromMsg(rspMsg, &output->ctbMeta)); } else { - strcpy(output->tbName, rspMsg->tbName); + TAOS_STRCPY(output->tbName, rspMsg->tbName); SET_META_TYPE_TABLE(output->metaType); @@ -348,14 +346,14 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq, _return: - ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); + (void)ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); // cache update not fatal error CTG_RET(code); } int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); CTG_ERR_RET(ctgReadTbTypeFromCache(pCtg, dbFName, pTableName->tname, tbType)); if (*tbType > 0) { return TSDB_CODE_SUCCESS; @@ -454,7 +452,11 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.pData = pJson; tagVal.nData = strlen(pJson); - taosArrayPush(pTagVals, &tagVal); + if (NULL == taosArrayPush(pTagVals, &tagVal)) { + taosMemoryFree(pJson); + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } else { CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals)); } @@ -484,7 +486,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTabl CTG_ERR_JRET(ctgGetTbMeta(pCtg, pConn, &ctx, &tbMeta)); char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SHashObj* vgHash = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, NULL)); @@ -555,7 +557,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* SCtgDBCache* dbCache = NULL; int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SDBVgInfo* vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, exists)); @@ -590,7 +592,7 @@ int32_t ctgGetTbsHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctI SCtgDBCache* dbCache = NULL; int32_t code = 0; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); + (void)snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); SDBVgInfo* vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo, NULL)); @@ -615,7 +617,7 @@ _return: int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; @@ -688,8 +690,8 @@ _return: void ctgProcessTimerEvent(void *param, void *tmrId) { CTG_API_NENTER(); - ctgdShowCacheInfo(); - ctgdShowStatInfo(); + (void)ctgdShowCacheInfo(); + (void)ctgdShowStatInfo(); int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize); if (cacheMaxSize >= 0) { @@ -703,7 +705,7 @@ void ctgProcessTimerEvent(void *param, void *tmrId) { int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); if (code) { qError("clear cache enqueue failed, error:%s", tstrerror(code)); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); } goto _return; @@ -711,7 +713,7 @@ void ctgProcessTimerEvent(void *param, void *tmrId) { } qTrace("reset catalog timer"); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); _return: @@ -723,10 +725,8 @@ int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName if (pDbCfg->cfgVersion < 0) { CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL)); - SDbCfgInfo *pCfg = ctgCloneDbCfgInfo(pDbCfg); - if (NULL == pCfg) { - return TSDB_CODE_OUT_OF_MEMORY; - } + SDbCfgInfo *pCfg = NULL; + CTG_ERR_RET(ctgCloneDbCfgInfo(pDbCfg, &pCfg)); CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pCtg, dbFName, pDbCfg->dbId, pCfg, false)); } @@ -735,6 +735,59 @@ int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName } +int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) { + STableTSMAInfoRsp tsmasRsp = {0}; + int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA); + if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { + code = 0; + goto _return; + } + + CTG_ERR_JRET(code); + + *ppRes = tsmasRsp.pTsmas; + tsmasRsp.pTsmas = NULL; + + for (int32_t i = 0; i < (*ppRes)->size; ++i) { + CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false)); + } + + return TSDB_CODE_SUCCESS; + +_return: + + if (tsmasRsp.pTsmas) { + tFreeTableTSMAInfoRsp(&tsmasRsp); + } + CTG_RET(code); +} + + +int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { + STableTSMAInfoRsp tsmaRsp = {0}; + int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA); + if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { + code = 0; + goto _return; + } + + CTG_ERR_JRET(code); + + ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1); + + *pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0); + taosArrayDestroy(tsmaRsp.pTsmas); + tsmaRsp.pTsmas = NULL; + +_return: + + if (tsmaRsp.pTsmas) { + tFreeTableTSMAInfoRsp(&tsmaRsp); + } + CTG_RET(code); +} + + int32_t catalogInit(SCatalogCfg* cfg) { qDebug("catalogInit start"); if (gCtgMgmt.pCluster) { @@ -742,10 +795,10 @@ int32_t catalogInit(SCatalogCfg* cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + TAOS_MEMSET(&gCtgMgmt, 0, sizeof(gCtgMgmt)); if (cfg) { - memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); + TAOS_MEMCPY(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); if (gCtgMgmt.cfg.maxDBCacheNum == 0) { gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; @@ -1129,7 +1182,7 @@ int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp) { CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pIndex, pRsp, sizeof(STableIndex)); + TAOS_MEMCPY(pIndex, pRsp, sizeof(STableIndex)); CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); @@ -1247,17 +1300,26 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = TSDB_CODE_SUCCESS; SName name = {0}; int32_t sver = 0; int32_t tver = 0; int32_t tbNum = taosArrayGetSize(pTables); for (int32_t i = 0; i < tbNum; ++i) { STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + if (NULL == pTb) { + ctgError("fail to get the %dth table, tbNum:%d", i, tbNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + if (NULL == pTb->tbFName || 0 == pTb->tbFName[0]) { continue; } - tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) { + ctgError("invalid tbFName format, tbFName:%s, idx:%d", pTb->tbFName, i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } if (IS_SYS_DBNAME(name.dbname)) { continue; @@ -1266,18 +1328,18 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* int32_t tbType = 0; uint64_t suid = 0; char stbName[TSDB_TABLE_FNAME_LEN]; - ctgReadTbVerFromCache(pCtg, &name, &sver, &tver, &tbType, &suid, stbName); + CTG_ERR_JRET(ctgReadTbVerFromCache(pCtg, &name, &sver, &tver, &tbType, &suid, stbName)); if ((sver >= 0 && sver < pTb->sver) || (tver >= 0 && tver < pTb->tver)) { switch (tbType) { case TSDB_CHILD_TABLE: { SName stb = name; tstrncpy(stb.tname, stbName, sizeof(stb.tname)); - ctgRemoveTbMeta(pCtg, &stb); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, &stb)); break; } case TSDB_SUPER_TABLE: case TSDB_NORMAL_TABLE: - ctgRemoveTbMeta(pCtg, &name); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, &name)); break; default: ctgError("ignore table type %d", tbType); @@ -1286,7 +1348,9 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* } } - CTG_API_LEAVE(TSDB_CODE_SUCCESS); +_return: + + CTG_API_LEAVE(code); } int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName) { @@ -1454,10 +1518,10 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SC _return: if (pJob) { - taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); if (code) { - taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); } } @@ -1558,7 +1622,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_ while (pAuth != NULL) { size_t len = 0; void* key = taosHashGetKey(pAuth, &len); - strncpy((*users)[i].user, key, len); + TAOS_STRNCPY((*users)[i].user, key, len); (*users)[i].user[len] = 0; (*users)[i].version = pAuth->userAuth.version; ++i; @@ -1795,10 +1859,12 @@ int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma, int32_t if (!pCtg || !ppTsma) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = 0; CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, ppTsma, tsmaVersion, false)); _return: + CTG_API_LEAVE(code); } @@ -1807,10 +1873,12 @@ int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma) { if (!pCtg || !pTsma) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = 0; CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, pTsma, 0, true)); _return: + CTG_API_LEAVE(code); } @@ -1823,36 +1891,14 @@ int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma) { } if (!pCtg->dbCache) { - return TSDB_CODE_SUCCESS; - } - CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true)); -_return: - CTG_API_LEAVE(code); -} - -int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) { - STableTSMAInfoRsp tsmasRsp = {0}; - int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA); - if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { - code = 0; goto _return; } - CTG_ERR_JRET(code); - assert(tsmasRsp.pTsmas); - assert(tsmasRsp.pTsmas->size > 0); - *ppRes = tsmasRsp.pTsmas; - tsmasRsp.pTsmas = NULL; - - for (int32_t i = 0; i < (*ppRes)->size; ++i) { - CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false)); - } - return TSDB_CODE_SUCCESS; - + + CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true)); + _return: - if (tsmasRsp.pTsmas) { - tFreeTableTSMAInfoRsp(&tsmasRsp); - } - CTG_RET(code); + + CTG_API_LEAVE(code); } int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) { @@ -1870,27 +1916,6 @@ _return: CTG_API_LEAVE(code); } -int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { - STableTSMAInfoRsp tsmaRsp = {0}; - int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA); - if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { - code = 0; - goto _return; - } - - CTG_ERR_JRET(code); - - ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1); - *pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0); - taosArrayDestroy(tsmaRsp.pTsmas); - tsmaRsp.pTsmas = NULL; - -_return: - if (tsmaRsp.pTsmas) { - tFreeTableTSMAInfoRsp(&tsmaRsp); - } - CTG_RET(code); -} int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { CTG_API_ENTER(); @@ -1903,6 +1928,7 @@ int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTs CTG_ERR_JRET(ctgGetTsma(pCtg, pConn, pTsmaName, pTsma)); _return: + CTG_API_LEAVE(code); } @@ -1930,7 +1956,7 @@ void catalogDestroy(void) { } if (gCtgMgmt.cacheTimer) { - taosTmrStop(gCtgMgmt.cacheTimer); + (void)taosTmrStop(gCtgMgmt.cacheTimer); gCtgMgmt.cacheTimer = NULL; taosTmrCleanUp(gCtgMgmt.timer); gCtgMgmt.timer = NULL; @@ -1939,8 +1965,8 @@ void catalogDestroy(void) { atomic_store_8((int8_t*)&gCtgMgmt.exit, true); if (!taosCheckCurrentInDll()) { - ctgClearCacheEnqueue(NULL, false, true, true, true); - taosThreadJoin(gCtgMgmt.updateThread, NULL); + (void)ctgClearCacheEnqueue(NULL, false, true, true, true); + (void)taosThreadJoin(gCtgMgmt.updateThread, NULL); } taosHashCleanup(gCtgMgmt.pCluster); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 601e01f7e9..a32d029016 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -65,10 +65,13 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); ctx->flag = pParam->flag | CTG_FLAG_UNKNOWN_STB; - taosArrayPush(pJob->pTasks, &task); + if (NULL != taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -91,8 +94,16 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgTbMetasCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); @@ -115,9 +126,12 @@ int32_t ctgInitGetDbVgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbVgCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -140,9 +154,12 @@ int32_t ctgInitGetDbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbCfgCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -165,9 +182,12 @@ int32_t ctgInitGetDbInfoTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbInfoCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -195,10 +215,13 @@ int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); - tNameGetFullDbName(ctx->pName, ctx->dbFName); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); + (void)tNameGetFullDbName(ctx->pName, ctx->dbFName); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -221,8 +244,16 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgTbHashsCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); @@ -238,7 +269,10 @@ int32_t ctgInitGetQnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.pJob = pJob; task.taskCtx = NULL; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -253,7 +287,10 @@ int32_t ctgInitGetDnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.pJob = pJob; task.taskCtx = NULL; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -277,7 +314,10 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { tstrncpy(ctx->indexFName, name, sizeof(ctx->indexFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); @@ -302,7 +342,10 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { tstrncpy(ctx->udfName, name, sizeof(ctx->udfName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); @@ -325,9 +368,12 @@ int32_t ctgInitGetUserTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgUserCtx* ctx = task.taskCtx; - memcpy(&ctx->user, user, sizeof(*user)); + TAOS_MEMCPY(&ctx->user, user, sizeof(*user)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); @@ -342,7 +388,10 @@ int32_t ctgInitGetSvrVerTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.taskId = taskIdx; task.pJob = pJob; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -369,9 +418,12 @@ int32_t ctgInitGetTbIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -399,9 +451,12 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -429,9 +484,12 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -454,8 +512,16 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgViewsCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->viewNum); @@ -463,19 +529,6 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } -int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { - int32_t viewNum = taosArrayGetSize(pReq->pView); - for (int32_t i = 0; i < viewNum; ++i) { - STablesReq* p = taosArrayGet(pReq->pView, i); - int32_t viewNum = taosArrayGetSize(p->pTables); - for (int32_t m = 0; m < viewNum; ++m) { - SName* name = taosArrayGet(p->pTables, m); - ctgDropViewMetaEnqueue(pCtg, p->dbFName, 0, name->tname, 0, true); - } - } - return TSDB_CODE_SUCCESS; -} - int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_TSMA; @@ -483,12 +536,24 @@ int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { task.pJob = pJob; SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx)); - if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pTaskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + task.taskCtx = pTaskCtx; pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == pTaskCtx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + return TSDB_CODE_SUCCESS; } @@ -499,16 +564,49 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { task.pJob = pJob; SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx)); - if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pTaskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } task.taskCtx = pTaskCtx; pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes)); + if (NULL == pTaskCtx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } - taosArrayPush(pJob->pTasks, &task); - return 0; + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; } + +int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { + int32_t viewNum = taosArrayGetSize(pReq->pView); + for (int32_t i = 0; i < viewNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pView, i); + if (NULL == p) { + qError("taosArrayGet the %dth view in req failed", i); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t viewNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < viewNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + CTG_ERR_RET(ctgDropViewMetaEnqueue(pCtg, p->dbFName, 0, name->tname, 0, true)); + } + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) { + int32_t code = TSDB_CODE_SUCCESS; SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == pDb || NULL == pTb) { @@ -519,101 +617,167 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con for (int32_t i = 0; i < pJob->dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbCfg, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbInfo, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } int32_t dbNum = taosArrayGetSize(pReq->pTableMeta); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pTableMeta, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); + int32_t tbNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < tbNum; ++m) { SName* name = taosArrayGet(p->pTables, m); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", m); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } } dbNum = taosArrayGetSize(pReq->pTableHash); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pTableHash, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); + int32_t tbNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < tbNum; ++m) { SName* name = taosArrayGet(p->pTables, m); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", m); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } } dbNum = taosArrayGetSize(pReq->pView); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pView, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + (void)tNameGetFullDbName(name, dbFName); + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } for (int32_t i = 0; i < pJob->tbTagNum; ++i) { SName* name = taosArrayGet(pReq->pTableTag, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + (void)tNameGetFullDbName(name, dbFName); + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } char* dbFName = taosHashIterate(pDb, NULL); while (dbFName) { - ctgDropDbVgroupEnqueue(pCtg, dbFName, true); + CTG_ERR_JRET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); dbFName = taosHashIterate(pDb, dbFName); } taosHashCleanup(pDb); + pDb = NULL; // REFRESH TABLE META for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } SName* name = taosHashIterate(pTb, NULL); while (name) { - ctgRemoveTbMeta(pCtg, name); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, name)); name = taosHashIterate(pTb, name); } taosHashCleanup(pTb); + pTb = NULL; for (int32_t i = 0; i < pJob->tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); - ctgDropTbIndexEnqueue(pCtg, name, true); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(ctgDropTbIndexEnqueue(pCtg, name, true)); } for (int32_t i = 0; i < pJob->tbTsmaNum; ++i) { STablesReq* pTbReq = taosArrayGet(pReq->pTableTSMAs, i); + if (NULL == pTbReq) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } for (int32_t j = 0; j < pTbReq->pTables->size; ++j) { SName* name = taosArrayGet(pTbReq->pTables, j); - ctgDropTSMAForTbEnqueue(pCtg, name, true); + CTG_ERR_JRET(ctgDropTSMAForTbEnqueue(pCtg, name, true)); } } // REFRESH VIEW META - return ctgHandleForceUpdateView(pCtg, pReq); + CTG_ERR_JRET(ctgHandleForceUpdateView(pCtg, pReq)); + +_return: + + taosHashCleanup(pDb); + taosHashCleanup(pTb); + + return code; } int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* taskId) { @@ -628,6 +792,7 @@ int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* tas } _return: + CTG_UNLOCK(CTG_WRITE, &pJob->taskLock); return code; @@ -714,16 +879,28 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const for (int32_t i = 0; i < dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbVgroup failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_VGROUP, dbFName, NULL)); } for (int32_t i = 0; i < dbCfgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbCfg, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbCfg failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_CFG, dbFName, NULL)); } for (int32_t i = 0; i < dbInfoNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbInfo, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbInfo failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_INFO, dbFName, NULL)); } @@ -751,54 +928,74 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const for (int32_t i = 0; i < tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableIndex failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_SMA_INDEX, name, NULL)); } for (int32_t i = 0; i < tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableCfg failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL)); } for (int32_t i = 0; i < tbTagNum; ++i) { SName* name = taosArrayGet(pReq->pTableTag, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableTag failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL)); } for (int32_t i = 0; i < indexNum; ++i) { char* indexName = taosArrayGet(pReq->pIndex, i); + if (NULL == indexName) { + qError("taosArrayGet the %dth index in pIndex failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_INDEX_INFO, indexName, NULL)); } for (int32_t i = 0; i < udfNum; ++i) { char* udfName = taosArrayGet(pReq->pUdf, i); + if (NULL == udfName) { + qError("taosArrayGet the %dth udf in pUdf failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_UDF, udfName, NULL)); } for (int32_t i = 0; i < userNum; ++i) { SUserAuthInfo* user = taosArrayGet(pReq->pUser, i); + if (NULL == user) { + qError("taosArrayGet the %dth user in pUser failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_USER, user, NULL)); } if (viewNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, pReq->pView, NULL)); } - if (tbTsmaNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TSMA, pReq->pTableTSMAs, NULL)); } if (tsmaNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TSMA, pReq->pTSMAs, NULL)); } - if (qnodeNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_QNODE, NULL, NULL)); } - if (dnodeNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DNODE, NULL, NULL)); } - if (svrVerNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_SVR_VER, NULL, NULL)); } @@ -809,7 +1006,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(terrno); } - taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); double el = (taosGetTimestampUs() - st) / 1000.0; qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", @@ -817,6 +1014,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const return TSDB_CODE_SUCCESS; _return: + ctgFreeJob(*job); CTG_RET(code); } @@ -835,8 +1033,10 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableMeta, &res); - + if (NULL == taosArrayPush(pJob->jobRes.pTableMeta, &res)) { + CTG_ERR_RET(terrno); + } + return TSDB_CODE_SUCCESS; } @@ -866,7 +1066,9 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbVgroup, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbVgroup, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -885,7 +1087,9 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableHash, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableHash, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -920,7 +1124,9 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableIndex, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableIndex, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -943,7 +1149,9 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableCfg, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableCfg, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -966,7 +1174,9 @@ int32_t ctgDumpTbTagRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableTag, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableTag, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -986,7 +1196,9 @@ int32_t ctgDumpIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pIndex, &res); + if (NULL == taosArrayPush(pJob->jobRes.pIndex, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1005,7 +1217,9 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pQnodeList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pQnodeList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1024,7 +1238,9 @@ int32_t ctgDumpDnodeRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDnodeList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDnodeList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1043,7 +1259,9 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbCfg, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbCfg, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1062,7 +1280,9 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbInfo, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbInfo, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1081,7 +1301,9 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pUdfList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pUdfList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1100,7 +1322,9 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pUser, &res); + if (NULL == taosArrayPush(pJob->jobRes.pUser, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1147,7 +1371,11 @@ int32_t ctgCallSubCb(SCtgTask* pTask) { for (int32_t i = 0; i < parentNum; ++i) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); SCtgTask* pParent = taosArrayGetP(pTask->pParents, i); - + if (NULL == pParent) { + qError("taosArrayGetP the %dth parent failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pParent->subRes.code = pTask->code; if (TSDB_CODE_SUCCESS == pTask->code) { code = (*gCtgAsyncFps[pTask->type].cloneFp)(pTask, &pParent->subRes.res); @@ -1178,7 +1406,7 @@ int32_t ctgCallUserCb(void* param) { qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); - taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); return TSDB_CODE_SUCCESS; } @@ -1204,7 +1432,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { pTask->code = rspCode; pTask->status = CTG_TASK_DONE; - ctgCallSubCb(pTask); + CTG_ERR_JRET(ctgCallSubCb(pTask)); int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); if (taskDone < taosArrayGetSize(pJob->pTasks)) { @@ -1220,12 +1448,11 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { _return: ctgUpdateJobErrCode(pJob, rspCode); - // pJob->jobResCode = code; - // taosSsleep(2); - // qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId); - - taosAsyncExec(ctgCallUserCb, pJob, NULL); + int32_t newCode = taosAsyncExec(ctgCallUserCb, pJob, NULL); + if (TSDB_CODE_SUCCESS == code && TSDB_CODE_SUCCESS != newCode) { + code = newCode; + } CTG_RET(code); } @@ -1242,6 +1469,11 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t flag = ctx->flag; int32_t* vgId = &ctx->vgId; + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { @@ -1264,7 +1496,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf if (CTG_IS_META_NULL(pOut->metaType)) { if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { @@ -1291,7 +1523,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf } ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // update cache not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1309,7 +1541,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf if (CTG_IS_META_NULL(pOut->metaType)) { ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // update cache not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1327,7 +1559,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1353,16 +1585,16 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP); + CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP)); if (CTG_IS_META_BOTH(pOut->metaType)) { - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } /* else if (CTG_IS_META_CTABLE(pOut->metaType)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1375,7 +1607,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf return TSDB_CODE_SUCCESS; } - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } */ @@ -1392,7 +1624,10 @@ _return: tstrerror(code)); } if (pTask->res || code) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (TSDB_CODE_SUCCESS == code && TSDB_CODE_SUCCESS != newCode) { + code = newCode; + } } CTG_RET(code); @@ -1406,12 +1641,24 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; + bool taskDone = false; + + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; - bool taskDone = false; - + SName* pName = NULL; + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, pFetch, &pName)); + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { @@ -1434,7 +1681,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu if (CTG_IS_META_NULL(pOut->metaType)) { if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { @@ -1461,7 +1708,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu } ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1479,7 +1726,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu if (CTG_IS_META_NULL(pOut->metaType)) { ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1497,13 +1744,13 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; STableMeta* stbMeta = NULL; - (void)ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta)); if (stbMeta && stbMeta->sversion >= pOut->tbMeta->sversion) { ctgTaskDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); exist = 1; @@ -1529,16 +1776,16 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - ctgUpdateTbMetaToCache(pCtg, pOut, false); + (void)ctgUpdateTbMetaToCache(pCtg, pOut, false); // cache update not fatal error if (CTG_IS_META_BOTH(pOut->metaType)) { - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } /* else if (CTG_IS_META_CTABLE(pOut->metaType)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1551,11 +1798,16 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu return TSDB_CODE_SUCCESS; } - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } */ SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->code = 0; pRes->pRes = pOut->tbMeta; pOut->tbMeta = NULL; @@ -1572,18 +1824,25 @@ _return: if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); - pRes->code = code; - pRes->pRes = NULL; - ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + } else { + pRes->code = code; + pRes->pRes = NULL; + ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1594,6 +1853,7 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgTask* pTask = tReq->pTask; SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1616,7 +1876,10 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1626,6 +1889,7 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf SCtgTask* pTask = tReq->pTask; SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1652,7 +1916,10 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1662,9 +1929,18 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SCtgTask* pTask = tReq->pTask; SCtgTbHashsCtx* ctx = (SCtgTbHashsCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); bool taskDone = false; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1673,6 +1949,11 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + if (NULL == pReq) { + ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(ctx->pNames)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); @@ -1694,21 +1975,32 @@ _return: if (code) { STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); - int32_t num = taosArrayGetSize(pReq->pTables); - for (int32_t i = 0; i < num; ++i) { - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); - pRes->code = code; - pRes->pRes = NULL; - } + if (NULL == pReq) { + ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(ctx->pNames)); + } else { + int32_t num = taosArrayGetSize(pReq->pTables); + for (int32_t i = 0; i < num; ++i) { + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); + if (NULL == pRes) { + ctgError("fail to get the %dth res in ctx->pResList, resNum:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(ctx->pResList)); + } else { + pRes->code = code; + pRes->pRes = NULL; + } + } - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1717,6 +2009,8 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; @@ -1731,7 +2025,11 @@ _return: if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { code = TSDB_CODE_SUCCESS; } - ctgHandleTaskEnd(pTask, code); + + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1739,13 +2037,18 @@ _return: int32_t ctgHandleGetTbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1755,6 +2058,8 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out; @@ -1778,7 +2083,11 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.pData = pJson; tagVal.nData = strlen(pJson); - taosArrayPush(pTagVals, &tagVal); + if (NULL == taosArrayPush(pTagVals, &tagVal)) { + taosMemoryFree(pJson); + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } else { CTG_ERR_JRET(tTagToValArray((const STag*)pRsp->pTags, &pTagVals)); } @@ -1787,7 +2096,10 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1797,10 +2109,12 @@ int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SCtgDbCfgCtx* ctx = pTask->taskCtx; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - SDbCfgInfo* pCfg = ctgCloneDbCfgInfo(pTask->msgCtx.out); + SDbCfgInfo* pCfg = NULL; + CTG_ERR_JRET(ctgCloneDbCfgInfo(pTask->msgCtx.out, &pCfg)); CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pTask->pJob->pCtg, ctx->dbFName, pCfg->dbId, pCfg, false)); @@ -1808,7 +2122,10 @@ int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1820,13 +2137,17 @@ int32_t ctgHandleGetDbInfoRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t ctgHandleGetQnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1834,13 +2155,18 @@ _return: int32_t ctgHandleGetDnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1848,13 +2174,17 @@ _return: int32_t ctgHandleGetIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1862,13 +2192,17 @@ _return: int32_t ctgHandleGetUdfRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1878,10 +2212,11 @@ int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - ctgUpdateUserEnqueue(pCtg, pOut, true); + (void)ctgUpdateUserEnqueue(pCtg, pOut, true); // cache update not fatal error taosMemoryFreeClear(pTask->msgCtx.out); CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); @@ -1890,7 +2225,10 @@ int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1898,6 +2236,7 @@ _return: int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1905,7 +2244,10 @@ int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1918,10 +2260,23 @@ int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgViewsCtx* ctx = (SCtgViewsCtx*)pTask->taskCtx; SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + bool taskDone = false; + + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; - bool taskDone = false; + SName* pName = NULL; + + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, pFetch, &pName)); CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1931,18 +2286,24 @@ int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - if (TSDB_CODE_SUCCESS != (code = dupViewMetaFromRsp(pRsp, pViewMeta))) { + code = dupViewMetaFromRsp(pRsp, pViewMeta); + if (TSDB_CODE_SUCCESS != code) { ctgFreeSViewMeta(pViewMeta); taosMemoryFree(pViewMeta); CTG_ERR_JRET(code); } ctgDebug("start to update view meta to cache, view:%s, querySQL:%s", pRsp->name, pRsp->querySql); - ctgUpdateViewMetaToCache(pCtg, pRsp, false); + (void)ctgUpdateViewMetaToCache(pCtg, pRsp, false); // cache update not fatal error taosMemoryFreeClear(pMsgCtx->out); pRsp = NULL; SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->code = 0; pRes->pRes = pViewMeta; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -1954,23 +2315,401 @@ _return: if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); - pRes->code = code; - pRes->pRes = NULL; - if (TSDB_CODE_MND_VIEW_NOT_EXIST == code) { - ctgTaskDebug("Get view %d.%s.%s meta failed with %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); } else { - ctgTaskError("Get view %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - } - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + pRes->code = code; + pRes->pRes = NULL; + if (TSDB_CODE_MND_VIEW_NOT_EXIST == code) { + ctgTaskDebug("Get view %d.%s.%s meta failed with %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } else { + ctgTaskError("Get view %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + } + + CTG_RET(code); +} + + +static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + int32_t subFetchIdx = 0; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SVgroupInfo* pVgInfo = NULL; + SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pTbName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pFetch->vgNum = taosHashGetSize(pVgHash); + for (int32_t i = 0; i < taosArrayGetSize(pTsmas->pTsmas); ++i) { + STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas->pTsmas, i); + if (NULL == pTsmaInfo) { + ctgError("fail to get the %dth STableTSMAInfo, totalNum:%d", i, (int32_t)taosArrayGetSize(pTsmas->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pVgInfo = taosHashIterate(pVgHash, NULL); + pTsmaInfo->reqTs = taosGetTimestampMs(); + while (pVgInfo) { + // make StreamProgressReq, send it + SStreamProgressReq req = {.fetchIdx = pFetch->fetchIdx, + .streamId = pTsmaInfo->streamUid, + .subFetchIdx = subFetchIdx++, + .vgId = pVgInfo->vgId}; + CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req)); + pFetch->subFetchNum++; + + pVgInfo = taosHashIterate(pVgHash, pVgInfo); + } + } + +_return: + + CTG_RET(code); +} + + +int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + int32_t newCode = TSDB_CODE_SUCCESS; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", tReq->msgIdx); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); + if (NULL == pRes) { + ctgError("fail to get the 0th SMetaRes, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); + if (NULL == pTbReq) { + ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SName* pName = taosArrayGet(pTbReq->pTables, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SRequestConnInfo* pConn = &pTask->pJob->conn; + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + if (!CTG_IS_META_NULL(pOut->metaType)) { + CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP)); + } + + break; + } + case TDMT_MND_GET_TSMA: { + STableTSMAInfoRsp* pOut = pMsgCtx->out; + pRes->code = 0; + if (pOut->pTsmas->size > 0) { + ASSERT(pOut->pTsmas->size == 1); + pRes->pRes = pOut; + pMsgCtx->out = NULL; + TSWAP(pTask->res, pCtx->pResList); + + STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0); + if (NULL == pTsma) { + ctgError("fail to get the 0th STableTSMAInfo, totalNum:%d", (int32_t)taosArrayGetSize(pOut->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + int32_t exists = false; + CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); + if (!exists) { + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); + } + } + + break; + } + default: + ASSERT(0); + } + +_return: + + if (code) { + if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } else { + ctgTaskError("Get tsma for %d.%s.%s failed with err: %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } + } + + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + + CTG_RET(code); +} + + +int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { + bool taskDone = false; + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + SArray* pTsmas = NULL; + SHashObj* pVgHash = NULL; + SCtgDBCache* pDbCache = NULL; + STableTSMAInfo* pTsma = NULL; + SRequestConnInfo* pConn = &pTask->pJob->conn; + + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", tReq->msgIdx); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, totalNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pTbName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (reqType != TDMT_VND_GET_STREAM_PROGRESS) + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_GET_TABLE_TSMA: { + STableTSMAInfoRsp* pOut = pMsgCtx->out; + pFetch->fetchType = FETCH_TSMA_STREAM_PROGRESS; + pRes->pRes = pOut; + pMsgCtx->out = NULL; + + if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) { + // fetch progress + (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore cache error + + if (!pDbCache) { + // do not know which vnodes to fetch, fetch vnode list first + SBuildUseDBInput input = {0}; + tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); + } else { + // fetch progress from every vnode + CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pDbCache->vgCache.vgInfo->vgHash, pOut)); + ctgReleaseVgInfoToCache(pCtg, pDbCache); + pDbCache = NULL; + } + } else { + // no tsmas + if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + break; + } + case TDMT_VND_GET_STREAM_PROGRESS: { + SStreamProgressRsp rsp = {0}; + CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + // update progress into res + STableTSMAInfoRsp* pTsmasRsp = pRes->pRes; + SArray* pTsmas = pTsmasRsp->pTsmas; + SStreamProgressRsp* pRsp = &rsp; + int32_t tsmaIdx = pRsp->subFetchIdx / pFetch->vgNum; + STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas, tsmaIdx); + if (NULL == pTsmaInfo) { + ctgError("fail to get the %dth STableTSMAInfo, totalNum:%d", tsmaIdx, (int32_t)taosArrayGetSize(pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (pTsmaInfo->rspTs == 0) { + pTsmaInfo->fillHistoryFinished = true; + } + + pTsmaInfo->rspTs = taosGetTimestampMs(); + pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration); + pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished; + + qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name, + pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay); + + if (atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) == pFetch->subFetchNum) { + // subfetch all finished + for (int32_t i = 0; i < taosArrayGetSize(pTsmas); ++i) { + STableTSMAInfo* pInfo = taosArrayGetP(pTsmas, i); + CTG_ERR_JRET(tCloneTbTSMAInfo(pInfo, &pTsma)); + CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, &pTsma, 0, false)); + } + + if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + break; + } + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + switch (pFetch->fetchType) { + case FETCH_TSMA_SOURCE_TB_META: { + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pTbName, &vgInfo)); + + pFetch->vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq)); + + break; + } + case FETCH_TSMA_STREAM_PROGRESS: { + STableTSMAInfoRsp* pTsmas = pRes->pRes; + TSWAP(pOut->dbVgroup->vgHash, pVgHash); + CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas)); + + break; + } + default: + ASSERT(0); + } + + break; + } + case TDMT_VND_TABLE_META: { + // handle source tb meta + ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META); + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + pFetch->fetchType = FETCH_TB_TSMA; + pFetch->tsmaSourceTbName = *pTbName; + + if (CTG_IS_META_NULL(pOut->metaType)) { + ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname); + (void)ctgRemoveTbMetaFromCache(pCtg, pTbName, false); // ignore cache error + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (META_TYPE_BOTH_TABLE == pOut->metaType) { + // rewrite tsma fetch table with it's super table name + (void)sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName); + } + + CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA)); + + break; + } + default: + ASSERT(0); + } + +_return: + + if (pDbCache) { + ctgReleaseVgInfoToCache(pCtg, pDbCache); + } + if (pTsma) { + tFreeTableTSMAInfo(pTsma); + pTsma = NULL; + } + if (pVgHash) { + taosHashCleanup(pVgHash); + } + if (code) { + int32_t newCode = TSDB_CODE_SUCCESS; + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, totalNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(pCtx->pResList)); + newCode = TSDB_CODE_CTG_INTERNAL_ERROR; + } else { + pRes->code = code; + if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } else { + ctgTaskError("Get tsma for %d.%s.%s faield with err: %s", pTbName->acctId, pTbName->dbname, pTbName->tname, + tstrerror(code)); + } + } + + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + + bool allSubFetchFinished = false; + if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) { + allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum; + } + if ((allSubFetchFinished || pFetch->subFetchNum == 0) && 0 == atomic_sub_fetch_32(&pCtx->fetchNum, 1)) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + if (pTask->res && taskDone) { + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1998,7 +2737,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq* tReq, int32_t flag, SName* pName, int SCtgDBCache* dbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { @@ -2032,6 +2771,11 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2056,12 +2800,18 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbMetasCtx* pCtx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; + SName* pName = NULL; int32_t dbNum = taosArrayGetSize(pCtx->pNames); int32_t fetchIdx = 0; int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, num:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + ctgDebug("start to check tb metas in db %s, tbNum %ld", pReq->dbFName, taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); @@ -2075,11 +2825,27 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } - pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + pTask->msgCtxs = taosArrayInit_s(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch in pCtx->pFetchs, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_RET(ctgGetFetchName(pCtx->pNames, pFetch, &pName)); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2101,6 +2867,11 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask* pTask) { SCtgDbVgCtx* pCtx = (SCtgDbVgCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2142,6 +2913,11 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask* pTask) { SCtgTbHashCtx* pCtx = (SCtgTbHashCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2192,6 +2968,10 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pReq->dbFName, &dbCache)); @@ -2207,12 +2987,14 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { baseResIdx += taosArrayGetSize(pReq->pTables); } else { - ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0); + CTG_ERR_JRET(ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0)); baseResIdx += taosArrayGetSize(pReq->pTables); int32_t inc = baseResIdx - taosArrayGetSize(pCtx->pResList); for(int32_t j = 0; j < inc; ++j) { - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } } } @@ -2221,22 +3003,41 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); - CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgMsgCtx", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } SBuildUseDBInput input = {0}; - strcpy(input.db, pReq->dbFName); + TAOS_STRCPY(input.db, pReq->dbFName); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; @@ -2263,6 +3064,11 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask* pTask) { SArray* pRes = NULL; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2285,10 +3091,16 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbCfgCtx* pCtx = (SCtgTbCfgCtx*)pTask->taskCtx; SArray* pRes = NULL; + SCtgJob* pJob = pTask->pJob; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pCtx->pName, dbFName); - SCtgJob* pJob = pTask->pJob; + (void)tNameGetFullDbName(pCtx->pName, dbFName); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2325,7 +3137,10 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { _return: if (CTG_TASK_LAUNCHED == pTask->status) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -2338,10 +3153,16 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; SArray* pRes = NULL; + SCtgJob* pJob = pTask->pJob; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pCtx->pName, dbFName); - SCtgJob* pJob = pTask->pJob; + (void)tNameGetFullDbName(pCtx->pName, dbFName); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2363,7 +3184,10 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { _return: if (CTG_TASK_LAUNCHED == pTask->status) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -2375,12 +3199,18 @@ int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } CTG_CACHE_NHIT_INC(CTG_CI_QNODE, 1); CTG_ERR_RET(ctgGetQnodeListFromMnode(pCtg, pConn, NULL, pTask)); + return TSDB_CODE_SUCCESS; } @@ -2389,12 +3219,18 @@ int32_t ctgLaunchGetDnodeTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } CTG_CACHE_NHIT_INC(CTG_CI_DNODE, 1); CTG_ERR_RET(ctgGetDnodeListFromMnode(pCtg, pConn, NULL, pTask)); + return TSDB_CODE_SUCCESS; } @@ -2404,11 +3240,16 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); - SDbCfgInfo cfgInfo; + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } + SDbCfgInfo cfgInfo; CTG_ERR_RET(ctgReadDBCfgFromCache(pCtg, pCtx->dbFName, &cfgInfo)); if (cfgInfo.cfgVersion < 0) { @@ -2419,7 +3260,7 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pTask->res, &cfgInfo, sizeof(cfgInfo)); + TAOS_MEMCPY(pTask->res, &cfgInfo, sizeof(cfgInfo)); CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); } @@ -2433,6 +3274,11 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { SCtgDbInfoCtx* pCtx = (SCtgDbInfoCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2473,6 +3319,11 @@ int32_t ctgLaunchGetIndexTask(SCtgTask* pTask) { SCtgIndexCtx* pCtx = (SCtgIndexCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2490,6 +3341,11 @@ int32_t ctgLaunchGetUdfTask(SCtgTask* pTask) { SCtgUdfCtx* pCtx = (SCtgUdfCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2511,6 +3367,11 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; bool tbNotExists = false; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2559,6 +3420,11 @@ int32_t ctgLaunchGetSvrVerTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2576,6 +3442,7 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { SCtgViewsCtx* pCtx = (SCtgViewsCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; bool tbMetaDone = false; + SName* pName = NULL; /* ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone); @@ -2593,6 +3460,11 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + ctgDebug("start to check views in db %s, viewNum %ld", pReq->dbFName, taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetViewsFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); @@ -2607,10 +3479,26 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_RET(ctgGetFetchName(pCtx->pNames, pFetch, &pName)); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2633,10 +3521,13 @@ int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) { SCtgDBCache* pDbCache = NULL; STablesReq* pTbReq = taosArrayGet(pTaskCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pTaskCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } - ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); + (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore error if (pDbCache) { - ctgReleaseVgInfoToCache(pCtg, pDbCache); } else { SBuildUseDBInput input = {0}; @@ -2645,7 +3536,9 @@ int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) { CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pReq)); } + _return: + return code; } @@ -2661,9 +3554,15 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { for (int32_t idx = 0; idx < dbNum; ++idx) { STablesReq* pReq = taosArrayGet(pCtx->pNames, idx); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", idx, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_RET(ctgGetTbTSMAFromCache(pCtg, pCtx, idx, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); } + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetches); if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); @@ -2672,12 +3571,39 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SName* pName = taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); - if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs; + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (!pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } SCtgTaskReq tReq; tReq.pTask = pTask; @@ -2686,11 +3612,13 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { switch (pFetch->fetchType) { case FETCH_TSMA_SOURCE_TB_META: { CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pFetch->flag, pName, &pFetch->vgId)); - } break; + break; + } case FETCH_TB_TSMA: { CTG_ERR_RET( ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, &tReq, TDMT_MND_GET_TABLE_TSMA)); - } break; + break; + } default: ASSERT(0); break; @@ -2706,24 +3634,58 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SArray* pRes = NULL; SCtgJob* pJob = pTask->pJob; + // currently, only support fetching one tsma ASSERT(pCtx->pNames->size == 1); STablesReq* pReq = taosArrayGet(pCtx->pNames, 0); + if (NULL == pReq) { + ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + ASSERT(pReq->pTables->size == 1); SName* pTsmaName = taosArrayGet(pReq->pTables, 0); + if (NULL == pReq) { + ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_RET(ctgGetTSMAFromCache(pCtg, pCtx, pTsmaName)); if (pCtx->pResList->size == 0) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, 0); - if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs; + if (NULL == pMsgCtx) { + ctgError("fail to get the 0th SCtgMsgCtx, taskType:%d", pTask->type); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (!pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } + SCtgTaskReq tReq = {.pTask = pTask, .msgIdx = 0}; - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + ctgError("taosArrayPush SMetaRes failed, code:%x", terrno); + CTG_ERR_RET(terrno); + } + CTG_ERR_RET(ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, NULL, &tReq, TDMT_MND_GET_TSMA)); } else { SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); + if (NULL == pRes) { + ctgError("fail to get the 0th SMetaRes, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STableTSMAInfoRsp* pRsp = (STableTSMAInfoRsp*)pRes->pRes; ASSERT(pRsp->pTsmas->size == 1); + const STSMACache* pTsma = taosArrayGetP(pRsp->pTsmas, 0); + if (NULL == pTsma) { + ctgError("fail to get the 0th STSMACache, totalNum:%d", (int32_t)taosArrayGetSize(pRsp->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + TSWAP(pTask->res, pCtx->pResList); // get tsma target stable meta if not existed in cache int32_t exists = false; @@ -2736,253 +3698,11 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { } else { CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); } + return TSDB_CODE_SUCCESS; } - return 0; -} - -int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); - SName* pName = taosArrayGet(pTbReq->pTables, 0); - SRequestConnInfo* pConn = &pTask->pJob->conn; - CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - - switch (reqType) { - case TDMT_MND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - if (!CTG_IS_META_NULL(pOut->metaType)) { - CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP)); - } - } break; - case TDMT_MND_GET_TSMA: { - STableTSMAInfoRsp* pOut = pMsgCtx->out; - pRes->code = 0; - if (pOut->pTsmas->size > 0) { - ASSERT(pOut->pTsmas->size == 1); - pRes->pRes = pOut; - pMsgCtx->out = NULL; - TSWAP(pTask->res, pCtx->pResList); - - STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0); - int32_t exists = false; - CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); - if (!exists) { - TSWAP(pMsgCtx->lastOut, pMsgCtx->out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); - } - } - } break; - default: - ASSERT(0); - } - -_return: - if (code) { - if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { - code = TSDB_CODE_SUCCESS; - } else { - ctgTaskError("Get tsma for %d.%s.%s failed with err: %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - } - } - ctgHandleTaskEnd(pTask, code); - CTG_RET(code); -} - -static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - int32_t subFetchIdx = 0; - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); - SRequestConnInfo* pConn = &pTask->pJob->conn; - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); - SVgroupInfo* pVgInfo = NULL; - - pFetch->vgNum = taosHashGetSize(pVgHash); - for (int32_t i = 0; i < taosArrayGetSize(pTsmas->pTsmas); ++i) { - STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas->pTsmas, i); - pVgInfo = taosHashIterate(pVgHash, NULL); - pTsmaInfo->reqTs = taosGetTimestampMs(); - while (pVgInfo) { - // make StreamProgressReq, send it - SStreamProgressReq req = {.fetchIdx = pFetch->fetchIdx, - .streamId = pTsmaInfo->streamUid, - .subFetchIdx = subFetchIdx++, - .vgId = pVgInfo->vgId}; - CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req)); - pFetch->subFetchNum++; - pVgInfo = taosHashIterate(pVgHash, pVgInfo); - } - } -_return: - CTG_RET(code); -} - -int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - bool taskDone = false; - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); - SArray* pTsmas = NULL; - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); - SHashObj* pVgHash = NULL; - SCtgDBCache* pDbCache = NULL; - STableTSMAInfo* pTsma = NULL; - SRequestConnInfo* pConn = &pTask->pJob->conn; - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); - - if (reqType != TDMT_VND_GET_STREAM_PROGRESS) - CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - - switch (reqType) { - case TDMT_MND_GET_TABLE_TSMA: { - STableTSMAInfoRsp* pOut = pMsgCtx->out; - pFetch->fetchType = FETCH_TSMA_STREAM_PROGRESS; - pRes->pRes = pOut; - pMsgCtx->out = NULL; - - if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) { - // fetch progress - ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); - if (!pDbCache) { - // do not know which vnodes to fetch, fetch vnode list first - SBuildUseDBInput input = {0}; - tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db)); - input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); - } else { - // fetch progress from every vnode - CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pDbCache->vgCache.vgInfo->vgHash, pOut)); - ctgReleaseVgInfoToCache(pCtg, pDbCache); - pDbCache = NULL; - } - } else { - // no tsmas - if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - } break; - case TDMT_VND_GET_STREAM_PROGRESS: { - SStreamProgressRsp rsp = {0}; - CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - // update progress into res - STableTSMAInfoRsp* pTsmasRsp = pRes->pRes; - SArray* pTsmas = pTsmasRsp->pTsmas; - SStreamProgressRsp* pRsp = &rsp; - int32_t tsmaIdx = pRsp->subFetchIdx / pFetch->vgNum; - STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas, tsmaIdx); - if (pTsmaInfo->rspTs == 0) pTsmaInfo->fillHistoryFinished = true; - pTsmaInfo->rspTs = taosGetTimestampMs(); - pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration); - pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished; - qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name, - pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay); - - if (atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) == pFetch->subFetchNum) { - // subfetch all finished - for (int32_t i = 0; i < taosArrayGetSize(pTsmas); ++i) { - STableTSMAInfo* pInfo = taosArrayGetP(pTsmas, i); - CTG_ERR_JRET(tCloneTbTSMAInfo(pInfo, &pTsma)); - CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, &pTsma, 0, false)); - } - - if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - } break; - case TDMT_MND_USE_DB: { - SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; - - switch (pFetch->fetchType) { - case FETCH_TSMA_SOURCE_TB_META: { - SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pTbName, &vgInfo)); - - pFetch->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq)); - } break; - case FETCH_TSMA_STREAM_PROGRESS: { - STableTSMAInfoRsp* pTsmas = pRes->pRes; - TSWAP(pOut->dbVgroup->vgHash, pVgHash); - CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas)); - } break; - default: - ASSERT(0); - } - } break; - case TDMT_VND_TABLE_META: { - // handle source tb meta - ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META); - STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - pFetch->fetchType = FETCH_TB_TSMA; - pFetch->tsmaSourceTbName = *pTbName; - if (CTG_IS_META_NULL(pOut->metaType)) { - ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname); - ctgRemoveTbMetaFromCache(pCtg, pTbName, false); - CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); - } - if (META_TYPE_BOTH_TABLE == pOut->metaType) { - // rewrite tsma fetch table with it's super table name - sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName); - } - CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA)); - } break; - default: - ASSERT(0); - } - -_return: - if (pDbCache) { - ctgReleaseVgInfoToCache(pCtg, pDbCache); - } - if (pTsma) { - tFreeTableTSMAInfo(pTsma); - pTsma = NULL; - } - if (pVgHash) { - taosHashCleanup(pVgHash); - } - if (code) { - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); - pRes->code = code; - if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { - code = TSDB_CODE_SUCCESS; - } else { - ctgTaskError("Get tsma for %d.%s.%s faield with err: %s", pTbName->acctId, pTbName->dbname, pTbName->tname, - tstrerror(code)); - } - bool allSubFetchFinished = false; - if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) { - allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum; - } - if ((allSubFetchFinished || pFetch->subFetchNum == 0) && 0 == atomic_sub_fetch_32(&pCtx->fetchNum, 1)) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); - } - - CTG_RET(code); + return TSDB_CODE_SUCCESS; } int32_t ctgDumpTbTSMARes(SCtgTask* pTask) { @@ -3025,6 +3745,10 @@ int32_t ctgGetTbCfgCb(SCtgTask* pTask) { SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(terrno); + } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } @@ -3045,6 +3769,10 @@ int32_t ctgGetTbTagCb(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(terrno); + } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } @@ -3148,6 +3876,11 @@ int32_t ctgSearchExistingTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, in int32_t taskNum = taosArrayGetSize(pJob->pTasks); for (int32_t i = 0; i < taskNum; ++i) { pTask = taosArrayGet(pJob->pTasks, i); + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (type != pTask->type) { continue; } @@ -3176,16 +3909,31 @@ int32_t ctgSetSubTaskCb(SCtgTask* pSub, SCtgTask* pTask) { pTask->subRes.code = pSub->code; CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].cloneFp)(pSub, &pTask->subRes.res)); SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + qError("fail to get the -1th SCtgMsgCtx"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); + if (NULL == pSubMsgCtx) { + qError("fail to get the -1th sub SCtgMsgCtx"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pMsgCtx->pBatchs = pSubMsgCtx->pBatchs; CTG_ERR_JRET(pTask->subRes.fp(pTask)); } else { if (NULL == pSub->pParents) { pSub->pParents = taosArrayInit(4, POINTER_BYTES); + if (NULL == pSub->pParents) { + CTG_ERR_JRET(terrno); + } } - taosArrayPush(pSub->pParents, &pTask); + if (NULL == taosArrayPush(pSub->pParents, &pTask)) { + CTG_ERR_JRET(terrno); + } } _return: @@ -3227,6 +3975,11 @@ int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp f } SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId); + if (NULL == pSub) { + qError("fail to get the %dth sub SCtgTask, taskNum:%d", subTaskId, (int32_t)taosArrayGetSize(pJob->pTasks)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (newTask) { pSub->subTask = true; } @@ -3235,7 +3988,17 @@ int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp f if (newTask) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(*ppTask, -1); + if (NULL == pMsgCtx) { + qError("fail to get the -1th SCtgMsgCtx"); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); + if (NULL == pSubMsgCtx) { + qError("fail to get the -1th sub SCtgMsgCtx"); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pSubMsgCtx->pBatchs = pMsgCtx->pBatchs; CTG_ERR_RET((*gCtgAsyncFps[pSub->type].launchFp)(pSub)); @@ -3250,21 +4013,30 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { for (int32_t i = 0; i < taskNum; ++i) { SCtgTask* pTask = taosArrayGet(pJob->pTasks, i); - + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask = taosArrayGet(pJob->pTasks, i); - atomic_val_compare_exchange_32((int32_t*)&pTask->status, 0, CTG_TASK_LAUNCHED); + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + (void)atomic_val_compare_exchange_32((int32_t*)&pTask->status, 0, CTG_TASK_LAUNCHED); } if (taskNum <= 0) { qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); - taosAsyncExec(ctgCallUserCb, pJob, NULL); + CTG_ERR_RET(taosAsyncExec(ctgCallUserCb, pJob, NULL)); #if CTG_BATCH_FETCH } else { - ctgLaunchBatchs(pJob->pCtg, pJob, pJob->pBatchs); + CTG_ERR_RET(ctgLaunchBatchs(pJob->pCtg, pJob, pJob->pBatchs)); #endif } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 9f9f783458..b023695937 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -764,12 +764,10 @@ int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* p CTG_LOCK(CTG_READ, &dbCache->cfgCache.cfgLock); if (dbCache->cfgCache.cfgInfo) { - SDbCfgInfo *pInfo = ctgCloneDbCfgInfo(dbCache->cfgCache.cfgInfo); - if (NULL == pInfo) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } + SDbCfgInfo *pInfo = NULL; + CTG_ERR_JRET(ctgCloneDbCfgInfo(dbCache->cfgCache.cfgInfo, &pInfo)); - memcpy(pDbCfg, pInfo, sizeof(*pInfo)); + TAOS_MEMCPY(pDbCfg, pInfo, sizeof(*pInfo)); taosMemoryFree(pInfo); CTG_CACHE_HIT_INC(CTG_CI_DB_CFG, 1); } else { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index bbd9b39f6c..f5de7fdc7a 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -556,7 +556,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - pName = ctgGetFetchName(ctx->pNames, fetch); + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); } else if (CTG_TASK_GET_TB_TSMA == pTask->type){ SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); @@ -616,7 +616,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - pName = ctgGetFetchName(ctx->pNames, fetch); + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); } else if (CTG_TASK_GET_TB_TSMA == pTask->type){ SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index eac716339b..0098c79384 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1627,23 +1627,41 @@ int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fet return TSDB_CODE_SUCCESS; } -SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { +int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName) { STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); - return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == pReq) { + qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + *ppName = (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == *ppName) { + qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; } static void* ctgCloneDbVgroup(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -void* ctgCloneDbCfgInfo(void* pSrc) { +int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst) { SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo)); if (NULL == pDst) { - return NULL; + return terrno; } - memcpy(pDst, pSrc, sizeof(SDbCfgInfo)); + TAOS_MEMCPY(pDst, pSrc, sizeof(SDbCfgInfo)); pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); - return pDst; + if (NULL == pDst->pRetensions) { + taosMemoryFree(pDst); + return terrno; + } + + *ppDst = pDst; + + return TSDB_CODE_SUCCESS; } static void ctgFreeDbCfgInfo(void* p) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index f906e3ec3e..37860ac202 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -77,10 +77,10 @@ void schtInitLogFile() { const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, TD_LOG_DIR_PATH); + TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); + (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } @@ -628,7 +628,7 @@ void schtFreeQueryJob(int32_t freeThread) { schedulerFreeJob(&job, 0); if (freeThread) { if (++freeNum % schtTestPrintNum == 0) { - printf("FreeNum:%d\n", freeNum); + (void)printf("FreeNum:%d\n", freeNum); } } } @@ -667,7 +667,7 @@ void *schtRunJobThread(void *aa) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; if (NULL == taosArrayPush(qnodeList, &load)) { assert(0); @@ -800,7 +800,7 @@ void *schtRunJobThread(void *aa) { schtFreeQueryDag(dag); if (++jobFinished % schtTestPrintNum == 0) { - printf("jobFinished:%d\n", jobFinished); + (void)printf("jobFinished:%d\n", jobFinished); } ++schtQueryId; @@ -834,9 +834,9 @@ TEST(queryTest, normalCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(taosArrayPush(qnodeList, &load) != NULL); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -872,7 +872,7 @@ TEST(queryTest, normalCase) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -888,7 +888,7 @@ TEST(queryTest, normalCase) { if (JOB_TASK_STATUS_EXEC == task->status) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -909,10 +909,10 @@ TEST(queryTest, normalCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -926,13 +926,13 @@ TEST(queryTest, normalCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(queryTest, readyFirstCase) { @@ -948,9 +948,9 @@ TEST(queryTest, readyFirstCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -985,7 +985,7 @@ TEST(queryTest, readyFirstCase) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1002,7 +1002,7 @@ TEST(queryTest, readyFirstCase) { if (JOB_TASK_STATUS_EXEC == task->status) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1023,10 +1023,10 @@ TEST(queryTest, readyFirstCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -1039,13 +1039,13 @@ TEST(queryTest, readyFirstCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(queryTest, flowCtrlCase) { @@ -1065,9 +1065,9 @@ TEST(queryTest, flowCtrlCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -1078,7 +1078,7 @@ TEST(queryTest, flowCtrlCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - initTaskQueue(); + assert(0 == initTaskQueue()); int32_t queryDone = 0; SRequestConnInfo conn = {0}; @@ -1106,7 +1106,7 @@ TEST(queryTest, flowCtrlCase) { if (JOB_TASK_STATUS_EXEC == task->status && 0 != task->lastMsgType) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1120,10 +1120,10 @@ TEST(queryTest, flowCtrlCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -1136,13 +1136,13 @@ TEST(queryTest, flowCtrlCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(insertTest, normalCase) { @@ -1158,9 +1158,9 @@ TEST(insertTest, normalCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -1171,12 +1171,12 @@ TEST(insertTest, normalCase) { schtSetAsyncSendMsgToServer(); TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); schtJobDone = false; TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId)); int32_t queryDone = 0; SRequestConnInfo conn = {0}; @@ -1204,17 +1204,17 @@ TEST(insertTest, normalCase) { schedulerDestroy(); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(multiThread, forceFree) { TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1, thread2, thread3; - taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL)); // taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL); - taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL); + assert(0 == taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL)); while (true) { if (schtTestDeadLoop) { @@ -1231,7 +1231,7 @@ TEST(multiThread, forceFree) { TEST(otherTest, otherCase) { // excpet test - schReleaseJob(0); + (void)schReleaseJob(0); schFreeRpcCtx(NULL); char* ep = NULL; From 43dc95869f801b1c81c2264b3ed654a1b6508617 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jul 2024 19:41:06 +0800 Subject: [PATCH 18/45] fix(stream):set the initialized value. --- source/libs/stream/src/streamHb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index d8ee15c909..84eb6b319f 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -287,6 +287,8 @@ int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes) { pInfo->stopFlag = 0; pInfo->msgSendTs = -1; pInfo->hbCount = 0; + + *pRes = pInfo; return TSDB_CODE_SUCCESS; } From c7e37a3a00cbf74e52c5636930d26aa3521db7e5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Jul 2024 19:44:41 +0800 Subject: [PATCH 19/45] opti:[TS-4593] transform offset from file to tdb in tmq --- source/dnode/vnode/src/tq/tqSnapshot.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index 6cb5734aa2..29b471b5db 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -193,7 +193,7 @@ int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData code = tDecodeSTqHandle(pDecoder, &handle); if (code) goto end; taosWLockLatch(&pTq->lock); - code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, (int)strlen(handle.subKey), pData, nData); + code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, (int)strlen(handle.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); taosWUnLockLatch(&pTq->lock); end: @@ -212,6 +212,7 @@ int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nD } code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.topic, strlen(info.topic), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + tDeleteSTqCheckInfo(&info); if (code) goto _err; return code; From decd68f811fce5cf75062c053536f161e7d7c449 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jul 2024 22:09:09 +0800 Subject: [PATCH 20/45] refact more code --- source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/inc/vnodeInt.h | 4 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 11 +- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 205 ++--- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 77 +- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 721 ++++++++---------- source/dnode/vnode/src/tsdb/tsdbDataFileRW.h | 16 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 92 +-- source/dnode/vnode/src/tsdb/tsdbFile2.c | 20 +- source/dnode/vnode/src/tsdb/tsdbMerge.c | 171 ++--- source/dnode/vnode/src/tsdb/tsdbOpen.c | 63 +- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 423 +++++----- source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 238 +++--- source/dnode/vnode/src/tsdb/tsdbWrite.c | 34 +- 14 files changed, 872 insertions(+), 1204 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 943ba099f6..ed6b3c2153 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -259,7 +259,6 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData); typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); -int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 70aa55915a..6b433e5958 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -222,8 +222,8 @@ typedef struct SMetaInfo { int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader); // tsdb -int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force); -int tsdbClose(STsdb** pTsdb); +int32_t tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force); +int32_t tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); // int32_t tsdbPrepareCommit(STsdb* pTsdb); // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f9a464b3bf..04da8df94a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -41,26 +41,23 @@ int32_t tsdbBegin(STsdb *pTsdb) { if (!pTsdb) return code; SMemTable *pMemTable; - code = tsdbMemTableCreate(pTsdb, &pMemTable); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMemTableCreate(pTsdb, &pMemTable), &lino, _exit); // lock if ((code = taosThreadMutexLock(&pTsdb->mutex))) { - code = TAOS_SYSTEM_ERROR(code); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(code), &lino, _exit); } pTsdb->mem = pMemTable; // unlock if ((code = taosThreadMutexUnlock(&pTsdb->mutex))) { - code = TAOS_SYSTEM_ERROR(code); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(code), &lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 998fba8722..728db63cab 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -92,12 +92,12 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { } } - code = tsdbFSetWriterOpen(&config, &committer->writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSetWriterOpen(&config, &committer->writer), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -122,32 +122,28 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { committer->ctx->tbid->uid = row->uid; if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { - code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid), &lino, _exit); continue; } } int64_t ts = TSDBROW_TS(&row->row); if (ts > committer->ctx->maxKey) { - code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid), &lino, _exit); continue; } committer->ctx->hasTSData = true; numOfRow++; - code = tsdbFSetWriteRow(committer->writer, row); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbIterMergerNext(committer->dataIterMerger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSetWriteRow(committer->writer, row), &lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerNext(committer->dataIterMerger), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid, numOfRow); @@ -176,8 +172,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { committer->ctx->tbid->uid = record->uid; if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { - code = tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid), &lino, _exit); continue; } } @@ -192,18 +187,17 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { if (!skip) { numRecord++; - code = tsdbFSetWriteTombRecord(committer->writer, record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSetWriteTombRecord(committer->writer, record), &lino, _exit); } } - code = tsdbIterMergerNext(committer->tombIterMerger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerNext(committer->tombIterMerger), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid, numRecord); @@ -241,11 +235,9 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { .file = fobj->f[0], }; - code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader), &lino, _exit); - code = TARRAY2_APPEND(committer->sttReaderArray, sttReader); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->sttReaderArray, sttReader), &lino, _exit); STFileOp op = { .optype = TSDB_FOP_REMOVE, @@ -253,15 +245,15 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { .of = fobj->f[0], }; - code = TARRAY2_APPEND(committer->fopArray, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->fopArray, op), &lino, _exit); } } _exit: if (code) { tsdbCommitCloseReader(committer); - TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -295,21 +287,15 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { .numOfPKs = 0, }; - code = tsdbIterOpen(&config, &iter); - TSDB_CHECK_CODE(code, lino, _exit); - - code = TARRAY2_APPEND(committer->dataIterArray, iter); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->dataIterArray, iter), &lino, _exit); // mem tomb iter config.type = TSDB_ITER_TYPE_MEMT_TOMB; config.memt = committer->tsdb->imem; - code = tsdbIterOpen(&config, &iter); - TSDB_CHECK_CODE(code, lino, _exit); - - code = TARRAY2_APPEND(committer->tombIterArray, iter); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->tombIterArray, iter), &lino, _exit); // STT SSttFileReader *sttReader; @@ -318,34 +304,27 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { config.type = TSDB_ITER_TYPE_STT; config.sttReader = sttReader; - code = tsdbIterOpen(&config, &iter); - TSDB_CHECK_CODE(code, lino, _exit); - - code = TARRAY2_APPEND(committer->dataIterArray, iter); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->dataIterArray, iter), &lino, _exit); // tomb iter config.type = TSDB_ITER_TYPE_STT_TOMB; config.sttReader = sttReader; - code = tsdbIterOpen(&config, &iter); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit); - code = TARRAY2_APPEND(committer->tombIterArray, iter); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->tombIterArray, iter), &lino, _exit); } // open merger - code = tsdbIterMergerOpen(committer->dataIterArray, &committer->dataIterMerger, false); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbIterMergerOpen(committer->tombIterArray, &committer->tombIterMerger, true); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerOpen(committer->dataIterArray, &committer->dataIterMerger, false), &lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerOpen(committer->tombIterArray, &committer->tombIterMerger, true), &lino, _exit); _exit: if (code) { tsdbCommitCloseIter(committer); - TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -361,8 +340,10 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now); tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey, &committer->ctx->maxKey); - code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did); - TSDB_CHECK_CODE(code, lino, _exit); + + TAOS_CHECK_GOTO(tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did), &lino, + _exit); + tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did); committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; @@ -371,18 +352,13 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { ASSERT(committer->dataIterMerger == NULL); ASSERT(committer->writer == NULL); - code = tsdbCommitOpenReader(committer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommitOpenIter(committer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommitOpenWriter(committer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitOpenReader(committer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitOpenIter(committer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitOpenWriter(committer), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode), __func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey, @@ -395,18 +371,14 @@ static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) { int32_t code = 0; int32_t lino = 0; - code = tsdbCommitCloseWriter(committer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommitCloseIter(committer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommitCloseReader(committer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitCloseWriter(committer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitCloseIter(committer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitCloseReader(committer), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); } @@ -417,24 +389,15 @@ static int32_t tsdbCommitFileSet(SCommitter2 *committer) { int32_t code = 0; int32_t lino = 0; - // fset commit start - code = tsdbCommitFileSetBegin(committer); - TSDB_CHECK_CODE(code, lino, _exit); - - // commit fset - code = tsdbCommitTSData(committer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommitTombData(committer); - TSDB_CHECK_CODE(code, lino, _exit); - - // fset commit end - code = tsdbCommitFileSetEnd(committer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitFileSetBegin(committer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitTSData(committer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitTombData(committer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitFileSetEnd(committer), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); } @@ -464,9 +427,6 @@ static uint32_t tFileSetCommitInfoHash(const void *arg) { } static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) { - int32_t code = 0; - int32_t lino = 0; - if (pTsdb->commitInfo) { for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); @@ -480,12 +440,7 @@ static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) { pTsdb->commitInfo->arr = NULL; taosMemoryFreeClear(pTsdb->commitInfo); } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; + return 0; } static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { @@ -494,11 +449,10 @@ static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo)); if (pTsdb->commitInfo == NULL) { - TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - code = vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare), &lino, _exit); pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *)); if (pTsdb->commitInfo->arr == NULL) { @@ -508,7 +462,7 @@ static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { _exit: if (code) { tsdbCommitInfoDestroy(pTsdb); - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -520,22 +474,21 @@ static int32_t tsdbCommitInfoAdd(STsdb *tsdb, int32_t fid) { SFileSetCommitInfo *tinfo; if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) { - TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } tinfo->fid = fid; tinfo->fset = NULL; - code = vHashPut(tsdb->commitInfo->ht, tinfo); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(vHashPut(tsdb->commitInfo->ht, tinfo), &lino, _exit); if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) { - TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -547,8 +500,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { STFileSet *fset = NULL; SRBTreeIter iter; - code = tsdbCommitInfoInit(tsdb); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitInfoInit(tsdb), &lino, _exit); // scan time-series data iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1); @@ -581,8 +533,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { }; vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); if (info == NULL) { - code = tsdbCommitInfoAdd(tsdb, fid); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitInfoAdd(tsdb, fid), &lino, _exit); } from.key.ts = maxKey + 1; @@ -640,7 +591,11 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset); if (fset) { - tsdbTFileSetInitCopy(tsdb, fset, &info->fset); + code = tsdbTFileSetInitCopy(tsdb, fset, &info->fset); + if (code) { + taosThreadMutexUnlock(&tsdb->mutex); + TAOS_CHECK_GOTO(code, &lino, _exit); + } } } @@ -649,7 +604,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { _exit: if (code) { tsdbCommitInfoDestroy(tsdb); - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -670,12 +625,11 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co committer->cid = tsdbFSAllocEid(tsdb->pFS); committer->now = taosGetTimestampSec(); - code = tsdbCommitInfoBuild(tsdb); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitInfoBuild(tsdb), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } @@ -687,8 +641,7 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) { int32_t lino = 0; if (eno == 0) { - code = tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT), &lino, _exit); } else { // TODO ASSERT(0); @@ -705,8 +658,8 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) { _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino, - tstrerror(code), committer->cid); + tsdbError("vgId:%d %s failed at %s:%d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, __FILE__, + lino, tstrerror(code), committer->cid); } else { tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid); } @@ -740,22 +693,19 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) { } else { SCommitter2 committer = {0}; - code = tsdbOpenCommitter(tsdb, info, &committer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenCommitter(tsdb, info, &committer), &lino, _exit); for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); - code = tsdbCommitFileSet(&committer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbCommitFileSet(&committer), &lino, _exit); } - code = tsdbCloseCommitter(&committer, code); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbCloseCommitter(&committer, code), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } else { tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(tsdb->pVnode), __func__, nRow, nDel); } @@ -792,7 +742,7 @@ int32_t tsdbCommitCommit(STsdb *tsdb) { _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } else { tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } @@ -805,8 +755,7 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) { if (pTsdb->imem == NULL) goto _exit; - code = tsdbFSEditAbort(pTsdb->pFS); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSEditAbort(pTsdb->pFS), &lino, _exit); taosThreadMutexLock(&pTsdb->mutex); for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index 107677ce1e..e358b8f87e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -23,8 +23,7 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader)); if (reader[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } reader[0]->config[0] = config[0]; @@ -32,25 +31,26 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon int32_t lcn = config->file.lcn; if (fname) { if (fname) { - code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn), &lino, _exit); } } else { char fname1[TSDB_FILENAME_LEN]; tsdbTFileName(config->tsdb, &config->file, fname1); - code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) { - if (reader[0] == NULL) return 0; + if (reader[0] == NULL) { + return 0; + } if (reader[0]->fd) { tsdbCloseFile(&reader[0]->fd); @@ -75,12 +75,14 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -91,18 +93,20 @@ int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataF int32_t lino = 0; SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter)); - if (!writer) return TSDB_CODE_OUT_OF_MEMORY; + if (!writer) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } writer->config[0] = config[0]; - code = tsdbDataFileRAWWriterDoOpen(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileRAWWriterDoOpen(writer), &lino, _exit); _exit: if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); taosMemoryFree(writer); writer = NULL; - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); } ppWriter[0] = writer; return code; @@ -126,21 +130,20 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil .fid = writer->config->fid, .nf = writer->file, }; - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; if (writer->fd) { - code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey), &lino, _exit); tsdbCloseFile(&writer->fd); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -157,12 +160,12 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) { } tsdbTFileName(writer->config->tsdb, &writer->file, fname); - code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -174,30 +177,30 @@ int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) { writer->file = writer->config->file; writer->ctx->offset = 0; - code = tsdbDataFileRAWWriterOpenDataFD(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileRAWWriterOpenDataFD(writer), &lino, _exit); writer->ctx->opened = true; _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) { - if (writer[0] == NULL) return 0; + if (writer[0] == NULL) { + return 0; + } int32_t code = 0; int32_t lino = 0; if (writer[0]->ctx->opened) { if (abort) { - code = tsdbDataFileRAWWriterCloseAbort(writer[0]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileRAWWriterCloseAbort(writer[0]), &lino, _exit); } else { - code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileRAWWriterCloseCommit(writer[0], opArr), &lino, _exit); } tsdbDataFileRAWWriterDoClose(writer[0]); } @@ -206,7 +209,8 @@ int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFil _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -216,15 +220,16 @@ int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDat int32_t code = 0; int32_t lino = 0; - code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, + pDataBlock->dataLength, encryptAlgorithm, encryptKey), + &lino, _exit); writer->ctx->offset += pDataBlock->dataLength; _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 18ddf74399..72379449fc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -39,7 +39,9 @@ struct SDataFileReader { }; static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) { - if (reader->ctx->headFooterLoaded) return 0; + if (reader->ctx->headFooterLoaded) { + return 0; + } int32_t code = 0; int32_t lino = 0; @@ -47,11 +49,11 @@ static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) { int32_t ftype = TSDB_FTYPE_HEAD; if (reader->fd[ftype]) { int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; #if 1 - code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter), - (uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter), + (uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0, encryptAlgorithm, encryptKey), + &lino, _exit); #else int64_t size = reader->config->files[ftype].file.size; for (; size > TSDB_FHDR_SIZE; size--) { @@ -72,13 +74,16 @@ static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) { _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) { - if (reader->ctx->tombFooterLoaded) return 0; + if (reader->ctx->tombFooterLoaded) { + return 0; + } int32_t code = 0; int32_t lino = 0; @@ -86,16 +91,17 @@ static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) { int32_t ftype = TSDB_FTYPE_TOMB; if (reader->fd[ftype]) { int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter), - (uint8_t *)reader->tombFooter, sizeof(STombFooter), 0, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + TAOS_CHECK_GOTO(tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter), + (uint8_t *)reader->tombFooter, sizeof(STombFooter), 0, encryptAlgorithm, encryptKey), + &lino, _exit); } reader->ctx->tombFooterLoaded = true; _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -104,10 +110,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig int32_t code = 0; int32_t lino = 0; - reader[0] = taosMemoryCalloc(1, sizeof(**reader)); - if (reader[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + if ((*reader = taosMemoryCalloc(1, sizeof(**reader))) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->local); i++) { @@ -124,8 +128,7 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { if (fname[i]) { int32_t lcn = config->files[i].file.lcn; - code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn), &lino, _exit); } } } else { @@ -134,21 +137,23 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig char fname1[TSDB_FILENAME_LEN]; tsdbTFileName(config->tsdb, &config->files[i].file, fname1); int32_t lcn = config->files[i].file.lcn; - code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn), &lino, _exit); } } } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } int32_t tsdbDataFileReaderClose(SDataFileReader **reader) { - if (reader[0] == NULL) return 0; + if (reader[0] == NULL) { + return 0; + } TARRAY2_DESTROY(reader[0]->tombBlkArray, NULL); TARRAY2_DESTROY(reader[0]->brinBlkArray, NULL); @@ -171,26 +176,23 @@ int32_t tsdbDataFileReaderClose(SDataFileReader **reader) { int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **brinBlkArray) { int32_t code = 0; int32_t lino = 0; + void *data = NULL; if (!reader->ctx->brinBlkLoaded) { - code = tsdbDataFileReadHeadFooter(reader); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReadHeadFooter(reader), &lino, _exit); if (reader->headFooter->brinBlkPtr->size > 0) { - void *data = taosMemoryMalloc(reader->headFooter->brinBlkPtr->size); + data = taosMemoryMalloc(reader->headFooter->brinBlkPtr->size); if (data == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data, - reader->headFooter->brinBlkPtr->size, 0, encryptAlgorithm, encryptKey); - if (code) { - taosMemoryFree(data); - TSDB_CHECK_CODE(code, lino, _exit); - } + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + TAOS_CHECK_GOTO(tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data, + reader->headFooter->brinBlkPtr->size, 0, encryptAlgorithm, encryptKey), + &lino, _exit); int32_t size = reader->headFooter->brinBlkPtr->size / sizeof(SBrinBlk); TARRAY2_INIT_EX(reader->brinBlkArray, size, size, data); @@ -204,7 +206,9 @@ int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **b _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); + taosMemoryFree(data); } return code; } @@ -217,12 +221,12 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB SBuffer *assist = reader->buffers + 1; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; // load data tBufferClear(buffer); - code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, brinBlk->dp->size, buffer, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, brinBlk->dp->size, buffer, 0, + encryptAlgorithm, encryptKey), + &lino, _exit); // decode brin block SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer); @@ -237,8 +241,7 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB .compressedSize = brinBlk->size[i], .originalSize = brinBlk->numRec * sizeof(int64_t), }; - code = tDecompressDataToBuffer(BR_PTR(&br), &cinfo, brinBlock->buffers + i, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &cinfo, brinBlock->buffers + i, assist), &lino, _exit); br.offset += brinBlk->size[i]; } @@ -249,8 +252,7 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB .compressedSize = brinBlk->size[i], .originalSize = brinBlk->numRec * sizeof(int32_t), }; - code = tDecompressDataToBuffer(BR_PTR(&br), &cinfo, brinBlock->buffers + i, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &cinfo, brinBlock->buffers + i, assist), &lino, _exit); br.offset += brinBlk->size[i]; } @@ -260,27 +262,23 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB SValueColumnCompressInfo lastInfos[TD_MAX_PK_COLS]; for (int32_t i = 0; i < brinBlk->numOfPKs; i++) { - code = tValueColumnCompressInfoDecode(&br, firstInfos + i); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, firstInfos + i), &lino, _exit); } for (int32_t i = 0; i < brinBlk->numOfPKs; i++) { - code = tValueColumnCompressInfoDecode(&br, lastInfos + i); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, lastInfos + i), &lino, _exit); } for (int32_t i = 0; i < brinBlk->numOfPKs; i++) { SValueColumnCompressInfo *info = firstInfos + i; - code = tValueColumnDecompress(BR_PTR(&br), info, brinBlock->firstKeyPKs + i, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), info, brinBlock->firstKeyPKs + i, assist), &lino, _exit); br.offset += (info->offsetCompressedSize + info->dataCompressedSize); } for (int32_t i = 0; i < brinBlk->numOfPKs; i++) { SValueColumnCompressInfo *info = lastInfos + i; - code = tValueColumnDecompress(BR_PTR(&br), info, brinBlock->lastKeyPKs + i, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), info, brinBlock->lastKeyPKs + i, assist), &lino, _exit); br.offset += (info->offsetCompressedSize + info->dataCompressedSize); } } @@ -289,7 +287,8 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -304,22 +303,22 @@ int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *re SBuffer *assist = reader->buffers + 1; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; // load data tBufferClear(buffer); - code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockSize, buffer, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockSize, buffer, 0, + encryptAlgorithm, encryptKey), + &lino, _exit); // decompress SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer); - code = tBlockDataDecompress(&br, bData, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataDecompress(&br, bData, assist), &lino, _exit); ASSERT(br.offset == buffer->size); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -335,17 +334,16 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe SBuffer *assist = reader->buffers + 2; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; // load key part tBufferClear(buffer0); - code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockKeySize, buffer0, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockKeySize, buffer0, + 0, encryptAlgorithm, encryptKey), + &lino, _exit); // SDiskDataHdr SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); - code = tGetDiskDataHdr(&br, &hdr); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); @@ -355,8 +353,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe bData->nRow = hdr.nRow; // Key part - code = tBlockDataDecompressKeyPart(&hdr, &br, bData, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit); ASSERT(br.offset == buffer0->size); int extraColIdx = -1; @@ -370,12 +367,12 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe if (extraColIdx < 0) { goto _exit; } - + // load SBlockCol part tBufferClear(buffer0); - code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, hdr.szBlkCol, - buffer0, 0, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, + hdr.szBlkCol, buffer0, 0, encryptAlgorithm, encryptKey), + &lino, _exit); // calc szHint int64_t szHint = 0; @@ -400,8 +397,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe break; } - code = tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg), &lino, _exit); } if (extraColCid == blockCol.cid || blockCol.cid == INT16_MAX) { @@ -423,8 +419,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe break; } - code = tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg), &lino, _exit); } if (extraColCid == blockCol.cid) { @@ -463,8 +458,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe break; } - code = tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg), &lino, _exit); } if (cid < blockCol.cid) { @@ -481,31 +475,30 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe .szValue = 0, .offset = 0, }; - code = tBlockDataDecompressColData(&hdr, &none, &br, bData, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &none, &br, bData, assist), &lino, _exit); } else if (cid == blockCol.cid) { int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - // load from file + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + // load from file tBufferClear(buffer1); - code = tsdbReadFileToBuffer( - reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset, - blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, firstRead ? szHint : 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], + record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset, + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, + firstRead ? szHint : 0, encryptAlgorithm, encryptKey), + &lino, _exit); firstRead = false; // decode the buffer SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1); - code = tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist), &lino, _exit); } } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -520,28 +513,26 @@ int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *rec if (record->smaSize > 0) { tBufferClear(buffer); int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, record->smaSize, buffer, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, record->smaSize, buffer, 0, + encryptAlgorithm, encryptKey), + &lino, _exit); // decode sma data SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer); while (br.offset < record->smaSize) { SColumnDataAgg sma[1]; - code = tGetColumnDataAgg(&br, sma); - TSDB_CHECK_CODE(code, lino, _exit); - - code = TARRAY2_APPEND_PTR(columnDataAggArray, sma); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tGetColumnDataAgg(&br, sma), &lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(columnDataAggArray, sma), &lino, _exit); } ASSERT(br.offset == record->smaSize); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -549,26 +540,21 @@ _exit: int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **tombBlkArray) { int32_t code = 0; int32_t lino = 0; + void *data = NULL; if (!reader->ctx->tombBlkLoaded) { - code = tsdbDataFileReadTombFooter(reader); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReadTombFooter(reader), &lino, _exit); if (reader->tombFooter->tombBlkPtr->size > 0) { - void *data = taosMemoryMalloc(reader->tombFooter->tombBlkPtr->size); - if (data == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + if ((data = taosMemoryMalloc(reader->tombFooter->tombBlkPtr->size)) == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data, - reader->tombFooter->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey); - if (code) { - taosMemoryFree(data); - TSDB_CHECK_CODE(code, lino, _exit); - } + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + TAOS_CHECK_GOTO(tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data, + reader->tombFooter->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey), + &lino, _exit); int32_t size = reader->tombFooter->tombBlkPtr->size / sizeof(STombBlk); TARRAY2_INIT_EX(reader->tombBlkArray, size, size, data); @@ -582,7 +568,9 @@ int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **t _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); + taosMemoryFree(data); } return code; } @@ -596,10 +584,10 @@ int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombB tBufferClear(buffer0); int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, + encryptAlgorithm, encryptKey), + &lino, _exit); int32_t size = 0; SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); @@ -612,14 +600,14 @@ int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombB .originalSize = tombBlk->numRec * sizeof(int64_t), .compressedSize = tombBlk->size[i], }; - code = tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tData->buffers + i, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tData->buffers + i, assist), &lino, _exit); br.offset += tombBlk->size[i]; } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -720,15 +708,15 @@ static int32_t tsdbDataFileWriterDoOpenReader(SDataFileWriter *writer) { } } - code = tsdbDataFileReaderOpen(NULL, config, &writer->ctx->reader); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReaderOpen(NULL, config, &writer->ctx->reader), &lino, _exit); break; } } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -746,8 +734,7 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { } // open reader - code = tsdbDataFileWriterDoOpenReader(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterDoOpenReader(writer), &lino, _exit); // .head ftype = TSDB_FTYPE_HEAD; @@ -814,7 +801,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -827,8 +815,10 @@ int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxV int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize, TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range, - int32_t encryptAlgorithm, char* encryptKey) { - if (brinBlock->numOfRecords == 0) return 0; + int32_t encryptAlgorithm, char *encryptKey) { + if (brinBlock->numOfRecords == 0) { + return 0; + } int32_t code; SBuffer *buffer0 = buffers + 0; @@ -878,10 +868,8 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr }; tBufferClear(buffer0); - code = tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist); - if (code) return code; - code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey); - if (code) return code; + TAOS_CHECK_RETURN(tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist)); + TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey)); brinBlk.size[i] = info.compressedSize; brinBlk.dp->size += info.compressedSize; *fileSize += info.compressedSize; @@ -894,10 +882,8 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr }; tBufferClear(buffer0); - code = tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist); - if (code) return code; - code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey); - if (code) return code; + TAOS_CHECK_RETURN(tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist)); + TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey)); brinBlk.size[i] = info.compressedSize; brinBlk.dp->size += info.compressedSize; *fileSize += info.compressedSize; @@ -911,27 +897,26 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr // encode for (int i = 0; i < brinBlock->numOfPKs; i++) { SValueColumnCompressInfo info = {.cmprAlg = cmprAlg}; - if ((code = tValueColumnCompress(&brinBlock->firstKeyPKs[i], &info, buffer1, assist))) return code; - if ((code = tValueColumnCompressInfoEncode(&info, buffer0))) return code; + TAOS_CHECK_RETURN(tValueColumnCompress(&brinBlock->firstKeyPKs[i], &info, buffer1, assist)); + TAOS_CHECK_RETURN(tValueColumnCompressInfoEncode(&info, buffer0)); } for (int i = 0; i < brinBlock->numOfPKs; i++) { SValueColumnCompressInfo info = {.cmprAlg = cmprAlg}; - if ((code = tValueColumnCompress(&brinBlock->lastKeyPKs[i], &info, buffer1, assist))) return code; - if ((code = tValueColumnCompressInfoEncode(&info, buffer0))) return code; + TAOS_CHECK_RETURN(tValueColumnCompress(&brinBlock->lastKeyPKs[i], &info, buffer1, assist)); + TAOS_CHECK_RETURN(tValueColumnCompressInfoEncode(&info, buffer0)); } // write to file - if ((code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey))) return code; + TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey)); *fileSize += buffer0->size; brinBlk.dp->size += buffer0->size; - if ((code = tsdbWriteFile(fd, *fileSize, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey))) return code; + TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey)); *fileSize += buffer1->size; brinBlk.dp->size += buffer1->size; } // append to brinBlkArray - code = TARRAY2_APPEND_PTR(brinBlkArray, &brinBlk); - if (code) return code; + TAOS_CHECK_RETURN(TARRAY2_APPEND_PTR(brinBlkArray, &brinBlk)); tBrinBlockClear(brinBlock); @@ -939,22 +924,25 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr } static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { - if (writer->brinBlock->numOfRecords == 0) return 0; + if (writer->brinBlock->numOfRecords == 0) { + return 0; + } int32_t code = 0; int32_t lino = 0; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, - &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->buffers, - &writer->ctx->range, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, + &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->buffers, + &writer->ctx->range, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -967,8 +955,7 @@ static int32_t tsdbDataFileWriteBrinRecord(SDataFileWriter *writer, const SBrinR code = tBrinBlockPut(writer->brinBlock, record); if (code == TSDB_CODE_INVALID_PARA) { // different records with different primary keys - code = tsdbDataFileWriteBrinBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteBrinBlock(writer), &lino, _exit); continue; } else { TSDB_CHECK_CODE(code, lino, _exit); @@ -977,19 +964,21 @@ static int32_t tsdbDataFileWriteBrinRecord(SDataFileWriter *writer, const SBrinR } if ((writer->brinBlock->numOfRecords) >= writer->config->maxRow) { - code = tsdbDataFileWriteBrinBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteBrinBlock(writer), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { - if (bData->nRow == 0) return 0; + if (bData->nRow == 0) { + return 0; + } ASSERT(bData->uid); @@ -1033,19 +1022,19 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, bData->suid != 0 ? bData->suid : bData->uid, &cmprInfo.pColCmpr); + ASSERT(code == TSDB_CODE_SUCCESS); - code = tBlockDataCompress(bData, &cmprInfo, buffers, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataCompress(bData, &cmprInfo, buffers, assist), &lino, _exit); record->blockKeySize = buffers[0].size + buffers[1].size; record->blockSize = record->blockKeySize + buffers[2].size + buffers[3].size; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; for (int i = 0; i < 4; i++) { - code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, buffers[i].data, - buffers[i].size, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, buffers[i].data, + buffers[i].size, encryptAlgorithm, encryptKey), + &lino, _exit); writer->files[TSDB_FTYPE_DATA].size += buffers[i].size; } @@ -1058,30 +1047,28 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData SColumnDataAgg sma[1] = {{.colId = colData->cid}}; tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull); - code = tPutColumnDataAgg(&buffers[0], sma); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tPutColumnDataAgg(&buffers[0], sma), &lino, _exit); } record->smaSize = buffers[0].size; if (record->smaSize > 0) { - code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, buffers[0].data, record->smaSize, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, buffers[0].data, record->smaSize, + encryptAlgorithm, encryptKey), + &lino, _exit); writer->files[TSDB_FTYPE_SMA].size += record->smaSize; } // append SBrinRecord - code = tsdbDataFileWriteBrinRecord(writer, record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteBrinRecord(writer, record), &lino, _exit); tBlockDataClear(bData); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } taosHashCleanup(cmprInfo.pColCmpr); - return code; } @@ -1091,8 +1078,9 @@ static int32_t tsdbDataFileDoWriteTSRow(SDataFileWriter *writer, TSDBROW *row) { // update/append if (row->type == TSDBROW_ROW_FMT) { - code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, TSDBROW_SVERSION(row), writer->config->skmRow); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, TSDBROW_SVERSION(row), writer->config->skmRow), &lino, + _exit); } if (TSDBROW_VERSION(row) <= writer->config->compactVersion // @@ -1100,21 +1088,21 @@ static int32_t tsdbDataFileDoWriteTSRow(SDataFileWriter *writer, TSDBROW *row) { && tsdbRowCompareWithoutVersion(row, &tsdbRowFromBlockData(writer->blockData, writer->blockData->nRow - 1)) == 0 // ) { - code = tBlockDataUpdateRow(writer->blockData, row, writer->config->skmRow->pTSchema); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataUpdateRow(writer->blockData, row, writer->config->skmRow->pTSchema), &lino, _exit); } else { if (writer->blockData->nRow >= writer->config->maxRow) { - code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteBlockData(writer, writer->blockData), &lino, _exit); } - code = tBlockDataAppendRow(writer->blockData, row, writer->config->skmRow->pTSchema, writer->ctx->tbid->uid); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tBlockDataAppendRow(writer->blockData, row, writer->config->skmRow->pTSchema, writer->ctx->tbid->uid), &lino, + _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1130,7 +1118,9 @@ static FORCE_INLINE int32_t tsdbRowKeyCmprNullAsLargest(const STsdbRowKey *key1, } static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const STsdbRowKey *key) { - if (writer->ctx->tbHasOldData == false) return 0; + if (writer->ctx->tbHasOldData == false) { + return 0; + } int32_t code = 0; int32_t lino = 0; @@ -1144,8 +1134,7 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const ST tsdbRowGetKey(&row, &rowKey); if (tsdbRowKeyCmprNullAsLargest(&rowKey, key) < 0) { // key <= rowKey - code = tsdbDataFileDoWriteTSRow(writer, &row); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTSRow(writer, &row), &lino, _exit); } else { goto _exit; } @@ -1171,15 +1160,13 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const ST tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, record); if (tsdbRowKeyCmprNullAsLargest(key, &record->lastKey) > 0) { // key > record->lastKey if (writer->blockData->nRow > 0) { - code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteBlockData(writer, writer->blockData), &lino, _exit); } - code = tsdbDataFileWriteBrinRecord(writer, record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteBrinRecord(writer, record), &lino, _exit); } else { - code = tsdbDataFileReadBlockData(writer->ctx->reader, record, writer->ctx->blockData); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReadBlockData(writer->ctx->reader, record, writer->ctx->blockData), &lino, + _exit); writer->ctx->blockDataIdx = 0; writer->ctx->brinBlockIdx++; @@ -1202,8 +1189,7 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const ST goto _exit; } - code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock), &lino, _exit); writer->ctx->brinBlockIdx = 0; writer->ctx->brinBlkArrayIdx++; @@ -1212,7 +1198,8 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const ST _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1224,40 +1211,38 @@ static int32_t tsdbDataFileDoWriteTSData(SDataFileWriter *writer, TSDBROW *row) if (writer->ctx->tbHasOldData) { STsdbRowKey key; tsdbRowGetKey(row, &key); - - code = tsdbDataFileDoWriteTableOldData(writer, &key); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTableOldData(writer, &key), &lino, _exit); } - code = tsdbDataFileDoWriteTSRow(writer, row); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTSRow(writer, row), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } static int32_t tsdbDataFileWriteTableDataEnd(SDataFileWriter *writer) { - if (writer->ctx->tbid->uid == 0) return 0; + if (writer->ctx->tbid->uid == 0) { + return 0; + } int32_t code = 0; int32_t lino = 0; if (writer->ctx->tbHasOldData) { - code = tsdbDataFileDoWriteTableOldData(writer, NULL /* as the largest key */); - TSDB_CHECK_CODE(code, lino, _exit); - + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTableOldData(writer, NULL /* as the largest key */), &lino, _exit); ASSERT(writer->ctx->tbHasOldData == false); } - code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteBlockData(writer, writer->blockData), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1276,7 +1261,7 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA while (writer->ctx->brinBlkArray) { // skip data of previous table for (; writer->ctx->brinBlockIdx < writer->ctx->brinBlock->numOfRecords; writer->ctx->brinBlockIdx++) { SBrinRecord record; - tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, &record); + TAOS_CHECK_GOTO(tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, &record), &lino, _exit); if (record.uid == tbid->uid) { writer->ctx->tbHasOldData = true; @@ -1299,8 +1284,7 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA } } - code = tsdbDataFileWriteBrinRecord(writer, &record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteBrinRecord(writer, &record), &lino, _exit); } } @@ -1310,8 +1294,7 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA } else { const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); - code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock), &lino, _exit); writer->ctx->brinBlockIdx = 0; writer->ctx->brinBlkArrayIdx++; @@ -1321,35 +1304,38 @@ static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TA _begin: writer->ctx->tbid[0] = *tbid; - if (tbid->uid == INT64_MAX) goto _exit; + if (tbid->uid == INT64_MAX) { + goto _exit; + } - code = tsdbUpdateSkmTb(writer->config->tsdb, tbid, writer->config->skmTb); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tBlockDataInit(writer->blockData, writer->ctx->tbid, writer->config->skmTb->pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpdateSkmTb(writer->config->tsdb, tbid, writer->config->skmTb), &lino, _exit); + TAOS_CHECK_GOTO(tBlockDataInit(writer->blockData, writer->ctx->tbid, writer->config->skmTb->pTSchema, NULL, 0), &lino, + _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } -int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm, - char* encryptKey) { - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey); - if (code) return code; +int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm, + char *encryptKey) { + TAOS_CHECK_RETURN( + tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey)); *fileSize += sizeof(*footer); return 0; } int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range, - int32_t encryptAlgorithm, char* encryptKey) { + int32_t encryptAlgorithm, char *encryptKey) { int32_t code; - if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; + if (TOMB_BLOCK_SIZE(tombBlock) == 0) { + return 0; + } SBuffer *buffer0 = buffers + 0; SBuffer *assist = buffers + 1; @@ -1395,19 +1381,15 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl .dataType = TSDB_DATA_TYPE_BIGINT, .originalSize = tombBlock->buffers[i].size, }; - code = tCompressDataToBuffer(tombBlock->buffers[i].data, &cinfo, buffer0, assist); - if (code) return code; - - code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey); - if (code) return code; + TAOS_CHECK_RETURN(tCompressDataToBuffer(tombBlock->buffers[i].data, &cinfo, buffer0, assist)); + TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey)); tombBlk.size[i] = cinfo.compressedSize; tombBlk.dp->size += tombBlk.size[i]; *fileSize += tombBlk.size[i]; } - code = TARRAY2_APPEND_PTR(tombBlkArray, &tombBlk); - if (code) return code; + TAOS_CHECK_RETURN(TARRAY2_APPEND_PTR(tombBlkArray, &tombBlk)); tTombBlockClear(tombBlock); return 0; @@ -1418,15 +1400,16 @@ static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) { int32_t lino = 0; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, + writer->headFooter, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1438,31 +1421,29 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { int32_t lino = 0; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, - &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->buffers, - &writer->ctx->tombRange, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, + &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->buffers, + &writer->ctx->tombRange, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } -int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize, - int32_t encryptAlgorithm, char* encryptKey) { +int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize, + int32_t encryptAlgorithm, char *encryptKey) { ptr->size = TARRAY2_DATA_LEN(tombBlkArray); if (ptr->size > 0) { ptr->offset = *fileSize; - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size, - encryptAlgorithm, encryptKey); - if (code) { - return code; - } + TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size, + encryptAlgorithm, encryptKey)); *fileSize += ptr->size; } @@ -1476,23 +1457,25 @@ static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) { int32_t lino = 0; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr, - &writer->files[TSDB_FTYPE_TOMB].size, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr, + &writer->files[TSDB_FTYPE_TOMB].size, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, - char* encryptKey) { - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey); - if (code) return code; + char *encryptKey) { + TAOS_CHECK_RETURN( + tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey)); *fileSize += sizeof(*footer); return 0; } @@ -1502,15 +1485,16 @@ static int32_t tsdbDataFileWriteTombFooter(SDataFileWriter *writer) { int32_t lino = 0; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, + &writer->files[TSDB_FTYPE_TOMB].size, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1528,8 +1512,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom if (c < 0) { goto _write; } else if (c > 0) { - code = tTombBlockPut(writer->tombBlock, record1); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tTombBlockPut(writer->tombBlock, record1), &lino, _exit); tsdbTrace("vgId:%d write tomb record to tomb file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 ", version:%" PRId64, @@ -1537,8 +1520,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom record1->suid, record1->uid, record1->version); if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { - code = tsdbDataFileDoWriteTombBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTombBlock(writer), &lino, _exit); } } else { ASSERT(0); @@ -1551,8 +1533,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom } else { const STombBlk *tombBlk = TARRAY2_GET_PTR(writer->ctx->tombBlkArray, writer->ctx->tombBlkArrayIdx); - code = tsdbDataFileReadTombBlock(writer->ctx->reader, tombBlk, writer->ctx->tombBlock); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReadTombBlock(writer->ctx->reader, tombBlk, writer->ctx->tombBlock), &lino, _exit); writer->ctx->tombBlockIdx = 0; writer->ctx->tombBlkArrayIdx++; @@ -1560,10 +1541,11 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom } _write: - if (record->suid == INT64_MAX) goto _exit; + if (record->suid == INT64_MAX) { + goto _exit; + } - code = tTombBlockPut(writer->tombBlock, record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tTombBlockPut(writer->tombBlock, record), &lino, _exit); tsdbTrace("vgId:%d write tomb record to tomb file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 ", version:%" PRId64, @@ -1571,26 +1553,25 @@ _write: record->uid, record->version); if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { - code = tsdbDataFileDoWriteTombBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTombBlock(writer), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } -int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize, - int32_t encryptAlgorithm, char* encryptKey) { +int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize, + int32_t encryptAlgorithm, char *encryptKey) { ASSERT(TARRAY2_SIZE(brinBlkArray) > 0); ptr->offset = *fileSize; ptr->size = TARRAY2_DATA_LEN(brinBlkArray); - int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size, encryptAlgorithm, - encryptKey); - if (code) return code; + TAOS_CHECK_RETURN( + tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size, encryptAlgorithm, encryptKey)); *fileSize += ptr->size; return 0; @@ -1601,15 +1582,17 @@ static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) { int32_t lino = 0; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr, - &writer->files[TSDB_FTYPE_HEAD].size, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr, + &writer->files[TSDB_FTYPE_HEAD].size, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1633,20 +1616,11 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .uid = INT64_MAX, }}; - code = tsdbDataFileWriteTableDataEnd(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileWriteTableDataBegin(writer, tbid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileWriteBrinBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileWriteBrinBlk(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileWriteHeadFooter(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteTableDataEnd(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteTableDataBegin(writer, tbid), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteBrinBlock(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteBrinBlk(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteHeadFooter(writer), &lino, _exit); SVersionRange ofRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; @@ -1659,8 +1633,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .of = writer->config->files[ftype].file, }; ofRange = (SVersionRange){.minVer = op.of.minVer, .maxVer = op.of.maxVer}; - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); } op = (STFileOp){ .optype = TSDB_FOP_CREATE, @@ -1669,8 +1642,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr }; tsdbTFileUpdVerRange(&op.nf, ofRange); tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); // .data ftype = TSDB_FTYPE_DATA; @@ -1681,8 +1653,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .nf = writer->files[ftype], }; tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { op = (STFileOp){ .optype = TSDB_FOP_MODIFY, @@ -1691,8 +1662,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .nf = writer->files[ftype], }; tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); } // .sma @@ -1704,8 +1674,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .nf = writer->files[ftype], }; tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { op = (STFileOp){ .optype = TSDB_FOP_MODIFY, @@ -1714,8 +1683,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .nf = writer->files[ftype], }; tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); } } @@ -1726,17 +1694,10 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .version = INT64_MAX, }}; - code = tsdbDataFileDoWriteTombRecord(writer, record); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileDoWriteTombBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileDoWriteTombBlk(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileWriteTombFooter(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTombRecord(writer, record), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTombBlock(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTombBlk(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteTombFooter(writer), &lino, _exit); SVersionRange ofRange = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; @@ -1748,8 +1709,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr .of = writer->config->files[ftype].file, }; ofRange = (SVersionRange){.minVer = op.of.minVer, .maxVer = op.of.maxVer}; - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); } op = (STFileOp){ .optype = TSDB_FOP_CREATE, @@ -1758,22 +1718,21 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr }; tsdbTFileUpdVerRange(&op.nf, ofRange); tsdbTFileUpdVerRange(&op.nf, writer->ctx->tombRange); - code = TARRAY2_APPEND(opArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit); } int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { if (writer->fd[i]) { - code = tsdbFsyncFile(writer->fd[i], encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd[i], encryptAlgorithm, encryptKey), &lino, _exit); tsdbCloseFile(&writer->fd[i]); } } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1796,37 +1755,38 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) { int32_t lcn = writer->files[ftype].lcn; tsdbTFileName(writer->config->tsdb, &writer->files[ftype], fname); - code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn), &lino, _exit); if (writer->files[ftype].size == 0) { uint8_t hdr[TSDB_FHDR_SIZE] = {0}; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey), &lino, + _exit); writer->files[ftype].size += TSDB_FHDR_SIZE; } } if (writer->ctx->reader) { - code = tsdbDataFileReadBrinBlk(writer->ctx->reader, &writer->ctx->brinBlkArray); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReadBrinBlk(writer->ctx->reader, &writer->ctx->brinBlkArray), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } int32_t tsdbDataFileWriterOpen(const SDataFileWriterConfig *config, SDataFileWriter **writer) { writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); - if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY; + if (!writer[0]) { + return TSDB_CODE_OUT_OF_MEMORY; + } writer[0]->config[0] = config[0]; return 0; @@ -1840,11 +1800,9 @@ int32_t tsdbDataFileWriterClose(SDataFileWriter **writer, bool abort, TFileOpArr if (writer[0]->ctx->opened) { if (abort) { - code = tsdbDataFileWriterCloseAbort(writer[0]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterCloseAbort(writer[0]), &lino, _exit); } else { - code = tsdbDataFileWriterCloseCommit(writer[0], opArr); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterCloseCommit(writer[0], opArr), &lino, _exit); } tsdbDataFileWriterDoClose(writer[0]); } @@ -1853,7 +1811,8 @@ int32_t tsdbDataFileWriterClose(SDataFileWriter **writer, bool abort, TFileOpArr _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1863,35 +1822,32 @@ int32_t tsdbDataFileWriteRow(SDataFileWriter *writer, SRowInfo *row) { int32_t lino = 0; if (!writer->ctx->opened) { - code = tsdbDataFileWriterDoOpen(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterDoOpen(writer), &lino, _exit); } if (writer->fd[TSDB_FTYPE_HEAD] == NULL) { - code = tsdbDataFileWriterOpenDataFD(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterOpenDataFD(writer), &lino, _exit); } if (row->uid != writer->ctx->tbid->uid) { - code = tsdbDataFileWriteTableDataEnd(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)row); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteTableDataEnd(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)row), &lino, _exit); } - code = tsdbDataFileDoWriteTSData(writer, &row->row); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTSData(writer, &row->row), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { - if (bData->nRow == 0) return 0; + if (bData->nRow == 0) { + return 0; + } int32_t code = 0; int32_t lino = 0; @@ -1899,48 +1855,41 @@ int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { ASSERT(bData->uid); if (!writer->ctx->opened) { - code = tsdbDataFileWriterDoOpen(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterDoOpen(writer), &lino, _exit); } if (writer->fd[TSDB_FTYPE_DATA] == NULL) { - code = tsdbDataFileWriterOpenDataFD(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterOpenDataFD(writer), &lino, _exit); } if (bData->uid != writer->ctx->tbid->uid) { - code = tsdbDataFileWriteTableDataEnd(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)bData); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteTableDataEnd(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)bData), &lino, _exit); } if (writer->ctx->tbHasOldData) { STsdbRowKey key; tsdbRowGetKey(&tsdbRowFromBlockData(bData, 0), &key); - code = tsdbDataFileDoWriteTableOldData(writer, &key); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTableOldData(writer, &key), &lino, _exit); } if (!writer->ctx->tbHasOldData // && writer->blockData->nRow == 0 // ) { - code = tsdbDataFileDoWriteBlockData(writer, bData); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteBlockData(writer, bData), &lino, _exit); } else { for (int32_t i = 0; i < bData->nRow; ++i) { TSDBROW row[1] = {tsdbRowFromBlockData(bData, i)}; - code = tsdbDataFileDoWriteTSData(writer, row); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTSData(writer, row), &lino, _exit); } } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1967,19 +1916,18 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) { int32_t lcn = writer->files[ftype].lcn; tsdbTFileName(writer->config->tsdb, writer->files + ftype, fname); - code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn); - TSDB_CHECK_CODE(code, lino, _exit); + + TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn), &lino, _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey), &lino, _exit); writer->files[ftype].size += TSDB_FHDR_SIZE; if (writer->ctx->reader) { - code = tsdbDataFileReadTombBlk(writer->ctx->reader, &writer->ctx->tombBlkArray); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileReadTombBlk(writer->ctx->reader, &writer->ctx->tombBlkArray), &lino, _exit); if (TARRAY2_SIZE(writer->ctx->tombBlkArray) > 0) { writer->ctx->hasOldTomb = true; @@ -1992,7 +1940,8 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) { _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -2002,21 +1951,19 @@ int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord * int32_t lino = 0; if (!writer->ctx->opened) { - code = tsdbDataFileWriterDoOpen(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterDoOpen(writer), &lino, _exit); } if (writer->fd[TSDB_FTYPE_TOMB] == NULL) { - code = tsdbDataFileWriterOpenTombFD(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileWriterOpenTombFD(writer), &lino, _exit); } - code = tsdbDataFileDoWriteTombRecord(writer, record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFileDoWriteTombRecord(writer, record), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h index dde33340f5..2b87818273 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -98,21 +98,21 @@ int32_t tsdbDataFileFlush(SDataFileWriter *writer); // head int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize, TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range, - int32_t encryptAlgorithm, char* encryptKey); + int32_t encryptAlgorithm, char *encryptKey); int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize, - int32_t encryptAlgorithm, char* encryptKey); -int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, - int32_t encryptAlgorithm, char* encryptKey); + int32_t encryptAlgorithm, char *encryptKey); +int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm, + char *encryptKey); // tomb int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range, - int32_t encryptAlgorithm, char* encryptKey); + int32_t encryptAlgorithm, char *encryptKey); int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize, - int32_t encryptAlgorithm, char* encryptKey); -int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, - int32_t encryptAlgorithm, char* encryptKey); + int32_t encryptAlgorithm, char *encryptKey); +int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, + char *encryptKey); // utils int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 3ee0c482a7..243ff662f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -93,11 +93,11 @@ static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) { } // EXPOSED APIS ================================================== -static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) { - const char* p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did); - int32_t len = strlen(p1); +static char *getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) { + const char *p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did); + int32_t len = strlen(p1); - char* p = memcpy(fname, p1, len); + char *p = memcpy(fname, p1, len); p += len; *(p++) = TD_DIRSEP[0]; @@ -125,99 +125,31 @@ static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t } void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]) { - char* p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname); + char *p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname); memcpy(p, ".head", 5); p[5] = 0; } void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]) { - char* p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname); + char *p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname); memcpy(p, ".data", 5); p[5] = 0; } void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) { - char* p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname); + char *p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname); memcpy(p, ".stt", 4); p[4] = 0; } void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { - char* p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname); + char *p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname); memcpy(p, ".sma", 4); p[4] = 0; } bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; } -int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { - int32_t code = 0; - int64_t size = 0; - int64_t n; - TdFilePtr pFD; - char fname[TSDB_FILENAME_LEN] = {0}; - char hdr[TSDB_FHDR_SIZE] = {0}; - - // truncate - switch (ftype) { - case TSDB_DATA_FILE: - size = pSet->pDataF->size; - tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); - tPutDataFile(hdr, pSet->pDataF); - break; - case TSDB_SMA_FILE: - size = pSet->pSmaF->size; - tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); - tPutSmaFile(hdr, pSet->pSmaF); - break; - default: - goto _err; // make the coverity scan happy - } - - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - // open - pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_WRITE_THROUGH); - if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // ftruncate - if (taosFtruncateFile(pFD, tsdbLogicToFileSize(size, pTsdb->pVnode->config.tsdbPageSize)) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // update header - n = taosLSeekFile(pFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // sync - if (taosFsyncFile(pFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // close - taosCloseFile(&pFD); - - return code; - -_err: - tsdbError("vgId:%d, tsdb rollback file failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t n = 0; @@ -249,7 +181,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { // head pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); if (pSet->pHeadF == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pSet->pHeadF->nRef = 1; n += tGetHeadFile(p + n, pSet->pHeadF); @@ -257,7 +189,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { // data pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile)); if (pSet->pDataF == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pSet->pDataF->nRef = 1; n += tGetDataFile(p + n, pSet->pDataF); @@ -265,7 +197,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { // sma pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile)); if (pSet->pSmaF == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pSet->pSmaF->nRef = 1; n += tGetSmaFile(p + n, pSet->pSmaF); @@ -275,7 +207,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile)); if (pSet->aSttF[iStt] == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pSet->aSttF[iStt]->nRef = 1; n += tGetSttFile(p + n, pSet->aSttF[iStt]); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 7d8b0a6c8b..a500074a65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -165,8 +165,7 @@ static int32_t data_to_json(const STFile *file, cJSON *json) { return tfile_to_j static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } static int32_t stt_to_json(const STFile *file, cJSON *json) { - int32_t code = tfile_to_json(file, json); - if (code) return code; + TAOS_CHECK_RETURN(tfile_to_json(file, json)); /* lvl */ if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) { @@ -181,8 +180,7 @@ static int32_t data_from_json(const cJSON *json, STFile *file) { return tfile_fr static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } static int32_t stt_from_json(const cJSON *json, STFile *file) { - int32_t code = tfile_from_json(json, file); - if (code) return code; + TAOS_CHECK_RETURN(tfile_from_json(json, file)); const cJSON *item; @@ -202,7 +200,9 @@ int32_t tsdbTFileToJson(const STFile *file, cJSON *json) { return g_tfile_info[file->type].to_json(file, json); } else { cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix); - if (item == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (item == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } return g_tfile_info[file->type].to_json(file, item); } } @@ -211,13 +211,11 @@ int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) { f[0] = (STFile){.type = ftype}; if (ftype == TSDB_FTYPE_STT) { - int32_t code = g_tfile_info[ftype].from_json(json, f); - if (code) return code; + TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(json, f)); } else { const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix); if (cJSON_IsObject(item)) { - int32_t code = g_tfile_info[ftype].from_json(item, f); - if (code) return code; + TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(item, f)); } else { return TSDB_CODE_NOT_FOUND; } @@ -228,7 +226,9 @@ int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) { int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { fobj[0] = taosMemoryMalloc(sizeof(*fobj[0])); - if (!fobj[0]) return TSDB_CODE_OUT_OF_MEMORY; + if (!fobj[0]) { + return TSDB_CODE_OUT_OF_MEMORY; + } taosThreadMutexInit(&fobj[0]->mutex, NULL); fobj[0]->f[0] = f[0]; diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index f2d4ab2cb2..599e7c1a6d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -66,7 +66,6 @@ static int32_t tsdbMergerOpen(SMerger *merger) { } static int32_t tsdbMergerClose(SMerger *merger) { - int32_t code = 0; int32_t lino = 0; SVnode *pVnode = merger->tsdb->pVnode; @@ -82,12 +81,7 @@ static int32_t tsdbMergerClose(SMerger *merger) { TARRAY2_DESTROY(merger->dataIterArr, NULL); TARRAY2_DESTROY(merger->sttReaderArr, NULL); TARRAY2_DESTROY(merger->fopArr, NULL); - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(pVnode), lino, code); - } - return code; + return 0; } static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) { @@ -126,8 +120,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { .fid = merger->ctx->fset->fid, .of = fobj->f[0], }; - code = TARRAY2_APPEND(merger->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->fopArr, op), &lino, _exit); SSttFileReader *reader; SSttFileReaderConfig config = { @@ -136,11 +129,9 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { .file[0] = fobj->f[0], }; - code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &reader), &lino, _exit); - code = TARRAY2_APPEND(merger->sttReaderArr, reader); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->sttReaderArr, reader), &lino, _exit); } } } else { @@ -212,8 +203,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { .fid = merger->ctx->fset->fid, .of = fobj->f[0], }; - code = TARRAY2_APPEND(merger->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->fopArr, op), &lino, _exit); SSttFileReader *reader; SSttFileReaderConfig config = { @@ -222,8 +212,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { .file[0] = fobj->f[0], }; - code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &reader), &lino, _exit); if ((code = TARRAY2_APPEND(merger->sttReaderArr, reader))) { tsdbSttFileReaderClose(&reader); @@ -239,8 +228,9 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { _exit: if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); tsdbMergeFileSetEndCloseReader(merger); - TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); } return code; } @@ -259,32 +249,25 @@ static int32_t tsdbMergeFileSetBeginOpenIter(SMerger *merger) { config.type = TSDB_ITER_TYPE_STT; config.sttReader = sttReader; - code = tsdbIterOpen(&config, &iter); - TSDB_CHECK_CODE(code, lino, _exit); - - code = TARRAY2_APPEND(merger->dataIterArr, iter); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->dataIterArr, iter), &lino, _exit); // tomb iter config.type = TSDB_ITER_TYPE_STT_TOMB; config.sttReader = sttReader; - code = tsdbIterOpen(&config, &iter); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit); - code = TARRAY2_APPEND(merger->tombIterArr, iter); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->tombIterArr, iter), &lino, _exit); } - code = tsdbIterMergerOpen(merger->dataIterArr, &merger->dataIterMerger, false); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerOpen(merger->dataIterArr, &merger->dataIterMerger, false), &lino, _exit); - code = tsdbIterMergerOpen(merger->tombIterArr, &merger->tombIterMerger, true); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerOpen(merger->tombIterArr, &merger->tombIterMerger, true), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(vid, lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", vid, __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -296,10 +279,9 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) { SDiskID did; int32_t level = tsdbFidLevel(merger->ctx->fset->fid, &merger->tsdb->keepCfg, merger->ctx->now); - if (tfsAllocDisk(merger->tsdb->pVnode->pTfs, level, &did) < 0) { - code = TSDB_CODE_FS_NO_VALID_DISK; - TSDB_CHECK_CODE(code, lino, _exit); - } + + TAOS_CHECK_GOTO(tfsAllocDisk(merger->tsdb->pVnode->pTfs, level, &did), &lino, _exit); + tfsMkdirRecurAt(merger->tsdb->pVnode->pTfs, merger->tsdb->path, did); SFSetWriterConfig config = { .tsdb = merger->tsdb, @@ -328,12 +310,11 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) { } } - code = tsdbFSetWriterOpen(&config, &merger->writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSetWriterOpen(&config, &merger->writer), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(vid, lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", vid, __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -353,20 +334,18 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) { merger->ctx->tbid->uid = 0; // open reader - code = tsdbMergeFileSetBeginOpenReader(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenReader(merger), &lino, _exit); // open iterator - code = tsdbMergeFileSetBeginOpenIter(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenIter(merger), &lino, _exit); // open writer - code = tsdbMergeFileSetBeginOpenWriter(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenWriter(merger), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -387,18 +366,14 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) { int32_t code = 0; int32_t lino = 0; - code = tsdbMergeFileSetEndCloseWriter(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseWriter(merger), &lino, _exit); - code = tsdbMergeFileSetEndCloseIter(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseIter(merger), &lino, _exit); - code = tsdbMergeFileSetEndCloseReader(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseReader(merger), &lino, _exit); // edit file system - code = tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE), &lino, _exit); taosThreadMutexLock(&merger->tsdb->mutex); code = tsdbFSEditCommit(merger->tsdb->pFS); @@ -410,6 +385,8 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) { _exit: if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); } return code; @@ -420,8 +397,7 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) { int32_t lino = 0; merger->ctx->fset = fset; - code = tsdbMergeFileSetBegin(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSetBegin(merger), &lino, _exit); // data SMetaInfo info; @@ -434,17 +410,14 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) { merger->ctx->tbid->suid = row->suid; if (metaGetInfo(merger->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { - code = tsdbIterMergerSkipTableData(merger->dataIterMerger, merger->ctx->tbid); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(merger->dataIterMerger, merger->ctx->tbid), &lino, _exit); continue; } } - code = tsdbFSetWriteRow(merger->writer, row); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSetWriteRow(merger->writer, row), &lino, _exit); - code = tsdbIterMergerNext(merger->dataIterMerger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerNext(merger->dataIterMerger), &lino, _exit); } // tomb @@ -456,24 +429,21 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) { merger->ctx->tbid->suid = record->suid; if (metaGetInfo(merger->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { - code = tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid), &lino, _exit); continue; } } - code = tsdbFSetWriteTombRecord(merger->writer, record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSetWriteTombRecord(merger->writer, record), &lino, _exit); - code = tsdbIterMergerNext(merger->tombIterMerger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbIterMergerNext(merger->tombIterMerger), &lino, _exit); } - code = tsdbMergeFileSetEnd(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSetEnd(merger), &lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(merger->tsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(merger->tsdb->pVnode), __func__, fset->fid); } @@ -487,20 +457,18 @@ static int32_t tsdbDoMerge(SMerger *merger) { if (TARRAY2_SIZE(merger->fset->lvlArr) == 0) return 0; SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr); - if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) return 0; + if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) { + return 0; + } - code = tsdbMergerOpen(merger); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbMergeFileSet(merger, merger->fset); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbMergerClose(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergerOpen(merger), &lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeFileSet(merger, merger->fset), &lino, _exit); + TAOS_CHECK_GOTO(tsdbMergerClose(merger), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbDebug("vgId:%d %s done", TD_VID(merger->tsdb->pVnode), __func__); } @@ -543,45 +511,12 @@ int32_t tsdbMerge(void *arg) { if (merger->sttTrigger <= 1) return 0; // copy snapshot - code = tsdbMergeGetFSet(merger); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMergeGetFSet(merger), &lino, _exit); - if (merger->fset == NULL) return 0; - /* - bool skipMerge = false; - { - extern int8_t tsS3Enabled; - extern int32_t tsS3UploadDelaySec; - long s3Size(const char *object_name); - int32_t nlevel = tfsGetLevel(merger->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1) { - STFileObj *fobj = merger->fset->farr[TSDB_FTYPE_DATA]; - if (fobj && fobj->f->did.level == nlevel - 1) { - // if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay - const char *object_name = taosDirEntryBaseName((char *)fobj->fname); - - if (taosCheckExistFile(fobj->fname)) { - int32_t now = taosGetTimestampSec(); - int32_t mtime = 0; - - taosStatFile(fobj->fname, NULL, &mtime, NULL); - if (mtime < now - tsS3UploadDelaySec) { - skipMerge = true; - } - } else // if (s3Size(object_name) > 0) - { - skipMerge = true; - } - } - // new fset can be written with ts data - } + if (merger->fset == NULL) { + return 0; } - if (skipMerge) { - code = 0; - goto _exit; - } - */ // do merge tsdbInfo("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid); code = tsdbDoMerge(merger); @@ -590,7 +525,7 @@ int32_t tsdbMerge(void *arg) { _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); tsdbFatal("vgId:%d, failed to merge stt files since %s. code:%d", TD_VID(tsdb->pVnode), terrstr(), code); taosMsleep(100); exit(EXIT_FAILURE); diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index cf8794770a..c75e0d92c0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,17 +38,11 @@ int64_t tsdbGetEarliestTs(STsdb *pTsdb) { return ts; } -/** - * @brief - * - * @param pVnode - * @param ppTsdb - * @param dir - * @return int 0: success, -1: failed - */ -int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) { - STsdb *pTsdb = NULL; - int slen = 0; +int32_t tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) { + STsdb *pTsdb = NULL; + int slen = 0; + int32_t code; + int32_t lino; *ppTsdb = NULL; slen = strlen(pVnode->path) + strlen(dir) + 2; @@ -56,8 +50,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); if (pTsdb == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TAOS_CHECK_RETURN(TSDB_CODE_OUT_OF_MEMORY); } pTsdb->path = (char *)&pTsdb[1]; @@ -79,42 +72,34 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } // open tsdb - int32_t code = tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback); - if (code < 0) { - terrno = code; - goto _err; - } + TAOS_CHECK_GOTO(tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback), &lino, _exit); if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) { - terrno = TSDB_CODE_NEED_RETRY; - goto _err; + TAOS_CHECK_GOTO(TSDB_CODE_NEED_RETRY, &lino, _exit); } - if (tsdbOpenCache(pTsdb) < 0) { - goto _err; - } + TAOS_CHECK_GOTO(tsdbOpenCache(pTsdb), &lino, _exit); #ifdef TD_ENTERPRISE - if (tsdbOpenCompMonitor(pTsdb) < 0) { - goto _err; - } + TAOS_CHECK_GOTO(tsdbOpenCompMonitor(pTsdb), &lino, _exit); #endif - tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path, - pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2, - pTsdb->keepCfg.keepTimeOffset); - - *ppTsdb = pTsdb; - return 0; - -_err: - tsdbCloseFS(&pTsdb->pFS); - taosThreadMutexDestroy(&pTsdb->mutex); - taosMemoryFree(pTsdb); - return -1; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, lino, tstrerror(code)); + tsdbCloseFS(&pTsdb->pFS); + taosThreadMutexDestroy(&pTsdb->mutex); + taosMemoryFree(pTsdb); + } else { + tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path, + pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2, + pTsdb->keepCfg.keepTimeOffset); + *ppTsdb = pTsdb; + } + return code; } -int tsdbClose(STsdb **pTsdb) { +int32_t tsdbClose(STsdb **pTsdb) { if (*pTsdb) { STsdb *pdb = *pTsdb; tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path, diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index f1a087e3c1..13dc06a4d5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -40,7 +40,9 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con int32_t lino = 0; reader[0] = taosMemoryCalloc(1, sizeof(*reader[0])); - if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (reader[0] == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } reader[0]->config[0] = config[0]; reader[0]->buffers = config->buffers; @@ -50,13 +52,11 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con // open file if (fname) { - code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit); } else { char fname1[TSDB_FILENAME_LEN]; tsdbTFileName(config->tsdb, config->file, fname1); - code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit); } // // open each segment reader @@ -64,17 +64,17 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con ASSERT(offset >= TSDB_FHDR_SIZE); int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey; #if 1 - code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm, - encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, + encryptAlgoirthm, encryptKey), + &lino, _exit); #else int64_t size = config->file->size; for (; size > TSDB_FHDR_SIZE; size--) { - code = tsdbReadFile(reader[0]->fd, size - sizeof(SSttFooter), (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm, - encryptKey); + code = tsdbReadFile(reader[0]->fd, size - sizeof(SSttFooter), (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, + encryptAlgoirthm, encryptKey); if (code) continue; if ((*reader)->footer->sttBlkPtr->offset + (*reader)->footer->sttBlkPtr->size + sizeof(SSttFooter) == size || (*reader)->footer->statisBlkPtr->offset + (*reader)->footer->statisBlkPtr->size + sizeof(SSttFooter) == size || @@ -89,7 +89,8 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); tsdbSttFileReaderClose(reader); } return code; @@ -118,13 +119,14 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray * int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk); void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size); - if (!data) return TSDB_CODE_OUT_OF_MEMORY; + if (!data) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - int32_t code = - tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0, - encryptAlgorithm, encryptKey); + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + int32_t code = tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, + reader->footer->statisBlkPtr->size, 0, encryptAlgorithm, encryptKey); if (code) { taosMemoryFree(data); return code; @@ -149,13 +151,14 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk); void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size); - if (!data) return TSDB_CODE_OUT_OF_MEMORY; + if (!data) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - int32_t code = - tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0, - encryptAlgorithm, encryptKey); + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + int32_t code = tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, + reader->footer->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey); if (code) { taosMemoryFree(data); return code; @@ -180,13 +183,14 @@ int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBl int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk); void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size); - if (!data) return TSDB_CODE_OUT_OF_MEMORY; + if (!data) { + return TSDB_CODE_OUT_OF_MEMORY; + } int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - int32_t code = - tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0, - encryptAlgorithm, encryptKey); + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + int32_t code = tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, + 0, encryptAlgorithm, encryptKey); if (code) { taosMemoryFree(data); return code; @@ -212,20 +216,20 @@ int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk, SBuffer *assist = reader->buffers + 1; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; // load data tBufferClear(buffer0); - code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0, + encryptAlgorithm, encryptKey), + &lino, _exit); SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); - code = tBlockDataDecompress(&br, bData, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataDecompress(&br, bData, assist), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -241,17 +245,16 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * SBuffer *assist = reader->buffers + 2; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; // load key part tBufferClear(buffer0); - code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0, + encryptAlgorithm, encryptKey), + &lino, _exit); // decode header SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); - code = tGetDiskDataHdr(&br, &hdr); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); @@ -262,8 +265,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * bData->nRow = hdr.nRow; // key part - code = tBlockDataDecompressKeyPart(&hdr, &br, bData, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit); ASSERT(br.offset == buffer0->size); bool loadExtra = false; @@ -280,9 +282,9 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * // load SBlockCol part tBufferClear(buffer0); - code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0, + encryptAlgorithm, encryptKey), + &lino, _exit); // load each column SBlockCol blockCol = { @@ -302,8 +304,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * break; } - code = tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg), &lino, _exit); } if (cid < blockCol.cid) { @@ -320,27 +321,26 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * .szValue = 0, .offset = 0, }; - code = tBlockDataDecompressColData(&hdr, &none, &br, bData, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &none, &br, bData, assist), &lino, _exit); } else if (cid == blockCol.cid) { // load from file tBufferClear(buffer1); - code = + TAOS_CHECK_GOTO( tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset, - blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, encryptAlgorithm, + encryptKey), + &lino, _exit); // decode the buffer SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1); - code = tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist), &lino, _exit); } } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -353,12 +353,12 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk SBuffer *assist = reader->buffers + 1; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; // load tBufferClear(buffer0); - code = tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, encryptAlgorithm, + encryptKey), + &lino, _exit); // decode int32_t size = 0; @@ -372,15 +372,15 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk .originalSize = tombBlk->numRec * sizeof(int64_t), .compressedSize = tombBlk->size[i], }; - code = tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tombBlock->buffers + i, assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tombBlock->buffers + i, assist), &lino, _exit); br.offset += tombBlk->size[i]; } ASSERT(br.offset == tombBlk->dp->size); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -393,12 +393,12 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta SBuffer *assist = reader->buffers + 1; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; // load data tBufferClear(buffer0); - code = tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0, + encryptAlgorithm, encryptKey), + &lino, _exit); // decode data tStatisBlockClear(statisBlock); @@ -413,8 +413,7 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta .originalSize = statisBlk->numRec * sizeof(int64_t), }; - code = tDecompressDataToBuffer(BR_PTR(&br), &info, &statisBlock->buffers[i], assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &info, &statisBlock->buffers[i], assist), &lino, _exit); br.offset += statisBlk->size[i]; } @@ -424,25 +423,23 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta // decode compress info for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { - code = tValueColumnCompressInfoDecode(&br, &firstKeyInfos[i]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &firstKeyInfos[i]), &lino, _exit); } for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { - code = tValueColumnCompressInfoDecode(&br, &lastKeyInfos[i]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &lastKeyInfos[i]), &lino, _exit); } // decode value columns for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { - code = tValueColumnDecompress(BR_PTR(&br), firstKeyInfos + i, &statisBlock->firstKeyPKs[i], assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), firstKeyInfos + i, &statisBlock->firstKeyPKs[i], assist), + &lino, _exit); br.offset += (firstKeyInfos[i].dataCompressedSize + firstKeyInfos[i].offsetCompressedSize); } for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { - code = tValueColumnDecompress(BR_PTR(&br), &lastKeyInfos[i], &statisBlock->lastKeyPKs[i], assist); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), &lastKeyInfos[i], &statisBlock->lastKeyPKs[i], assist), &lino, + _exit); br.offset += (lastKeyInfos[i].dataCompressedSize + lastKeyInfos[i].offsetCompressedSize); } } @@ -451,7 +448,8 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -486,7 +484,7 @@ struct SSttFileWriter { static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, SColCompressInfo *info, int64_t *fileSize, TSttBlkArray *sttBlkArray, SBuffer *buffers, - SVersionRange *range, int32_t encryptAlgorithm, char* encryptKey) { + SVersionRange *range, int32_t encryptAlgorithm, char *encryptKey) { if (blockData->nRow == 0) return 0; int32_t code = 0; @@ -510,25 +508,21 @@ static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, S } tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer); - code = tBlockDataCompress(blockData, info, buffers, buffers + 4); - if (code) return code; + TAOS_CHECK_RETURN(tBlockDataCompress(blockData, info, buffers, buffers + 4)); sttBlk->bInfo.offset = *fileSize; sttBlk->bInfo.szKey = buffers[0].size + buffers[1].size; sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey; for (int i = 0; i < 4; i++) { if (buffers[i].size) { - code = tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey); - if (code) return code; + TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey)); *fileSize += buffers[i].size; } } - code = TARRAY2_APPEND_PTR(sttBlkArray, sttBlk); - if (code) return code; + TAOS_CHECK_RETURN(TARRAY2_APPEND_PTR(sttBlkArray, sttBlk)); tBlockDataClear(blockData); - return 0; } @@ -541,24 +535,28 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { tb_uid_t uid = writer->blockData->suid == 0 ? writer->blockData->uid : writer->blockData->suid; SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL}; code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr)); + ASSERT(code == TSDB_CODE_SUCCESS); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray, - writer->buffers, &writer->ctx->range, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + TAOS_CHECK_GOTO( + tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray, + writer->buffers, &writer->ctx->range, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } taosHashCleanup(info.pColCmpr); return code; } static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { - if (writer->staticBlock->numOfRecords == 0) return 0; + if (writer->staticBlock->numOfRecords == 0) { + return 0; + } int32_t code = 0; int32_t lino = 0; @@ -586,7 +584,7 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { statisBlk.maxTbid.uid = record.uid; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; // compress each column for (int32_t i = 0; i < ARRAY_SIZE(statisBlk.size); i++) { @@ -597,12 +595,10 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { }; tBufferClear(buffer0); - code = tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist), &lino, _exit); + TAOS_CHECK_GOTO( + tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize, encryptAlgorithm, encryptKey), + &lino, _exit); statisBlk.size[i] = info.compressedSize; statisBlk.dp->size += info.compressedSize; @@ -617,74 +613,72 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { tBufferClear(buffer1); for (int32_t i = 0; i < statisBlk.numOfPKs; i++) { - code = tValueColumnCompress(&statisBlock->firstKeyPKs[i], &compressInfo, buffer1, assist); - TSDB_CHECK_CODE(code, lino, _exit); - code = tValueColumnCompressInfoEncode(&compressInfo, buffer0); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->firstKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit); + TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit); } for (int32_t i = 0; i < statisBlk.numOfPKs; i++) { - code = tValueColumnCompress(&statisBlock->lastKeyPKs[i], &compressInfo, buffer1, assist); - TSDB_CHECK_CODE(code, lino, _exit); - code = tValueColumnCompressInfoEncode(&compressInfo, buffer0); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->lastKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit); + TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit); } - code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey), + &lino, _exit); writer->file->size += buffer0->size; statisBlk.dp->size += buffer0->size; - code = tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey), + &lino, _exit); writer->file->size += buffer1->size; statisBlk.dp->size += buffer1->size; } - code = TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk), &lino, _exit); tStatisBlockClear(writer->staticBlock); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { - if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; + if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) { + return 0; + } int32_t code = 0; int32_t lino = 0; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, - writer->tombBlkArray, writer->buffers, &writer->ctx->range, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, + writer->tombBlkArray, writer->buffers, &writer->ctx->range, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize, - int32_t encryptAlgorithm, char* encryptKey) { + int32_t encryptAlgorithm, char *encryptKey) { ptr->size = TARRAY2_DATA_LEN(sttBlkArray); if (ptr->size > 0) { ptr->offset = *fileSize; - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, encryptAlgorithm, - encryptKey); - if (code) { - return code; - } + TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, + encryptAlgorithm, encryptKey)); *fileSize += ptr->size; } @@ -696,15 +690,16 @@ static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) { int32_t lino; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size, + encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -713,20 +708,21 @@ static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) { int32_t code = 0; int32_t lino; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray); if (writer->footer->statisBlkPtr->size) { writer->footer->statisBlkPtr->offset = writer->file->size; - code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray), - writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray), + writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey), + &lino, _exit); writer->file->size += writer->footer->statisBlkPtr->size; } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -736,30 +732,31 @@ static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) { int32_t lino = 0; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size, - encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, + &writer->file->size, encryptAlgorithm, encryptKey), + &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } -int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, - char* encryptKey) { - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey); - if (code) return code; +int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, + char *encryptKey) { + TAOS_CHECK_RETURN( + tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey)); *fileSize += sizeof(*footer); return 0; } static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) { int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, encryptAlgorithm, encryptKey); } @@ -795,15 +792,13 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { char fname[TSDB_FILENAME_LEN]; tsdbTFileName(writer->config->tsdb, writer->file, fname); - code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0), &lino, _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey), &lino, _exit); writer->file->size += sizeof(hdr); // range @@ -813,7 +808,8 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -843,34 +839,19 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o int32_t lino; int32_t code; - code = tsdbSttFileDoWriteBlockData(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSttFileDoWriteStatisBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSttFileDoWriteTombBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSttFileDoWriteSttBlk(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSttFileDoWriteStatisBlk(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSttFileDoWriteTombBlk(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSttFileDoWriteFooter(writer); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSttFileDoUpdateHeader(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteSttBlk(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlk(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlk(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteFooter(writer), &lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoUpdateHeader(writer), &lino, _exit); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey), &lino, _exit); tsdbCloseFile(&writer->fd); @@ -882,12 +863,12 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o }; tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); - code = TARRAY2_APPEND(opArray, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(opArray, op), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -902,7 +883,9 @@ static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) { int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) { writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); - if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (writer[0] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } writer[0]->config[0] = config[0]; writer[0]->ctx->opened = false; @@ -915,11 +898,9 @@ int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArr if (writer[0]->ctx->opened) { if (abort) { - code = tsdbSttFWriterCloseAbort(writer[0]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFWriterCloseAbort(writer[0]), &lino, _exit); } else { - code = tsdbSttFWriterCloseCommit(writer[0], opArray); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFWriterCloseCommit(writer[0], opArray), &lino, _exit); } tsdbSttFWriterDoClose(writer[0]); } @@ -928,7 +909,8 @@ int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArr _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -938,20 +920,19 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) { int32_t lino = 0; if (!writer->ctx->opened) { - code = tsdbSttFWriterDoOpen(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit); } if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) { - code = tsdbSttFileDoWriteBlockData(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit); - code = tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb), &lino, _exit); - TABLEID id = {.suid = row->suid, .uid = row->suid ? 0 : row->uid}; - code = tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); + TABLEID id = { + .suid = row->suid, + .uid = row->suid ? 0 : row->uid, + }; + TAOS_CHECK_GOTO(tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0), &lino, _exit); } if (writer->ctx->tbid->uid != row->uid) { @@ -965,8 +946,7 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) { for (;;) { code = tStatisBlockPut(writer->staticBlock, row, writer->config->maxRow); if (code == TSDB_CODE_INVALID_PARA) { - code = tsdbSttFileDoWriteStatisBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit); continue; } else { TSDB_CHECK_CODE(code, lino, _exit); @@ -975,9 +955,9 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) { } if (row->row.type == TSDBROW_ROW_FMT) { - code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, // - TSDBROW_SVERSION(&row->row), writer->config->skmRow); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, // + TSDBROW_SVERSION(&row->row), writer->config->skmRow), + &lino, _exit); } // row to col conversion @@ -989,21 +969,20 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) { && tsdbRowCompareWithoutVersion(&row->row, &tsdbRowFromBlockData(writer->blockData, writer->blockData->nRow - 1)) == 0 // ) { - code = tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema), &lino, _exit); } else { if (writer->blockData->nRow >= writer->config->maxRow) { - code = tsdbSttFileDoWriteBlockData(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit); } - code = tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid), + &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1018,13 +997,13 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *bdata) { row->uid = bdata->uid ? bdata->uid : bdata->aUid[i]; row->row = tsdbRowFromBlockData(bdata, i); - code = tsdbSttFileWriteRow(writer, row); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileWriteRow(writer, row), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -1034,31 +1013,27 @@ int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *re int32_t lino; if (!writer->ctx->opened) { - code = tsdbSttFWriterDoOpen(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit); } else { if (writer->blockData->nRow > 0) { - code = tsdbSttFileDoWriteBlockData(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit); } if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) { - code = tsdbSttFileDoWriteStatisBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit); } } - code = tTombBlockPut(writer->tombBlock, record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tTombBlockPut(writer->tombBlock, record), &lino, _exit); if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { - code = tsdbSttFileDoWriteTombBlock(writer); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 ", version:%" PRId64, diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index 09ab2243a0..ef116e2642 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -37,8 +37,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * int8_t cmprAlg; int32_t szPage; SBuffer buffers[10]; - int32_t encryptAlgorithm; - char* encryptKey; + int32_t encryptAlgorithm; + char *encryptKey; // reader SArray *aBlockIdx; SMapData mDataBlk[1]; @@ -58,12 +58,10 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * // read SBlockIdx array if ((ctx->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - code = tsdbReadBlockIdx(reader, ctx->aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadBlockIdx(reader, ctx->aBlockIdx), &lino, _exit); if (taosArrayGetSize(ctx->aBlockIdx) > 0) { // init/open file fd @@ -77,22 +75,19 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * .maxVer = VERSION_MIN, }; - code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]), &lino, _exit); // open fd char fname[TSDB_FILENAME_LEN]; tsdbTFileName(tsdb, &file, fname); - code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0), &lino, _exit); // convert for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(ctx->aBlockIdx); ++iBlockIdx) { SBlockIdx *pBlockIdx = taosArrayGet(ctx->aBlockIdx, iBlockIdx); - code = tsdbReadDataBlk(reader, pBlockIdx, ctx->mDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadDataBlk(reader, pBlockIdx, ctx->mDataBlk), &lino, _exit); for (int32_t iDataBlk = 0; iDataBlk < ctx->mDataBlk->nItem; ++iDataBlk) { SDataBlk dataBlk[1]; @@ -134,13 +129,13 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * record.count = 0; } - code = tBrinBlockPut(ctx->brinBlock, &record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tBrinBlockPut(ctx->brinBlock, &record), &lino, _exit); if (ctx->brinBlock->numOfRecords >= ctx->maxRow) { SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; - code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, - ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey); + code = + tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, + ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -148,28 +143,28 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * if (ctx->brinBlock->numOfRecords > 0) { SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; - code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, - ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, + ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey), + &lino, _exit); } - code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr, - &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey); + TAOS_CHECK_GOTO(tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr, + &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey), + &lino, _exit); + + code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm, + ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm, - ctx->encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit); tsdbCloseFile(&ctx->fd); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } TARRAY2_DESTROY(ctx->brinBlkArray, NULL); tBrinBlockDestroy(ctx->brinBlock); @@ -200,12 +195,11 @@ static int32_t tsdbUpgradeData(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * .maxVer = VERSION_MIN, }; - code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -228,12 +222,11 @@ static int32_t tsdbUpgradeSma(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *r .maxVer = VERSION_MIN, }; - code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -245,12 +238,10 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade SArray *aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (aSttBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - code = tsdbReadSttBlk(reader, iStt, aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadSttBlk(reader, iStt, aSttBlk), &lino, _exit); if (taosArrayGetSize(aSttBlk) > 0) { SSttFile *pSttF = pDFileSet->aSttF[iStt]; @@ -258,7 +249,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade struct { int32_t szPage; int32_t encryptAlgorithm; - char* encryptKey; + char *encryptKey; // writer STsdbFD *fd; TSttBlkArray sttBlkArray[1]; @@ -278,31 +269,27 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade .minVer = VERSION_MAX, .maxVer = VERSION_MIN, }; - code = tsdbTFileObjInit(tsdb, &file, &fobj); - TSDB_CHECK_CODE(code, lino, _exit1); + TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fobj), &lino, _exit); - code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0); - TSDB_CHECK_CODE(code, lino, _exit1); + TAOS_CHECK_GOTO(tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0), &lino, _exit); for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) { - code = TARRAY2_APPEND_PTR(ctx->sttBlkArray, (SSttBlk *)taosArrayGet(aSttBlk, iSttBlk)); - TSDB_CHECK_CODE(code, lino, _exit1); + TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(ctx->sttBlkArray, (SSttBlk *)taosArrayGet(aSttBlk, iSttBlk)), &lino, _exit); } code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm, - ctx->encryptKey); + ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit1); - code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey); - TSDB_CHECK_CODE(code, lino, _exit1); + TAOS_CHECK_GOTO( + tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey), &lino, + _exit); - code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); - TSDB_CHECK_CODE(code, lino, _exit1); + TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit); tsdbCloseFile(&ctx->fd); - code = TARRAY2_APPEND(lvl->fobjArr, fobj); - TSDB_CHECK_CODE(code, lino, _exit1); + TAOS_CHECK_GOTO(TARRAY2_APPEND(lvl->fobjArr, fobj), &lino, _exit); _exit1: TARRAY2_DESTROY(ctx->sttBlkArray, NULL); @@ -310,7 +297,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } taosArrayDestroy(aSttBlk); return code; @@ -325,24 +312,21 @@ static int32_t tsdbUpgradeStt(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *r } SSttLvl *lvl; - code = tsdbSttLvlInit(0, &lvl); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbSttLvlInit(0, &lvl), &lino, _exit); for (int32_t iStt = 0; iStt < pDFileSet->nSttF; ++iStt) { - code = tsdbUpgradeSttFile(tsdb, pDFileSet, reader, fset, iStt, lvl); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeSttFile(tsdb, pDFileSet, reader, fset, iStt, lvl), &lino, _exit); } if (TARRAY2_SIZE(lvl->fobjArr) > 0) { - code = TARRAY2_APPEND(fset->lvlArr, lvl); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(fset->lvlArr, lvl), &lino, _exit); } else { tsdbSttLvlClear(&lvl); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -356,40 +340,33 @@ static int32_t tsdbUpgradeFileSet(STsdb *tsdb, SDFileSet *pDFileSet, TFileSetArr SDataFReader *reader; STFileSet *fset; - code = tsdbTFileSetInit(pDFileSet->fid, &fset); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbTFileSetInit(pDFileSet->fid, &fset), &lino, _exit); - code = tsdbDataFReaderOpen(&reader, tsdb, pDFileSet); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDataFReaderOpen(&reader, tsdb, pDFileSet), &lino, _exit); // .head - code = tsdbUpgradeHead(tsdb, pDFileSet, reader, fset); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeHead(tsdb, pDFileSet, reader, fset), &lino, _exit); // .data - code = tsdbUpgradeData(tsdb, pDFileSet, reader, fset); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeData(tsdb, pDFileSet, reader, fset), &lino, _exit); // .sma - code = tsdbUpgradeSma(tsdb, pDFileSet, reader, fset); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeSma(tsdb, pDFileSet, reader, fset), &lino, _exit); // .stt if (pDFileSet->nSttF > 0) { - code = tsdbUpgradeStt(tsdb, pDFileSet, reader, fset); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeStt(tsdb, pDFileSet, reader, fset), &lino, _exit); } tsdbDataFReaderClose(&reader); - code = TARRAY2_APPEND(fileSetArray, fset); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(fileSetArray, fset), &lino, _exit); tsdbInfo("vgId:%d upgrade file set end, fid:%d", TD_VID(tsdb->pVnode), pDFileSet->fid); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -411,8 +388,7 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f .maxVer = VERSION_MIN, }; - code = tsdbTFileObjInit(tsdb, &file, fobj); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, fobj), &lino, _exit); fset->farr[TSDB_FTYPE_TOMB] = *fobj; } else { // to .stt file @@ -429,28 +405,26 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f .maxVer = VERSION_MIN, }; - code = tsdbTFileObjInit(tsdb, &file, fobj); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, fobj), &lino, _exit); - code = TARRAY2_APPEND(lvl->fobjArr, fobj[0]); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(lvl->fobjArr, fobj[0]), &lino, _exit); } char fname[TSDB_FILENAME_LEN] = {0}; - code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0), &lino, + _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey; - code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey), &lino, _exit); fobj[0]->f->size += TSDB_FHDR_SIZE; _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -467,8 +441,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * int64_t minKey; int64_t maxKey; SBuffer buffers[10]; - int32_t encryptAlgorithm; - char* encryptKey; + int32_t encryptAlgorithm; + char *encryptKey; // reader SArray *aDelData; // writer @@ -488,15 +462,13 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey); if ((ctx->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(aDelIdx, iDelIdx); - code = tsdbReadDelData(reader, pDelIdx, ctx->aDelData); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadDelData(reader, pDelIdx, ctx->aDelData), &lino, _exit); for (int32_t iDelData = 0; iDelData < taosArrayGetSize(ctx->aDelData); iDelData++) { SDelData *pDelData = (SDelData *)taosArrayGet(ctx->aDelData, iDelData); @@ -509,61 +481,59 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * .ekey = pDelData->eKey, }; - code = tTombBlockPut(ctx->tombBlock, &record); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tTombBlockPut(ctx->tombBlock, &record), &lino, _exit); if (TOMB_BLOCK_SIZE(ctx->tombBlock) > ctx->maxRow) { if (ctx->fd == NULL) { - code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt), &lino, _exit); } SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; - code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, - ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, + ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey), + &lino, _exit); } } } if (TOMB_BLOCK_SIZE(ctx->tombBlock) > 0) { if (ctx->fd == NULL) { - code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt), &lino, _exit); } SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; - code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, - ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO( + tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, + ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey), + &lino, _exit); } if (ctx->fd != NULL) { if (ctx->toStt) { - code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size, + code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, - ctx->encryptKey); + code = + tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } else { - code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size, + code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, - ctx->encryptKey); + code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, + ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit); tsdbCloseFile(&ctx->fd); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } for (int32_t i = 0; i < ARRAY_SIZE(ctx->buffers); i++) { tBufferDestroy(ctx->buffers + i); @@ -582,27 +552,23 @@ static int32_t tsdbUpgradeTombFile(STsdb *tsdb, SDelFile *pDelFile, TFileSetArra SArray *aDelIdx = NULL; if ((aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - code = tsdbDelFReaderOpen(&reader, pDelFile, tsdb); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDelFReaderOpen(&reader, pDelFile, tsdb), &lino, _exit); - code = tsdbReadDelIdx(reader, aDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbReadDelIdx(reader, aDelIdx), &lino, _exit); if (taosArrayGetSize(aDelIdx) > 0) { STFileSet *fset; TARRAY2_FOREACH(fileSetArray, fset) { - code = tsdbDumpTombDataToFSet(tsdb, reader, aDelIdx, fset); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDumpTombDataToFSet(tsdb, reader, aDelIdx, fset), &lino, _exit); } } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } tsdbDelFReaderClose(&reader); taosArrayDestroy(aDelIdx); @@ -615,19 +581,17 @@ static int32_t tsdbDoUpgradeFileSystem(STsdb *tsdb, TFileSetArray *fileSetArray) // upgrade each file set for (int32_t i = 0; i < taosArrayGetSize(tsdb->fs.aDFileSet); i++) { - code = tsdbUpgradeFileSet(tsdb, taosArrayGet(tsdb->fs.aDFileSet, i), fileSetArray); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeFileSet(tsdb, taosArrayGet(tsdb->fs.aDFileSet, i), fileSetArray), &lino, _exit); } // upgrade tomb file if (tsdb->fs.pDelFile != NULL) { - code = tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray), &lino, _exit); } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -639,24 +603,21 @@ static int32_t tsdbUpgradeFileSystem(STsdb *tsdb, int8_t rollback) { TFileSetArray fileSetArray[1] = {0}; // open old file system - code = tsdbFSOpen(tsdb, rollback); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSOpen(tsdb, rollback), &lino, _exit); - code = tsdbDoUpgradeFileSystem(tsdb, fileSetArray); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoUpgradeFileSystem(tsdb, fileSetArray), &lino, _exit); // close file system - code = tsdbFSClose(tsdb); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSClose(tsdb), &lino, _exit); // save new file system char fname[TSDB_FILENAME_LEN]; current_fname(tsdb, fname, TSDB_FCURRENT); - code = save_fs(fileSetArray, fname); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(save_fs(fileSetArray, fname), &lino, _exit); _exit: if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } TARRAY2_DESTROY(fileSetArray, tsdbTFileSetClear); @@ -669,9 +630,8 @@ int32_t tsdbCheckAndUpgradeFileSystem(STsdb *tsdb, int8_t rollback) { tsdbGetCurrentFName(tsdb, fname, NULL); if (!taosCheckExistFile(fname)) return 0; - int32_t code = tsdbUpgradeFileSystem(tsdb, rollback); - if (code) return code; + TAOS_CHECK_RETURN(tsdbUpgradeFileSystem(tsdb, rollback)); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index e8be48499b..0b32c96d2e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -28,6 +28,7 @@ int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400 // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) { + int32_t code; int32_t arrSize = 0; int32_t affectedrows = 0; int32_t numOfRows = 0; @@ -39,18 +40,16 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 arrSize = taosArrayGetSize(pMsg->aSubmitTbData); // scan and convert - if ((terrno = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) { - if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { + if ((code = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) { + if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE) { tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno)); } - return -1; + return code; } // loop to insert for (int32_t i = 0; i < arrSize; ++i) { - if ((terrno = tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)) < 0) { - return -1; - } + TAOS_CHECK_RETURN(tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)); } if (pRsp != NULL) { @@ -74,22 +73,12 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowK } int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { - int32_t code = 0; STsdbKeepCfg *pCfg = &pTsdb->keepCfg; TSKEY now = taosGetTimestamp(pCfg->precision); TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); - /* - int32_t nlevel = tfsGetLevel(pTsdb->pVnode->pTfs); - if (nlevel > 1 && tsS3Enabled) { - if (nlevel == 3) { - minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1; - } else if (nlevel == 2) { - minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep0; - } - } - */ + for (int32_t i = 0; i < size; ++i) { SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { @@ -99,22 +88,17 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { int32_t nRows = aColData[0].nVal; TSKEY *aKey = (TSKEY *)aColData[0].pData; for (int32_t r = 0; r < nRows; ++r) { - if ((code = tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now)) < 0) { - goto _exit; - } + TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now)); } } } else { int32_t nRows = taosArrayGetSize(pData->aRowP); for (int32_t r = 0; r < nRows; ++r) { SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r); - if ((code = tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now)) < 0) { - goto _exit; - } + TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now)); } } } -_exit: - return code; + return 0; } From a8433ec6eb4156c403d5df05330ecdd47a382eab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Jul 2024 00:31:28 +0800 Subject: [PATCH 21/45] fix(tsdb):set the initialized value. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 9 ++++++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 9ee5e1c7ba..2474666e93 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -296,8 +296,11 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo pIter->pSttBlockReader->uid = 0; tMergeTreeClose(&pIter->pSttBlockReader->mergeTree); destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); - pReader->status.pLDataIterArray = NULL; + pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + if (pReader->status.pLDataIterArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } // check file the time range of coverage STimeWindow win = {0}; @@ -711,6 +714,10 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S return code; } + if (pRecord == NULL) { + break; + } + if (pRecord->suid > pReader->info.suid) { break; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index f937dac606..3051b097be 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -461,7 +461,7 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) { pIter->blockIndex += 1; if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); @@ -470,7 +470,7 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) { int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); if (code != TSDB_CODE_SUCCESS) { tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); - return TSDB_CODE_FAILED; + return code; } pIter->recordIndex = -1; From 80a7b7da5fb0635d25e704011ba31be483f14eb3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Jul 2024 08:58:39 +0800 Subject: [PATCH 22/45] fix(tsdb): fix invalid return value. --- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 3051b097be..d7da6cbd9d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -656,7 +656,7 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 } void* p = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); - if (p != NULL) { + if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } From 66245a54f0cc5a47f6a71143263bcab99d051e55 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 Jul 2024 09:06:52 +0800 Subject: [PATCH 23/45] fix: taos array init issue --- source/libs/catalog/src/ctgAsync.c | 2 +- source/libs/catalog/src/ctgUtil.c | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index a32d029016..4580cd4fcd 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -2825,7 +2825,7 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } - pTask->msgCtxs = taosArrayInit_s(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); if (NULL == pTask->msgCtxs) { ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 0098c79384..ffe894abad 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1653,10 +1653,12 @@ int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst) { return terrno; } TAOS_MEMCPY(pDst, pSrc, sizeof(SDbCfgInfo)); - pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); - if (NULL == pDst->pRetensions) { - taosMemoryFree(pDst); - return terrno; + if (((SDbCfgInfo *)pSrc)->pRetensions) { + pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); + if (NULL == pDst->pRetensions) { + taosMemoryFree(pDst); + return terrno; + } } *ppDst = pDst; From 438921e9cc9cf12cdc33eb56e980d8ead3a8782b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Jul 2024 09:30:58 +0800 Subject: [PATCH 24/45] refactor: check return value for each function. --- include/libs/stream/tstream.h | 2 + source/libs/stream/src/streamHb.c | 82 +++++++++++++++++++--------- source/libs/stream/src/streamMeta.c | 13 +++-- source/libs/stream/src/streamTimer.c | 11 ++++ 4 files changed, 78 insertions(+), 30 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 068f0a282e..5327428f5b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -771,6 +771,8 @@ int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask); // timer tmr_h streamTimerGetInstance(); +void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, + const char* pMsg); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 84eb6b319f..bcfe2c4243 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -54,7 +54,7 @@ static bool existInHbMsg(SStreamHbMsg* pMsg, SDownstreamTaskEpset* pTaskEpset) { static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { SStreamMeta* pMeta = pTask->pMeta; - taosThreadMutexLock(&pTask->lock); + (void) taosThreadMutexLock(&pTask->lock); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); for (int j = 0; j < num; ++j) { @@ -62,14 +62,18 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { bool exist = existInHbMsg(pMsg, pTaskEpset); if (!exist) { - taosArrayPush(pMsg->pUpdateNodes, &pTaskEpset->nodeId); + void* p = taosArrayPush(pMsg->pUpdateNodes, &pTaskEpset->nodeId); + if (p == NULL) { + stError("failed to set the updateNode info in hbMsg, vgId:%d", pMeta->vgId); + } + stDebug("vgId:%d nodeId:%d added into hbMsg update list, total:%d", pMeta->vgId, pTaskEpset->nodeId, (int32_t)taosArrayGetSize(pMsg->pUpdateNodes)); } } taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList); - taosThreadMutexUnlock(&pTask->lock); + (void) taosThreadMutexUnlock(&pTask->lock); } static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* pEpset) { @@ -101,9 +105,7 @@ static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* p SRpcMsg msg = {0}; initRpcMsg(&msg, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); - tmsgSendReq(pEpset, &msg); - - return TSDB_CODE_SUCCESS; + return tmsgSendReq(pEpset, &msg); } // NOTE: this task should be executed within the SStreamMeta lock region. @@ -112,6 +114,7 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { bool hasMnodeEpset = false; int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); SMetaHbInfo* pInfo = pMeta->pHbInfo; + int32_t code = 0; // not recv the hb msg rsp yet, send current hb msg again if (pInfo->msgSendTs > 0) { @@ -135,8 +138,7 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { } pInfo->msgSendTs = taosGetTimestampMs(); - doSendHbMsgInfo(&pInfo->hbMsg, pMeta, &epset); - return TSDB_CODE_SUCCESS; + return doSendHbMsgInfo(&pInfo->hbMsg, pMeta, &epset); } SStreamHbMsg* pMsg = &pInfo->hbMsg; @@ -168,9 +170,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { continue; } - taosThreadMutexLock(&(*pTask)->lock); + (void) taosThreadMutexLock(&(*pTask)->lock); STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask); - taosThreadMutexUnlock(&(*pTask)->lock); + (void) taosThreadMutexUnlock(&(*pTask)->lock); entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { @@ -188,9 +190,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { stInfo("s-task:%s set kill checkpoint trans in hbMsg, transId:%d, clear the active checkpointInfo", (*pTask)->id.idStr, p->transId); - taosThreadMutexLock(&(*pTask)->lock); + (void) taosThreadMutexLock(&(*pTask)->lock); streamTaskClearCheckInfo((*pTask), true); - taosThreadMutexUnlock(&(*pTask)->lock); + (void) taosThreadMutexUnlock(&(*pTask)->lock); } } @@ -210,7 +212,11 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { } addUpdateNodeIntoHbMsg(*pTask, pMsg); - taosArrayPush(pMsg->pTaskStatus, &entry); + p = taosArrayPush(pMsg->pTaskStatus, &entry); + if (p == NULL) { + stError("failed to add taskInfo:0x%x in hbMsg, vgId:%d", (*pTask)->id.taskId, pMeta->vgId); + } + if (!hasMnodeEpset) { epsetAssign(&epset, &(*pTask)->info.mnodeEpset); hasMnodeEpset = true; @@ -221,18 +227,19 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { if (hasMnodeEpset) { pInfo->msgSendTs = taosGetTimestampMs(); - doSendHbMsgInfo(pMsg, pMeta, &epset); + code = doSendHbMsgInfo(pMsg, pMeta, &epset); } else { stDebug("vgId:%d no tasks or no mnd epset, not send stream hb to mnode", pMeta->vgId); tCleanupStreamHbMsg(&pInfo->hbMsg); pInfo->msgSendTs = -1; } - return TSDB_CODE_SUCCESS; + return code; } void streamMetaHbToMnode(void* param, void* tmrId) { int64_t rid = *(int64_t*)param; + int32_t code = 0; SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid); if (pMeta == NULL) { @@ -243,15 +250,25 @@ void streamMetaHbToMnode(void* param, void* tmrId) { // need to stop, stop now if (pMeta->pHbInfo->stopFlag == STREAM_META_WILL_STOP) { // todo refactor: not need this now, use closeFlag in Meta pMeta->pHbInfo->stopFlag = STREAM_META_OK_TO_STOP; - stDebug("vgId:%d jump out of meta timer", pMeta->vgId); - taosReleaseRef(streamMetaId, rid); + code = taosReleaseRef(streamMetaId, rid); + if (code == TSDB_CODE_SUCCESS) { + stDebug("vgId:%d jump out of meta timer", pMeta->vgId); + } else { + stError("vgId:%d jump out of meta timer, failed to release the meta rid:%d", pMeta->vgId, rid); + } return; } // not leader not send msg if (pMeta->role != NODE_ROLE_LEADER) { - stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role); - taosReleaseRef(streamMetaId, rid); + code = taosReleaseRef(streamMetaId, rid); + if (code == TSDB_CODE_SUCCESS) { + stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role); + } else { + stError("vgId:%d role:%d not leader not send hb to mnodefailed to release the meta rid:%d", pMeta->vgId, + pMeta->role, rid); + } + pMeta->pHbInfo->hbStart = 0; return; } @@ -262,17 +279,30 @@ void streamMetaHbToMnode(void* param, void* tmrId) { } if (!waitForEnoughDuration(pMeta->pHbInfo)) { - taosTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); - taosReleaseRef(streamMetaId, rid); + streamTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, pMeta->vgId, + "meta-hb-tmr"); + + code = taosReleaseRef(streamMetaId, rid); + if (code) { + stError("vgId:%d in meta timer, failed to release the meta rid:%d", pMeta->vgId, rid); + } return; } streamMetaRLock(pMeta); - streamMetaSendHbHelper(pMeta); - streamMetaRUnLock(pMeta); + code = streamMetaSendHbHelper(pMeta); + if (code) { + stError("vgId:%d failed to send hmMsg to mnode, try again in 5s, code:%s", pMeta->vgId, strerror(code)); + } - taosTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); - taosReleaseRef(streamMetaId, rid); + streamMetaRUnLock(pMeta); + streamTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, pMeta->vgId, + "meta-hb-tmr"); + + code = taosReleaseRef(streamMetaId, rid); + if (code) { + stError("vgId:%d in meta timer, failed to release the meta rid:%d", pMeta->vgId, rid); + } } int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes) { @@ -297,7 +327,7 @@ void destroyMetaHbInfo(SMetaHbInfo* pInfo) { tCleanupStreamHbMsg(&pInfo->hbMsg); if (pInfo->hbTmr != NULL) { - taosTmrStop(pInfo->hbTmr); + (void) taosTmrStop(pInfo->hbTmr); pInfo->hbTmr = NULL; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a2b1d39e0c..0a627034b1 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1106,7 +1106,12 @@ int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, voi int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { *pList = NULL; + int32_t code = 0; SArray* pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + if (pTaskList == NULL) { + stError("failed to generate the task list during send hbMsg to mnode, vgId:%d, code: out of memory", pMeta->vgId); + return TSDB_CODE_OUT_OF_MEMORY; + } bool sendMsg = pMeta->sendMsgBeforeClosing; if (!sendMsg) { @@ -1122,8 +1127,8 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTask* pTask = NULL; - int32_t code = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); - if (code != TSDB_CODE_SUCCESS) { + code = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); + if (code != TSDB_CODE_SUCCESS) { // this error is ignored continue; } @@ -1140,9 +1145,9 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { streamMetaReleaseTask(pMeta, pTask); } - streamMetaSendHbHelper(pMeta); + code = streamMetaSendHbHelper(pMeta); pMeta->sendMsgBeforeClosing = false; - return TSDB_CODE_SUCCESS; + return code; } void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) { diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 6e956e2682..c76ec92e33 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -38,3 +38,14 @@ void streamTimerCleanUp() { tmr_h streamTimerGetInstance() { return streamTimer; } + +void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, + const char* pMsg) { + while (1) { + bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId); + if (ret) { + break; + } + stError("vgId:%d failed to reset %s, try again", vgId, pMsg); + } +} From 24183d3f86abff528100fb4fdb8330e289ce0dba Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 18 Jul 2024 09:58:23 +0800 Subject: [PATCH 25/45] fix: start udfd with env var --- source/libs/function/src/tudf.c | 2 ++ source/libs/function/src/udfd.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5f7764f342..a38dc4e42a 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -71,6 +71,7 @@ void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { } } +extern char **environ; static int32_t udfSpawnUdfd(SUdfdData *pData) { fnInfo("start to init udfd"); uv_process_options_t options = {0}; @@ -117,6 +118,7 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { child_stdio[2].data.fd = 2; options.stdio_count = 3; options.stdio = child_stdio; + options.env = environ; options.flags = UV_PROCESS_DETACHED; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index df97e873aa..75bed73bb3 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1421,22 +1421,26 @@ int main(int argc, char *argv[]) { if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { fnError("failed to start since read config error"); + taosCloseLog(); return -2; } initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp); if (udfdOpenClientRpc() != 0) { fnError("open rpc connection to mnode failed"); + taosCloseLog(); return -3; } if (udfdCreateUdfSourceDir() != 0) { fnError("create udf source directory failed"); + taosCloseLog(); return -4; } if (udfdUvInit() != 0) { fnError("uv init failure"); + taosCloseLog(); return -5; } @@ -1452,6 +1456,7 @@ int main(int argc, char *argv[]) { udfdDeinitScriptPlugins(); + taosCloseLog(); udfdCleanup(); return 0; } From fd3a06586bcbce1c2f5f55d3b269686ca2947fb2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Jul 2024 10:49:46 +0800 Subject: [PATCH 26/45] fix(stream): fix syntax error. --- source/libs/stream/src/streamHb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index bcfe2c4243..6f5c1b7981 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -284,7 +284,7 @@ void streamMetaHbToMnode(void* param, void* tmrId) { code = taosReleaseRef(streamMetaId, rid); if (code) { - stError("vgId:%d in meta timer, failed to release the meta rid:%d", pMeta->vgId, rid); + stError("vgId:%d in meta timer, failed to release the meta rid:%" PRId64, pMeta->vgId, rid); } return; } @@ -301,7 +301,7 @@ void streamMetaHbToMnode(void* param, void* tmrId) { code = taosReleaseRef(streamMetaId, rid); if (code) { - stError("vgId:%d in meta timer, failed to release the meta rid:%d", pMeta->vgId, rid); + stError("vgId:%d in meta timer, failed to release the meta rid:%" PRId64, pMeta->vgId, rid); } } From 69e79966b6b83077be92fa6f5a4c1ea482a90b1c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jul 2024 11:32:13 +0800 Subject: [PATCH 27/45] enh: check return code --- source/dnode/vnode/src/tsdb/tsdbDef.h | 12 +- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 8 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 12 +- source/dnode/vnode/src/tsdb/tsdbRetention.c | 248 +++++++++--------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 26 +- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 37 +-- 6 files changed, 144 insertions(+), 199 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDef.h b/source/dnode/vnode/src/tsdb/tsdbDef.h index d843a0af75..ff59c8b808 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDef.h +++ b/source/dnode/vnode/src/tsdb/tsdbDef.h @@ -24,7 +24,7 @@ extern "C" { #endif #define TSDB_ERROR_LOG(vid, lino, code) \ - tsdbError("vgId:%d %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)) + tsdbError("vgId:%d %s failed at %s:%d since %s", vid, __func__, __FILE__, lino, tstrerror(code)) typedef struct SFDataPtr { int64_t offset; @@ -34,13 +34,13 @@ typedef struct SFDataPtr { extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn); extern void tsdbCloseFile(STsdbFD **ppFD); -extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, - int32_t encryptAlgorithm, char* encryptKey); +extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, + char *encryptKey); extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, - int32_t encryptAlgorithm, char* encryptKey); + int32_t encryptAlgorithm, char *encryptKey); extern int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint, - int32_t encryptAlgorithm, char* encryptKey); -extern int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey); + int32_t encryptAlgorithm, char *encryptKey); +extern int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey); typedef struct SColCompressInfo SColCompressInfo; struct SColCompressInfo { diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 598ca0aec3..2c0662b83b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -17,7 +17,9 @@ #include "vnd.h" int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl) { - if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) return TSDB_CODE_OUT_OF_MEMORY; + if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) { + return TSDB_CODE_OUT_OF_MEMORY; + } lvl[0]->level = level; TARRAY2_INIT(lvl[0]->fobjArr); return 0; @@ -81,14 +83,14 @@ static int32_t tsdbSttLvlFilteredInitEx(STsdb *pTsdb, const SSttLvl *lvl1, int64 return code; } - TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + TAOS_CHECK_RETURN(TARRAY2_APPEND(lvl[0]->fobjArr, fobj)); } else { STFileOp op = { .optype = TSDB_FOP_REMOVE, .fid = fobj1->f->fid, .of = fobj1->f[0], }; - TARRAY2_APPEND(fopArr, op); + TAOS_CHECK_RETURN(TARRAY2_APPEND(fopArr, op)); } } return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 211855b245..865e04cc2d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -370,7 +370,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } pTbData->suid = suid; pTbData->uid = uid; @@ -415,11 +415,11 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid taosWUnLockLatch(&pMemTable->latch); _exit: - *ppTbData = pTbData; - return code; - -_err: - *ppTbData = NULL; + if (code) { + *ppTbData = NULL; + } else { + *ppTbData = pTbData; + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 9d56d7b117..d189b5979b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -38,8 +38,7 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { return TARRAY2_APPEND(&rtner->fopArr, op); } -static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) { - int64_t total = 0; +static int32_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) { int64_t interval = 1000; // 1s int64_t limit = limitMB ? limitMB * 1024 * 1024 : INT64_MAX; int64_t offset = 0; @@ -49,10 +48,9 @@ static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_ int64_t n; int64_t last = taosGetTimestampMs(); if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) { - return -1; + TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno)); } - total += n; remain -= n; if (remain > 0) { @@ -63,7 +61,7 @@ static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_ } } - return total; + return 0; } static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) { @@ -77,32 +75,32 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi tsdbTFileLastChunkName(rtner->tsdb, to, fname_to); fdFrom = taosOpenFile(fname_from, TD_FILE_READ); - if (fdFrom == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + if (fdFrom == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size); fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (fdTo == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + if (fdTo == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config; int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1); - int64_t n = taosFSendFile(fdTo, fdFrom, 0, lc_size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); + + if (taosFSendFile(fdTo, fdFrom, 0, lc_size) < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } - taosCloseFile(&fdFrom); - taosCloseFile(&fdTo); _exit: if (code) { - tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); - if (fdFrom) taosCloseFile(&fdFrom); - if (fdTo) taosCloseFile(&fdTo); + tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); return code; } @@ -117,30 +115,29 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile tsdbTFileName(rtner->tsdb, to, fname); fdFrom = taosOpenFile(from->fname, TD_FILE_READ); - if (fdFrom == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + if (fdFrom == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, from->f->size); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (fdTo == NULL) code = terrno; + if (fdTo == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } TSDB_CHECK_CODE(code, lino, _exit); - int64_t n = tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage), - tsRetentionSpeedLimitMB); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - taosCloseFile(&fdFrom); - taosCloseFile(&fdTo); + TAOS_CHECK_GOTO(tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage), + tsRetentionSpeedLimitMB), + &lino, _exit); _exit: if (code) { - tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); - taosCloseFile(&fdFrom); - taosCloseFile(&fdTo); + tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); return code; } @@ -157,8 +154,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const .of = fobj->f[0], }; - code = TARRAY2_APPEND(&rtner->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit); // create new op = (STFileOp){ @@ -181,22 +177,20 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const }, }; - code = TARRAY2_APPEND(&rtner->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit); // do copy the file if (lcn < 1) { - code = tsdbDoCopyFile(rtner, fobj, &op.nf); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoCopyFile(rtner, fobj, &op.nf), &lino, _exit); } else { - code = tsdbDoCopyFileLC(rtner, fobj, &op.nf); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoCopyFileLC(rtner, fobj, &op.nf), &lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); + tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -213,8 +207,7 @@ static int32_t tsdbDoRetentionEnd(SRTNer *rtner) { int32_t lino = 0; if (TARRAY2_SIZE(&rtner->fopArr) > 0) { - code = tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION), &lino, _exit); taosThreadMutexLock(&rtner->tsdb->mutex); @@ -231,7 +224,8 @@ static int32_t tsdbDoRetentionEnd(SRTNer *rtner) { _exit: if (code) { - tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); + tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); } @@ -248,26 +242,19 @@ static int32_t tsdbDoRetention(SRTNer *rtner) { if (expLevel < 0) { // remove the fileset for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) { if (fobj == NULL) continue; - code = tsdbDoRemoveFileObject(rtner, fobj); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoRemoveFileObject(rtner, fobj), &lino, _exit); } SSttLvl *lvl; TARRAY2_FOREACH(fset->lvlArr, lvl) { - TARRAY2_FOREACH(lvl->fobjArr, fobj) { - code = tsdbDoRemoveFileObject(rtner, fobj); - TSDB_CHECK_CODE(code, lino, _exit); - } + TARRAY2_FOREACH(lvl->fobjArr, fobj) { TAOS_CHECK_GOTO(tsdbDoRemoveFileObject(rtner, fobj), &lino, _exit); } } } else if (expLevel == 0) { // only migrate to upper level return 0; } else { // migrate SDiskID did; - if (tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } + TAOS_CHECK_GOTO(tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did), &lino, _exit); tfsMkdirRecurAt(rtner->tsdb->pVnode->pTfs, rtner->tsdb->path, did); // data @@ -284,25 +271,26 @@ static int32_t tsdbDoRetention(SRTNer *rtner) { tsdbInfo("file:%s size: %" PRId64 " do migrate from %d to %d", fobj->fname, fobj->f->size, fobj->f->did.level, did.level); - code = tsdbDoMigrateFileObj(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoMigrateFileObj(rtner, fobj, &did), &lino, _exit); } // stt SSttLvl *lvl; TARRAY2_FOREACH(fset->lvlArr, lvl) { TARRAY2_FOREACH(lvl->fobjArr, fobj) { - if (fobj->f->did.level == did.level) continue; + if (fobj->f->did.level == did.level) { + continue; + } - code = tsdbDoMigrateFileObj(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoMigrateFileObj(rtner, fobj, &did), &lino, _exit); } } } _exit: if (code) { - tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); + tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -338,15 +326,12 @@ static int32_t tsdbRetention(void *arg) { // do retention if (rtner.fset) { if (rtnArg->s3Migrate) { - code = tsdbDoS3Migrate(&rtner); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoS3Migrate(&rtner), &lino, _exit); } else { - code = tsdbDoRetention(&rtner); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoRetention(&rtner), &lino, _exit); } - code = tsdbDoRetentionEnd(&rtner); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbDoRetentionEnd(&rtner), &lino, _exit); } _exit: @@ -361,7 +346,7 @@ _exit: TARRAY2_DESTROY(&rtner.fopArr, NULL); taosMemoryFree(arg); if (code) { - tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(pTsdb->pVnode), __func__, code, lino); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -374,12 +359,11 @@ static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate) if (!tsdb->bgTaskDisabled) { TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { - code = tsdbTFileSetOpenChannel(fset); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbTFileSetOpenChannel(fset), &lino, _exit); SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); if (arg == NULL) { - TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } arg->tsdb = tsdb; @@ -396,7 +380,7 @@ static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate) _exit: if (code) { - tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(tsdb->pVnode), __func__, code, lino); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } return code; } @@ -446,20 +430,19 @@ static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile tsdbTFileName(rtner->tsdb, to, fname); fdFrom = taosOpenFile(from->fname, TD_FILE_READ); - if (fdFrom == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + if (fdFrom == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } char *object_name = taosDirEntryBaseName(fname); - code = s3PutObjectFromFile2(from->fname, object_name, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - taosCloseFile(&fdFrom); + TAOS_CHECK_GOTO(s3PutObjectFromFile2(from->fname, object_name, 1), &lino, _exit); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - taosCloseFile(&fdFrom); + tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } + taosCloseFile(&fdFrom); return code; } @@ -477,8 +460,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int .of = fobj->f[0], }; - code = TARRAY2_APPEND(&rtner->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit); // create new op = (STFileOp){ @@ -501,8 +483,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int }, }; - code = TARRAY2_APPEND(&rtner->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit); char fname[TSDB_FILENAME_LEN]; tsdbTFileName(rtner->tsdb, &op.nf, fname); @@ -514,13 +495,13 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int char *dot = strrchr(object_name_prefix, '.'); if (!dot) { tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); - return -1; + TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit); } char *dot2 = strchr(object_name, '.'); if (!dot) { tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); - return -1; + TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit); } snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn); @@ -529,8 +510,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn); int64_t c_offset = chunksize * (cn - fobj->f->lcn); - code = s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize), &lino, _exit); } // copy last chunk @@ -540,30 +520,30 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn); fdFrom = taosOpenFile(fname, TD_FILE_READ); - if (fdFrom == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + if (fdFrom == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } tsdbInfo("vgId:%d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size); snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (fdTo == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + if (fdTo == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } - taosCloseFile(&fdFrom); - taosCloseFile(&fdTo); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - if (fdFrom) taosCloseFile(&fdFrom); - if (fdTo) taosCloseFile(&fdTo); + tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); return code; } @@ -581,8 +561,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 .of = fobj->f[0], }; - code = TARRAY2_APPEND(&rtner->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit); // create new op = (STFileOp){ @@ -605,8 +584,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 }, }; - code = TARRAY2_APPEND(&rtner->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit); char fname[TSDB_FILENAME_LEN]; tsdbTFileName(rtner->tsdb, &op.nf, fname); @@ -618,7 +596,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 char *dot = strrchr(object_name_prefix, '.'); if (!dot) { tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); - return -1; + TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit); } // do copy the file @@ -626,8 +604,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn); int64_t c_offset = chunksize * (cn - 1); - code = s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize), &lino, _exit); } // copy last chunk @@ -637,34 +614,34 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 dot = strchr(object_name, '.'); if (!dot) { tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); - return -1; + TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit); } snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name), "%d.data", lcn); fdFrom = taosOpenFile(fobj->fname, TD_FILE_READ); - if (fdFrom == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + if (fdFrom == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, fobj->f->size); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (fdTo == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); + if (fdTo == NULL) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); + } int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit); } - taosCloseFile(&fdFrom); - taosCloseFile(&fdTo); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - taosCloseFile(&fdFrom); - taosCloseFile(&fdTo); + tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); return code; } @@ -674,15 +651,21 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { STFileSet *fset = rtner->fset; STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA]; - if (!fobj) return code; + if (!fobj) { + return 0; + } int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now); - if (expLevel < 0) return code; // expired + if (expLevel < 0) { // expired + return 0; + } SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config; int32_t s3KeepLocal = pCfg->s3KeepLocal; int32_t s3ExpLevel = tsdbS3FidLevel(fset->fid, &rtner->tsdb->keepCfg, s3KeepLocal, rtner->now); - if (s3ExpLevel < 1) return code; // keep on local storage + if (s3ExpLevel < 1) { // keep on local storage + return 0; + } int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; int32_t lcn = fobj->f->lcn; @@ -698,19 +681,18 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { STimeWindow win = {0}; tsdbFidKeyRange(fset->fid, rtner->tsdb->keepCfg.days, rtner->tsdb->keepCfg.precision, &win.skey, &win.ekey); - tsdbInfo("vgId:%d, compact begin lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn); - tsdbAsyncCompact(rtner->tsdb, &win, pCfg->sttTrigger == 1); - tsdbInfo("vgId:%d, compact end lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn); + tsdbInfo("vgId:%d, async compact begin lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn); + code = tsdbAsyncCompact(rtner->tsdb, &win, pCfg->sttTrigger == 1); + tsdbInfo("vgId:%d, async compact end lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn); + goto _exit; return code; } - code = tsdbMigrateDataFileS3(rtner, fobj, size, chunksize); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMigrateDataFileS3(rtner, fobj, size, chunksize), &lino, _exit); } } else { if (lcn <= 1) { - tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); - return code; + TAOS_CHECK_GOTO(TSDB_CODE_INVALID_PARA, &lino, _exit); } char fname1[TSDB_FILENAME_LEN]; tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1); @@ -720,8 +702,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { int64_t size = 0; taosStatFile(fname1, &size, &mtime, NULL); if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { - code = tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize); - TSDB_CHECK_CODE(code, lino, _exit); + TAOS_CHECK_GOTO(tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize), &lino, _exit); } } else { tsdbError("vgId:%d, file: %s not found, %s at line %d", TD_VID(rtner->tsdb->pVnode), fname1, __func__, lino); @@ -731,7 +712,8 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) { _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } return code; } @@ -750,11 +732,15 @@ int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) { } if (!tsS3Enabled) { - return code; + return 0; } taosThreadMutexLock(&tsdb->mutex); code = tsdbAsyncRetentionImpl(tsdb, now, true); taosThreadMutexUnlock(&tsdb->mutex); + + if (code) { + tsdbError("vgId:%d, %s failed, reason:%s", TD_VID(tsdb->pVnode), __func__, tstrerror(code)); + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index df89746c31..c7c0135270 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -58,17 +58,9 @@ struct STsdbSnapReader { }; static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) { - int32_t code = 0; - int32_t lino = 0; - TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader->dataReader); - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); - } - return code; + return 0; } static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { @@ -255,7 +247,6 @@ static int32_t tsdbSnapCmprData(STsdbSnapReader* reader, uint8_t** data) { SColCompressInfo cmprInfo = {.pColCmpr = NULL, .defaultCmprAlg = NO_COMPRESSION}; code = tBlockDataCompress(reader->blockData, (void*)&cmprInfo, reader->buffers, reader->buffers + 4); TSDB_CHECK_CODE(code, lino, _exit); - // TSDB_CHECK_CODE(code, lino, _exit); int32_t size = 0; for (int i = 0; i < 4; i++) { @@ -448,8 +439,8 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), - __func__, lino, tstrerror(code), sver, ever, type); + tsdbError("vgId:%d %s failed at %s:%d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), + __func__, __FILE__, lino, tstrerror(code), sver, ever, type); tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); taosMemoryFree(reader[0]); reader[0] = NULL; @@ -461,10 +452,11 @@ _exit: } int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { - if (reader[0] == NULL) return 0; + if (reader[0] == NULL) { + return 0; + } int32_t code = 0; - int32_t lino = 0; STsdb* tsdb = reader[0]->tsdb; @@ -488,12 +480,6 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { taosMemoryFree(reader[0]); reader[0] = NULL; -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); - } else { - tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); - } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index e4f48745e7..d7c919bb4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -148,30 +148,14 @@ _exit: } static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader) { - int32_t code = 0; - int32_t lino = 0; - TARRAY2_CLEAR(reader->dataReaderArr, tsdbDataFileRAWReaderClose); - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); - } - return code; + return 0; } static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) { - int32_t code = 0; - int32_t lino = 0; - reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr); reader->dataIter->idx = 0; - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); - } - return code; + return 0; } static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { @@ -392,17 +376,6 @@ _exit: return code; } -static int32_t tsdbSnapRAWWriteFileSetOpenIter(STsdbSnapRAWWriter* writer) { - int32_t code = 0; - int32_t lino = 0; - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); - } - return code; -} - static int32_t tsdbSnapRAWWriteFileSetCloseIter(STsdbSnapRAWWriter* writer) { return 0; } static int32_t tsdbSnapRAWWriteFileSetOpenWriter(STsdbSnapRAWWriter* writer) { @@ -445,10 +418,8 @@ static int32_t tsdbSnapRAWWriteFileSetBegin(STsdbSnapRAWWriter* writer, int32_t writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec()); - if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) { - code = TSDB_CODE_NO_AVAIL_DISK; - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did); + TSDB_CHECK_CODE(code, lino, _exit); tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did); code = tsdbSnapRAWWriteFileSetOpenWriter(writer); From e8218f7f6389ad6fc6e9ebb607ccffb288090270 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 18 Jul 2024 12:18:17 +0800 Subject: [PATCH 28/45] fix: set fqdn for udfd --- source/libs/function/src/tudf.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index a38dc4e42a..6b6eb0f439 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -71,7 +71,6 @@ void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { } } -extern char **environ; static int32_t udfSpawnUdfd(SUdfdData *pData) { fnInfo("start to init udfd"); uv_process_options_t options = {0}; @@ -118,7 +117,6 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { child_stdio[2].data.fd = 2; options.stdio_count = 3; options.stdio = child_stdio; - options.env = environ; options.flags = UV_PROCESS_DETACHED; @@ -145,14 +143,29 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { udfdPathLdLib[udfdLdLibPathLen] = ':'; strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { - fnInfo("udfd LD_LIBRARY_PATH: %s", udfdPathLdLib); + fnInfo("[UDFD]udfd LD_LIBRARY_PATH: %s", udfdPathLdLib); } else { - fnError("can not set correct udfd LD_LIBRARY_PATH"); + fnError("[UDFD]can not set correct udfd LD_LIBRARY_PATH"); } char ldLibPathEnvItem[1024 + 32] = {0}; snprintf(ldLibPathEnvItem, 1024 + 32, "%s=%s", "LD_LIBRARY_PATH", udfdPathLdLib); - char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, ldLibPathEnvItem, NULL}; + char *taosFqdnEnvItem = NULL; + char *taosFqdn = getenv("TAOS_FQDN"); + if (taosFqdn != NULL) { + taosFqdnEnvItem = taosMemoryMalloc(strlen("TAOS_FQDN=") + strlen(taosFqdn) + 1); + if (taosFqdnEnvItem != NULL) { + strcpy(taosFqdnEnvItem, "TAOS_FQDN="); + strcat(taosFqdnEnvItem, taosFqdn); + fnInfo("[UDFD]Succsess to set TAOS_FQDN:%s", taosFqdn); + } else { + fnError("[UDFD]Failed to allocate memory for TAOS_FQDN"); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, ldLibPathEnvItem,taosFqdnEnvItem, NULL}; + options.env = envUdfd; int err = uv_spawn(&pData->loop, &pData->process, &options); From 5d797ce7fd004338e57ce22b0068e02fafec4ade Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Jul 2024 13:24:45 +0800 Subject: [PATCH 29/45] fix(stream): fix syntax error. --- source/libs/stream/src/streamHb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 6f5c1b7981..e8caa7e1a2 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -265,7 +265,7 @@ void streamMetaHbToMnode(void* param, void* tmrId) { if (code == TSDB_CODE_SUCCESS) { stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role); } else { - stError("vgId:%d role:%d not leader not send hb to mnodefailed to release the meta rid:%d", pMeta->vgId, + stError("vgId:%d role:%d not leader not send hb to mnodefailed to release the meta rid:%" PRId64, pMeta->vgId, pMeta->role, rid); } From 123b0e937840e2cc554b74cae87f4dd24f799670 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 Jul 2024 13:25:56 +0800 Subject: [PATCH 30/45] fix: catalog free issue --- include/os/osMemory.h | 1 + source/libs/catalog/inc/catalogInt.h | 1 + source/libs/catalog/src/ctgAsync.c | 2 + source/libs/catalog/src/ctgCache.c | 498 +++++++++++++++++++-------- 4 files changed, 362 insertions(+), 140 deletions(-) diff --git a/include/os/osMemory.h b/include/os/osMemory.h index c6a5ce27c4..7f2963f15a 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -55,6 +55,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); #define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) #define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) +#define TAOS_MEMMOVE(_d, _s, _n) ((void)memmove(_d, _s, _n)) #define taosMemoryFreeClear(ptr) \ do { \ diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 4f5e1b069b..afadc8d8c2 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -25,6 +25,7 @@ extern "C" { #include "tcommon.h" #include "ttimer.h" #include "tglobal.h" +#include "os.h" #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 4580cd4fcd..d8dcf963ae 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1016,6 +1016,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const _return: ctgFreeJob(*job); + *job = NULL; + CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index b023695937..da7411921a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -224,15 +224,16 @@ void ctgReleaseTSMAToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTSMACache* } int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { + int32_t code = TSDB_CODE_SUCCESS; SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; } bool inCache = false; - ctgRLockVgInfo(pCtg, dbCache, &inCache); + CTG_ERR_JRET(ctgRLockVgInfo(pCtg, dbCache, &inCache)); if (!inCache) { ctgDebug("vgInfo of db %s not in cache", dbFName); goto _return; @@ -256,13 +257,15 @@ _return: CTG_CACHE_NHIT_INC(CTG_CI_DB_VGROUP, 1); - return TSDB_CODE_SUCCESS; + return code; } int32_t ctgAcquireTbMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + int32_t code = TSDB_CODE_SUCCESS; + + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; @@ -303,15 +306,16 @@ int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const cha SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; bool vgInCache = false; + int32_t code = TSDB_CODE_SUCCESS; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); CTG_CACHE_NHIT_INC(CTG_CI_DB_VGROUP, 1); goto _return; } - ctgRLockVgInfo(pCtg, dbCache, &vgInCache); + CTG_ERR_JRET(ctgRLockVgInfo(pCtg, dbCache, &vgInCache)); if (!vgInCache) { ctgDebug("vgInfo of db %s not in cache", dbFName); CTG_CACHE_NHIT_INC(CTG_CI_DB_VGROUP, 1); @@ -466,7 +470,9 @@ _return: int32_t ctgAcquireTbIndexFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + int32_t code = TSDB_CODE_SUCCESS; + + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; @@ -506,7 +512,8 @@ _return: int32_t ctgTbMetaExistInCache(SCatalog *pCtg, const char *dbFName, const char *tbName, int32_t *exist) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; - ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); + + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -541,10 +548,10 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*pTableMeta, tbMeta, metaSize); + TAOS_MEMCPY(*pTableMeta, tbMeta, metaSize); if (tbMeta->schemaExt != NULL) { (*pTableMeta)->schemaExt = (SSchemaExt *)((char *)*pTableMeta + metaSize); - memcpy((*pTableMeta)->schemaExt, tbMeta->schemaExt, schemaExtSize); + TAOS_MEMCPY((*pTableMeta)->schemaExt, tbMeta->schemaExt, schemaExtSize); } else { (*pTableMeta)->schemaExt = NULL; } @@ -561,7 +568,7 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*pTableMeta, tbMeta, metaSize); + TAOS_MEMCPY(*pTableMeta, tbMeta, metaSize); // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -572,7 +579,7 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); + CTG_ERR_RET(ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache)); if (NULL == tbCache) { taosMemoryFreeClear(*pTableMeta); *pDb = NULL; @@ -595,7 +602,7 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); + TAOS_MEMCPY(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); (*pTableMeta)->schemaExt = NULL; return TSDB_CODE_SUCCESS; @@ -609,12 +616,12 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - strcpy(dbFName, ctx->pName->dbname); + TAOS_STRCPY(dbFName, ctx->pName->dbname); } else { - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); } - ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache); + CTG_ERR_JRET(ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -645,9 +652,9 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); - ctgAcquireTbMetaFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -678,7 +685,7 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); - ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); + CTG_ERR_RET(ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache)); if (NULL == tbCache) { // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); @@ -695,7 +702,7 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, size_t nameLen = 0; char *name = taosHashGetKey(tbCache, &nameLen); - strncpy(stbName, name, nameLen); + TAOS_STRNCPY(stbName, name, nameLen); stbName[nameLen] = 0; *sver = stbMeta->sversion; @@ -731,11 +738,11 @@ int32_t ctgReadTbIndexFromCache(SCatalog *pCtg, SName *pTableName, SArray **pRes SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); *pRes = NULL; - ctgAcquireTbIndexFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + CTG_ERR_RET(ctgAcquireTbIndexFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -753,7 +760,8 @@ _return: int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* pDbCfg) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + + CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); pDbCfg->cfgVersion = -1; @@ -786,10 +794,11 @@ _return: } int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName) { + int32_t code = TSDB_CODE_SUCCESS; *stbName = NULL; SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); return TSDB_CODE_SUCCESS; @@ -803,11 +812,14 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid } *stbName = taosStrdup(stb); + if (NULL == *stbName) { + code = TSDB_CODE_OUT_OF_MEMORY; + } taosHashRelease(dbCache->stbCache, stb); ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; + return code; } @@ -831,7 +843,7 @@ int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool tbNotExist req.tbNotExists = tbNotExists; CTG_LOCK(CTG_READ, &pUser->lock); - memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); + TAOS_MEMCPY(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); code = ctgChkSetAuthRes(pCtg, &req, pRes); CTG_UNLOCK(CTG_READ, &pUser->lock); CTG_ERR_JRET(code); @@ -865,6 +877,7 @@ void ctgDequeue(SCtgCacheOperation **op) { } int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { + int32_t code = TSDB_CODE_SUCCESS; SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); @@ -873,13 +886,19 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { CTG_RET(TSDB_CODE_OUT_OF_MEMORY); } + node->op = operation; + bool syncOp = operation->syncOp; char *opName = gCtgCacheOperation[operation->opId].name; if (operation->syncOp) { - tsem_init(&operation->rspSem, 0, 0); + code = tsem_init(&operation->rspSem, 0, 0); + if (TSDB_CODE_SUCCESS != code) { + qError("tsem_init failed, code:%x", code); + ctgFreeQNode(node); + CTG_RET(code); + } } - node->op = operation; CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); @@ -901,25 +920,34 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { CTG_QUEUE_INC(); CTG_STAT_RT_INC(numOfOpEnqueue, 1); - tsem_post(&gCtgMgmt.queue.reqSem); + code = tsem_post(&gCtgMgmt.queue.reqSem); + if (TSDB_CODE_SUCCESS != code) { + qError("tsem_post failed, code:%x", code); + CTG_RET(code); + } if (syncOp) { if (!operation->unLocked) { CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); } - tsem_wait(&operation->rspSem); + code = tsem_wait(&operation->rspSem); if (!operation->unLocked) { CTG_LOCK(CTG_READ, &gCtgMgmt.lock); } taosMemoryFree(operation); } - return TSDB_CODE_SUCCESS; + return code; } int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_DB_CACHE; op->syncOp = true; @@ -953,6 +981,11 @@ _return: int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_DB_VGROUP; op->syncOp = syncOp; @@ -986,6 +1019,11 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_STB_META; op->syncOp = syncOp; @@ -1016,6 +1054,11 @@ _return: int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_TB_META; op->syncOp = syncOp; @@ -1045,6 +1088,11 @@ _return: int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDBVgInfo *dbInfo, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_VGROUP; op->syncOp = syncOp; @@ -1090,6 +1138,11 @@ _return: int32_t ctgUpdateDbCfgEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDbCfgInfo *cfgInfo, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_DB_CFG; op->syncOp = syncOp; @@ -1127,6 +1180,11 @@ _return: int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_TB_META; op->syncOp = syncOp; @@ -1140,7 +1198,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool sy char *p = strchr(output->dbFName, '.'); if (p && IS_SYS_DBNAME(p + 1)) { int32_t len = strlen(p + 1); - memmove(output->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); + TAOS_MEMMOVE(output->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); } msg->pCtg = pCtg; @@ -1165,6 +1223,11 @@ _return: int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEpSet *pEpSet) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_VG_EPSET; SCtgUpdateEpsetMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateEpsetMsg)); @@ -1193,6 +1256,11 @@ _return: int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_USER; op->syncOp = syncOp; @@ -1222,6 +1290,11 @@ _return: int32_t ctgUpdateTbIndexEnqueue(SCatalog *pCtg, STableIndex **pIndex, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_TB_INDEX; op->syncOp = syncOp; @@ -1253,6 +1326,11 @@ _return: int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_TB_INDEX; op->syncOp = syncOp; @@ -1264,8 +1342,8 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { } msg->pCtg = pCtg; - tNameGetFullDbName(pName, msg->dbFName); - strcpy(msg->tbName, pName->tname); + (void)tNameGetFullDbName(pName, msg->dbFName); + TAOS_STRCPY(msg->tbName, pName->tname); op->data = msg; @@ -1281,6 +1359,11 @@ _return: int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool clearMeta, bool freeCtg, bool stopQueue, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_CLEAR_CACHE; op->syncOp = syncOp; op->stopQueue = stopQueue; @@ -1310,6 +1393,11 @@ _return: int32_t ctgUpdateViewMetaEnqueue(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_VIEW_META; op->syncOp = syncOp; @@ -1323,7 +1411,7 @@ int32_t ctgUpdateViewMetaEnqueue(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncOp char *p = strchr(pRsp->dbFName, '.'); if (p && IS_SYS_DBNAME(p + 1)) { int32_t len = strlen(p + 1); - memmove(pRsp->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); + TAOS_MEMMOVE(pRsp->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); } msg->pCtg = pCtg; @@ -1349,6 +1437,11 @@ _return: int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_VIEW_META; op->syncOp = syncOp; @@ -1476,7 +1569,7 @@ int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *d int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgGetDBCache(pCtg, dbFName, &dbCache)); if (dbCache) { // TODO OPEN IT @@ -1507,7 +1600,7 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); - ctgGetDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgGetDBCache(pCtg, dbFName, &dbCache)); *pCache = dbCache; @@ -1545,7 +1638,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); } else { ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); - atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); } } } @@ -1560,16 +1653,19 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(meta)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(meta)); pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { CTG_LOCK(CTG_WRITE, &pCache->metaLock); if (orig) { CTG_META_NUM_DEC(origType); } - atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(meta) - ctgGetTbMetaCacheSize(pCache->pMeta)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(meta) - ctgGetTbMetaCacheSize(pCache->pMeta)); taosMemoryFree(pCache->pMeta); pCache->pMeta = meta; @@ -1591,7 +1687,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - atomic_add_fetch_64(&dbCache->dbCacheSize, sizeof(meta->suid) + strlen(tbName) + 1); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, sizeof(meta->suid) + strlen(tbName) + 1); ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); @@ -1623,7 +1719,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbIndexCacheSize(pIndex)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbIndexCacheSize(pIndex)); CTG_DB_NUM_INC(CTG_CI_TBL_SMA); @@ -1641,7 +1737,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa CTG_LOCK(CTG_WRITE, &pCache->indexLock); if (pCache->pIndex) { - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pCache->pIndex)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pCache->pIndex)); if (0 == suid) { suid = pCache->pIndex->suid; } @@ -1652,7 +1748,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa pCache->pIndex = pIndex; CTG_UNLOCK(CTG_WRITE, &pCache->indexLock); - atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pIndex)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pIndex)); *index = NULL; @@ -1683,7 +1779,7 @@ int32_t ctgWriteViewMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFN CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(viewName) + sizeof(SCtgViewCache) + ctgGetViewMetaCacheSize(pMeta)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(viewName) + sizeof(SCtgViewCache) + ctgGetViewMetaCacheSize(pMeta)); CTG_DB_NUM_INC(CTG_CI_VIEW); @@ -1698,7 +1794,7 @@ int32_t ctgWriteViewMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFN CTG_LOCK(CTG_WRITE, &pCache->viewLock); if (pCache->pMeta) { - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pCache->pMeta)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pCache->pMeta)); ctgFreeSViewMeta(pCache->pMeta); taosMemoryFree(pCache->pMeta); } @@ -1706,7 +1802,7 @@ int32_t ctgWriteViewMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFN pCache->pMeta = pMeta; CTG_UNLOCK(CTG_WRITE, &pCache->viewLock); - atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pMeta)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pMeta)); ctgDebug("view meta updated to cache, view:%s, id:%" PRIu64 ", ver:%d, effectiveUser:%s, querySQL:%s", viewName, pMeta->viewId, pMeta->version, pMeta->user, pMeta->querySql); @@ -1847,7 +1943,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { uint64_t groupCacheSize = ctgGetDbVgroupCacheSize(vgCache->vgInfo); ctgDebug("sub dbGroupCacheSize %" PRIu64 " from db, dbFName:%s", groupCacheSize, dbFName); - atomic_sub_fetch_64(&dbCache->dbCacheSize, groupCacheSize); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, groupCacheSize); freeVgInfo(vgInfo); CTG_DB_NUM_RESET(CTG_CI_DB_VGROUP); @@ -1868,7 +1964,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { ctgWUnlockVgInfo(dbCache); uint64_t groupCacheSize = ctgGetDbVgroupCacheSize(vgCache->vgInfo); - atomic_add_fetch_64(&dbCache->dbCacheSize, groupCacheSize); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, groupCacheSize); ctgDebug("add dbGroupCacheSize %" PRIu64 " from db, dbFName:%s", groupCacheSize, dbFName); dbCache = NULL; @@ -1959,7 +2055,7 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + CTG_ERR_RET(ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } @@ -1989,14 +2085,14 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + CTG_ERR_RET(ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetDbVgroupCacheSize(dbCache->vgCache.vgInfo)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetDbVgroupCacheSize(dbCache->vgCache.vgInfo)); freeVgInfo(dbCache->vgCache.vgInfo); dbCache->vgCache.vgInfo = NULL; @@ -2051,7 +2147,7 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { if (NULL == ctbMeta) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); + TAOS_MEMCPY(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->ctbName, (STableMeta *)ctbMeta)); } @@ -2077,7 +2173,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } @@ -2095,7 +2191,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); } } @@ -2106,7 +2202,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { } tblType = pTbCache->pMeta->tableType; - atomic_sub_fetch_64(&dbCache->dbCacheSize, + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex)); ctgFreeTbCacheImpl(pTbCache, true); @@ -2114,7 +2210,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_META_NUM_DEC(tblType); - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->stbName)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->stbName)); } ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -2141,7 +2237,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } @@ -2159,7 +2255,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } tblType = pTbCache->pMeta->tableType; - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex)); ctgFreeTbCacheImpl(pTbCache, true); @@ -2167,7 +2263,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->tbName)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->tbName)); CTG_META_NUM_DEC(tblType); } @@ -2197,7 +2293,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { SCtgUserAuth userAuth = {0}; - memcpy(&userAuth.userAuth, &msg->userAuth, sizeof(msg->userAuth)); + TAOS_MEMCPY(&userAuth.userAuth, &msg->userAuth, sizeof(msg->userAuth)); userAuth.userCacheSize = ctgGetUserCacheSize(&userAuth.userAuth); if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { @@ -2230,7 +2326,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { taosHashCleanup(pUser->userAuth.alterViews); taosHashCleanup(pUser->userAuth.useDbs); - memcpy(&pUser->userAuth, &msg->userAuth, sizeof(msg->userAuth)); + TAOS_MEMCPY(&pUser->userAuth, &msg->userAuth, sizeof(msg->userAuth)); msg->userAuth.createdDbs = NULL; msg->userAuth.readDbs = NULL; @@ -2245,7 +2341,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { CTG_UNLOCK(CTG_WRITE, &pUser->lock); - atomic_store_64(&pUser->userCacheSize, ctgGetUserCacheSize(&pUser->userAuth)); + (void)atomic_store_64(&pUser->userCacheSize, ctgGetUserCacheSize(&pUser->userAuth)); _return: @@ -2367,8 +2463,8 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) { if (NULL == pIndex) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - strcpy(pIndex->tbName, msg->tbName); - strcpy(pIndex->dbFName, msg->dbFName); + TAOS_STRCPY(pIndex->tbName, msg->tbName); + TAOS_STRCPY(pIndex->dbFName, msg->dbFName); pIndex->version = -1; CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); @@ -2437,7 +2533,7 @@ int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } @@ -2460,7 +2556,7 @@ int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { goto _return; } - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); ctgFreeViewCacheImpl(pViewCache, true); if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) { @@ -2527,21 +2623,21 @@ void ctgClearMetaCache(SCtgCacheOperation *operation) { if (CTG_CACHE_LOW(remainSize, cacheMaxSize)) { qDebug("catalog finish meta clear, remainSize:%" PRId64 ", cacheMaxSize:%dMB", remainSize, cacheMaxSize); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); return; } if (!roundDone) { qDebug("catalog all meta cleared, remainSize:%" PRId64 ", cacheMaxSize:%dMB, to clear handle", remainSize, cacheMaxSize); ctgClearFreeCache(operation); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); return; } int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); if (code) { qError("clear cache enqueue failed, error:%s", tstrerror(code)); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); } } @@ -2646,7 +2742,7 @@ void ctgCleanupCacheQueue(void) { if (op->stopQueue) { SCatalog *pCtg = ((SCtgUpdateMsgHeader *)op->data)->pCtg; ctgDebug("process [%s] operation", gCtgCacheOperation[op->opId].name); - (*gCtgCacheOperation[op->opId].func)(op); + (void)(*gCtgCacheOperation[op->opId].func)(op); // ignore any error stopQueue = true; CTG_STAT_RT_INC(numOfOpDequeue, 1); } else { @@ -2655,7 +2751,7 @@ void ctgCleanupCacheQueue(void) { } if (op->syncOp) { - tsem_post(&op->rspSem); + (void)tsem_post(&op->rspSem); } else { taosMemoryFree(op); } @@ -2699,18 +2795,18 @@ void *ctgUpdateThreadFunc(void *param) { ctgDebug("process [%s] operation", gCtgCacheOperation[operation->opId].name); - (*gCtgCacheOperation[operation->opId].func)(operation); + (void)(*gCtgCacheOperation[operation->opId].func)(operation); // ignore any error if (operation->syncOp) { - tsem_post(&operation->rspSem); + (void)tsem_post(&operation->rspSem); } else { taosMemoryFreeClear(operation); } CTG_STAT_RT_INC(numOfOpDequeue, 1); - ctgdShowCacheInfo(); - ctgdShowStatInfo(); + (void)ctgdShowCacheInfo(); + (void)ctgdShowStatInfo(); } qInfo("catalog update thread stopped"); @@ -2719,9 +2815,10 @@ void *ctgUpdateThreadFunc(void *param) { } int32_t ctgStartUpdateThread() { + int32_t code = TSDB_CODE_SUCCESS; TdThreadAttr thAttr; - taosThreadAttrInit(&thAttr); - taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + CTG_ERR_JRET(taosThreadAttrInit(&thAttr)); + CTG_ERR_JRET(taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE)); if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -2729,6 +2826,14 @@ int32_t ctgStartUpdateThread() { } taosThreadAttrDestroy(&thAttr); + +_return: + + if (code) { + terrno = TAOS_SYSTEM_ERROR(errno); + CTG_ERR_RET(terrno); + } + return TSDB_CODE_SUCCESS; } @@ -2816,28 +2921,35 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx *ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray *pList) { int32_t tbNum = taosArrayGetSize(pList); - SName *pName = taosArrayGet(pList, 0); char dbFName[TSDB_DB_FNAME_LEN] = {0}; int32_t flag = CTG_FLAG_UNKNOWN_STB; + int32_t code = TSDB_CODE_SUCCESS; uint64_t lastSuid = 0; STableMeta *lastTableMeta = NULL; - + SName *pName = taosArrayGet(pList, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName from tableList, tableNum:%d", (int32_t)taosArrayGetSize(pList)); + return TSDB_CODE_CTG_INVALID_INPUT; + } + if (IS_SYS_DBNAME(pName->dbname)) { CTG_FLAG_SET_SYS_DB(flag); - strcpy(dbFName, pName->dbname); + TAOS_STRCPY(dbFName, pName->dbname); } else { - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); } SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaData){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } return TSDB_CODE_SUCCESS; @@ -2845,12 +2957,19 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe for (int32_t i = 0; i < tbNum; ++i) { pName = taosArrayGet(pList, i); + if (NULL == pName) { + ctgError("fail to get the %dth SName from tableList, tableNum:%d", i, (int32_t)taosArrayGetSize(pList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", pName->tname, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_META_NHIT_INC(); continue; @@ -2860,9 +2979,14 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe if (NULL == pCache->pMeta) { CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); + ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_META_NHIT_INC(); continue; @@ -2887,16 +3011,17 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe if (tbMeta->schemaExt != NULL) { schemaExtSize = tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); } + pTableMeta = taosMemoryCalloc(1, metaSize + schemaExtSize); if (NULL == pTableMeta) { ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pTableMeta, tbMeta, metaSize); + TAOS_MEMCPY(pTableMeta, tbMeta, metaSize); if (tbMeta->schemaExt != NULL) { pTableMeta->schemaExt = (SSchemaExt *)((char *)pTableMeta + metaSize); - memcpy(pTableMeta->schemaExt, tbMeta->schemaExt, schemaExtSize); + TAOS_MEMCPY(pTableMeta->schemaExt, tbMeta->schemaExt, schemaExtSize); } else { pTableMeta->schemaExt = NULL; } @@ -2907,7 +3032,9 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, pTableMeta->tableType, dbFName); res.pRes = pTableMeta; - taosArrayPush(ctx->pResList, &res); + if (NULL == taosArrayPush(ctx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } continue; } @@ -2915,8 +3042,14 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe // PROCESS FOR CHILD TABLE if (lastSuid && tbMeta->suid == lastSuid && lastTableMeta) { - cloneTableMeta(lastTableMeta, &pTableMeta); - memcpy(pTableMeta, tbMeta, sizeof(SCTableMeta)); + code = cloneTableMeta(lastTableMeta, &pTableMeta); + if (code) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + CTG_ERR_JRET(code); + } + + TAOS_MEMCPY(pTableMeta, tbMeta, sizeof(SCTableMeta)); CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); @@ -2924,7 +3057,9 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, pTableMeta->tableType, dbFName); res.pRes = pTableMeta; - taosArrayPush(ctx->pResList, &res); + if (NULL == taosArrayPush(ctx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } continue; } @@ -2936,7 +3071,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pTableMeta, tbMeta, metaSize); + TAOS_MEMCPY(pTableMeta, tbMeta, metaSize); CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); @@ -2947,8 +3082,10 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe char *stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", pTableMeta->suid, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } taosMemoryFreeClear(pTableMeta); CTG_META_NHIT_INC(); @@ -2960,11 +3097,14 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", pTableMeta->suid, stName, dbFName); taosHashRelease(dbCache->stbCache, stName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } taosMemoryFreeClear(pTableMeta); CTG_META_NHIT_INC(); + continue; } @@ -2976,8 +3116,11 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + taosMemoryFreeClear(pTableMeta); CTG_META_NHIT_INC(); @@ -2992,8 +3135,11 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%" PRIx64, stbMeta->suid, nctx.tbInfo.suid); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + taosMemoryFreeClear(pTableMeta); CTG_META_NHIT_INC(); @@ -3007,7 +3153,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); + TAOS_MEMCPY(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); pTableMeta->schemaExt = NULL; CTG_UNLOCK(CTG_READ, &pCache->metaLock); @@ -3016,15 +3162,19 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_META_HIT_INC(pTableMeta->tableType); res.pRes = pTableMeta; - taosArrayPush(ctx->pResList, &res); + if (NULL == taosArrayPush(ctx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } lastSuid = pTableMeta->suid; lastTableMeta = pTableMeta; } +_return: + ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; + return code; } int32_t ctgRemoveTbMetaFromCache(SCatalog *pCtg, SName *pTableName, bool syncReq) { @@ -3038,7 +3188,7 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog *pCtg, SName *pTableName, bool syncReq if (NULL != tblMeta) { char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); if (TSDB_SUPER_TABLE == tblMeta->tableType) { CTG_ERR_JRET(ctgDropStbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq)); @@ -3067,7 +3217,7 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg SCtgDBCache *dbCache = NULL; int32_t code = 0; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); @@ -3096,28 +3246,35 @@ _return: int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray *pList) { int32_t tbNum = taosArrayGetSize(pList); - SName *pName = taosArrayGet(pList, 0); char dbFName[TSDB_DB_FNAME_LEN] = {0}; int32_t flag = CTG_FLAG_UNKNOWN_STB; uint64_t lastSuid = 0; STableMeta *lastTableMeta = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SName *pName = taosArrayGet(pList, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName from viewList, viewNum:%d", (int32_t)taosArrayGetSize(pList)); + return TSDB_CODE_CTG_INVALID_INPUT; + } if (IS_SYS_DBNAME(pName->dbname)) { CTG_FLAG_SET_SYS_DB(flag); - strcpy(dbFName, pName->dbname); + TAOS_STRCPY(dbFName, pName->dbname); } else { - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); } SCtgDBCache *dbCache = NULL; SCtgViewCache *pCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaData){0}); + CTG_ERR_RET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } return TSDB_CODE_SUCCESS; @@ -3125,12 +3282,18 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC for (int32_t i = 0; i < tbNum; ++i) { pName = taosArrayGet(pList, i); + if (NULL == pName) { + ctgError("fail to get the %dth SName from viewList, viewNum:%d", i, (int32_t)taosArrayGetSize(pList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } pCache = taosHashAcquire(dbCache->viewCache, pName->tname, strlen(pName->tname)); if (NULL == pCache) { ctgDebug("view %s not in cache, dbFName:%s", pName->tname, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_CACHE_NHIT_INC(CTG_CI_VIEW, 1); continue; @@ -3141,8 +3304,10 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC CTG_UNLOCK(CTG_READ, &pCache->viewLock); taosHashRelease(dbCache->viewCache, pCache); ctgDebug("view %s meta not in cache, dbFName:%s", pName->tname, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_CACHE_NHIT_INC(CTG_CI_VIEW, 1); continue; @@ -3157,7 +3322,7 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pViewMeta, pCache->pMeta, sizeof(*pViewMeta)); + TAOS_MEMCPY(pViewMeta, pCache->pMeta, sizeof(*pViewMeta)); pViewMeta->querySql = tstrdup(pCache->pMeta->querySql); pViewMeta->user = tstrdup(pCache->pMeta->user); if (NULL == pViewMeta->querySql || NULL == pViewMeta->user) { @@ -3175,7 +3340,7 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC taosMemoryFree(pViewMeta); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pViewMeta->pSchema, pCache->pMeta->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); + TAOS_MEMCPY(pViewMeta->pSchema, pCache->pMeta->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); CTG_UNLOCK(CTG_READ, &pCache->viewLock); taosHashRelease(dbCache->viewCache, pCache); @@ -3183,12 +3348,16 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC ctgDebug("Got view %s meta from cache, dbFName:%s", pName->tname, dbFName); res.pRes = pViewMeta; - taosArrayPush(ctx->pResList, &res); + if (NULL == taosArrayPush(ctx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } +_return: + ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; + return code; } int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx, @@ -3200,48 +3369,70 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx int32_t flag = CTG_FLAG_UNKNOWN_STB; uint64_t lastSuid = 0; STableMeta * pTableMeta = NULL; - SName * pName = taosArrayGet(pList, 0); int32_t tbNum = taosArrayGetSize(pList); SCtgTbCache * pTbCache = NULL; + SName * pName = taosArrayGet(pList, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName from tbTSMAList, num:%d", (int32_t)taosArrayGetSize(pList)); + return TSDB_CODE_CTG_INVALID_INPUT; + } if (IS_SYS_DBNAME(pName->dbname)) { return TSDB_CODE_SUCCESS; } - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); // get db cache CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (!dbCache) { ctgDebug("DB %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); - taosArrayPush(pCtx->pResList, &(SMetaData){0}); + CTG_ERR_RET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaData){0})) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } + return TSDB_CODE_SUCCESS; } for (int32_t i = 0; i < tbNum; ++i) { // get tb cache pName = taosArrayGet(pList, i); + if (NULL == pName) { + ctgError("fail to get the %dth SName from tbTSMAList, num:%d", i, (int32_t)taosArrayGetSize(pList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); if (!pTbCache) { ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname); - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + continue; } + CTG_LOCK(CTG_READ, &pTbCache->metaLock); if (!pTbCache->pMeta) { CTG_UNLOCK(CTG_READ, &pTbCache->metaLock); ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname); - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + + CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + taosHashRelease(dbCache->tbCache, pTbCache); + continue; } uint64_t suid = pTbCache->pMeta->suid; int8_t tbType = pTbCache->pMeta->tableType; CTG_UNLOCK(CTG_READ, &pTbCache->metaLock); + taosHashRelease(dbCache->tbCache, pTbCache); SName tsmaSourceTbName = *pName; @@ -3249,12 +3440,16 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx if (tbType == TSDB_CHILD_TABLE) { char* stbName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(uint64_t)); if (stbName) { - snprintf(tsmaSourceTbName.tname, TMIN(TSDB_TABLE_NAME_LEN, strlen(stbName) + 1), "%s", stbName); + (void)snprintf(tsmaSourceTbName.tname, TMIN(TSDB_TABLE_NAME_LEN, strlen(stbName) + 1), "%s", stbName); taosHashRelease(dbCache->stbCache, stbName); } else { ctgDebug("stb in db: %s, uid: %" PRId64 " not in cache", dbFName, suid); - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + + CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + continue; } } @@ -3262,7 +3457,11 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx // get tsma cache pCache = taosHashAcquire(dbCache->tsmaCache, tsmaSourceTbName.tname, strlen(tsmaSourceTbName.tname)); if (!pCache || !pCache->pTsmas || pCache->pTsmas->size == 0) { - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + ctgReleaseTSMAToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + continue; } @@ -3270,9 +3469,14 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx if (hasOutOfDateTSMACache(pCache->pTsmas)) { CTG_UNLOCK(CTG_READ, &pCache->tsmaLock); taosHashRelease(dbCache->tsmaCache, pCache); + ctgDebug("tsma for tb: %s.%s not in cache", tsmaSourceTbName.tname, dbFName); - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TB_TSMA, &tsmaSourceTbName); - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + + CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TB_TSMA, &tsmaSourceTbName)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1); continue; } @@ -3284,31 +3488,45 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx ctgReleaseTSMAToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pRsp->pTsmas = taosArrayInit(pCache->pTsmas->size, POINTER_BYTES); if (!pRsp->pTsmas) { ctgReleaseTSMAToCache(pCtg, dbCache, pCache); taosMemoryFreeClear(pRsp); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + SMetaRes res = {0}; for (int32_t i = 0; i < pCache->pTsmas->size; ++i) { STSMACache *pTsmaOut = NULL; STSMACache *pTsmaCache = taosArrayGetP(pCache->pTsmas, i); code = tCloneTbTSMAInfo(pTsmaCache, &pTsmaOut); - if (code) { + if (TSDB_CODE_SUCCESS != code) { ctgReleaseTSMAToCache(pCtg, dbCache, pCache); tFreeTableTSMAInfoRsp(pRsp); taosMemoryFreeClear(pRsp); CTG_ERR_RET(code); } - taosArrayPush(pRsp->pTsmas, &pTsmaOut); + + if (NULL == taosArrayPush(pRsp->pTsmas, &pTsmaOut)) { + ctgReleaseTSMAToCache(pCtg, dbCache, pCache); + tFreeTableTSMAInfoRsp(pRsp); + taosMemoryFreeClear(pRsp); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } res.pRes = pRsp; - taosArrayPush(pCtx->pResList, &res); CTG_UNLOCK(CTG_READ, &pCache->tsmaLock); taosHashRelease(dbCache->tsmaCache, pCache); + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } + +_return: + ctgReleaseDBCache(pCtg, dbCache); + CTG_RET(code); } From 3277496f84c01a287e3af26d8c017b9a08e1a537 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Jul 2024 13:26:14 +0800 Subject: [PATCH 31/45] fix(stream): fix syntax error. --- source/libs/stream/src/streamHb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index e8caa7e1a2..4aaaed615c 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -254,7 +254,7 @@ void streamMetaHbToMnode(void* param, void* tmrId) { if (code == TSDB_CODE_SUCCESS) { stDebug("vgId:%d jump out of meta timer", pMeta->vgId); } else { - stError("vgId:%d jump out of meta timer, failed to release the meta rid:%d", pMeta->vgId, rid); + stError("vgId:%d jump out of meta timer, failed to release the meta rid:%" PRId64, pMeta->vgId, rid); } return; } From 960103a6ef1b7c377fc3791ba71be5fcfeffecf3 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 18 Jul 2024 13:58:47 +0800 Subject: [PATCH 32/45] fix: mem leak --- source/libs/function/src/tudf.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 6b6eb0f439..3c5e4014b3 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -195,6 +195,7 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { } else { fnInfo("udfd is initialized"); } + if(taosFqdnEnvItem) taosMemoryFree(taosFqdnEnvItem); return err; } From 927300c924a43a7d5c5d55591c4b8904ea4c7c76 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jul 2024 14:07:26 +0800 Subject: [PATCH 33/45] fix: compile error --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 865e04cc2d..e5bcf65bd7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -401,7 +401,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid code = tsdbMemTableRehash(pMemTable); if (code) { taosWUnLockLatch(&pMemTable->latch); - goto _err; + goto _exit; } } @@ -798,4 +798,4 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { _exit: return aTbDataP; -} +} \ No newline at end of file From 5cd03bb2ceefb5589272db605dca107b0cd02565 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 Jul 2024 14:13:29 +0800 Subject: [PATCH 34/45] fix: compile issues --- include/os/osDef.h | 2 +- source/libs/catalog/src/ctgCache.c | 733 +++++++++++++++++------------ 2 files changed, 431 insertions(+), 304 deletions(-) diff --git a/include/os/osDef.h b/include/os/osDef.h index 335b151cac..1052722492 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -220,7 +220,7 @@ void syslog(int unused, const char *format, ...); // Linux, length of name must <= 16 (the last '\0' included) #define setThreadName(name) \ do { \ - prctl(PR_SET_NAME, (name)); \ + (void)prctl(PR_SET_NAME, (name)); \ } while (0) #define getThreadName(name) \ do { \ diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index da7411921a..565dcd1739 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1469,6 +1469,165 @@ _return: CTG_RET(code); } +int32_t ctgUpdateTbTSMAEnqueue(SCatalog *pCtg, STSMACache **pTsma, int32_t tsmaVersion, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + op->opId = CTG_OP_UPDATE_TB_TSMA; + op->syncOp = syncOp; + + SCtgUpdateTbTSMAMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbTSMAMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbTSMAMsg)); + taosMemoryFree(op); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + msg->pCtg = pCtg; + msg->pTsma = *pTsma; + msg->dbTsmaVersion = tsmaVersion; + msg->dbId = (*pTsma)->dbId; + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + *pTsma = NULL; + return TSDB_CODE_SUCCESS; + +_return: + + CTG_RET(code); +} + +int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation* op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + op->opId = CTG_OP_DROP_TB_TSMA; + op->syncOp = syncOp; + + SCtgDropTbTSMAMsg* msg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg)); + if (!msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbTSMAMsg)); + taosMemoryFree(op); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + msg->pCtg = pCtg; + msg->dbId = pTsma->dbId; + msg->tbId = pTsma->suid; + msg->tsmaId = pTsma->tsmaId; + tstrncpy(msg->dbFName, pTsma->dbFName, TSDB_DB_FNAME_LEN); + tstrncpy(msg->tbName, pTsma->tb, TSDB_TABLE_NAME_LEN); + tstrncpy(msg->tsmaName, pTsma->name, TSDB_TABLE_NAME_LEN); + + op->data = msg; + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + CTG_RET(code); +} + + +static int32_t createDropAllTbTsmaCtgCacheOp(SCatalog* pCtg, const STSMACache* pCache, bool syncOp, SCtgCacheOperation** ppOp) { + SCtgCacheOperation* pOp = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == pOp) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgDropTbTSMAMsg* pMsg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg)); + if (NULL == pMsg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbTSMAMsg)); + taosMemoryFree(pOp); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + pOp->opId = CTG_OP_DROP_TB_TSMA; + pOp->syncOp = syncOp; + pMsg->pCtg = pCtg; + pMsg->dbId = pCache->dbId; + pMsg->tbId = pCache->suid; + pMsg->tsmaId = pCache->tsmaId; + pMsg->dropAllForTb = true; + tstrncpy(pMsg->tsmaName, pCache->name, TSDB_TABLE_NAME_LEN); + tstrncpy(pMsg->dbFName, pCache->dbFName, TSDB_DB_FNAME_LEN); + tstrncpy(pMsg->tbName, pCache->tb, TSDB_TABLE_NAME_LEN); + pOp->data = pMsg; + + *ppOp = pOp; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { + ctgDebug("drop tsma meta for tb: %s.%s", pName->dbname, pName->tname); + + int32_t code = 0; + SCtgDBCache *pDbCache = NULL; + SCtgCacheOperation *pOp = NULL; + char dbFName[TSDB_DB_FNAME_LEN]; + SCtgTSMACache *pCtgCache = NULL; + (void)tNameGetFullDbName(pName, dbFName); + + CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache)); + if (NULL == pDbCache || !pDbCache->tsmaCache) { + goto _return; + } + + pCtgCache = taosHashAcquire(pDbCache->tsmaCache, pName->tname, strlen(pName->tname)); + if (!pCtgCache) { + goto _return; + } + + CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); + if (!pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) { + CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + goto _return; + } + + STSMACache *pCache = taosArrayGetP(pCtgCache->pTsmas, 0); + if (NULL == pCache) { + ctgError("fail to get the 0th STSMACache, total:%d", (int32_t)pCtgCache->pTsmas->size); + code = TSDB_CODE_CTG_INTERNAL_ERROR; + } + if (TSDB_CODE_SUCCESS == code) { + code = createDropAllTbTsmaCtgCacheOp(pCtg, pCache, syncOp, &pOp); + } + CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + + CTG_ERR_JRET(code); + + CTG_ERR_JRET(ctgEnqueue(pCtg, pOp)); + taosHashRelease(pDbCache->tsmaCache, pCtgCache); + + return TSDB_CODE_SUCCESS; + +_return: + + if (pCtgCache) { + taosHashRelease(pDbCache->tsmaCache, pCtgCache); + } + if (pOp) { + taosMemoryFree(pOp->data); + taosMemoryFree(pOp); + } + + CTG_RET(code); +} + int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { int32_t code = 0; @@ -1888,6 +2047,103 @@ int32_t ctgVgInfoIdComp(void const *lp, void const *rp) { return 0; } + + + +int32_t ctgWriteTbTSMAToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, char *tbName, + STSMACache **ppTsmaCache) { + if (NULL == dbCache->tsmaCache) { + ctgError("db is dropping, dbId:0x%" PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + STSMACache *pTsmaCache = *ppTsmaCache; + int32_t code = TSDB_CODE_SUCCESS; + + SCtgTSMACache* pCache = taosHashGet(dbCache->tsmaCache, tbName, strlen(tbName)); + if (!pCache) { + SCtgTSMACache cache = {0}; + cache.pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES)); + if (NULL == cache.pTsmas) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(cache.pTsmas, &pTsmaCache)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (taosHashPut(dbCache->tsmaCache, tbName, strlen(tbName), &cache, sizeof(cache))) { + ctgError("taosHashPut new tsmacache for tb: %s.%s failed", dbFName, tbName); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(STSMACache) + ctgGetTbTSMACacheSize(pTsmaCache)); + + CTG_DB_NUM_INC(CTG_CI_TBL_TSMA); + ctgDebug("tb %s tsma updated to cache, name: %s", tbName, pTsmaCache->name); + + CTG_ERR_JRET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache)); + *ppTsmaCache = NULL; + + goto _return; + } + + CTG_LOCK(CTG_WRITE, &pCache->tsmaLock); + + if (pCache->pTsmas) { + uint64_t cacheSize = 0; + for (int32_t i = 0; i < pCache->pTsmas->size; ++i) { + STableTSMAInfo* pInfo = taosArrayGetP(pCache->pTsmas, i); + if (NULL == pInfo) { + ctgError("fail to get the %dth STableTSMAInfo, total:%d", i, (int32_t)pCache->pTsmas->size); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (pInfo->tsmaId == pTsmaCache->tsmaId) { + ctgDebug("tsma: %s removed from cache, history from %d to %d, reqTs from %" PRId64 " to %" PRId64 + "rspTs from %" PRId64 " to %" PRId64 " delay from %" PRId64 " to %" PRId64, + pInfo->name, pInfo->fillHistoryFinished, pTsmaCache->fillHistoryFinished, pInfo->reqTs, + pTsmaCache->reqTs, pInfo->rspTs, pTsmaCache->rspTs, pInfo->delayDuration, pTsmaCache->delayDuration); + + cacheSize = ctgGetTbTSMACacheSize(pInfo); + taosArrayRemove(pCache->pTsmas, i); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); + + tFreeTableTSMAInfo(pInfo); + taosMemoryFreeClear(pInfo); + + break; + } + } + } else { + pCache->pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES)); + if (!pCache->pTsmas) { + CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + // push the new cache + if (NULL == taosArrayPush(pCache->pTsmas, &pTsmaCache)) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + *ppTsmaCache = NULL; + + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbTSMACacheSize(pTsmaCache)); + + CTG_ERR_RET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache)); + + ctgDebug("table %s tsma updated to cache, tsma: %s", tbName, pTsmaCache->name); + } + + CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); + +_return: + + CTG_RET(code); +} + + int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; @@ -2563,7 +2819,7 @@ int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); CTG_DB_NUM_DEC(CTG_CI_VIEW); } @@ -2658,6 +2914,150 @@ _return: CTG_RET(code); } + +int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropTbTSMAMsg * msg = operation->data; + SCatalog *pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + + if (pCtg->stopUpdate) { + goto _return; + } + + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); + if (NULL == dbCache || !dbCache->tsmaCache || (msg->dbId != dbCache->dbId && msg->dbId != 0)) { + goto _return; + } + + SCtgTSMACache* pCtgCache = taosHashGet(dbCache->tsmaCache, msg->tbName, strlen(msg->tbName)); + if (!pCtgCache || !pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) { + goto _return; + } + + uint64_t cacheSize = 0; + STSMACache *pCache = NULL; + if (msg->dropAllForTb) { + CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock); + + for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) { + pCache = taosArrayGetP(pCtgCache->pTsmas, i); + if (NULL == pCache) { + ctgError("fail to the %dth tsma in pTsmas, total:%d", i, (int32_t)pCtgCache->pTsmas->size); + continue; + } + + cacheSize += ctgGetTbTSMACacheSize(pCache); + (void)ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare); + + CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA); + } + + taosArrayDestroyP(pCtgCache->pTsmas, tFreeAndClearTableTSMAInfo); + pCtgCache->pTsmas = NULL; + + ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName); + (void)taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN); + + CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); + } else { + CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock); + + pCache = taosArrayGetP(pCtgCache->pTsmas, 0); + if (NULL == pCache) { + ctgError("fail to the 0th tsma in pTsmas, total:%d", (int32_t)pCtgCache->pTsmas->size); + code = TSDB_CODE_CTG_INTERNAL_ERROR; + } else { + if (msg->tbId != 0 && pCache->suid != msg->tbId) { + // table id mismatch, skip drops + CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); + goto _return; + } + + for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) { + pCache = taosArrayGetP(pCtgCache->pTsmas, i); + if (NULL == pCache) { + ctgError("fail to the %dth tsma in pTsmas, total:%d", i, (int32_t)pCtgCache->pTsmas->size); + code = TSDB_CODE_CTG_INTERNAL_ERROR; + continue; + } + + if (pCache->tsmaId != msg->tsmaId) { + continue; + } + + cacheSize = ctgGetTbTSMACacheSize(pCache); + (void)ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare); + + taosArrayRemove(pCtgCache->pTsmas, i); + tFreeAndClearTableTSMAInfo(pCache); + + CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA); + + break; + } + } + + CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); + } + + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgOpUpdateTbTSMA(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateTbTSMAMsg *msg = operation->data; + SCatalog * pCtg = msg->pCtg; + STableTSMAInfo * pTsmaInfo = msg->pTsma; + SCtgDBCache * dbCache = NULL; + + if (pCtg->stopUpdate) { + goto _return; + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pTsmaInfo->dbFName, pTsmaInfo->dbId, &dbCache)); + CTG_ERR_JRET(ctgWriteTbTSMAToCache(pCtg, dbCache, pTsmaInfo->dbFName, pTsmaInfo->tb, &pTsmaInfo)); + + if (dbCache && msg->dbTsmaVersion > 0) { + dbCache->tsmaVersion = msg->dbTsmaVersion; + SDbCacheInfo cacheInfo = {0}; + cacheInfo.dbId = dbCache->dbId; + + if (dbCache->cfgCache.cfgInfo) { + cacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion; + tstrncpy(cacheInfo.dbFName, dbCache->cfgCache.cfgInfo->db, TSDB_DB_FNAME_LEN); + } + + if (dbCache->vgCache.vgInfo) { + cacheInfo.vgVersion = dbCache->vgCache.vgInfo->vgVersion; + cacheInfo.numOfTable = dbCache->vgCache.vgInfo->numOfTable; + cacheInfo.stateTs = dbCache->vgCache.vgInfo->stateTs; + } + + cacheInfo.tsmaVersion = dbCache->tsmaVersion; + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &cacheInfo, cacheInfo.dbId, sizeof(SDbCacheInfo), + ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); + } + +_return: + + if (pTsmaInfo) { + tFreeTableTSMAInfo(pTsmaInfo); + taosMemoryFreeClear(pTsmaInfo); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + void ctgFreeCacheOperationData(SCtgCacheOperation *op) { if (NULL == op || NULL == op->data) { return; @@ -2825,7 +3225,7 @@ int32_t ctgStartUpdateThread() { CTG_ERR_RET(terrno); } - taosThreadAttrDestroy(&thAttr); + (void)taosThreadAttrDestroy(&thAttr); _return: @@ -3538,7 +3938,7 @@ int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaNam bool found = false; STSMACache * pTsmaOut = NULL; - tNameGetFullDbName(pTsmaName, dbFName); + (void)tNameGetFullDbName(pTsmaName, dbFName); CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &pDbCache)); if (!pDbCache) { @@ -3550,10 +3950,16 @@ int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaNam while (pIter && !found) { SCtgTSMACache* pCtgCache = pIter; + CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); int32_t size = pCtgCache ? (pCtgCache->pTsmas ? pCtgCache->pTsmas->size : 0) : 0; for (int32_t i = 0; i < size; ++i) { STSMACache* pCache = taosArrayGetP(pCtgCache->pTsmas, i); + if (NULL == pCache) { + ctgError("fail to the %dth tsma in pTsmas, total:%d", i, size); + code = TSDB_CODE_CTG_INTERNAL_ERROR; + break; + } if (memcmp(pCache->name, pTsmaName->tname, TSDB_TABLE_NAME_LEN) == 0) { found = true; CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1); @@ -3562,323 +3968,44 @@ int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaNam } } CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + + if (TSDB_CODE_SUCCESS != code) { + break; + } + pIter = taosHashIterate(pDbCache->tsmaCache, pIter); } + taosHashCancelIterate(pDbCache->tsmaCache, pIter); + if (found && code == TSDB_CODE_SUCCESS) { res.pRes = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp)); if (!res.pRes) { tFreeAndClearTableTSMAInfo(pTsmaOut); - CTG_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + STableTSMAInfoRsp* pRsp = res.pRes; pRsp->pTsmas = taosArrayInit(1, POINTER_BYTES); if (!pRsp->pTsmas) { tFreeAndClearTableTSMAInfo(pTsmaOut); - CTG_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pRsp->pTsmas, &pTsmaOut); - taosArrayPush(pCtx->pResList, &res); + if (NULL == taosArrayPush(pRsp->pTsmas, &pTsmaOut)) { + tFreeAndClearTableTSMAInfo(pTsmaOut); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } +_return: + ctgReleaseDBCache(pCtg, pDbCache); + CTG_RET(code); } -int32_t ctgUpdateTbTSMAEnqueue(SCatalog *pCtg, STSMACache **pTsma, int32_t tsmaVersion, bool syncOp) { - int32_t code = 0; - SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - op->opId = CTG_OP_UPDATE_TB_TSMA; - op->syncOp = syncOp; - - SCtgUpdateTbTSMAMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbTSMAMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbTSMAMsg)); - taosMemoryFree(op); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - msg->pCtg = pCtg; - msg->pTsma = *pTsma; - msg->dbTsmaVersion = tsmaVersion; - msg->dbId = (*pTsma)->dbId; - - op->data = msg; - - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - - *pTsma = NULL; - return TSDB_CODE_SUCCESS; - -_return: - CTG_RET(code); -} - -int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool syncOp) { - int32_t code = 0; - SCtgCacheOperation* op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - if (!op) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - - op->opId = CTG_OP_DROP_TB_TSMA; - op->syncOp = syncOp; - - SCtgDropTbTSMAMsg* msg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg)); - if (!msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbTSMAMsg)); - taosMemoryFree(op); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - msg->pCtg = pCtg; - msg->dbId = pTsma->dbId; - msg->tbId = pTsma->suid; - msg->tsmaId = pTsma->tsmaId; - tstrncpy(msg->dbFName, pTsma->dbFName, TSDB_DB_FNAME_LEN); - tstrncpy(msg->tbName, pTsma->tb, TSDB_TABLE_NAME_LEN); - tstrncpy(msg->tsmaName, pTsma->name, TSDB_TABLE_NAME_LEN); - - op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - return TSDB_CODE_SUCCESS; -_return: - CTG_RET(code); -} - -static SCtgCacheOperation* createDropAllTbTsmaCtgCacheOp(SCatalog* pCtg, const STSMACache* pCache, bool syncOp) { - SCtgCacheOperation* pOp = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - if (!pOp) return NULL; - - SCtgDropTbTSMAMsg* pMsg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg)); - if (!pMsg) { - taosMemoryFree(pOp); - return NULL; - } - pOp->opId = CTG_OP_DROP_TB_TSMA; - pOp->syncOp = syncOp; - pMsg->pCtg = pCtg; - pMsg->dbId = pCache->dbId; - pMsg->tbId = pCache->suid; - pMsg->tsmaId = pCache->tsmaId; - pMsg->dropAllForTb = true; - tstrncpy(pMsg->tsmaName, pCache->name, TSDB_TABLE_NAME_LEN); - tstrncpy(pMsg->dbFName, pCache->dbFName, TSDB_DB_FNAME_LEN); - tstrncpy(pMsg->tbName, pCache->tb, TSDB_TABLE_NAME_LEN); - pOp->data = pMsg; - return pOp; -} - -int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { - ctgDebug("drop tsma meta for tb: %s.%s", pName->dbname, pName->tname); - int32_t code = 0; - SCtgDBCache *pDbCache = NULL; - SCtgCacheOperation *pOp = NULL; - char dbFName[TSDB_DB_FNAME_LEN]; - SCtgTSMACache *pCtgCache = NULL; - tNameGetFullDbName(pName, dbFName); - CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache)); - if (NULL == pDbCache || !pDbCache->tsmaCache) { - goto _return; - } - - pCtgCache = taosHashAcquire(pDbCache->tsmaCache, pName->tname, strlen(pName->tname)); - if (!pCtgCache) goto _return; - - CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); - if (!pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) { - CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); - goto _return; - } - STSMACache *pCache = taosArrayGetP(pCtgCache->pTsmas, 0); - pOp = createDropAllTbTsmaCtgCacheOp(pCtg, pCache, syncOp); - if (!pOp) { - code = TSDB_CODE_OUT_OF_MEMORY; - CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); - goto _return; - } - CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); - CTG_ERR_JRET(ctgEnqueue(pCtg, pOp)); - taosHashRelease(pDbCache->tsmaCache, pCtgCache); - return TSDB_CODE_SUCCESS; - -_return: - if (pCtgCache) taosHashRelease(pDbCache->tsmaCache, pCtgCache); - if (pOp) { - taosMemoryFree(pOp->data); - taosMemoryFree(pOp); - } - CTG_RET(code); -} - -int32_t ctgWriteTbTSMAToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, char *tbName, - STSMACache **ppTsmaCache) { - if (NULL == dbCache->tsmaCache) { - ctgError("db is dropping, dbId:0x%" PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - STSMACache *pTsmaCache = *ppTsmaCache; - int32_t code = TSDB_CODE_SUCCESS; - - SCtgTSMACache* pCache = taosHashGet(dbCache->tsmaCache, tbName, strlen(tbName)); - if (!pCache) { - SCtgTSMACache cache = {0}; - cache.pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES)); - if (!cache.pTsmas) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - taosArrayPush(cache.pTsmas, &pTsmaCache); - if (taosHashPut(dbCache->tsmaCache, tbName, strlen(tbName), &cache, sizeof(cache))) { - ctgError("taosHashPut new tsmacache for tb: %s.%s failed", dbFName, tbName); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(STSMACache) + ctgGetTbTSMACacheSize(pTsmaCache)); - CTG_DB_NUM_INC(CTG_CI_TBL_TSMA); - ctgDebug("tb %s tsma updated to cache, name: %s", tbName, pTsmaCache->name); - CTG_ERR_JRET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache)); - *ppTsmaCache = NULL; - goto _return; - } - - CTG_LOCK(CTG_WRITE, &pCache->tsmaLock); - - if (pCache->pTsmas) { - uint64_t cacheSize = 0; - for (int32_t i = 0; i < pCache->pTsmas->size; ++i) { - STableTSMAInfo* pInfo = taosArrayGetP(pCache->pTsmas, i); - if (pInfo->tsmaId == pTsmaCache->tsmaId) { - ctgDebug("tsma: %s removed from cache, history from %d to %d, reqTs from %" PRId64 " to %" PRId64 - "rspTs from %" PRId64 " to %" PRId64 " delay from %" PRId64 " to %" PRId64, - pInfo->name, pInfo->fillHistoryFinished, pTsmaCache->fillHistoryFinished, pInfo->reqTs, - pTsmaCache->reqTs, pInfo->rspTs, pTsmaCache->rspTs, pInfo->delayDuration, pTsmaCache->delayDuration); - cacheSize = ctgGetTbTSMACacheSize(pInfo); - taosArrayRemove(pCache->pTsmas, i); - atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); - tFreeTableTSMAInfo(pInfo); - taosMemoryFreeClear(pInfo); - break; - } - } - } else { - pCache->pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES)); - if (!pCache->pTsmas) { - CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - } - // push the new cache - taosArrayPush(pCache->pTsmas, &pTsmaCache); - *ppTsmaCache = NULL; - - atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbTSMACacheSize(pTsmaCache)); - CTG_ERR_RET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache)); - CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); - ctgDebug("table %s tsma updated to cache, tsma: %s", tbName, pTsmaCache->name); -_return: - CTG_RET(code); -} - -int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgDropTbTSMAMsg * msg = operation->data; - SCatalog *pCtg = msg->pCtg; - SCtgDBCache *dbCache = NULL; - - if (pCtg->stopUpdate) { - goto _return; - } - - CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); - if (NULL == dbCache || !dbCache->tsmaCache || (msg->dbId != dbCache->dbId && msg->dbId != 0)) { - goto _return; - } - - SCtgTSMACache* pCtgCache = taosHashGet(dbCache->tsmaCache, msg->tbName, strlen(msg->tbName)); - if (!pCtgCache || !pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) { - goto _return; - } - - uint64_t cacheSize = 0; - STSMACache *pCache = NULL; - if (msg->dropAllForTb) { - CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock); - for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) { - pCache = taosArrayGetP(pCtgCache->pTsmas, i); - cacheSize += ctgGetTbTSMACacheSize(pCache); - ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare); - CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA); - } - taosArrayDestroyP(pCtgCache->pTsmas, tFreeAndClearTableTSMAInfo); - pCtgCache->pTsmas = NULL; - ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName); - taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN); - CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); - } else { - CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock); - pCache = taosArrayGetP(pCtgCache->pTsmas, 0); - if (msg->tbId != 0 && pCache->suid != msg->tbId) { - // table id mismatch, skip drops - CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); - goto _return; - } - for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) { - pCache = taosArrayGetP(pCtgCache->pTsmas, i); - if (pCache->tsmaId != msg->tsmaId) { - continue; - } - cacheSize = ctgGetTbTSMACacheSize(pCache); - ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare); - taosArrayRemove(pCtgCache->pTsmas, i); - tFreeAndClearTableTSMAInfo(pCache); - CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA); - break; - } - CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); - } - atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); - -_return: - - taosMemoryFreeClear(msg); - CTG_RET(code); -} - -int32_t ctgOpUpdateTbTSMA(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgUpdateTbTSMAMsg *msg = operation->data; - SCatalog * pCtg = msg->pCtg; - STableTSMAInfo * pTsmaInfo = msg->pTsma; - SCtgDBCache * dbCache = NULL; - - if (pCtg->stopUpdate) { - goto _return; - } - - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pTsmaInfo->dbFName, pTsmaInfo->dbId, &dbCache)); - CTG_ERR_JRET(ctgWriteTbTSMAToCache(pCtg, dbCache, pTsmaInfo->dbFName, pTsmaInfo->tb, &pTsmaInfo)); - if (dbCache && msg->dbTsmaVersion > 0) { - dbCache->tsmaVersion = msg->dbTsmaVersion; - SDbCacheInfo cacheInfo = {0}; - cacheInfo.dbId = dbCache->dbId; - if (dbCache->cfgCache.cfgInfo) { - cacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion; - tstrncpy(cacheInfo.dbFName, dbCache->cfgCache.cfgInfo->db, TSDB_DB_FNAME_LEN); - } - if (dbCache->vgCache.vgInfo) { - cacheInfo.vgVersion = dbCache->vgCache.vgInfo->vgVersion; - cacheInfo.numOfTable = dbCache->vgCache.vgInfo->numOfTable; - cacheInfo.stateTs = dbCache->vgCache.vgInfo->stateTs; - } - cacheInfo.tsmaVersion = dbCache->tsmaVersion; - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &cacheInfo, cacheInfo.dbId, sizeof(SDbCacheInfo), - ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); - } - -_return: - - if (pTsmaInfo) { - tFreeTableTSMAInfo(pTsmaInfo); - taosMemoryFreeClear(pTsmaInfo); - } - - taosMemoryFreeClear(msg); - CTG_RET(code); -} From 5c73e1636951a1593b5009e4e14d533beb2f60c6 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 18 Jul 2024 14:30:59 +0800 Subject: [PATCH 35/45] hash key exist code --- include/util/thash.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/util/thash.h b/include/util/thash.h index c6275d276c..475d03f533 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -28,8 +28,7 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *); -#define HASH_KEY_ALREADY_EXISTS (-2) -#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS) +#define HASH_NODE_EXIST(code) (code == TSDB_CODE_DUP_KEY) /** * murmur hash algorithm From 1d8791a106c2f6d6499f9f9af70aec1fff820d18 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 Jul 2024 14:32:04 +0800 Subject: [PATCH 36/45] opti:[TS-4593] transform offset from file to tdb in tmq --- include/common/tmsg.h | 1 + source/client/src/clientMonitor.c | 2 +- source/common/src/tmsg.c | 6 +++++ source/dnode/vnode/src/inc/tq.h | 3 ++- source/dnode/vnode/src/tq/tq.c | 3 ++- source/dnode/vnode/src/tq/tqMeta.c | 35 +++++++++++++++++----------- source/dnode/vnode/src/tq/tqOffset.c | 20 ++++++---------- 7 files changed, 40 insertions(+), 30 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 40fce1d67b..a5dea8a44e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3612,6 +3612,7 @@ typedef struct { int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset); +void tDeleteSTqOffset(void* val); typedef struct SMqVgOffset { int64_t consumerId; diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 304f18cd68..b8b57160fe 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -691,7 +691,7 @@ static void* monitorThreadFunc(void* param) { tscDebug("monitorThreadFunc start"); int64_t quitTime = 0; while (1) { - if (atomic_load_32(&slowLogFlag) > 0 > 0) { + if (atomic_load_32(&slowLogFlag) > 0) { if (quitCnt == 0) { monitorSendAllSlowLogAtQuit(); if (quitCnt == 0) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e93de486e0..98e6530be0 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9258,6 +9258,12 @@ void tOffsetDestroy(void *param) { taosMemoryFreeClear(pVal->primaryKey.pData); } } + +void tDeleteSTqOffset(void *param) { + STqOffset *pVal = (STqOffset *)param; + tOffsetDestroy(&pVal->val); +} + int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1; if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f946dc76f8..d3582ab8f3 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -131,6 +131,7 @@ int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen); int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen); +int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen); int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle); void* tqMetaGetOffset(STQ* pTq, const char* subkey); int32_t tqMetaTransform(STQ* pTq); @@ -141,7 +142,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data); // tqOffset int32_t tqBuildFName(char** data, const char* path, char* name); -int32_t tqOffsetRestoreFromFile(SHashObj* pOffset, char* name); +int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name); // tq util int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0d167ccfa6..dd33d4ed65 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -80,6 +80,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); pTq->pOffset = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_ENTRY_LOCK); + taosHashSetFreeFp(pTq->pOffset, (FDelete)tDeleteSTqOffset); int32_t code = tqInitialize(pTq); if (code != TSDB_CODE_SUCCESS) { @@ -228,7 +229,7 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - if (tqMetaSaveInfo(pTq, pTq->pOffsetStore, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset)) < 0) { + if (tqMetaSaveInfo(pTq, pTq->pOffsetStore, pOffset->subKey, strlen(pOffset->subKey), msg, msgLen - sizeof(vgOffset.consumerId)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 2ab459416d..a985b804be 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -87,6 +87,18 @@ int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){ return code; } +int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen){ + SDecoder decoder = {0}; + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); + int32_t code = tDecodeSTqOffset(&decoder, info); + if (code != 0) { + tDeleteSTqOffset(info); + return TSDB_CODE_OUT_OF_MEMORY; + } + tDecoderClear(&decoder); + return code; +} + int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) { int32_t code = TDB_CODE_SUCCESS; TXN* txn = NULL; @@ -113,21 +125,9 @@ int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) { static int32_t tqMetaTransformOffsetInfo(STQ* pTq, char* path) { int32_t code = TDB_CODE_SUCCESS; - void* pIter = NULL; - - TQ_ERR_RETURN(tqOffsetRestoreFromFile(pTq->pOffset, path)); - while (1) { - pIter = taosHashIterate(pTq->pOffset, pIter); - if (pIter == NULL) { - break; - } - - STqOffset* offset = (STqOffset*)pIter; - TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset->subKey, strlen(offset->subKey), offset, sizeof(STqOffset))); - } + TQ_ERR_RETURN(tqOffsetRestoreFromFile(pTq, path)); END: - taosHashCancelIterate(pTq->pOffset, pIter); return code; } @@ -140,7 +140,14 @@ void* tqMetaGetOffset(STQ* pTq, const char* subkey){ return NULL; } - if(taosHashPut(pTq->pOffset, subkey, strlen(subkey), data, sizeof(STqOffset)) != 0){ + STqOffset offset = {0}; + if (tqMetaDecodeOffsetInfo(&offset, data, vLen) != TDB_CODE_SUCCESS) { + tdbFree(data); + return NULL; + } + + if(taosHashPut(pTq->pOffset, subkey, strlen(subkey), &offset, sizeof(STqOffset)) != 0){ + tDeleteSTqOffset(&offset); tdbFree(data); return NULL; } diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index e37cdfc697..f9faf611e1 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -32,10 +32,9 @@ int32_t tqBuildFName(char** data, const char* path, char* name) { return TDB_CODE_SUCCESS; } -int32_t tqOffsetRestoreFromFile(SHashObj* pOffset, char* name) { +int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) { int32_t code = TDB_CODE_SUCCESS; void* pMemBuf = NULL; - SDecoder decoder = {0}; TdFilePtr pFile = taosOpenFile(name, TD_FILE_READ); if (pFile == NULL) { @@ -65,19 +64,15 @@ int32_t tqOffsetRestoreFromFile(SHashObj* pOffset, char* name) { goto END; } - STqOffset offset; - tDecoderInit(&decoder, pMemBuf, size); - if (tDecodeSTqOffset(&decoder, &offset) < 0) { - code = TSDB_CODE_INVALID_MSG; + STqOffset offset = {0}; + TQ_ERR_GO_TO_END(tqMetaDecodeOffsetInfo(&offset, pMemBuf, size)); + code = taosHashPut(pTq->pOffset, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)); + if (code != TDB_CODE_SUCCESS) { + tDeleteSTqOffset(&offset); goto END; } + TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset.subKey, strlen(offset.subKey), pMemBuf, size)); - if (taosHashPut(pOffset, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto END; - } - - tDecoderClear(&decoder); taosMemoryFree(pMemBuf); pMemBuf = NULL; } @@ -85,7 +80,6 @@ int32_t tqOffsetRestoreFromFile(SHashObj* pOffset, char* name) { END: taosCloseFile(&pFile); taosMemoryFree(pMemBuf); - tDecoderClear(&decoder); return code; } From 3de0b18d8e982f8177ca63b221405fbfb073dad6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jul 2024 16:12:19 +0800 Subject: [PATCH 37/45] make CI pass --- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 13dc06a4d5..a552f9bcca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -535,7 +535,6 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { tb_uid_t uid = writer->blockData->suid == 0 ? writer->blockData->uid : writer->blockData->suid; SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL}; code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr)); - ASSERT(code == TSDB_CODE_SUCCESS); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; @@ -1043,4 +1042,4 @@ _exit: return code; } -bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer) { return writer->ctx->opened; } +bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer) { return writer->ctx->opened; } \ No newline at end of file From d8b476c4493bb567fe6b6ade046dfedeff4cac15 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 17:12:29 +0800 Subject: [PATCH 38/45] add count window --- docs/en/12-taos-sql/12-distinguished.md | 2 +- docs/zh/12-taos-sql/12-distinguished.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md index 5dca92a35c..d8f582fcab 100644 --- a/docs/en/12-taos-sql/12-distinguished.md +++ b/docs/en/12-taos-sql/12-distinguished.md @@ -38,7 +38,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10 ## Windowed Queries -Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are four kinds of windows: time window, status window, session window, and event window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows: +Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are five kinds of windows: time window, status window, session window, event window, and count window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows: ```sql window_clause: { diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md index b979d44a5e..d6dcd95555 100755 --- a/docs/zh/12-taos-sql/12-distinguished.md +++ b/docs/zh/12-taos-sql/12-distinguished.md @@ -39,7 +39,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10 ## 窗口切分查询 -TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）四种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。 +TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）、计数窗口（count window）五种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。 窗口子句语法如下： From ecc2ae77579192b7fab3c2afce25c456cd55c570 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 Jul 2024 18:14:43 +0800 Subject: [PATCH 39/45] enh: add catalog return code processing --- source/libs/catalog/src/ctgAsync.c | 2 +- source/libs/catalog/src/ctgRemote.c | 237 ++++++++++++++++------ source/libs/catalog/src/ctgRent.c | 17 +- source/libs/catalog/src/ctgUtil.c | 292 ++++++++++++++++++++++------ 4 files changed, 425 insertions(+), 123 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index d8dcf963ae..a416c0c77d 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -68,7 +68,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); ctx->flag = pParam->flag | CTG_FLAG_UNKNOWN_STB; - if (NULL != taosArrayPush(pJob->pTasks, &task)) { + if (NULL == taosArrayPush(pJob->pTasks, &task)) { ctgFreeTask(&task, true); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index f5de7fdc7a..a312dce164 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -57,8 +57,23 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu for (int32_t i = 0; i < taskNum; ++i) { int32_t* taskId = taosArrayGet(cbParam->taskId, i); + if (NULL == taskId) { + ctgError("taosArrayGet %d taskId failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->taskId)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); + if (NULL == msgIdx) { + ctgError("taosArrayGet %d msgIdx failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->msgIdx)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); + if (NULL == pTask) { + ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (msgNum > 0) { pRsp = taosArrayGet(batchRsp.pRsps, i); @@ -89,12 +104,17 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu tReq.pTask = pTask; tReq.msgIdx = pRsp->msgIdx; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq.msgIdx); + if (NULL == pMsgCtx) { + ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq.msgIdx, pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pMsgCtx->pBatchs = pBatchs; ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); - (*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); + (void)(*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal } CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -398,7 +418,16 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode)); } else { int32_t* taskId = taosArrayGet(cbParam->taskId, 0); + if (NULL == taskId) { + ctgError("taosArrayGet %d taskId failed, total:%d", 0, (int32_t)taosArrayGetSize(cbParam->taskId)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); + if (NULL == pTask) { + ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); @@ -412,6 +441,11 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { } SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("get task %d SCtgMsgCtx failed, taskType:%d", -1, pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pMsgCtx->pBatchs = pBatchs; #endif @@ -432,7 +466,7 @@ _return: taosMemoryFree(pMsg->pEpSet); if (pJob) { - taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); + (void)taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); } CTG_API_LEAVE(code); @@ -450,6 +484,7 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArr SCtgTaskCallbackParam* param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); if (NULL == param) { qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam)); + taosMemoryFree(msgSendInfo); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -482,7 +517,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob, SMsgSendInfo* pMsgSendInfo = NULL; CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, pMsgIdx, msgType, &pMsgSendInfo)); - ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId); + CTG_ERR_JRET(ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId)); pMsgSendInfo->requestId = pConn->requestId; pMsgSendInfo->requestObjRefId = pConn->requestObjRefId; @@ -515,18 +550,25 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT void* msg, uint32_t msgSize) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SHashObj* pBatchs = pMsgCtx->pBatchs; SCtgJob* pJob = pTask->pJob; - SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); SCtgBatch newBatch = {0}; SBatchMsg req = {0}; - + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq->msgIdx, pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SHashObj* pBatchs = pMsgCtx->pBatchs; + SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); if (NULL == pBatch) { newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); newBatch.pMsgIdxs = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds || NULL == newBatch.pMsgIdxs) { + taosArrayDestroy(newBatch.pMsgs); + taosArrayDestroy(newBatch.pTaskIds); + taosArrayDestroy(newBatch.pMsgIdxs); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -562,6 +604,10 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; pName = ctx->pName; @@ -569,14 +615,26 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT } else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) { SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } - tNameGetFullDbName(pName, newBatch.dbFName); + (void)tNameGetFullDbName(pName, newBatch.dbFName); } newBatch.msgType = (vgId > 0) ? TDMT_VND_BATCH_META : TDMT_MND_BATCH_META; @@ -622,6 +680,10 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; pName = ctx->pName; @@ -629,14 +691,26 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT } else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) { SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } - tNameGetFullDbName(pName, pBatch->dbFName); + (void)tNameGetFullDbName(pName, pBatch->dbFName); } ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId, @@ -752,7 +826,10 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -765,7 +842,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); @@ -801,7 +878,10 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -814,7 +894,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); @@ -854,7 +934,10 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -867,7 +950,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, input->db)); @@ -909,7 +992,10 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -922,7 +1008,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)dbFName)); @@ -964,7 +1050,10 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -977,7 +1066,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)indexName)); @@ -993,7 +1082,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n int32_t reqType = TDMT_MND_GET_TABLE_INDEX; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(name, tbFName); + (void)tNameExtractFullName(name, tbFName); ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); @@ -1021,7 +1110,10 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1034,7 +1126,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); @@ -1076,7 +1168,10 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1089,7 +1184,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)funcName)); @@ -1131,7 +1226,10 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1144,7 +1242,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)user)); @@ -1162,7 +1260,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const int32_t msgLen = 0; int32_t reqType = TDMT_MND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; - sprintf(tbFName, "%s.%s", dbFName, tbName); + (void)sprintf(tbFName, "%s.%s", dbFName, tbName); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); @@ -1188,7 +1286,10 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1201,7 +1302,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); @@ -1213,7 +1314,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out, SCtgTaskReq* tReq) { char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, tReq); } @@ -1222,10 +1323,10 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa STableMetaOutput* out, SCtgTaskReq* tReq) { SCtgTask* pTask = tReq ? tReq->pTask : NULL; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); int32_t reqType = TDMT_VND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; - sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); + (void)sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; @@ -1261,12 +1362,15 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa #else SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->vgId, reqType, msg, msgLen)); #endif @@ -1279,7 +1383,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); @@ -1294,10 +1398,10 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t msgLen = 0; int32_t reqType = TDMT_VND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFName); + (void)tNameExtractFullName(pTableName, tbFName); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; @@ -1325,12 +1429,15 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S #else SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); @@ -1344,7 +1451,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); @@ -1359,10 +1466,10 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t msgLen = 0; int32_t reqType = TDMT_MND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFName); + (void)tNameExtractFullName(pTableName, tbFName); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; ctgDebug("try to get table cfg from mnode, tbFName:%s", tbFName); @@ -1386,7 +1493,10 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1399,7 +1509,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); @@ -1435,7 +1545,10 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1448,7 +1561,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); @@ -1465,7 +1578,7 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); + (void)tNameExtractFullName(pName, fullName); ctgDebug("try to get view info from mnode, viewFName:%s", fullName); @@ -1490,7 +1603,10 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1503,7 +1619,7 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, fullName)); @@ -1519,7 +1635,7 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(name, tbFName); + (void)tNameExtractFullName(name, tbFName); ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); @@ -1544,7 +1660,10 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1557,7 +1676,7 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); @@ -1573,7 +1692,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c int32_t msgLen = 0; int32_t reqType = TDMT_VND_GET_STREAM_PROGRESS; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTbName, tbFName); + (void)tNameExtractFullName(pTbName, tbFName); SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; @@ -1602,15 +1721,17 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen)); #else char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTbName, dbFName); + (void)tNameGetFullDbName(pTbName, dbFName); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } - CTG_RET( - ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen)); #endif } @@ -1621,7 +1742,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 67b85a7825..329aee1703 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -100,7 +100,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - memcpy(orig, meta, size); + TAOS_MEMCPY(orig, meta, size); qDebug("meta in rent updated, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); @@ -185,8 +185,12 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_ } void *meta = taosArrayGet(slot->meta, 0); + if (NULL == meta) { + qError("get the 0th meta in slot failed, total:%d", (int32_t)metaNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } - memcpy(*res, meta, msize); + TAOS_MEMCPY(*res, meta, msize); *num = (uint32_t)metaNum; @@ -227,13 +231,14 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache) { return; } + int32_t code = TSDB_CODE_SUCCESS; void *pIter = taosHashIterate(dbCache->stbCache, NULL); while (pIter) { uint64_t *suid = NULL; suid = taosHashGetKey(pIter, NULL); - if (TSDB_CODE_SUCCESS == - ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + code = ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); + if (TSDB_CODE_SUCCESS == code) { ctgDebug("stb removed from rent, suid:0x%" PRIx64, *suid); } @@ -265,6 +270,7 @@ void ctgRemoveTSMARent(SCatalog *pCtg, SCtgDBCache *dbCache) { void* pIter = taosHashIterate(dbCache->tsmaCache, NULL); while (pIter) { SCtgTSMACache* pCtgCache = pIter; + CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); int32_t size = (pCtgCache && pCtgCache->pTsmas) ? pCtgCache->pTsmas->size : 0; for (int32_t i = 0; i < size; ++i) { @@ -274,6 +280,7 @@ void ctgRemoveTSMARent(SCatalog *pCtg, SCtgDBCache *dbCache) { } } CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + pIter = taosHashIterate(dbCache->tsmaCache, pIter); } } @@ -325,8 +332,10 @@ int32_t ctgUpdateRentTSMAVersion(SCatalog *pCtg, char *dbFName, const STSMACache tstrncpy(tsmaRent.name, pTsmaInfo->name, TSDB_TABLE_NAME_LEN); tstrncpy(tsmaRent.dbFName, dbFName, TSDB_DB_FNAME_LEN); tstrncpy(tsmaRent.tbName, pTsmaInfo->tb, TSDB_TABLE_NAME_LEN); + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->tsmaRent, &tsmaRent, tsmaRent.tsmaId, sizeof(STSMAVersion), ctgTSMAVersionSortCompare, ctgTSMAVersionSearchCompare)); + ctgDebug("db %s, 0x%" PRIx64 " tsma %s, 0x%" PRIx64 "version %d updated to tsmaRent", dbFName, tsmaRent.dbId, pTsmaInfo->name, pTsmaInfo->tsmaId, pTsmaInfo->version); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index ffe894abad..c46ded17b8 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -107,6 +107,10 @@ char* ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get table tag]"; case CTG_TASK_GET_VIEW: return "[get view]"; + case CTG_TASK_GET_TB_TSMA: + return "[get table TSMA]"; + case CTG_TASK_GET_TSMA: + return "[get TSMA]"; default: return "unknown"; } @@ -443,10 +447,10 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardN continue; } - taosHashRemove(dbCache->tbCache, key, len); - cacheSize = - len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); - atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); + (void)taosHashRemove(dbCache->tbCache, key, len); + + cacheSize = len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); *pClearedSize += cacheSize; (*pCleardNum)++; @@ -508,10 +512,10 @@ void ctgClearHandle(SCatalog* pCtg) { ctgFreeInstDbCache(pCtg->dbCache); ctgFreeInstUserCache(pCtg->userCache); - ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)); - ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)); - ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion)); - ctgMetaRentInit(&pCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion)); + (void)ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)); + (void)ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)); + (void)ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion)); + (void)ctgMetaRentInit(&pCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion)); pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); @@ -525,7 +529,7 @@ void ctgClearHandle(SCatalog* pCtg) { ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); } - memset(pCtg->cacheStat.cacheNum, 0, sizeof(pCtg->cacheStat.cacheNum)); + TAOS_MEMSET(pCtg->cacheStat.cacheNum, 0, sizeof(pCtg->cacheStat.cacheNum)); CTG_STAT_RT_INC(numOfOpClearCache, 1); @@ -668,7 +672,7 @@ void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) { void ctgResetTbMetaTask(SCtgTask* pTask) { SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx; - memset(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo)); + TAOS_MEMSET(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo)); taskCtx->flag = CTG_FLAG_UNKNOWN_STB; if (pTask->msgCtx.lastOut) { @@ -1087,7 +1091,10 @@ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) { } } - taosArrayPush(pCtxs, &ctx); + if (NULL == taosArrayPush(pCtxs, &ctx)) { + ctgFreeMsgCtx(&ctx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } return TSDB_CODE_SUCCESS; } @@ -1174,13 +1181,14 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); if (IS_SYS_DBNAME(pTableName->dbname)) { pVgroup->vgId = MNODE_HANDLE; if (pMgmtEps) { - memcpy(&pVgroup->epSet, pMgmtEps, sizeof(pVgroup->epSet)); + TAOS_MEMCPY(&pVgroup->epSet, pMgmtEps, sizeof(pVgroup->epSet)); } + return TSDB_CODE_SUCCESS; } @@ -1191,7 +1199,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d SVgroupInfo* vgInfo = NULL; char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + (void)tNameExtractFullName(pTableName, tbFullName); uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1243,28 +1251,41 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR SVgroupInfo mgmtInfo = {0}; mgmtInfo.vgId = MNODE_HANDLE; if (pMgmgEpSet) { - memcpy(&mgmtInfo.epSet, pMgmgEpSet, sizeof(mgmtInfo.epSet)); + TAOS_MEMCPY(&mgmtInfo.epSet, pMgmgEpSet, sizeof(mgmtInfo.epSet)); } + for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == vgInfo) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(vgInfo, &mgmtInfo, sizeof(mgmtInfo)); + TAOS_MEMCPY(vgInfo, &mgmtInfo, sizeof(mgmtInfo)); ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); if (update) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->pRes = vgInfo; } else { res.pRes = vgInfo; - taosArrayPush(pCtx->pResList, &res); + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } } + return TSDB_CODE_SUCCESS; } @@ -1281,18 +1302,35 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - *vgInfo = *(SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0); + SVgroupInfo* pSrcVg = (SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0); + if (NULL == pSrcVg) { + ctgError("fail to get the 0th SVgroupInfo, total:%d", (int32_t)taosArrayGetSize(dbInfo->vgArray)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + TAOS_MEMCPY(vgInfo, pSrcVg, sizeof(*pSrcVg)); ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); if (update) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->pRes = vgInfo; } else { res.pRes = vgInfo; - taosArrayPush(pCtx->pResList, &res); + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -1300,16 +1338,20 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR } char tbFullName[TSDB_TABLE_FNAME_LEN]; - sprintf(tbFullName, "%s.", dbFName); + (void)sprintf(tbFullName, "%s.", dbFName); int32_t offset = strlen(tbFullName); SName* pName = NULL; int32_t tbNameLen = 0; for (int32_t i = 0; i < tbNum; ++i) { pName = taosArrayGet(pNames, i); + if (NULL == pName) { + ctgError("fail to get the %dth SName, total:%d", i, (int32_t)taosArrayGetSize(pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } tbNameLen = offset + strlen(pName->tname); - strcpy(tbFullName + offset, pName->tname); + TAOS_STRCPY(tbFullName + offset, pName->tname); uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1334,11 +1376,22 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR if (update) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->pRes = pNewVg; } else { res.pRes = pNewVg; - taosArrayPush(pCtx->pResList, &res); + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -1351,7 +1404,6 @@ int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFNam CTG_ERR_RET(ctgMakeVgArray(dbInfo)); int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); - if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); @@ -1359,11 +1411,11 @@ int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFNam SVgroupInfo* vgInfo = NULL; char tbFullName[TSDB_TABLE_FNAME_LEN]; - snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); + (void)snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); int32_t offset = strlen(tbFullName); for (int32_t i = 0; i < tbNum; ++i) { - snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); + (void)snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1475,7 +1527,11 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { void* pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { - taosArrayPush(dbInfo->vgArray, pIter); + if (NULL == taosArrayPush(dbInfo->vgArray, pIter)) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + pIter = taosHashIterate(dbInfo->vgHash, pIter); } @@ -1494,7 +1550,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*dst, src, sizeof(SDBVgInfo)); + TAOS_MEMCPY(*dst, src, sizeof(SDBVgInfo)); size_t hashSize = taosHashGetSize(src->vgHash); (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -1522,6 +1578,11 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { if (src->vgArray) { (*dst)->vgArray = taosArrayDup(src->vgArray, NULL); + if (NULL == (*dst)->vgArray) { + taosHashCleanup((*dst)->vgHash); + taosMemoryFreeClear(*dst); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } return TSDB_CODE_SUCCESS; @@ -1534,7 +1595,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*pOutput, output, sizeof(STableMetaOutput)); + TAOS_MEMCPY(*pOutput, output, sizeof(STableMetaOutput)); if (output->tbMeta) { int32_t metaSize = CTG_META_SIZE(output->tbMeta); @@ -1542,6 +1603,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { schemaExtSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); } + (*pOutput)->tbMeta = taosMemoryMalloc(metaSize + schemaExtSize); qDebug("tbMeta cloned, size:%d, p:%p", metaSize, (*pOutput)->tbMeta); if (NULL == (*pOutput)->tbMeta) { @@ -1550,10 +1612,10 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); + TAOS_MEMCPY((*pOutput)->tbMeta, output->tbMeta, metaSize); if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { (*pOutput)->tbMeta->schemaExt = (SSchemaExt *)((char *)(*pOutput)->tbMeta + metaSize); - memcpy((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize); + TAOS_MEMCPY((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize); } else { (*pOutput)->tbMeta->schemaExt = NULL; } @@ -1576,8 +1638,18 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { for (int32_t i = 0; i < num; ++i) { STableIndexInfo* pInfo = taosArrayGet(pIndex, i); + if (NULL == pInfo) { + qError("fail to get the %dth STableIndexInfo, total:%d", i, (int32_t)taosArrayGetSize(pIndex)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } pInfo = taosArrayPush(*pRes, pInfo); + if (NULL == pInfo) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } pInfo->expr = taosStrdup(pInfo->expr); + if (NULL == pInfo->expr) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } return TSDB_CODE_SUCCESS; @@ -1588,6 +1660,9 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, cha pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = vgId; pMsgSendInfo->target.dbFName = taosStrdup(dbFName); + if (NULL == pMsgSendInfo->target.dbFName) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } else { pMsgSendInfo->target.type = TARGET_TYPE_MNODE; } @@ -1604,6 +1679,11 @@ int32_t ctgGetTablesReqNum(SArray* pList) { int32_t n = taosArrayGetSize(pList); for (int32_t i = 0; i < n; ++i) { STablesReq* pReq = taosArrayGet(pList, i); + if (NULL == pReq) { + qError("fail to get the %dth STablesReq, total:%d", i, (int32_t)taosArrayGetSize(pList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + total += taosArrayGetSize(pReq->pTables); } @@ -1613,6 +1693,9 @@ int32_t ctgGetTablesReqNum(SArray* pList) { int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag) { if (NULL == (*pFetchs)) { *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgFetch)); + if (NULL == *pFetchs) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } SCtgFetch fetch = {0}; @@ -1622,7 +1705,9 @@ int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fet fetch.resIdx = resIdx; fetch.flag = flag; - taosArrayPush(*pFetchs, &fetch); + if (NULL == taosArrayPush(*pFetchs, &fetch)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } return TSDB_CODE_SUCCESS; } @@ -1643,7 +1728,19 @@ int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName) { return TSDB_CODE_SUCCESS; } -static void* ctgCloneDbVgroup(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } +static int32_t ctgCloneDbVgroup(void* pSrc, void** ppDst) { +#if 0 + if (NULL == pSrc) { + *ppDst = NULL; + return TSDB_CODE_SUCCESS; + } + + *ppDst = taosArrayDup((const SArray*)pSrc, NULL); + return (*ppDst) ? TSDB_CODE_SUCCESS : TSDB_CODE_OUT_OF_MEMORY; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif +} static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } @@ -1652,6 +1749,7 @@ int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst) { if (NULL == pDst) { return terrno; } + TAOS_MEMCPY(pDst, pSrc, sizeof(SDbCfgInfo)); if (((SDbCfgInfo *)pSrc)->pRetensions) { pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); @@ -1671,13 +1769,19 @@ static void ctgFreeDbCfgInfo(void* p) { freeDbCfgInfo(pDst); } -static void* ctgCloneDbInfo(void* pSrc) { +static int32_t ctgCloneDbInfo(void* pSrc, void** ppDst) { +#if 0 SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo)); if (NULL == pDst) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(pDst, pSrc, sizeof(SDbInfo)); - return pDst; + + TAOS_MEMCPY(pDst, pSrc, sizeof(SDbInfo)); + + return TSDB_CODE_SUCCESS; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeDbInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } @@ -1710,55 +1814,83 @@ static void ctgFreeDbInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeTableMeta(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneVgroupInfo(void* pSrc) { +static int32_t ctgCloneVgroupInfo(void* pSrc, void** ppDst) { +#if 0 SVgroupInfo* pDst = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, sizeof(SVgroupInfo)); return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeVgroupInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneTableIndexs(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } +static int32_t ctgCloneTableIndexs(void* pSrc, void** ppDst) { +#if 0 + return taosArrayDup((const SArray*)pSrc, NULL); +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif +} static void ctgFreeTableIndexs(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -static void* ctgCloneFuncInfo(void* pSrc) { +static int32_t ctgCloneFuncInfo(void* pSrc, void** ppDst) { +#if 0 SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo)); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, sizeof(SFuncInfo)); return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeFuncInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneIndexInfo(void* pSrc) { +static int32_t ctgCloneIndexInfo(void* pSrc) { +#if 0 SIndexInfo* pDst = taosMemoryMalloc(sizeof(SIndexInfo)); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, sizeof(SIndexInfo)); return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeIndexInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneUserAuth(void* pSrc) { +static int32_t ctgCloneUserAuth(void* pSrc) { +#if 0 bool* pDst = taosMemoryMalloc(sizeof(bool)); if (NULL == pDst) { return NULL; } *pDst = *(bool*)pSrc; return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeUserAuth(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneQnodeList(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } +static int32_t ctgCloneQnodeList(void* pSrc) { +#if 0 + return taosArrayDup((const SArray*)pSrc, NULL); +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif +} static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } @@ -1773,11 +1905,18 @@ static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p) static void ctgFreeTableCfg(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } +static int32_t ctgCloneDnodeList(void* pSrc) { +#if 0 + return taosArrayDup((const SArray*)pSrc, NULL); +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif +} static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -static void* ctgCloneViewMeta(void* pSrc) { +static int32_t ctgCloneViewMeta(void* pSrc) { +#if 0 SViewMeta* pSrcMeta = pSrc; SViewMeta* pDst = taosMemoryMalloc(sizeof(SViewMeta)); if (NULL == pDst) { @@ -1791,6 +1930,9 @@ static void* ctgCloneViewMeta(void* pSrc) { } memcpy(pDst->pSchema, pSrcMeta->pSchema, pSrcMeta->numOfCols * sizeof(*pSrcMeta->pSchema)); return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeViewMeta(void* p) { @@ -1818,8 +1960,8 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char tbFName[TSDB_TABLE_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; - tNameExtractFullName(&req->pRawReq->tbName, tbFName); - tNameGetFullDbName(&req->pRawReq->tbName, dbFName); + (void)tNameExtractFullName(&req->pRawReq->tbName, tbFName); + (void)tNameGetFullDbName(&req->pRawReq->tbName, dbFName); while (true) { taosMemoryFreeClear(pMeta); @@ -1871,7 +2013,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { continue; } - sprintf(tbFName, "%s.%s", dbFName, stbName); + (void)sprintf(tbFName, "%s.%s", dbFName, stbName); continue; } @@ -1918,7 +2060,7 @@ int32_t ctgChkSetBasicAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res } char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(&pReq->tbName, dbFName); + (void)tNameGetFullDbName(&pReq->tbName, dbFName); // since that we add read/write previliges when create db, there is no need to check createdDbs #if 0 @@ -2002,9 +2144,9 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) char viewFName[TSDB_VIEW_FNAME_LEN]; if (IS_SYS_DBNAME(req->pRawReq->tbName.dbname)) { - snprintf(viewFName, sizeof(viewFName), "%s.%s", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + (void)snprintf(viewFName, sizeof(viewFName), "%s.%s", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); } else { - tNameExtractFullName(&req->pRawReq->tbName, viewFName); + (void)tNameExtractFullName(&req->pRawReq->tbName, viewFName); } int32_t len = strlen(viewFName) + 1; @@ -2378,7 +2520,7 @@ void ctgGetGlobalCacheStat(SCtgCacheStat* pStat) { pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); } - memcpy(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache)); + TAOS_MEMCPY(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache)); } void ctgGetGlobalCacheSize(uint64_t *pSize) { @@ -2405,12 +2547,19 @@ int32_t ctgBuildViewNullRes(SCtgTask* pTask, SCtgViewsCtx* pCtx) { int32_t dbNum = taosArrayGetSize(pCtx->pNames); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + qError("fail to get the %dth STablesReq, total:%d", i, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t viewNum = taosArrayGetSize(pReq->pTables); ctgDebug("start to check views in db %s, viewNum %d", pReq->dbFName, viewNum); for (int32_t m = 0; m < viewNum; ++m) { - taosArrayPush(pCtx->pResList, &(SMetaData){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaData){0})) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -2435,17 +2584,27 @@ int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) { if (pViewMeta->pSchema == NULL) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pViewMeta->pSchema, pRsp->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); + + TAOS_MEMCPY(pViewMeta->pSchema, pRsp->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); return TSDB_CODE_SUCCESS; } uint64_t ctgGetTbTSMACacheSize(STableTSMAInfo* pTsmaInfo) { - if (!pTsmaInfo) return 0; + if (!pTsmaInfo) { + return 0; + } uint64_t size = sizeof(STableTSMAInfo); - if (pTsmaInfo->pFuncs) size += sizeof(STableTSMAFuncInfo) * pTsmaInfo->pFuncs->size; - if (pTsmaInfo->pTags) size += sizeof(SSchema) * pTsmaInfo->pTags->size; - if (pTsmaInfo->pUsedCols) size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size; + if (pTsmaInfo->pFuncs) { + size += sizeof(STableTSMAFuncInfo) * pTsmaInfo->pFuncs->size; + } + if (pTsmaInfo->pTags) { + size += sizeof(SSchema) * pTsmaInfo->pTags->size; + } + if (pTsmaInfo->pUsedCols) { + size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size; + } + return size; } @@ -2455,8 +2614,14 @@ bool hasOutOfDateTSMACache(SArray* pTsmas) { } for (int32_t i = 0; i < pTsmas->size; ++i) { STSMACache* pTsmaInfo = taosArrayGetP(pTsmas, i); - if (isCtgTSMACacheOutOfDate(pTsmaInfo)) return true; + if (NULL == pTsmaInfo) { + ASSERT(0); + } + if (isCtgTSMACacheOutOfDate(pTsmaInfo)) { + return true; + } } + return false; } @@ -2477,6 +2642,9 @@ int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName) { if (NULL == (*pFetchs)) { *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgTSMAFetch)); + if (NULL == *pFetchs) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } SCtgTSMAFetch fetch = {0}; @@ -2487,9 +2655,13 @@ int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetch.flag = flag; fetch.fetchType = fetchType; - if (sourceTbName) fetch.tsmaSourceTbName = *sourceTbName; + if (sourceTbName) { + fetch.tsmaSourceTbName = *sourceTbName; + } - taosArrayPush(*pFetchs, &fetch); + if (NULL == taosArrayPush(*pFetchs, &fetch)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } return TSDB_CODE_SUCCESS; } From 6da9bfbf18fd1ae66a727697215087923903922c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 18 Jul 2024 18:26:24 +0800 Subject: [PATCH 40/45] test:fix case for default debugflag --- tests/pytest/util/dnodes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index bb6f8ff030..b90e62cf62 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -30,6 +30,7 @@ class TDSimClient: def __init__(self, path): self.testCluster = False self.path = path + # cdebugflag 135 for tmqClientConsLog self.cfgDict = { "fqdn": "localhost", "numOfLogLines": "100000000", @@ -39,7 +40,7 @@ class TDSimClient: "rpcDebugFlag": "135", "tmrDebugFlag": "131", "dDebugFlag":"131", - "cDebugFlag": "131", + "cDebugFlag": "135", "uDebugFlag": "131", "jniDebugFlag": "131", "qDebugFlag": "135", From 5eb3251a27f83d5fd37869bcee0329a1637671b8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 Jul 2024 18:38:09 +0800 Subject: [PATCH 41/45] opti:[TS-4593] transform offset from file to tdb in tmq --- source/dnode/vnode/src/tq/tqSnapshot.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index 29b471b5db..1731759dae 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -225,9 +225,14 @@ int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nD int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; STQ* pTq = pWriter->pTq; - STqOffset *info = (STqOffset*)(pData + sizeof(SSnapDataHdr)); - code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info->subKey, strlen(info->subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + STqOffset info = {0}; + if(tqMetaDecodeOffsetInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)) != 0){ + goto _err; + } + + code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info.subKey, strlen(info.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + tDeleteSTqOffset(&info); if (code) goto _err; return code; From 97b6c43c9a532da6ea2351424242e869652237b3 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 18 Jul 2024 19:15:44 +0800 Subject: [PATCH 42/45] test:fix case for default debugflag --- tests/pytest/util/dnodes.py | 2 +- tests/system-test/7-tmq/tmqClientConsLog.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index b90e62cf62..070e56114a 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -40,7 +40,7 @@ class TDSimClient: "rpcDebugFlag": "135", "tmrDebugFlag": "131", "dDebugFlag":"131", - "cDebugFlag": "135", + "cDebugFlag": "131", "uDebugFlag": "131", "jniDebugFlag": "131", "qDebugFlag": "135", diff --git a/tests/system-test/7-tmq/tmqClientConsLog.py b/tests/system-test/7-tmq/tmqClientConsLog.py index ab0c5f6a72..83d6f93be1 100644 --- a/tests/system-test/7-tmq/tmqClientConsLog.py +++ b/tests/system-test/7-tmq/tmqClientConsLog.py @@ -16,6 +16,9 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: + + clientCfgDict = {'debugFlag': 135} + updatecfgDict = {'debugFlag': 131, 'clientCfg':clientCfgDict} def __init__(self): self.vgroups = 3 self.ctbNum = 10 From cf91c71e1b9996b64e2f990abc0715072b6e7c3f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 18 Jul 2024 19:16:14 +0800 Subject: [PATCH 43/45] test:fix case for default debugflag --- tests/pytest/util/dnodes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 070e56114a..bb6f8ff030 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -30,7 +30,6 @@ class TDSimClient: def __init__(self, path): self.testCluster = False self.path = path - # cdebugflag 135 for tmqClientConsLog self.cfgDict = { "fqdn": "localhost", "numOfLogLines": "100000000", From ad1d85bb130810c0b5ed0f4075c56100851afc19 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 Jul 2024 19:26:50 +0800 Subject: [PATCH 44/45] fix: add hash duplicated issue --- source/libs/catalog/src/ctgAsync.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index a416c0c77d..13bf2d08e6 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -607,8 +607,8 @@ int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) { int32_t code = TSDB_CODE_SUCCESS; - SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (NULL == pDb || NULL == pTb) { taosHashCleanup(pDb); taosHashCleanup(pTb); @@ -2414,12 +2414,6 @@ int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); - if (NULL == pRes) { - ctgError("fail to get the 0th SMetaRes, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pResList)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); if (NULL == pTbReq) { ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); @@ -2445,6 +2439,12 @@ int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* break; } case TDMT_MND_GET_TSMA: { + SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); + if (NULL == pRes) { + ctgError("fail to get the 0th SMetaRes, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STableTSMAInfoRsp* pOut = pMsgCtx->out; pRes->code = 0; if (pOut->pTsmas->size > 0) { From ee2ec0e7eee3c26d77c5ddb5cf2de7f39b62e6c3 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 19 Jul 2024 10:13:00 +0800 Subject: [PATCH 45/45] merge packaging script from main branch --- packaging/deb/DEBIAN/postinst | 1 + packaging/deb/DEBIAN/prerm | 1 + packaging/deb/makedeb.sh | 13 +- packaging/rpm/makerpm.sh | 4 +- packaging/rpm/tdengine.spec | 8 + packaging/tools/install.sh | 63 +++--- packaging/tools/makepkg.sh | 2 +- packaging/tools/post.sh | 127 ++++++----- packaging/tools/remove.sh | 359 ++++++++++--------------------- packaging/tools/remove_client.sh | 7 +- 10 files changed, 240 insertions(+), 345 deletions(-) diff --git a/packaging/deb/DEBIAN/postinst b/packaging/deb/DEBIAN/postinst index 2638f09625..8c2b8e78ce 100644 --- a/packaging/deb/DEBIAN/postinst +++ b/packaging/deb/DEBIAN/postinst @@ -8,6 +8,7 @@ if command -v sudo > /dev/null; then csudo="sudo " fi +${csudo}mkdir -p ${insmetaPath} ${csudo}chmod -R 744 ${insmetaPath} cd ${insmetaPath} ${csudo}./post.sh diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index a474dc4c80..2bbc6ff855 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -33,6 +33,7 @@ else ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || : + ${csudo}rm -f ${bin_link_dir}/taos-explorer || : ${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 337407f8d2..906a227ad5 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -76,11 +76,11 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || : fi -if [ -f "%{_compiledir}/../../../explorer/target/taos-explorer.service" ]; then - cp %{_compiledir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || : +if [ -f "${compile_dir}/../../../explorer/target/taos-explorer.service" ]; then + cp ${compile_dir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || : fi -if [ -f "%{_compiledir}/../../../explorer/server/example/explorer.toml" ]; then - cp %{_compiledir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || : +if [ -f "${compile_dir}/../../../explorer/server/example/explorer.toml" ]; then + cp ${compile_dir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || : fi cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin @@ -90,6 +90,11 @@ cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin +cp ${top_dir}/../enterprise/packaging/start-all.sh ${pkg_dir}${install_home_path}/bin +cp ${top_dir}/../enterprise/packaging/stop-all.sh ${pkg_dir}${install_home_path}/bin +sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" ${pkg_dir}${install_home_path}/bin/start-all.sh +sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" ${pkg_dir}${install_home_path}/bin/stop-all.sh + cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/udfd ${pkg_dir}${install_home_path}/bin diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh index 9cf00364aa..f895193b6b 100755 --- a/packaging/rpm/makerpm.sh +++ b/packaging/rpm/makerpm.sh @@ -70,8 +70,8 @@ else fi cd ${top_dir} -echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper" -taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper` +echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper -t ver-${tdengine_ver}" +taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper -t ver-${tdengine_ver}` echo "taoskeeper_binary: ${taoskeeper_binary}" cd ${package_dir} diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 6b324486b2..0a107a4108 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -56,6 +56,10 @@ mkdir -p %{buildroot}%{homepath}/include #mkdir -p %{buildroot}%{homepath}/init.d mkdir -p %{buildroot}%{homepath}/script +if [ -f %{_compiledir}/../packaging/cfg/taosd.service ]; then + cp %{_compiledir}/../packaging/cfg/taosd.service %{buildroot}%{homepath}/cfg +fi + cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg if [ -f %{_compiledir}/test/cfg/taosadapter.toml ]; then cp %{_compiledir}/test/cfg/taosadapter.toml %{buildroot}%{homepath}/cfg @@ -91,6 +95,10 @@ cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin +cp %{_compiledir}/../../enterprise/packaging/start-all.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../../enterprise/packaging/stop-all.sh %{buildroot}%{homepath}/bin +sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" %{buildroot}%{homepath}/bin/start-all.sh +sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" %{buildroot}%{homepath}/bin/stop-all.sh if [ -f %{_compiledir}/../../../explorer/target/release/taos-explorer ]; then cp %{_compiledir}/../../../explorer/target/release/taos-explorer %{buildroot}%{homepath}/bin diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 03e0a0b5f5..cc3868d2f0 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -156,7 +156,7 @@ done #echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" -tools=(${clientName} ${benchmarkName} ${dumpName} ${demoName} remove.sh udfd set_core.sh TDinsight.sh start_pre.sh) +tools=(${clientName} ${benchmarkName} ${dumpName} ${demoName} remove.sh udfd set_core.sh TDinsight.sh start_pre.sh start-all.sh stop-all.sh) if [ "${verMode}" == "cluster" ]; then services=(${serverName} ${adapterName} ${xname} ${explorerName} ${keeperName}) elif [ "${verMode}" == "edge" ]; then @@ -221,6 +221,8 @@ function install_bin() { ${csudo}cp -r ${script_dir}/bin/remove.sh ${install_main_dir}/bin else ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin + ${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/bin + ${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/bin fi if [[ "${verMode}" == "cluster" && "${verType}" != "client" ]]; then @@ -576,7 +578,9 @@ function install_taosd_config() { ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$serverFqdn/" ${script_dir}/cfg/${configFile} ${csudo}echo "monitor 1" >>${script_dir}/cfg/${configFile} ${csudo}echo "monitorFQDN ${serverFqdn}" >>${script_dir}/cfg/${configFile} - ${csudo}echo "audit 1" >>${script_dir}/cfg/${configFile} + if [ "$verMode" == "cluster" ]; then + ${csudo}echo "audit 1" >>${script_dir}/cfg/${configFile} + fi if [ -f "${configDir}/${configFile}" ]; then ${csudo}cp ${fileName} ${configDir}/${configFile}.new @@ -594,6 +598,7 @@ function install_config() { [ ! -z $1 ] && return 0 || : # only install client if ((${update_flag} == 1)); then + install_taosd_config return 0 fi @@ -651,9 +656,7 @@ function install_data() { function install_connector() { if [ -d "${script_dir}/connector/" ]; then - ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector" - ${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/ || echo "failed to copy start-all.sh" - ${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/ || echo "failed to copy stop-all.sh" + ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector" ${csudo}cp ${script_dir}/README.md ${install_main_dir}/ || echo "failed to copy README.md" fi } @@ -910,39 +913,36 @@ function updateProduct() { echo echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}" [ -f ${configDir}/${adapterName}.toml ] && [ -f ${installDir}/bin/${adapterName} ] && - echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${configDir}/${adapterName}.toml" - if [ "$verMode" == "cluster" ]; then - echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${configDir}/explorer.toml" - fi + echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${configDir}/${adapterName}.toml" + echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${configDir}/explorer.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}systemctl start ${serverName}${NC}" + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}" [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}service ${serverName} start${NC}" + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}" [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ./${serverName}${NC}" + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ./${serverName}${NC}" [ -f ${installDir}/bin/${clientName}adapter ] && echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}" fi - echo -e "${GREEN_DARK}To enable ${clientName}keeper ${NC}\t\t: sudo systemctl enable ${clientName}keeper ${NC}" + echo -e "${GREEN_DARK}To start ${clientName}keeper ${NC}\t\t: sudo systemctl start ${clientName}keeper ${NC}" if [ "$verMode" == "cluster" ]; then - echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}" - echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}" + echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}" fi + echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}" echo echo "${productName} is updated successfully!" echo + + echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}" + echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : ${clientName} -h $serverFqdn${NC}" + echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}" if [ "$verMode" == "cluster" ]; then - echo -e "\033[44;32;1mTo start all the components : ./start-all.sh${NC}" - fi - echo -e "\033[44;32;1mTo access ${productName} : ${clientName} -h $serverFqdn${NC}" - if [ "$verMode" == "cluster" ]; then - echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}" echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs${NC}" fi else @@ -1007,39 +1007,36 @@ function installProduct() { echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}" [ -f ${configDir}/${clientName}adapter.toml ] && [ -f ${installDir}/bin/${clientName}adapter ] && echo -e "${GREEN_DARK}To configure ${clientName}Adapter ${NC}\t: edit ${configDir}/${clientName}adapter.toml" - if [ "$verMode" == "cluster" ]; then - echo -e "${GREEN_DARK}To configure ${clientName}-explorer ${NC}\t: edit ${configDir}/explorer.toml" - fi + echo -e "${GREEN_DARK}To configure ${clientName}-explorer ${NC}\t: edit ${configDir}/explorer.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}systemctl start ${serverName}${NC}" + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}" [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}service ${serverName} start${NC}" + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}" [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${serverName}${NC}" + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${serverName}${NC}" [ -f ${installDir}/bin/${clientName}adapter ] && echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}" fi - echo -e "${GREEN_DARK}To enable ${clientName}keeper ${NC}\t\t: sudo systemctl enable ${clientName}keeper ${NC}" + echo -e "${GREEN_DARK}To start ${clientName}keeper ${NC}\t\t: sudo systemctl start ${clientName}keeper ${NC}" if [ "$verMode" == "cluster" ]; then echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}" - echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}" fi + echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}" echo echo "${productName} is installed successfully!" echo + + echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}" + echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : ${clientName} -h $serverFqdn${NC}" + echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}" if [ "$verMode" == "cluster" ]; then - echo -e "\033[44;32;1mTo start all the components : sudo ./start-all.sh${NC}" - fi - echo -e "\033[44;32;1mTo access ${productName} : ${clientName} -h $serverFqdn${NC}" - if [ "$verMode" == "cluster" ]; then - echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}" echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs-en${NC}" fi echo diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 5733c35ebe..9e1cc73238 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -232,7 +232,7 @@ fi if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh - sed -i "s/PREFIX=\"taos\"/PREFIX=\"${serverName2}\"/g" remove_temp.sh + sed -i "s/PREFIX=\"taos\"/PREFIX=\"${clientName2}\"/g" remove_temp.sh sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" remove_temp.sh mv remove_temp.sh ${install_dir}/bin/remove.sh fi diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index ceaebfdc7c..6b4c96e8c4 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -104,6 +104,8 @@ fi function log_print(){ now=$(date +"%D %T") + ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} + ${csudo} touch ${install_log_path} echo "$now $1" >> ${install_log_path} } @@ -235,11 +237,14 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || : + ${csudo}rm -f ${bin_link_dir}/taos-explorer || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/rmtaos || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/*explorer || : + ${csudo}rm -f ${bin_link_dir}/start-all.sh || : + ${csudo}rm -f ${bin_link_dir}/stop-all.sh || : ${csudo}chmod 0555 ${bin_dir}/* @@ -275,8 +280,14 @@ function install_bin() { if [ -x ${bin_dir}/taoskeeper ]; then ${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper 2>>${install_log_path} || return 1 fi - if [ -x ${bin_dir}/*explorer ]; then - ${csudo}ln -s ${bin_dir}/*explorer ${bin_link_dir}/*explorer 2>>${install_log_path} || return 1 + if [ -x ${bin_dir}/taos-explorer ]; then + ${csudo}ln -s ${bin_dir}/taos-explorer ${bin_link_dir}/taos-explorer 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/start-all.sh ]; then + ${csudo}ln -s ${bin_dir}/start-all.sh ${bin_link_dir}/start-all.sh 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/stop-all.sh ]; then + ${csudo}ln -s ${bin_dir}/stop-all.sh ${bin_link_dir}/stop-all.sh 2>>${install_log_path} || return 1 fi log_print "install bin success" } @@ -474,6 +485,22 @@ function install_taoskeeper_config() { ${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir} } +function install_taos-explorer_config() { + if [ ! -f "${cfg_install_dir}/explorer.toml" ]; then + [ ! -d ${cfg_install_dir} ] && + ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/explorer.toml ] && ${csudo}cp ${cfg_dir}/explorer.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/explorer.toml ] && + ${csudo}chmod 644 ${cfg_install_dir}/explorer.toml + fi + + [ -f ${cfg_dir}/explorer.toml ] && + ${csudo}mv ${cfg_dir}/explorer.toml ${cfg_dir}/explorer.toml.new + + [ -f ${cfg_install_dir}/explorer.toml ] && + ${csudo}ln -s ${cfg_install_dir}/explorer.toml ${cfg_dir} +} + function install_config() { log_print "start install config from ${cfg_dir} to ${cfg_install_dir}" if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then @@ -609,30 +636,12 @@ function clean_service_on_systemd() { function install_service_on_systemd() { clean_service_on_systemd + + [ -f ${script_dir}/../cfg/taosd.service ] &&\ + ${csudo}cp ${script_dir}/../cfg/taosd.service \ + ${service_config_dir}/ || : - taosd_service_config="${service_config_dir}/taosd.service" - - ${csudo}bash -c "echo '[Unit]' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'After=network-online.target' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'Wants=network-online.target' >> ${taosd_service_config}" - ${csudo}bash -c "echo >> ${taosd_service_config}" - ${csudo}bash -c "echo '[Service]' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'Type=simple' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'Restart=always' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" - ${csudo}bash -c "echo >> ${taosd_service_config}" - ${csudo}bash -c "echo '[Install]' >> ${taosd_service_config}" - ${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" + ${csudo}systemctl daemon-reload ${csudo}systemctl enable taosd } @@ -652,6 +661,11 @@ function install_service_on_launchctl() { ${csudo}cp ${install_main_dir}/service/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || : ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || : fi + if [ -f ${install_main_dir}/service/com.taosdata.taos-explorer.plist ]; then + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taos-explorer.plist > /dev/null 2>&1 || : + ${csudo}cp ${install_main_dir}/service/com.taosdata.taos-explorer.plist /Library/LaunchDaemons/com.taosdata.taos-explorer.plist || : + ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taos-explorer.plist || : + fi } function install_taosadapter_service() { @@ -672,6 +686,15 @@ function install_taoskeeper_service() { fi } +function install_taos-explorer_service() { + if ((${service_mod}==0)); then + [ -f ${script_dir}/../cfg/taos-explorer.service ] &&\ + ${csudo}cp ${script_dir}/../cfg/taos-explorer.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + fi +} + function install_service() { log_print "start install service" if [ "$osType" != "Darwin" ]; then @@ -748,8 +771,10 @@ function install_TDengine() { install_config install_taosadapter_config install_taoskeeper_config + install_taos-explorer_config install_taosadapter_service install_taoskeeper_service + install_taos-explorer_service install_service install_app @@ -757,40 +782,38 @@ function install_TDengine() { #echo #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo - echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + echo -e "${GREEN_DARK}To configure TDengine ${NC}\t: edit /etc/taos/taos.cfg" + echo -e "${GREEN_DARK}To configure taosAdapter ${NC}\t: edit /etc/taos/taosadapter.toml" + echo -e "${GREEN_DARK}To configure taos-explorer ${NC}\t: edit /etc/taos/explorer.toml" if ((${service_mod}==0)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}systemctl start taosd${NC}" + echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ${csudo}systemctl start taosd${NC}" + echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ${csudo}systemctl start taosadapter${NC}" + echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ${csudo}systemctl start taoskeeper${NC}" + echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ${csudo}systemctl start taos-explorer${NC}" elif ((${service_mod}==1)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}" + echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}" echo -e " : ${csudo}service taosd start ${RED} after${NC}" + echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ${csudo}update-rc.d taosadapter default ${RED} for the first time${NC}" + echo -e " : ${csudo}service taosd taosadapter ${RED} after${NC}" + echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ${csudo}update-rc.d taoskeeper default ${RED} for the first time${NC}" + echo -e " : ${csudo}service taosd taoskeeper ${RED} after${NC}" + echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ${csudo}update-rc.d taos-explorer default ${RED} for the first time${NC}" + echo -e " : ${csudo}service taosd taos-explorer ${RED} after${NC}" else - echo -e "${GREEN_DARK}To start TDengine ${NC}: ./taosd${NC}" - fi - - - - if [ ! -z "$firstEp" ]; then - tmpFqdn=${firstEp%%:*} - substr=":" - if [[ $firstEp =~ $substr ]];then - tmpPort=${firstEp#*:} - else - tmpPort="" - fi - if [[ "$tmpPort" != "" ]];then - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" - else - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" - fi - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo - elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" - echo + echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ./taosd${NC}" + echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ./taosadapter${NC}" + echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ./taoskeeper${NC}" + echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ./taos-explorer${NC}" fi + log_print "install TDengine successfully!" echo - echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" + echo "TDengine is installed successfully!" + echo + echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}" + echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : taos -h $serverFqdn${NC}" + echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}" + } diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 36298a291e..378949591f 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -31,25 +31,22 @@ else lib64_link_dir="/usr/local/lib" inc_link_dir="/usr/local/include" fi -serverName="taosd" -clientName="taos" -uninstallScript="rmtaos" + +PREFIX="taos" +serverName="${PREFIX}d" +clientName="${PREFIX}" +uninstallScript="rm${PREFIX}" +adapterName="${PREFIX}adapter" +demoName="${PREFIX}demo" +benchmarkName="${PREFIX}Benchmark" +dumpName="${PREFIX}dump" +keeperName="${PREFIX}keeper" +xName="${PREFIX}x" +explorerName="${PREFIX}-explorer" +tarbitratorName="tarbitratord" productName="TDengine" -serverName2="taosd" -clientName2="taos" -productName2="TDengine" - -adapterName2="${clientName2}adapter" -demoName2="${clientName2}demo" -benchmarkName2="${clientName2}Benchmark" -dumpName2="${clientName2}dump" -keeperName2="${clientName2}keeper" -xName2="${clientName2}x" -explorerName2="${clientName2}-explorer" -uninstallScript2="rm${clientName2}" - -installDir="/usr/local/${clientName2}" +installDir="/usr/local/${PREFIX}" #install main path install_main_dir=${installDir} @@ -57,15 +54,11 @@ data_link_dir=${installDir}/data log_link_dir=${installDir}/log cfg_link_dir=${installDir}/cfg local_bin_link_dir="/usr/local/bin" - - service_config_dir="/etc/systemd/system" -taos_service_name=${serverName2} -taosadapter_service_name="${clientName2}adapter" -tarbitrator_service_name="tarbitratord" - -config_dir="/etc/${clientName2}" +config_dir="/etc/${PREFIX}" +services=(${PREFIX}"d" ${PREFIX}"adapter" ${PREFIX}"x" ${PREFIX}"-explorer" ${PREFIX}"keeper") +tools=(${PREFIX} ${PREFIX}"Benchmark" ${PREFIX}"dump" ${PREFIX}"demo" udfd set_core.sh TDinsight.sh $uninstallScript start-all.sh stop-all.sh) csudo="" if command -v sudo >/dev/null; then @@ -92,215 +85,26 @@ else service_mod=2 fi -function kill_taosadapter() { - pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}') - if [ -n "$pid" ]; then - ${csudo}kill -9 $pid || : - fi -} - -function kill_taosd() { - pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}') - if [ -n "$pid" ]; then - ${csudo}kill -9 $pid || : - fi -} - -function kill_tarbitrator() { - pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') - if [ -n "$pid" ]; then - ${csudo}kill -9 $pid || : - fi -} - -function clean_bin() { - # Remove link - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/${serverName} || : - echo "${serverName} is removed successfully" - ${csudo}rm -f ${bin_link_dir}/udfd || : - ${csudo}rm -f ${bin_link_dir}/${adapterName2} || : - echo "${adapterName2} is removed successfully" - ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : - ${csudo}rm -f ${bin_link_dir}/${demoName2} || : - ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : - ${csudo}rm -f ${bin_link_dir}/tarbitrator || : - ${csudo}rm -f ${bin_link_dir}/set_core || : - ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : - ${csudo}rm -f ${bin_link_dir}/${keeperName2} || : - - - if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - ${csudo}rm -f ${bin_link_dir}/${xName2} || : - ${csudo}rm -f ${bin_link_dir}/${explorerName2} || : - ${csudo}rm -f ${bin_link_dir}/${clientName2} || : - ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : - ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : - fi -} - -function clean_local_bin() { - ${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || : - ${csudo}rm -f ${local_bin_link_dir}/${demoName2} || : -} - -function clean_lib() { - # Remove link - ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : - - ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : - [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : - #${csudo}rm -rf ${v15_java_app_dir} || : - -} - -function clean_header() { - # Remove link - ${csudo}rm -f ${inc_link_dir}/taos.h || : - ${csudo}rm -f ${inc_link_dir}/taosdef.h || : - ${csudo}rm -f ${inc_link_dir}/taoserror.h || : - ${csudo}rm -f ${inc_link_dir}/tdef.h || : - ${csudo}rm -f ${inc_link_dir}/taosudf.h || : - - [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : -} - -function clean_config() { - # Remove link - ${csudo}rm -f ${cfg_link_dir}/* || : -} - -function clean_log() { - # Remove link - ${csudo}rm -rf ${log_link_dir} || : -} - -function clean_service_on_systemd() { - taosd_service_config="${service_config_dir}/${taos_service_name}.service" - if systemctl is-active --quiet ${taos_service_name}; then - echo "${taos_service_name} is running, stopping it..." - ${csudo}systemctl stop ${taos_service_name} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${taosd_service_config} - - taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service" - if systemctl is-active --quiet ${taosadapter_service_name}; then - echo "${clientName2}Adapter is running, stopping it..." - ${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${taosadapter_service_name} &>/dev/null || echo &>/dev/null - [ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config} - - tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" - if systemctl is-active --quiet ${tarbitrator_service_name}; then - echo "${productName2} tarbitrator is running, stopping it..." - ${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null - -} - -function clean_service_on_sysvinit() { - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then - echo "${serverName2} is running, stopping it..." - ${csudo}service ${serverName} stop || : - fi - - if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then - echo "${productName2} tarbitrator is running, stopping it..." - ${csudo}service tarbitratord stop || : - fi - - if ((${initd_mod} == 1)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}chkconfig --del ${serverName} || : - fi - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}chkconfig --del tarbitratord || : - fi - elif ((${initd_mod} == 2)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}insserv -r ${serverName} || : - fi - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}insserv -r tarbitratord || : - fi - elif ((${initd_mod} == 3)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}update-rc.d -f ${serverName} remove || : - fi - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}update-rc.d -f tarbitratord remove || : - fi - fi - - ${csudo}rm -f ${service_config_dir}/${serverName} || : - ${csudo}rm -f ${service_config_dir}/tarbitratord || : - - if $(which init &>/dev/null); then - ${csudo}init q || : - fi -} - -function clean_service_on_launchctl() { - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : -} - -function clean_service() { - if ((${service_mod} == 0)); then - clean_service_on_systemd - elif ((${service_mod} == 1)); then - clean_service_on_sysvinit - else - if [ "$osType" = "Darwin" ]; then - clean_service_on_launchctl - fi - kill_taosadapter - kill_taosd - kill_tarbitrator - fi -} - -function remove_data_and_config() { - data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}` - if [ X"$data_dir" == X"" ]; then - data_dir="/var/lib/taos" - fi - log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}` - if [ X"$log_dir" == X"" ]; then - log_dir="/var/log/taos" - fi - [ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir}/* - [ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir}/* - [ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}/* -} - -_kill_service_of() { +kill_service_of() { _service=$1 - pid=$(ps -ef | grep "$_service" | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep $_service | grep -v grep | grep -v $uninstallScript | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi } -_clean_service_on_systemd_of() { +clean_service_on_systemd_of() { _service=$1 _service_config="${service_config_dir}/${_service}.service" if systemctl is-active --quiet ${_service}; then - echo "taoskeeper is running, stopping it..." + echo "${_service} is running, stopping it..." ${csudo}systemctl stop ${_service} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${_service} &>/dev/null || echo &>/dev/null ${csudo}rm -f ${_service_config} } -_clean_service_on_sysvinit_of() { + +clean_service_on_sysvinit_of() { _service=$1 if pidof ${_service} &>/dev/null; then echo "${_service} is running, stopping it..." @@ -327,45 +131,99 @@ _clean_service_on_sysvinit_of() { fi } -_clean_service_of() { +clean_service_of() { _service=$1 if ((${service_mod} == 0)); then - _clean_service_on_systemd_of $_service + clean_service_on_systemd_of $_service elif ((${service_mod} == 1)); then - _clean_service_on_sysvinit_of $_service + clean_service_on_sysvinit_of $_service else - _kill_service_of $_service + kill_service_of $_service fi } -remove_taoskeeper() { - # remove taoskeeper bin - _clean_service_of taoskeeper - [ -e "${bin_link_dir}/taoskeeper" ] && ${csudo}rm -rf ${bin_link_dir}/taoskeeper - [ -e "${installDir}/taoskeeper" ] && ${csudo}rm -rf ${installDir}/taoskeeper - [ -e "${cfg_link_dir}/metrics.toml" ] || ${csudo}rm -rf ${cfg_link_dir}/metrics.toml - echo "taosKeeper is removed successfully!" -} - -function uninstall_taosx() { - if [ -f ${installDir}/uninstall.sh ]; then - cd ${installDir} - bash uninstall.sh +remove_service_of() { + _service=$1 + clean_service_of ${_service} + if [[ -e "${bin_link_dir}/${_service}" || -e "${installDir}/bin/${_service}" || -e "${local_bin_link_dir}/${_service}" ]]; then + ${csudo}rm -rf ${bin_link_dir}/${_service} + ${csudo}rm -rf ${installDir}/bin/${_service} + ${csudo}rm -rf ${local_bin_link_dir}/${_service} + echo "${_service} is removed successfully!" fi } -if [ "$verMode" == "cluster" ]; then - uninstall_taosx -fi +remove_tools_of() { + _tool=$1 + kill_service_of ${_tool} + [ -e "${bin_link_dir}/${_tool}" ] && ${csudo}rm -rf ${bin_link_dir}/${_tool} || : + [ -e "${installDir}/bin/${_tool}" ] && ${csudo}rm -rf ${installDir}/bin/${_tool} || : + [ -e "${local_bin_link_dir}/${_tool}" ] && ${csudo}rm -rf ${local_bin_link_dir}/${_tool} || : +} -remove_taoskeeper -# Stop service and disable booting start. -clean_service -# Remove binary file and links -clean_bin -# Remove links of local bin -clean_local_bin -# Remove header file. +remove_bin() { + for _service in "${services[@]}"; do + remove_service_of ${_service} + done + + for _tool in "${tools[@]}"; do + remove_tools_of ${_tool} + done +} + +function clean_lib() { + # Remove link + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : + + ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : + #${csudo}rm -rf ${v15_java_app_dir} || : +} + +function clean_header() { + # Remove link + ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosdef.h || : + ${csudo}rm -f ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${inc_link_dir}/tdef.h || : + ${csudo}rm -f ${inc_link_dir}/taosudf.h || : + + [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : +} + +function clean_config() { + # Remove link + ${csudo}rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo}rm -rf ${log_link_dir} || : +} + +function clean_service_on_launchctl() { + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : +} + +function remove_data_and_config() { + data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}` + if [ X"$data_dir" == X"" ]; then + data_dir="/var/lib/${PREFIX}" + fi + log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}` + if [ X"$log_dir" == X"" ]; then + log_dir="/var/log/${PREFIX}" + fi + [ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir} + [ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir} + [ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir} +} + +remove_bin clean_header # Remove lib file clean_lib @@ -375,8 +233,7 @@ clean_log clean_config # Remove data link directory ${csudo}rm -rf ${data_link_dir} || : - -${csudo}rm -rf ${install_main_dir} +${csudo}rm -rf ${install_main_dir} || : if [[ -e /etc/os-release ]]; then osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) else @@ -394,6 +251,7 @@ elif echo $osinfo | grep -qwi "centos"; then ${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || : fi if [ "$osType" = "Darwin" ]; then + clean_service_on_launchctl ${csudo}rm -rf /Applications/TDengine.app fi @@ -411,6 +269,7 @@ if [ X$answer == X"y" ] || [ X$answer == X"Y" ]; then fi fi +command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true echo -echo "${productName2} is removed successfully!" +echo "${productName} is removed successfully!" echo diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 695307254d..d183c10a31 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -38,9 +38,10 @@ if command -v sudo > /dev/null; then fi function kill_client() { - if [ -n "$(ps aux | grep -v grep | grep ${clientName})" ]; then - ${csudo}kill -9 $pid || : - fi + pid=$(ps -ef | grep ${clientName2} | grep -v grep | grep -v $uninstallScript2 | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi } function clean_bin() {