diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e2bedc258a..c61949b316 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -44,11 +44,16 @@ static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash); static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) { + int32_t code = 0; SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - return 0; + if (pCommitRaw == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw)); + TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)); + TAOS_RETURN(code); } int32_t mndInitSubscribe(SMnode *pMnode) { @@ -75,6 +80,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { } static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) { + int32_t code = 0; SMqSubscribeObj *pSub = tNewSubscribeObj(subKey); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -86,7 +92,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj pSub->subType = pTopic->subType; pSub->withMeta = pTopic->withMeta; - if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { + if ((terrno = mndSchedInitSubEp(pMnode, pTopic, pSub)) < 0) { tDeleteSubscribeObj(pSub); taosMemoryFree(pSub); return NULL; @@ -97,6 +103,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + int32_t code = 0; SMqRebVgReq req = {0}; req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; @@ -106,8 +113,8 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj pPlan->execNode.nodeId = pRebVg->pVgEp->vgId; int32_t msgLen; if (qSubPlanToString(pPlan, &req.qmsg, &msgLen) < 0) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + code = TSDB_CODE_QRY_INVALID_INPUT; + TAOS_RETURN(code); } } else { req.qmsg = taosStrdup(""); @@ -122,7 +129,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret); if (ret < 0) { taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(ret); } tlen += sizeof(SMsgHead); @@ -130,7 +137,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(ret); } SMsgHead *pMsgHead = (SMsgHead *)buf; @@ -140,40 +147,41 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj SEncoder encoder = {0}; tEncoderInit(&encoder, POINTER_SHIFT(buf, sizeof(SMsgHead)), tlen); - if (tEncodeSMqRebVgReq(&encoder, &req) < 0) { + if ((code = tEncodeSMqRebVgReq(&encoder, &req)) < 0) { taosMemoryFreeClear(buf); tEncoderClear(&encoder); taosMemoryFree(req.qmsg); - return -1; + TAOS_RETURN(code); } tEncoderClear(&encoder); *pBuf = buf; *pLen = tlen; taosMemoryFree(req.qmsg); - return 0; + TAOS_RETURN(code); } static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + int32_t code = 0; if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { if (pRebVg->oldConsumerId == -1) return 0; // drop stream, no consumer, while split vnode,all consumerId is -1 - terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; - return -1; + code = TSDB_CODE_MND_INVALID_SUB_OPTION; + TAOS_RETURN(code); } void *buf; int32_t tlen; - if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan) < 0) { - return -1; + if ((code = mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan)) < 0) { + TAOS_RETURN(code); } int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); if (pVgObj == NULL) { taosMemoryFree(buf); - terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; - return -1; + code = TSDB_CODE_MND_VGROUP_NOT_EXIST; + TAOS_RETURN(code); } STransAction action = {0}; @@ -183,11 +191,11 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubsc action.msgType = TDMT_VND_TMQ_SUBSCRIBE; mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(buf); - return -1; + TAOS_RETURN(code); } - return 0; + TAOS_RETURN(code); } static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, bool fullName) { @@ -209,6 +217,7 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, } static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + terrno = 0; SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); if (pRebInfo == NULL) { pRebInfo = tNewSMqRebSubscribe(key); @@ -612,7 +621,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu if (strcmp(pOutput->pSub->qmsg, "") != 0) { code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan); if (code != 0) { - terrno = code; goto END; } } @@ -623,7 +631,8 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb"); if (pTrans == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; goto END; } @@ -665,7 +674,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu END: nodesDestroyNode((SNode *)pPlan); mndTransDrop(pTrans); - return code; + TAOS_RETURN(code); } static void freeRebalanceItem(void *param) { @@ -827,6 +836,7 @@ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { } static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { + int32_t code = 0; const char *key = rebInput->pRebInfo->key; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key); @@ -838,8 +848,10 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); if (pTopic == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; mError("[rebalance] mq rebalance %s ignored since topic %s doesn't exist", key, topic); - return -1; + TAOS_RETURN(code); } taosRLockLatch(&pTopic->lock); @@ -848,10 +860,12 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu rebOutput->pSub = mndCreateSubscription(pMnode, pTopic, key); if (rebOutput->pSub == NULL) { - mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, terrstr()); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, tstrerror(code)); taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); - return -1; + TAOS_RETURN(code); } memcpy(rebOutput->pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN); @@ -869,7 +883,7 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu mInfo("[rebalance] sub topic:%s has %d consumers sub till now", key, rebInput->oldConsumerNum); mndReleaseSubscribe(pMnode, pSub); } - return 0; + TAOS_RETURN(code); } static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { @@ -885,6 +899,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); if (rebSubHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + if (terrno != 0) code = terrno; goto END; } taosHashSetFreeFp(rebSubHash, freeRebalanceItem); @@ -915,8 +930,8 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mndDoRebalance(pMnode, &rebInput, &rebOutput); - if (mndPersistRebResult(pMnode, pMsg, &rebOutput) != 0) { - mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr()) + if ((code = mndPersistRebResult(pMnode, pMsg, &rebOutput)) != 0) { + mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", tstrerror(code)) } clearRebOutput(&rebOutput); @@ -931,7 +946,7 @@ END: taosHashCleanup(rebSubHash); mndRebCntDec(); - return code; + TAOS_RETURN(code); } static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) { @@ -950,8 +965,7 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran } SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - ret = -1; + ret = TSDB_CODE_OUT_OF_MEMORY; goto END; } pReq->head.vgId = htonl(pVgObj->vgId); @@ -968,15 +982,14 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST; sdbRelease(pMnode->pSdb, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - ret = -1; + if ((ret = mndTransAppendRedoAction(pTrans, &action)) != 0) { goto END; } } END: sdbRelease(pMnode->pSdb, pVgObj); sdbCancelFetch(pMnode->pSdb, pIter); - return ret; + TAOS_RETURN(ret); } static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { @@ -1012,7 +1025,7 @@ static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgro END: sdbRelease(pMnode->pSdb, pConsumer); sdbCancelFetch(pMnode->pSdb, pIter); - return ret; + TAOS_RETURN(ret); } static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { @@ -1022,8 +1035,8 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { int32_t code = TSDB_CODE_ACTION_IN_PROGRESS; if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; + code = TSDB_CODE_INVALID_MSG; + TAOS_RETURN(code); } SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, dropReq.cgroup, dropReq.topic); @@ -1032,24 +1045,24 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { mInfo("cgroup:%s on topic:%s, not exist, ignore not exist is set", dropReq.cgroup, dropReq.topic); return 0; } else { - terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; - mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, terrstr()); - return -1; + code = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; + mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, tstrerror(code)); + TAOS_RETURN(code); } } taosWLockLatch(&pSub->lock); if (taosHashGetSize(pSub->consumerHash) != 0) { - terrno = TSDB_CODE_MND_CGROUP_USED; - mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - code = -1; + code = TSDB_CODE_MND_CGROUP_USED; + mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code)); goto end; } pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup"); if (pTrans == NULL) { - mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - code = -1; + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code)); goto end; } @@ -1088,9 +1101,9 @@ end: if (code != 0) { mError("cgroup %s on topic:%s, failed to drop", dropReq.cgroup, dropReq.topic); - return code; + TAOS_RETURN(code); } - return TSDB_CODE_ACTION_IN_PROGRESS; + TAOS_RETURN(TSDB_CODE_ACTION_IN_PROGRESS); } void mndCleanupSubscribe(SMnode *pMnode) {} @@ -1340,7 +1353,7 @@ END: sdbRelease(pSdb, pSub); sdbCancelFetch(pSdb, pIter); - return code; + TAOS_RETURN(code); } static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic, diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 573b75ff5a..70d0b858f6 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -123,15 +123,16 @@ _OUT: } static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) { + int32_t code = 0; if (pTrans->stage == TRN_STAGE_PREPARE) { - if (mndTransCheckConflict(pMnode, pTrans) < 0) { + if ((code = mndTransCheckConflict(pMnode, pTrans)) < 0) { mError("trans:%d, failed to validate trans conflicts.", pTrans->id); - return -1; + TAOS_RETURN(code); } return mndTransValidatePrepareStage(pMnode, pTrans); } - return 0; + TAOS_RETURN(code); } static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { @@ -139,10 +140,18 @@ static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { int32_t code = -1; SSdbRow *pRow = mndTransDecode(pRaw); - if (pRow == NULL) goto _OUT; + if (pRow == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OUT; + } pTrans = sdbGetRowObj(pRow); - if (pTrans == NULL) goto _OUT; + if (pTrans == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + goto _OUT; + } code = mndTransValidateImp(pMnode, pTrans); @@ -150,11 +159,10 @@ _OUT: if (pTrans) mndTransDropData(pTrans); if (pRow) taosMemoryFreeClear(pRow); if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT); - return code; + TAOS_RETURN(code); } int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { - terrno = TSDB_CODE_SUCCESS; SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; STrans *pTrans = NULL; @@ -163,7 +171,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (transId <= 0) { mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; goto _OUT; } @@ -176,7 +184,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (code != 0) { mError("trans:%d, failed to validate requested trans since %s", transId, terrstr()); code = 0; - pMeta->code = terrno; + pMeta->code = code; goto _OUT; } @@ -184,13 +192,15 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (code != 0) { mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); code = 0; - pMeta->code = terrno; + pMeta->code = code; goto _OUT; } pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { - mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + mError("trans:%d, not found while execute in mnode since %s", transId, tstrerror(code)); goto _OUT; } @@ -207,7 +217,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { _OUT: if (pTrans) mndReleaseTrans(pMnode, pTrans); - return code; + TAOS_RETURN(code); } static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { @@ -258,7 +268,7 @@ _OUT: mndPostMgmtCode(pMnode, code ? code : pMeta->code); rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; - return code; + TAOS_RETURN(code); } SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) { @@ -466,16 +476,18 @@ int32_t mndInitSync(SMnode *pMnode) { pNode->clusterId); } + int32_t code = 0; tsem_init(&pMgmt->syncSem, 0, 0); pMgmt->sync = syncOpen(&syncInfo, true); if (pMgmt->sync <= 0) { - mError("failed to open sync since %s", terrstr()); - return -1; + if (terrno != 0) code = terrno; + mError("failed to open sync since %s", tstrerror(code)); + TAOS_RETURN(code); } pMnode->pSdb->sync = pMgmt->sync; mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); - return 0; + TAOS_RETURN(code); } void mndCleanupSync(SMnode *pMnode) { @@ -518,10 +530,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; - if (req.contLen <= 0) return -1; + if (req.contLen <= 0) return TSDB_CODE_OUT_OF_MEMORY; req.pCont = rpcMallocCont(req.contLen); - if (req.pCont == NULL) return -1; + if (req.pCont == NULL) return TSDB_CODE_OUT_OF_MEMORY; memcpy(req.pCont, pRaw, req.contLen); taosThreadMutexLock(&pMgmt->lock); @@ -531,8 +543,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); taosThreadMutexUnlock(&pMgmt->lock); rpcFreeCont(req.pCont); - terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; - return terrno; + TAOS_RETURN(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED); } mInfo("trans:%d, will be proposed", transId); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 400725b041..f70a8844ba 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -22,7 +22,6 @@ set( # meta "src/meta/metaOpen.c" - "src/meta/metaIdx.c" "src/meta/metaTable.c" "src/meta/metaSma.c" "src/meta/metaQuery.c" diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0dc53afc48..32e4fea336 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -318,7 +318,7 @@ void* tdUidStoreFree(STbUidStore* pStore); // SMetaSnapReader ======================================== int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader); -int32_t metaSnapReaderClose(SMetaSnapReader** ppReader); +void metaSnapReaderClose(SMetaSnapReader** ppReader); int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData); // SMetaSnapWriter ======================================== int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 64cee3513c..3d4757d3e4 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -117,91 +117,81 @@ static void freeCacheEntryFp(void* param) { } int32_t metaCacheOpen(SMeta* pMeta) { - int32_t code = 0; - SMetaCache* pCache = NULL; + int32_t code = 0; + int32_t lino; - pCache = (SMetaCache*)taosMemoryMalloc(sizeof(SMetaCache)); - if (pCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + pMeta->pCache = (SMetaCache*)taosMemoryCalloc(1, sizeof(SMetaCache)); + if (pMeta->pCache == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // open entry cache - pCache->sEntryCache.nEntry = 0; - pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET; - pCache->sEntryCache.aBucket = - (SMetaCacheEntry**)taosMemoryCalloc(pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*)); - if (pCache->sEntryCache.aBucket == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + pMeta->pCache->sEntryCache.nEntry = 0; + pMeta->pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET; + pMeta->pCache->sEntryCache.aBucket = + (SMetaCacheEntry**)taosMemoryCalloc(pMeta->pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*)); + if (pMeta->pCache->sEntryCache.aBucket == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // open stats cache - pCache->sStbStatsCache.nEntry = 0; - pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET; - pCache->sStbStatsCache.aBucket = - (SMetaStbStatsEntry**)taosMemoryCalloc(pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*)); - if (pCache->sStbStatsCache.aBucket == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + pMeta->pCache->sStbStatsCache.nEntry = 0; + pMeta->pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET; + pMeta->pCache->sStbStatsCache.aBucket = + (SMetaStbStatsEntry**)taosMemoryCalloc(pMeta->pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*)); + if (pMeta->pCache->sStbStatsCache.aBucket == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } - pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); - if (pCache->sTagFilterResCache.pUidResCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + pMeta->pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); + if (pMeta->pCache->sTagFilterResCache.pUidResCache == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - pCache->sTagFilterResCache.accTimes = 0; - pCache->sTagFilterResCache.pTableEntry = + pMeta->pCache->sTagFilterResCache.accTimes = 0; + pMeta->pCache->sTagFilterResCache.pTableEntry = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); - if (pCache->sTagFilterResCache.pTableEntry == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + if (pMeta->pCache->sTagFilterResCache.pTableEntry == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp); - taosThreadMutexInit(&pCache->sTagFilterResCache.lock, NULL); + taosHashSetFreeFp(pMeta->pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp); + taosThreadMutexInit(&pMeta->pCache->sTagFilterResCache.lock, NULL); - pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); - if (pCache->STbGroupResCache.pResCache == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + pMeta->pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); + if (pMeta->pCache->STbGroupResCache.pResCache == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - pCache->STbGroupResCache.accTimes = 0; - pCache->STbGroupResCache.pTableEntry = + pMeta->pCache->STbGroupResCache.accTimes = 0; + pMeta->pCache->STbGroupResCache.pTableEntry = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); - if (pCache->STbGroupResCache.pTableEntry == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + if (pMeta->pCache->STbGroupResCache.pTableEntry == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); - taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); + taosHashSetFreeFp(pMeta->pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); + taosThreadMutexInit(&pMeta->pCache->STbGroupResCache.lock, NULL); - pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - if (pCache->STbFilterCache.pStb == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + pMeta->pCache->STbFilterCache.pStb = + taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pMeta->pCache->STbFilterCache.pStb == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - pCache->STbFilterCache.pStbName = + pMeta->pCache->STbFilterCache.pStbName = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); - if (pCache->STbFilterCache.pStbName == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + if (pMeta->pCache->STbFilterCache.pStbName == NULL) { + TSDB_CHECK_CODE(code = terrno, lino, _exit); } - pMeta->pCache = pCache; - return code; - -_err2: - entryCacheClose(pMeta); - -_err: - taosMemoryFree(pCache); - metaError("vgId:%d, meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); +_exit: + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code)); + metaCacheClose(pMeta); + } else { + metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__); + } return code; } @@ -289,8 +279,7 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { } } else { // insert if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) { - code = metaRehashCache(pCache, 1); - if (code) goto _exit; + TAOS_UNUSED(metaRehashCache(pCache, 1)); iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket; } @@ -328,8 +317,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) { pCache->sEntryCache.nEntry--; if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 && pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) { - code = metaRehashCache(pCache, 0); - if (code) goto _exit; + TAOS_UNUSED(metaRehashCache(pCache, 0)); } } else { code = TSDB_CODE_NOT_FOUND; @@ -351,7 +339,9 @@ int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo) { } if (pEntry) { - *pInfo = pEntry->info; + if (pInfo) { + *pInfo = pEntry->info; + } } else { code = TSDB_CODE_NOT_FOUND; } @@ -415,9 +405,7 @@ int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) { (*ppEntry)->info.ctbNum = pInfo->ctbNum; } else { // insert if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) { - code = metaRehashStatsCache(pCache, 1); - if (code) goto _exit; - + TAOS_UNUSED(metaRehashStatsCache(pCache, 1)); iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket; } @@ -454,8 +442,7 @@ int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid) { pCache->sStbStatsCache.nEntry--; if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 && pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) { - code = metaRehashStatsCache(pCache, 0); - if (code) goto _exit; + TAOS_UNUSED(metaRehashStatsCache(pCache, 0)); } } else { code = TSDB_CODE_NOT_FOUND; @@ -477,7 +464,9 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) { } if (pEntry) { - *pInfo = pEntry->info; + if (pInfo) { + *pInfo = pEntry->info; + } } else { code = TSDB_CODE_NOT_FOUND; } @@ -502,7 +491,9 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv // check whether it is existed in LRU cache, and remove it from linked list if not. LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len); if (pRes == NULL) { // remove the item in the linked list - taosArrayPush(pInvalidRes, &pNode); + if (taosArrayPush(pInvalidRes, &pNode) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { taosLRUCacheRelease(pCache, pRes, false); } @@ -626,7 +617,7 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL p->hitTimes = 0; tdListInit(&p->list, keyLen); - taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES); + TAOS_CHECK_RETURN(taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES)); tdListAppend(&p->list, pKey); return 0; } @@ -956,9 +947,7 @@ int32_t metaInitTbFilterCache(SMeta* pMeta) { } if (tbNum && pTbArr) { for (int32_t i = 0; i < tbNum; ++i) { - if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) { - return terrno ? terrno : -1; - } + TAOS_CHECK_RETURN(metaPutTbToFilterCache(pMeta, pTbArr[i], 1)); } } #else diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index f8b41e413b..bbf2217b31 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -21,7 +21,10 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } // begin a meta txn -int metaBegin(SMeta *pMeta, int8_t heap) { +int32_t metaBegin(SMeta *pMeta, int8_t heap) { + int32_t code = 0; + int32_t lino; + void *(*xMalloc)(void *, size_t) = NULL; void (*xFree)(void *, void *) = NULL; void *xArg = NULL; @@ -36,12 +39,19 @@ int metaBegin(SMeta *pMeta, int8_t heap) { xArg = pMeta->pVnode->inUse; } - if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - return -1; + code = tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdbCommit(pMeta->pEnv, pMeta->txn); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, + tstrerror(terrno)); + } else { + metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__); } - - tdbCommit(pMeta->pEnv, pMeta->txn); - return 0; } @@ -49,20 +59,36 @@ int metaBegin(SMeta *pMeta, int8_t heap) { TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; } int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); } int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); } -int metaPrepareAsyncCommit(SMeta *pMeta) { - // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); - int code = 0; + +int metaPrepareAsyncCommit(SMeta *pMeta) { + // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); + int code = 0; + int32_t lino; + metaWLock(pMeta); - code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn); + TAOS_UNUSED(ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn)); metaULock(pMeta); + code = tdbCommit(pMeta->pEnv, pMeta->txn); + TSDB_CHECK_CODE(code, lino, _exit); pMeta->changed = false; + +_exit: + if (code) { + metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, + tstrerror(terrno)); + } else { + metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__); + } return code; } // abort the meta txn int metaAbort(SMeta *pMeta) { - if (!pMeta->txn) return 0; + if (!pMeta->txn) { + return 0; + } + int code = tdbAbort(pMeta->pEnv, pMeta->txn); if (code) { metaError("vgId:%d, failed to abort meta since %s", TD_VID(pMeta->pVnode), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 9a111ae2d4..6e94cca390 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -17,159 +17,166 @@ int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) { const SColCmprWrapper *pw = &pME->colCmpr; - if (tEncodeI32v(pCoder, pw->nCols) < 0) return -1; - if (tEncodeI32v(pCoder, pw->version) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols)); + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->version)); uDebug("encode cols:%d", pw->nCols); for (int32_t i = 0; i < pw->nCols; i++) { SColCmpr *p = &pw->pColCmpr[i]; - if (tEncodeI16v(pCoder, p->id) < 0) return -1; - if (tEncodeU32(pCoder, p->alg) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI16v(pCoder, p->id)); + TAOS_CHECK_RETURN(tEncodeU32(pCoder, p->alg)); } return 0; } int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) { SColCmprWrapper *pWrapper = &pME->colCmpr; - if (tDecodeI32v(pDecoder, &pWrapper->nCols) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->nCols)); if (pWrapper->nCols == 0) { return 0; } - if (tDecodeI32v(pDecoder, &pWrapper->version) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->version)); uDebug("dencode cols:%d", pWrapper->nCols); pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr)); - if (pWrapper->pColCmpr == NULL) return -1; + if (pWrapper->pColCmpr == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (int i = 0; i < pWrapper->nCols; i++) { SColCmpr *p = &pWrapper->pColCmpr[i]; - if (tDecodeI16v(pDecoder, &p->id) < 0) return -1; - if (tDecodeU32(pDecoder, &p->alg) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &p->id)); + TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &p->alg)); } return 0; } -static FORCE_INLINE void metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr, - SSchemaWrapper *pSchema) { +static FORCE_INLINE int32_t metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr, + SSchemaWrapper *pSchema) { pCmpr->nCols = pSchema->nCols; - pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr)); + if ((pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr))) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; i < pCmpr->nCols; i++) { SColCmpr *pColCmpr = &pCmpr->pColCmpr[i]; SSchema *pColSchema = &pSchema->pSchema[i]; pColCmpr->id = pColSchema->colId; pColCmpr->alg = createDefaultColCmprByType(pColSchema->type); } + return 0; } int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { - if (tStartEncode(pCoder) < 0) return -1; + TAOS_CHECK_RETURN(tStartEncode(pCoder)); + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->version)); + TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->type)); + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->uid)); - if (tEncodeI64(pCoder, pME->version) < 0) return -1; - if (tEncodeI8(pCoder, pME->type) < 0) return -1; - if (tEncodeI64(pCoder, pME->uid) < 0) return -1; - if (pME->name == NULL || tEncodeCStr(pCoder, pME->name) < 0) return -1; + if (pME->name == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->name)); if (pME->type == TSDB_SUPER_TABLE) { - if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor? - if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; - if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->flags)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag)); if (TABLE_IS_ROLLUP(pME->flags)) { - if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam)); } } else if (pME->type == TSDB_CHILD_TABLE) { - if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1; - if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; - if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.btime)); + TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ctbEntry.ttlDays)); + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ctbEntry.commentLen)); if (pME->ctbEntry.commentLen > 0) { - if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ctbEntry.comment)); } - if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; - if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.suid)); + TAOS_CHECK_RETURN(tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags)); } else if (pME->type == TSDB_NORMAL_TABLE) { - if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1; - if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; - if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ntbEntry.btime)); + TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ntbEntry.ttlDays)); + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.commentLen)); if (pME->ntbEntry.commentLen > 0) { - if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ntbEntry.comment)); } - if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1; - if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.ncid)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow)); } else if (pME->type == TSDB_TSMA_TABLE) { - if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; + TAOS_CHECK_RETURN(tEncodeTSma(pCoder, pME->smaEntry.tsma)); } else { metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); - - return -1; + return TSDB_CODE_INVALID_PARA; } - if (meteEncodeColCmprEntry(pCoder, pME) < 0) return -1; + TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME)); tEndEncode(pCoder); return 0; } int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { - if (tStartDecode(pCoder) < 0) return -1; - - if (tDecodeI64(pCoder, &pME->version) < 0) return -1; - if (tDecodeI8(pCoder, &pME->type) < 0) return -1; - if (tDecodeI64(pCoder, &pME->uid) < 0) return -1; - if (tDecodeCStr(pCoder, &pME->name) < 0) return -1; + TAOS_CHECK_RETURN(tStartDecode(pCoder)); + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->version)); + TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->type)); + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->uid)); + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name)); if (pME->type == TSDB_SUPER_TABLE) { - if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor? - if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; - if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->flags)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag)); if (TABLE_IS_ROLLUP(pME->flags)) { - if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam)); } } else if (pME->type == TSDB_CHILD_TABLE) { - if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; - if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.btime)); + TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ctbEntry.ttlDays)); + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ctbEntry.commentLen)); if (pME->ctbEntry.commentLen > 0) { - if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ctbEntry.comment)); } - if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; - if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.suid)); + TAOS_CHECK_RETURN(tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags)); } else if (pME->type == TSDB_NORMAL_TABLE) { - if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1; - if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; - if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ntbEntry.btime)); + TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ntbEntry.ttlDays)); + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.commentLen)); if (pME->ntbEntry.commentLen > 0) { - if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ntbEntry.comment)); } - if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1; - if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.ncid)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow)); } else if (pME->type == TSDB_TSMA_TABLE) { pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma)); if (!pME->smaEntry.tsma) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } - if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1; + TAOS_CHECK_RETURN(tDecodeTSma(pCoder, pME->smaEntry.tsma, true)); } else { metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); - return -1; + return TSDB_CODE_INVALID_PARA; } if (pME->type == TSDB_SUPER_TABLE) { if (TABLE_IS_COL_COMPRESSED(pME->flags)) { - if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; + TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME)); if (pME->colCmpr.nCols == 0) { - metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow)); } } else { - metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow)); TABLE_SET_COL_COMPRESSED(pME->flags); } } else if (pME->type == TSDB_NORMAL_TABLE) { if (!tDecodeIsEnd(pCoder)) { uDebug("set type: %d, tableName:%s", pME->type, pME->name); - if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; + TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME)); if (pME->colCmpr.nCols == 0) { - metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow)); } } else { uDebug("set default type: %d, tableName:%s", pME->type, pME->name); - metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow)); } TABLE_SET_COL_COMPRESSED(pME->flags); } diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c deleted file mode 100644 index dc62ab2b9f..0000000000 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifdef USE_INVERTED_INDEX -#include "index.h" -#endif -#include "meta.h" - -struct SMetaIdx { -#ifdef USE_INVERTED_INDEX - SIndex *pIdx; -#endif - /* data */ -#ifdef WINDOWS - size_t avoidCompilationErrors; -#endif -}; - -int metaOpenIdx(SMeta *pMeta) { -#if 0 - char idxDir[128]; // TODO - char * err = NULL; - rocksdb_options_t *options = rocksdb_options_create(); - - // TODO - sprintf(idxDir, "%s/index", pMeta->path); - - if (pMeta->pCache) { - rocksdb_options_set_row_cache(options, pMeta->pCache); - } - rocksdb_options_set_create_if_missing(options, 1); - - pMeta->pIdx = rocksdb_open(options, idxDir, &err); - if (pMeta->pIdx == NULL) { - // TODO: handle error - rocksdb_options_destroy(options); - return -1; - } - - rocksdb_options_destroy(options); -#endif - -#ifdef USE_INVERTED_INDEX - // SIndexOpts opts; - // if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { - // return -1; - //} - -#endif - return 0; -} - -#ifdef BUILD_NO_CALL -void metaCloseIdx(SMeta *pMeta) { /* TODO */ -#if 0 - if (pMeta->pIdx) { - rocksdb_close(pMeta->pIdx); - pMeta->pIdx = NULL; - } -#endif - -#ifdef USE_INVERTED_INDEX - // SIndexOpts opts; - // if (indexClose(pMeta->pIdx->pIdx) != 0) { - // return -1; - //} - // return 0; - -#endif -} - -int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) { -#ifdef USE_INVERTED_INDEX - // if (pTbCfgs->type == META_CHILD_TABLE) { - // char buf[8] = {0}; - // int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId; - // sprintf(buf, "%d", colId); // colname - - // char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); - - // tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id - // tb_uid_t tuid = 0; // child table uid - // SIndexMultiTerm *terms = indexMultiTermCreate(); - // SIndexTerm *term = - // indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid); - // indexMultiTermAdd(terms, term); - - // int ret = indexPut(pMeta->pIdx->pIdx, terms); - // indexMultiTermDestroy(terms); - // return ret; - //} else { - // return DB_DONOTINDEX; - //} -#endif - // TODO - return 0; -} - -int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) { -#ifdef USE_INVERTED_INDEX - -#endif - // TODO - return 0; -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index e3b3de6cd6..ec26e94c5a 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -39,13 +39,13 @@ static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&p static void metaCleanup(SMeta **ppMeta); -int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { - SMeta *pMeta = NULL; - int ret; - int offset; - char path[TSDB_FILENAME_LEN] = {0}; - - *ppMeta = NULL; +int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { + SMeta *pMeta = NULL; + int32_t code = 0; + int32_t lino; + int32_t offset; + char path[TSDB_FILENAME_LEN] = {0}; + char indexFullPath[128] = {0}; // create handle vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); @@ -53,8 +53,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR); if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } metaInitLock(pMeta); @@ -69,163 +68,104 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { taosMkDir(pMeta->path); // open env - ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback, - pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey); - if (ret < 0) { - metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback, + pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey); + TSDB_CHECK_CODE(code, lino, _exit); // open pTbDb - ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pSkmDb - ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pUidIdx - ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pNameIdx - ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pCtbIdx - ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pSuidIdx - ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); - char indexFullPath[128] = {0}; sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); - taosMkDir(indexFullPath); + TAOS_UNUSED(taosMkDir(indexFullPath)); SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; - ret = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); - if (ret < 0) { - metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); + TSDB_CHECK_CODE(code, lino, _exit); - ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // open pTtlMgr ("ttlv1.idx") char logPrefix[128] = {0}; sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); - ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); - if (ret < 0) { - metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); + TSDB_CHECK_CODE(code, lino, _exit); // open pSmaIdx - ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // idx table create time - ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); // idx num of col, normal table only - ret = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta ncol index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0); + TSDB_CHECK_CODE(code, lino, _exit); - ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0); - if (ret < 0) { - metaError("vgId:%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0); + TSDB_CHECK_CODE(code, lino, _exit); - // open index - if (metaOpenIdx(pMeta) < 0) { - metaError("vgId:%d, failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + code = metaCacheOpen(pMeta); + TSDB_CHECK_CODE(code, lino, _exit); - int32_t code = metaCacheOpen(pMeta); + code = metaInitTbFilterCache(pMeta); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: if (code) { - terrno = code; - metaError("vgId:%d, failed to open meta cache since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; + metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code)); + metaCleanup(&pMeta); + *ppMeta = NULL; + } else { + metaDebug("vgId:%d %s success", TD_VID(pVnode), __func__); + *ppMeta = pMeta; } - - if (metaInitTbFilterCache(pMeta) != 0) { - goto _err; - } - - metaDebug("vgId:%d, meta is opened", TD_VID(pVnode)); - - *ppMeta = pMeta; - return 0; - -_err: - metaCleanup(&pMeta); - return -1; + return code; } -int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { - int code = TSDB_CODE_SUCCESS; - SMeta *pMeta = *ppMeta; +int32_t metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino; + SMeta *pMeta = *ppMeta; if (ttlMgrNeedUpgrade(pMeta->pEnv)) { code = metaBegin(pMeta, META_BEGIN_HEAP_OS); - if (code < 0) { - metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta); - if (code < 0) { - metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); code = metaCommit(pMeta, pMeta->txn); - if (code < 0) { - metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); } - return TSDB_CODE_SUCCESS; - -_err: - metaCleanup(ppMeta); +_exit: + if (code) { + metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code)); + metaCleanup(ppMeta); + } return code; } @@ -235,33 +175,42 @@ int metaClose(SMeta **ppMeta) { } int metaAlterCache(SMeta *pMeta, int32_t nPage) { + int32_t code = 0; metaWLock(pMeta); - - if (tdbAlter(pMeta->pEnv, nPage) < 0) { - metaULock(pMeta); - return -1; - } - + code = tdbAlter(pMeta->pEnv, nPage); metaULock(pMeta); - return 0; + + if (code) { + metaError("vgId:%d %s failed since %s", TD_VID(pMeta->pVnode), __func__, tstrerror(code)); + } + return code; } int32_t metaRLock(SMeta *pMeta) { metaTrace("meta rlock %p", &pMeta->lock); - int32_t ret = taosThreadRwlockRdlock(&pMeta->lock); - return ret; + int32_t code = taosThreadRwlockRdlock(&pMeta->lock); + if (code) { + return TAOS_SYSTEM_ERROR(code); + } + return 0; } int32_t metaWLock(SMeta *pMeta) { metaTrace("meta wlock %p", &pMeta->lock); - int32_t ret = taosThreadRwlockWrlock(&pMeta->lock); - return ret; + int32_t code = taosThreadRwlockWrlock(&pMeta->lock); + if (code) { + return TAOS_SYSTEM_ERROR(code); + } + return 0; } int32_t metaULock(SMeta *pMeta) { metaTrace("meta ulock %p", &pMeta->lock); - int32_t ret = taosThreadRwlockUnlock(&pMeta->lock); - return ret; + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code) { + return TAOS_SYSTEM_ERROR(code); + } + return 0; } static void metaCleanup(SMeta **ppMeta) { diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 34d4f8e845..95c65e5e80 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -25,14 +25,14 @@ struct SMetaSnapReader { int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) { int32_t code = 0; + int32_t lino; int32_t c = 0; SMetaSnapReader* pReader = NULL; // alloc pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } pReader->pMeta = pMeta; pReader->sver = sver; @@ -40,36 +40,29 @@ int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapRe // impl code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL); - if (code) { - taosMemoryFree(pReader); - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: if (code) { - taosMemoryFree(pReader); - goto _err; + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code)); + metaSnapReaderClose(&pReader); + *ppReader = NULL; + } else { + metaInfo("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__); + *ppReader = pReader; } - - metaInfo("vgId:%d, vnode snapshot meta reader opened", TD_VID(pMeta->pVnode)); - - *ppReader = pReader; - return code; - -_err: - metaError("vgId:%d, vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); - *ppReader = NULL; return code; } -int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) { - int32_t code = 0; - - tdbTbcClose((*ppReader)->pTbc); - taosMemoryFree(*ppReader); - *ppReader = NULL; - - return code; +void metaSnapReaderClose(SMetaSnapReader** ppReader) { + if (ppReader && *ppReader) { + tdbTbcClose((*ppReader)->pTbc); + taosMemoryFree(*ppReader); + *ppReader = NULL; + } } int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { @@ -106,7 +99,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); @@ -122,10 +115,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { } _exit: - return code; - -_err: - metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code)); + if (code) { + metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), + tstrerror(code)); + } return code; } @@ -138,26 +131,30 @@ struct SMetaSnapWriter { int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) { int32_t code = 0; + int32_t lino; SMetaSnapWriter* pWriter; // alloc pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } pWriter->pMeta = pMeta; pWriter->sver = sver; pWriter->ever = ever; - metaBegin(pMeta, META_BEGIN_HEAP_NIL); + code = metaBegin(pMeta, META_BEGIN_HEAP_NIL); + TSDB_CHECK_CODE(code, lino, _exit); - *ppWriter = pWriter; - return code; - -_err: - metaError("vgId:%d, meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); - *ppWriter = NULL; +_exit: + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code)); + taosMemoryFree(pWriter); + *ppWriter = NULL; + } else { + metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__); + *ppWriter = pWriter; + } return code; } @@ -189,25 +186,24 @@ _err: int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; - int32_t line = 0; + int32_t lino = 0; SMeta* pMeta = pWriter->pMeta; SMetaEntry metaEntry = {0}; SDecoder* pDecoder = &(SDecoder){0}; tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); code = metaDecodeEntry(pDecoder, &metaEntry); - VND_CHECK_CODE(code, line, _err); + TSDB_CHECK_CODE(code, lino, _exit); code = metaHandleEntry(pMeta, &metaEntry); - VND_CHECK_CODE(code, line, _err); + TSDB_CHECK_CODE(code, lino, _exit); +_exit: + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code)); + } tDecoderClear(pDecoder); return code; - -_err: - tDecoderClear(pDecoder); - metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line); - return code; } typedef struct STableInfoForChildTable { @@ -468,19 +464,17 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) { return c; } -void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid){ - bool ret = false; - SSchemaWrapper *schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); - if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){ +void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) { + bool ret = false; + SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); + if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } tDeleteSchemaWrapper(schema); ctx->hasPrimaryKey = ret; } -bool taosXGetTablePrimaryKey(SSnapContext* ctx){ - return ctx->hasPrimaryKey; -} +bool taosXGetTablePrimaryKey(SSnapContext* ctx) { return ctx->hasPrimaryKey; } int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { int32_t ret = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 75ba2be100..046f12f9e7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -312,8 +312,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) goto _exit; } else { pReader->metaDone = 1; - code = metaSnapReaderClose(&pReader->pMetaReader); - TSDB_CHECK_CODE(code, lino, _exit); + metaSnapReaderClose(&pReader->pMetaReader); } } diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index cc50919413..bb8d33366c 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -10,7 +10,6 @@ target_sources(tdb "src/db/tdbTable.c" "src/db/tdbTxn.c" "src/db/tdbPage.c" - "src/db/tdbOs.c" ) target_include_directories( diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4cca63a966..4dd3d3298c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -76,14 +76,14 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg if (keyLen == 0) { tdbError("tdb/btree-open: key len cannot be zero."); - return -1; + return TSDB_CODE_INVALID_PARA; } *ppBt = NULL; pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt)); if (pBt == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } // pBt->keyLen diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 9a1d89eeac..b9ba9de51f 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,8 +15,8 @@ #include "tdbInt.h" -int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm, - char *encryptKey) { +int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, + int32_t encryptAlgorithm, char *encryptKey) { TDB *pDb; int dsize; int zsize; @@ -31,7 +31,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pDb = (TDB *)pPtr; @@ -51,38 +51,38 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i pDb->jfd = -1; pDb->encryptAlgorithm = encryptAlgorithm; - if(encryptKey != NULL){ - strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); + if (encryptKey != NULL) { + strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); } ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); if (ret < 0) { - return -1; + return ret; } pDb->nPgrHash = 8; tsize = sizeof(SPager *) * pDb->nPgrHash; pDb->pgrHash = tdbOsMalloc(tsize); if (pDb->pgrHash == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } memset(pDb->pgrHash, 0, tsize); ret = taosMulModeMkDir(dbname, 0755, false); if (ret < 0) { - return -1; + return TAOS_SYSTEM_ERROR(ret); } #ifdef USE_MAINDB // open main db ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback); if (ret < 0) { - return -1; + return ret; } ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback); if (ret < 0) { - return -1; + return ret; } #endif @@ -125,12 +125,13 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void ( TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); if (!pTxn) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } - if (tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags) < 0) { + ret = tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags); + if (ret < 0) { tdbOsFree(pTxn); - return -1; + return ret; } for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { @@ -139,7 +140,7 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void ( tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); tdbTxnClose(pTxn); - return -1; + return ret; } } @@ -157,7 +158,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { if (ret < 0) { tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); - return -1; + return ret; } } @@ -173,7 +174,7 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) { if (ret < 0) { tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); - return -1; + return ret; } } @@ -191,7 +192,7 @@ int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn) { if (ret < 0) { tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); - return -1; + return ret; } } @@ -207,7 +208,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { if (ret < 0) { tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); - return -1; + return ret; } } diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c deleted file mode 100644 index a2fce8925b..0000000000 --- a/source/libs/tdb/src/db/tdbOs.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbOs.h" - -#ifndef TDB_FOR_TDENGINE - -// tdbOsRead -i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes) { - i64 nRead = 0; - i64 iRead = 0; - u8 *pBuf = (u8 *)pData; - - while (nBytes > 0) { - iRead = read(fd, pBuf, nBytes); - if (iRead < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (iRead == 0) { - break; - } - - nRead += iRead; - pBuf += iRead; - nBytes -= iRead; - } - - return nRead; -} - -// tdbOsPRead -i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset) { - i64 nRead = 0; - i64 iRead = 0; - i64 iOffset = offset; - u8 *pBuf = (u8 *)pData; - - while (nBytes > 0) { - iRead = pread(fd, pBuf, nBytes, iOffset); - if (iRead < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (iRead == 0) { - break; - } - - nRead += iRead; - pBuf += iRead; - iOffset += iRead; - nBytes -= iRead; - } - - return nRead; -} - -// tdbOsWrite -i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes) { - i64 nWrite = 0; - i64 iWrite = 0; - u8 *pBuf = (u8 *)pData; - - while (nBytes > 0) { - iWrite = write(fd, pBuf, nBytes); - if (iWrite < 0) { - if (errno == EINTR) { - continue; - } - - return -1; - } - - nWrite += iWrite; - pBuf += iWrite; - nBytes -= iWrite; - } - - return nWrite; -} - -#endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 455128e6ec..6cb5f89758 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -50,30 +50,36 @@ static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); } static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); } int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { + int32_t code = 0; + int32_t lino; SPCache *pCache; void *pPtr; SPage *pPgHdr; pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize); if (pCache == NULL) { - return -1; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } pCache->szPage = pageSize; pCache->nPages = cacheSize; pCache->aPage = (SPage **)tdbOsCalloc(cacheSize, sizeof(SPage *)); if (pCache->aPage == NULL) { - tdbOsFree(pCache); - return -1; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } - if (tdbPCacheOpenImpl(pCache) < 0) { - tdbOsFree(pCache); - return -1; - } + code = tdbPCacheOpenImpl(pCache); + TSDB_CHECK_CODE(code, lino, _exit); - *ppCache = pCache; - return 0; +_exit: + if (code) { + tdbError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(code)); + tdbPCacheClose(pCache); + *ppCache = NULL; + } else { + *ppCache = pCache; + } + return code; } int tdbPCacheClose(SPCache *pCache) { @@ -99,14 +105,14 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) { } else if (pCache->nPages < nPage) { SPage **aPage = tdbOsCalloc(nPage, sizeof(SPage *)); if (aPage == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) { - if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) { - // TODO: handle error + int32_t code = tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL); + if (code) { tdbOsFree(aPage); - return -1; + return code; } // pPage->pgid = 0; @@ -156,15 +162,11 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) { } int tdbPCacheAlter(SPCache *pCache, int32_t nPage) { - int ret = 0; - + int code; tdbPCacheLock(pCache); - - ret = tdbPCacheAlterImpl(pCache, nPage); - + code = tdbPCacheAlterImpl(pCache, nPage); tdbPCacheUnlock(pCache); - - return ret; + return code; } SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { @@ -180,9 +182,6 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { tdbPCacheUnlock(pCache); - // printf("thread %" PRId64 " fetch page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id, - // TDB_PAGE_PGNO(pPage), pPage, nRef); - if (pPage) { tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); } else { @@ -285,6 +284,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pTxn) { tdbError("tdb/pcache: null ptr pTxn, fetch impl failed."); + terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -327,7 +327,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); - // TODO: recycle other backup pages + terrno = ret; return NULL; } @@ -475,10 +475,8 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nFree = 0; pCache->pFree = NULL; for (int i = 0; i < pCache->nPages; i++) { - if (tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL) < 0) { - // TODO: handle error - return -1; - } + ret = tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL); + if (ret) return ret; // pPage->pgid = 0; pPage->isAnchor = 0; @@ -504,8 +502,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages; pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *)); if (pCache->pgHash == NULL) { - // TODO - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } // Open LRU list diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index c6310f4985..322b735163 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -45,12 +45,12 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) if (!xMalloc) { tdbError("tdb/page-create: null xMalloc."); - return -1; + return TSDB_CODE_INVALID_PARA; } if (!TDB_IS_PGSIZE_VLD(pageSize)) { tdbError("tdb/page-create: invalid pageSize: %d.", pageSize); - return -1; + return TSDB_CODE_INVALID_PARA; } *ppPage = NULL; @@ -58,7 +58,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) ptr = (u8 *)(xMalloc(arg, size)); if (ptr == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } memset(ptr, 0, size); @@ -86,12 +86,12 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) if (pPage->isDirty) { tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty); - return -1; + return TSDB_CODE_INVALID_PARA; } if (!xFree) { tdbError("tdb/page-destroy: null xFree."); - return -1; + return TSDB_CODE_INVALID_PARA; } for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { @@ -129,7 +129,8 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); pPage->pPageHdr = pPage->pData + szAmHdr; if (TDB_PAGE_NCELLS(pPage) == 0) { - return tdbPageZero(pPage, szAmHdr, xCellSize); + tdbPageZero(pPage, szAmHdr, xCellSize); + return; } pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); @@ -159,7 +160,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) { tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell, TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } nFree = TDB_PAGE_NFREE(pPage); @@ -207,7 +208,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) { tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart, pPage->pCellIdx, nCells); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } } @@ -234,7 +235,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { if (idx < 0 || idx >= nCells + pPage->nOverflow) { tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow); - return -1; + return TSDB_CODE_INVALID_PARA; } iOvfl = 0; @@ -265,14 +266,14 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { pPage->aiOvfl[iOvfl]--; if (pPage->aiOvfl[iOvfl] <= 0) { tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } } return 0; } -void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { +int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int delta, nFree; pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); @@ -280,7 +281,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { if (pToPage->pFreeEnd < pToPage->pFreeStart) { tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd); - return; + return TSDB_CODE_INVALID_DATA_FMT; } memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); @@ -289,7 +290,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) { tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage), pToPage->pFreeEnd - pToPage->pData); - return; + return TSDB_CODE_INVALID_DATA_FMT; } delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); @@ -304,6 +305,10 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { if (deepCopyOvfl) { int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); pNewCell = (SCell *)tdbOsMalloc(szCell); + if (pNewCell == NULL) { + tdbError("tdb/page-copy: out of memory, size: %d", szCell); + return TSDB_CODE_OUT_OF_MEMORY; + } memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); } @@ -312,6 +317,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; } pToPage->nOverflow = pFromPage->nOverflow; + return 0; } int tdbPageCapacity(int pageSize, int amHdrSize) { @@ -343,12 +349,12 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell, TDB_PAGE_OFFSET_SIZE(pPage)); - return -1; + return TSDB_CODE_INVALID_PARA; } if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) { tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage), pPage->pFreeEnd - pPage->pData); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } // 1. Try to allocate from the free space block area @@ -363,7 +369,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { cellFree = TDB_PAGE_FCELL(pPage); if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) { tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { SCell *pPrevFreeCell = NULL; @@ -408,19 +414,19 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { } // 3. Try to dfragment and allocate again - tdbPageDefragment(pPage); + TAOS_CHECK_RETURN(tdbPageDefragment(pPage)); if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } if (TDB_PAGE_NFREE(pPage) != nFree) { tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage)); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) { tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart, pPage->pData); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } pPage->pFreeEnd -= szCell; @@ -430,7 +436,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { _alloc_finish: if (NULL == pCell) { tdbError("tdb/page-allocate: null ptr pCell."); - return -1; + return TSDB_CODE_OUT_OF_BUFFER; } pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); @@ -447,15 +453,15 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { if (pCell < pPage->pFreeEnd) { tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd); - return -1; + return TSDB_CODE_INVALID_PARA; } if (pCell + szCell > (u8 *)(pPage->pPageFtr)) { tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd); - return -1; + return TSDB_CODE_INVALID_PARA; } if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) { tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx)); - return -1; + return TSDB_CODE_INVALID_PARA; } nFree = TDB_PAGE_NFREE(pPage); @@ -470,7 +476,7 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); } else { tdbError("tdb/page-free: invalid cell size: %d", szCell); - return -1; + return TSDB_CODE_INVALID_PARA; } } @@ -502,7 +508,9 @@ static int tdbPageDefragment(SPage *pPage) { int32_t nCell = TDB_PAGE_NCELLS(pPage); SCellIdx *aCellIdx = (SCellIdx *)tdbOsMalloc(sizeof(SCellIdx) * nCell); - if (aCellIdx == NULL) return -1; + if (aCellIdx == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } for (int32_t iCell = 0; iCell < nCell; iCell++) { aCellIdx[iCell].iCell = iCell; aCellIdx[iCell].offset = TDB_PAGE_CELL_OFFSET_AT(pPage, iCell); diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 9dbac11b02..14165045ca 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "tdbInt.h" #include "crypt.h" +#include "tdbInt.h" #include "tglobal.h" /* #pragma pack(push, 1) @@ -41,9 +41,10 @@ struct hashset_st { static const unsigned int prime = 39; static const unsigned int prime2 = 5009; -hashset_t hashset_create(void) { +static hashset_t hashset_create(void) { hashset_t set = tdbOsCalloc(1, sizeof(struct hashset_st)); if (!set) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -52,6 +53,7 @@ hashset_t hashset_create(void) { set->items = tdbOsCalloc(set->capacity, sizeof(size_t)); if (!set->items) { tdbOsFree(set); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } set->mask = set->capacity - 1; @@ -69,7 +71,7 @@ void hashset_destroy(hashset_t set) { } } -int hashset_add_member(hashset_t set, void *item) { +static int hashset_add_member(hashset_t set, void *item) { size_t value = (size_t)item; size_t h; @@ -88,7 +90,7 @@ int hashset_add_member(hashset_t set, void *item) { return 1; } -int hashset_add(hashset_t set, void *item) { +static int hashset_add(hashset_t set, void *item) { int ret = hashset_add_member(set, item); size_t old_capacity = set->capacity; @@ -113,7 +115,7 @@ int hashset_add(hashset_t set, void *item) { return ret; } -int hashset_remove(hashset_t set, void *item) { +static int hashset_remove(hashset_t set, void *item) { size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { @@ -127,7 +129,7 @@ int hashset_remove(hashset_t set, void *item) { return 0; } -int hashset_contains(hashset_t set, void *item) { +static int hashset_contains(hashset_t set, void *item) { size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { @@ -177,7 +179,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { + fsize + 8 + 1; /* jFileName */ pPtr = (uint8_t *)tdbOsCalloc(1, zsize); if (pPtr == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pPager = (SPager *)pPtr; @@ -198,12 +200,12 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pPager->fd)) { // if (pPager->fd < 0) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } ret = tdbGnrtFileID(pPager->fd, pPager->fid, false); if (ret < 0) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } // pPager->jfd = -1; @@ -221,11 +223,6 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { int tdbPagerClose(SPager *pPager) { if (pPager) { - /* - if (pPager->inTran) { - tdbOsClose(pPager->jfd); - } - */ tdbOsClose(pPager->fd); tdbOsFree(pPager); } @@ -254,8 +251,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { !hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { - tdbError("failed to write page to journal since %s", tstrerror(terrno)); - return -1; + tdbError("failed to write page to journal since %s", tstrerror(ret)); + return ret; } if (pPager->pActiveTxn->jPageSet) { @@ -278,11 +275,13 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (TDB_FD_INVALID(pTxn->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } pTxn->jPageSet = hashset_create(); + if (pTxn->jPageSet == NULL) { + return terrno; + } pPager->pActiveTxn = pTxn; @@ -319,8 +318,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } // loop to write the dirty pages to file @@ -331,13 +329,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { if (pPage->nOverflow != 0) { tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); - return -1; + return ret; } } @@ -368,8 +366,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the db file if (tdbOsFSync(pPager->fd) < 0) { tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } return 0; @@ -382,14 +379,12 @@ int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { // remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } // pPager->inTran = 0; @@ -408,8 +403,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } // loop to write the dirty pages to file @@ -426,7 +420,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); - return -1; + return ret; } } @@ -445,36 +439,28 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { tdbPCacheRelease(pPager->pCache, pPage, pTxn); } - /* - tdbTrace("reset dirty tree: %p", &pPager->rbt); - tRBTreeCreate(&pPager->rbt, pageCmpFn); - - // sync the db file - if (tdbOsFSync(pPager->fd) < 0) { - tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - */ return 0; } -static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, const char* function, - int64_t offset){ +static char *tdbEncryptPage(SPager *pPager, char *pPageData, int32_t pageSize, const char *function, int64_t offset) { int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; - char* encryptKey = pPager->pEnv->encryptKey; + char *encryptKey = pPager->pEnv->encryptKey; - char* buf = pPageData; + char *buf = pPageData; - if(encryptAlgorithm == DND_CA_SM4){ - //tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); - //ASSERT(strlen(encryptKey) > 0); + if (encryptAlgorithm == DND_CA_SM4) { + // tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); + // ASSERT(strlen(encryptKey) > 0); - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]); + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]); buf = taosMemoryMalloc(pageSize); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - unsigned char packetData[128]; + unsigned char packetData[128]; int32_t count = 0; while (count < pageSize) { @@ -488,19 +474,19 @@ static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, c int32_t newLen = CBC_Encrypt(&opts); memcpy(buf + count, packetData, newLen); - count += newLen; + count += newLen; } - //tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function); + // tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function); - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]); + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]); } return buf; } -void tdbFreeEncryptBuf(SPager *pPager, char* buf){ +void tdbFreeEncryptBuf(SPager *pPager, char *buf) { int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; - if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf); + if (encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf); } // recovery dirty pages int tdbPagerAbort(SPager *pPager, TXN *pTxn) { @@ -518,26 +504,24 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdb_fd_t jfd = pTxn->jfd; ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); if (ret < 0) { - return -1; + return ret; } if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { - return -1; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } tdbDebug("pager/abort: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId); @@ -549,7 +533,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbTrace("pager/abort: restore pgno:%d,", pgno); @@ -559,27 +543,28 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } - char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); + char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); + if (buf == NULL) { + return terrno; + } ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); if (ret < 0) { tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); tdbFreeEncryptBuf(pPager, buf); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbFreeEncryptBuf(pPager, buf); @@ -587,9 +572,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { if (tdbOsFSync(pPager->fd) < 0) { tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbOsFree(pageBuf); @@ -617,8 +601,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 4, remove the journal file if (tdbOsClose(pTxn->jfd) < 0) { tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } char jTxnFileName[TDB_FILENAME_LEN]; @@ -626,8 +609,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } // pPager->inTran = 0; @@ -658,7 +640,7 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) { ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); - return -1; + return ret; } tdbTrace("tdb/flush:%p, pgno:%d, %d/%d/%d", pPager, pgno, pPager->dbOrigSize, pPager->dbFileSize, maxPgno); @@ -717,13 +699,13 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa ret = tdbPagerAllocPage(pPager, &pgno, pTxn); if (ret < 0) { tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); - return -1; + return ret; } } if (pgno == 0) { tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } // fetch a page container @@ -739,7 +721,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); if (ret < 0) { tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage); - return -1; + return ret; } } @@ -748,11 +730,11 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa if (!TDB_PAGE_INITIALIZED(pPage)) { tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } if (pPage->pPager != pPager) { tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager); - return -1; + return TSDB_CODE_INVALID_DATA_FMT; } *ppgno = pgno; @@ -771,12 +753,17 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { SPgno pgno = TDB_PAGE_PGNO(pPage); if (pPager->frps) { - taosArrayPush(pPager->frps, &pgno); + if (taosArrayPush(pPager->frps, &pgno) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } pPage->pPager = NULL; return code; } pPager->frps = taosArrayInit(8, sizeof(SPgno)); + if (pPager->frps == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } // memset(pPage->pData, 0, pPage->pageSize); tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno); // printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno); @@ -785,7 +772,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); taosArrayDestroy(pPager->frps); pPager->frps = NULL; - return -1; + return code; } while (TARRAY_SIZE(pPager->frps) > 0) { @@ -796,7 +783,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); taosArrayDestroy(pPager->frps); pPager->frps = NULL; - return -1; + return code; } } @@ -822,7 +809,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn); if (code < 0) { - return 0; + return code; } code = tdbTbcMoveToFirst(pCur); @@ -924,20 +911,19 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage } int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; - char* encryptKey = pPager->pEnv->encryptKey; + char *encryptKey = pPager->pEnv->encryptKey; - if(encryptAlgorithm == DND_CA_SM4){ - //tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); - //ASSERT(strlen(encryptKey) > 0); + if (encryptAlgorithm == DND_CA_SM4) { + // tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); + // ASSERT(strlen(encryptKey) > 0); - //uint8_t flags = pPage->pData[0]; - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags); + // uint8_t flags = pPage->pData[0]; + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags); - unsigned char packetData[128]; + unsigned char packetData[128]; int32_t count = 0; - while(count < pPage->pageSize) - { + while (count < pPage->pageSize) { SCryptOpts opts = {0}; opts.len = 128; opts.source = pPage->pData + count; @@ -945,20 +931,23 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage opts.unitLen = 128; strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); - int newLen = CBC_Decrypt(&opts); + int newLen = CBC_Decrypt(&opts); memcpy(pPage->pData + count, packetData, newLen); count += newLen; } - //tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, __FUNCTION__); + // tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, + // __FUNCTION__); - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__); + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), + // pPage->pData[0], __FUNCTION__); } } else { init = 0; } - //tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__); + // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], + // __FUNCTION__); ret = (*initPage)(pPage, arg, init); if (ret < 0) { @@ -1001,16 +990,14 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { if (ret < 0) { tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName, pgno, pPager->pActiveTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno), pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } return 0; @@ -1044,15 +1031,14 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); - char* buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset); + char *buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset); ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset); if (ret < 0) { tdbFreeEncryptBuf(pPager, buf); tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, pPage->pageSize); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbFreeEncryptBuf(pPager, buf); @@ -1072,18 +1058,17 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); if (ret < 0) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName); @@ -1095,7 +1080,7 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { tdbOsFree(pageBuf); - return -1; + return TAOS_SYSTEM_ERROR(errno); } tdbTrace("pager/restore: restore pgno:%d,", pgno); @@ -1103,27 +1088,28 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { tdbOsFree(pageBuf); - return -1; + return TAOS_SYSTEM_ERROR(errno); } i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } - char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); + char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); + if (buf == NULL) { + return terrno; + } ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); if (ret < 0) { tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); tdbFreeEncryptBuf(pPager, buf); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbFreeEncryptBuf(pPager, buf); @@ -1131,9 +1117,8 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { if (tdbOsFSync(pPager->fd) < 0) { tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); - terrno = TAOS_SYSTEM_ERROR(errno); tdbOsFree(pageBuf); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } tdbOsFree(pageBuf); @@ -1160,21 +1145,27 @@ static int32_t txnIdCompareDesc(const void *pLeft, const void *pRight) { } int tdbPagerRestoreJournals(SPager *pPager) { + int32_t code = 0; tdbDirEntryPtr pDirEntry; tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); if (pDir == NULL) { tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); - return -1; + return TAOS_SYSTEM_ERROR(errno); } SArray *pTxnList = taosArrayInit(16, sizeof(int64_t)); + if (pTxnList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } while ((pDirEntry = tdbReadDir(pDir)) != NULL) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { int64_t txnId = -1; sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId); - taosArrayPush(pTxnList, &txnId); + if (taosArrayPush(pTxnList, &txnId) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } } taosArraySort(pTxnList, txnIdCompareDesc); @@ -1185,12 +1176,12 @@ int tdbPagerRestoreJournals(SPager *pPager) { memcpy(jname, pPager->pEnv->dbName, dirLen); jname[dirLen] = '/'; sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); - if (tdbPagerRestore(pPager, jname) < 0) { + code = tdbPagerRestore(pPager, jname); + if (code) { taosArrayDestroy(pTxnList); tdbCloseDir(&pDir); - - tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname); - return -1; + tdbError("failed to restore file due to %s. jFileName:%s", strerror(code), jname); + return code; } } @@ -1205,7 +1196,7 @@ int tdbPagerRollback(SPager *pPager) { tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); if (pDir == NULL) { tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } while ((pDirEntry = tdbReadDir(pDir)) != NULL) { @@ -1221,8 +1212,7 @@ int tdbPagerRollback(SPager *pPager) { tdbCloseDir(&pDir); tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } } } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 3bc6f75bef..365be222ef 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -41,7 +41,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb)); if (pTb == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } // pTb->pEnv @@ -54,7 +54,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF pPager = tdbEnvGetPager(pEnv, fFullName); if (!pPager) { tdbOsFree(pTb); - return -1; + return terrno; } ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData); @@ -74,7 +74,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { tdbOsFree(pTb); - return -1; + return ret; } tdbEnvAddPager(pEnv, pPager); @@ -109,7 +109,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { tdbOsFree(pTb); - return -1; + return ret; } } else { tdbPagerRollback(pPager); @@ -119,7 +119,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); if (ret < 0) { tdbOsFree(pTb); - return -1; + return ret; } *ppTb = pTb; diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index 0aeed3c140..24a70f62b2 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -20,7 +20,7 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void // not support read-committed version at the moment if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) { tdbError("tdb/txn: invalid txn flags: %" PRId32, flags); - return -1; + return TSDB_CODE_INVALID_PARA; } pTxn->flags = flags; @@ -39,7 +39,7 @@ int tdbTxnCloseImpl(TXN *pTxn) { } if (pTxn->jfd) { - tdbOsClose(pTxn->jfd); + TAOS_UNUSED(tdbOsClose(pTxn->jfd)); ASSERT(pTxn->jfd == NULL); } diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 9021e08ffe..d6d33bc7d8 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -38,9 +38,8 @@ void tdbFree(void *p) { int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) { int64_t stDev = 0, stIno = 0; - if (taosDevInoFile(fd, &stDev, &stIno) < 0) { - return -1; - } + int32_t code = taosDevInoFile(fd, &stDev, &stIno); + return code; memset(fileid, 0, TDB_FILE_ID_LEN); @@ -59,7 +58,7 @@ int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) { ret = tdbOsFileSize(fd, &szBytes); if (ret < 0) { - return -1; + return TAOS_SYSTEM_ERROR(errno); } *size = szBytes / szPage; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index b359f8a4cc..605fe6a1a4 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -17,6 +17,7 @@ #define _TD_TDB_INTERNAL_H_ #include "tdb.h" +#include "tutil.h" #include "tdef.h" #include "tlog.h" @@ -338,15 +339,15 @@ static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { ((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset) #define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); -void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); -int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt); -int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt); -void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int copyOvflCells); -int tdbPageCapacity(int pageSize, int amHdrSize); +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); +int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt); +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt); +int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl); +int tdbPageCapacity(int pageSize, int amHdrSize); static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { SCell *pCell; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index f63339bcfc..79acec1211 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -67,7 +67,7 @@ typedef struct TdFile { void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #ifdef WINDOWS - char tmpPath[PATH_MAX]; + char tmpPath[PATH_MAX]; int32_t len = (int32_t)strlen(inputTmpDir); memcpy(tmpPath, inputTmpDir, len); @@ -269,13 +269,13 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #else if (pFile == NULL || pFile->fd < 0) { - return -1; + return TSDB_CODE_INVALID_PARA; } struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { printf("taosFStatFile run fstat fail."); - return code; + return TAOS_SYSTEM_ERROR(errno); } if (stDev != NULL) { @@ -1239,7 +1239,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { #ifdef WINDOWS size_t bufferSize = 512; *ptrBuf = taosMemoryMalloc(bufferSize); - if (*ptrBuf == NULL) goto END; + if (*ptrBuf == NULL) goto END; size_t bytesRead = 0; size_t totalBytesRead = 0; @@ -1274,7 +1274,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { ret = getline(ptrBuf, &len, pFile->fp); #endif - END: +END: #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -1413,34 +1413,30 @@ int32_t taosLinkFile(char *src, char *dst) { return 0; } -FILE* taosOpenCFile(const char* filename, const char* mode) { - return fopen(filename, mode); -} +FILE *taosOpenCFile(const char *filename, const char *mode) { return fopen(filename, mode); } -int taosSeekCFile(FILE* file, int64_t offset, int whence) { +int taosSeekCFile(FILE *file, int64_t offset, int whence) { #ifdef WINDOWS return _fseeki64(file, offset, whence); #else return fseeko(file, offset, whence); -#endif +#endif } -size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ) { +size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) { return fread(buffer, size, count, stream); } -size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream) { +size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) { return fwrite(ptr, size, nitems, stream); } -int taosCloseCFile(FILE *f) { - return fclose(f); -} +int taosCloseCFile(FILE *f) { return fclose(f); } -int taosSetAutoDelFile(char* path) { +int taosSetAutoDelFile(char *path) { #ifdef WINDOWS return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY); #else return unlink(path); -#endif +#endif } \ No newline at end of file diff --git a/tests/ci/ci_deploy_dependency_file_list.txt b/tests/ci/ci_deploy_dependency_file_list.txt new file mode 100644 index 0000000000..1e7afdd7db --- /dev/null +++ b/tests/ci/ci_deploy_dependency_file_list.txt @@ -0,0 +1,8 @@ +/home/m.json +/home/log_server.json +/var/lib/jenkins/workspace/restore.sh +/var/lib/jenkins/workspace/start_http.sh +/var/lib/jenkins/workspace/TDinternal +/var/lib/jenkins/workspace/remove_corefile.sh +/var/lib/jenkins/workspace/CI_disk_Monitor.py +/var/lib/jenkins/workspace/start_CI_Monitor.sh diff --git a/tests/ci/container_build_newmachine.sh b/tests/ci/container_build_newmachine.sh new file mode 100755 index 0000000000..3c87cf156f --- /dev/null +++ b/tests/ci/container_build_newmachine.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +function usage() { + echo "$0" + echo -e "\t -w work dir" + echo -e "\t -e enterprise edition" + echo -e "\t -t make thread count" + echo -e "\t -h help" +} + +ent=0 +while getopts "w:t:eh" opt; do + case $opt in + w) + WORKDIR=$OPTARG + ;; + e) + ent=1 + ;; + t) + THREAD_COUNT=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi +# if [ -z "$THREAD_COUNT" ]; then +# THREAD_COUNT=1 +# fi + +ulimit -c unlimited + +if [ $ent -eq 0 ]; then + REP_DIR=/home/TDengine + REP_REAL_PATH=$WORKDIR/TDengine + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDengine +else + REP_DIR=/home/TDinternal + REP_REAL_PATH=$WORKDIR/TDinternal + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal + +fi +date +docker run \ + -v $REP_MOUNT_PARAM \ + -v /root/.cargo/registry:/root/.cargo/registry \ + -v /root/.cargo/git:/root/.cargo/git \ + -v /root/go/pkg/mod:/root/go/pkg/mod \ + -v /root/.cache/go-build:/root/.cache/go-build \ + -v /root/.cos-local.1:/root/.cos-local.2 \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " + # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ + +if [[ -d ${WORKDIR}/debugNoSan ]] ;then + echo "delete ${WORKDIR}/debugNoSan" + rm -rf ${WORKDIR}/debugNoSan +fi +if [[ -d ${WORKDIR}/debugSan ]] ;then + echo "delete ${WORKDIR}/debugSan" + rm -rf ${WORKDIR}/debugSan +fi + +if [ "$(uname -m)" = "aarch64" ] ;then + CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Debug" +else + CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Release" +fi + + +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +date + +ret=$? +exit $ret + diff --git a/tests/ci/deploy_ci_machines.sh b/tests/ci/deploy_ci_machines.sh new file mode 100644 index 0000000000..e34ec1d6e5 --- /dev/null +++ b/tests/ci/deploy_ci_machines.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +function scp_file_from_host { + # check at least three parameters + if [ "$#" -lt 3 ]; then + echo "Usage: $0 host passwd source_filename [dest_filename]" + exit 1 + fi + + host=$1 + passwd=$2 + source_filename=$3 + # If the fourth parameter is not provided, use the third parameter as the default value + dest_filename=${4:-$3} + + # use sshpass and scp for secure file transfer + sshpass -p "$passwd" scp -o StrictHostKeyChecking=no -r "$host":"$source_filename" "$dest_filename" +} + + +# install docker and sshpass +curl -fsSL https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository "deb [arch=amd64] https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu $(lsb_release -cs) stable" +sudo apt-get update +sudo apt-get install -y docker-ce sshpass jq +sudo systemctl enable docker +sudo systemctl start docker + +# create a log directory +mkdir -p /var/lib/jenkins/workspace/log + +# Assuming you have a file called 'file_list.txt' with one filename per line +file_list="ci_deploy_dependency_file_list.txt" +monitorip="192.168.1.59" +passwd_all="abcdefg" + +# Read the file list and call scp_file_from_host for each file +while IFS= read -r source_filename; do + scp_file_from_host "$monitorip" "$passwd_all" "$source_filename" +done < "$file_list" + +# modify the configuration file +ip=$(ifconfig |grep inet|grep 192 |awk '{print $2}') +sed -i "s/${monitorip}/$ip/" /home/log_server.json +sed -i "s/${monitorip}/$ip/" /home/m.json + +#mkdir corefile dir and configure the system to automatically set corefile dir at startup +mkdir -p /home/coredump/ && echo "echo '/home/coredump/core_%e-%p' | sudo tee /proc/sys/kernel/core_pattern " >> /root/.bashrc + + +# get image from 0.212 +image_ip="192.168.0.212" +scp_file_from_host $image_ip $passwd_all "/home/tang/work/image/taos_image.tar " "/home/taos_image.tar" +docker load -i /home/taos_image.tar + +#start http server +nohup /var/lib/jenkins/workspace/log/start_http.sh & + +# start CI monitor and remove corefile in crontable +(crontab -l;echo "0 1 * * * /usr/bin/bash /var/lib/jenkins/workspace/remove_corefile.sh") | crontab +(crontab -l;echo "@reboot /usr/bin/bash /var/lib/jenkins/workspace/start_CI_Monitor.sh") | crontab + + +# generate cache dir +cd /var/lib/jenkins/workspace/TDinternal/community/tests/parallel_test || exit +time ./container_build_newmachine.sh -w /var/lib/jenkins/workspace -e + +# test if the CI machine compilation is successful +time ./container_build.sh -w /var/lib/jenkins/workspace -e \ No newline at end of file diff --git a/tests/ci/remove_corefile.sh b/tests/ci/remove_corefile.sh new file mode 100644 index 0000000000..7374101547 --- /dev/null +++ b/tests/ci/remove_corefile.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +date_str=$(date -d "4 day ago" +%Y%m%d) +if [ ! -z "$1" ]; then + date_str="$1" +fi +script_dir=$(dirname $0) +cd "${script_dir}"/log || exit +# date >>core.list +# find . -name "core.*" | grep "$date_str" >>core.list +# find . -name "core.*" | grep "$date_str" | xargs rm -rf +# find . -name "build_*" | grep "$date_str" | xargs rm -rf +for file in *; do + if [[ $file == *"$date_str"* ]]; then + rm -rf "$file" + fi +done diff --git a/tests/ci/restore.sh b/tests/ci/restore.sh new file mode 100644 index 0000000000..4c9daf73fe --- /dev/null +++ b/tests/ci/restore.sh @@ -0,0 +1,159 @@ +#!/bin/bash + set -x +JENKINS_LOG=jenkins.log +# pr_num=14228 +# n=1 +function usage() { + echo "$0" + echo -e "\t -p PR number" + echo -e "\t -n build number" + echo -e "\t -c container name" + echo -e "\t -h help" +} +while getopts "p:n:c:h" opt; do + case $opt in + p) + pr_num=$OPTARG + ;; + n) + n=$OPTARG + ;; + c) + container_name=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 1 + ;; + esac +done +if [ -z "$container_name" ]; then + echo "container name not specified" + usage + exit 1 +fi +if [ -z "$pr_num" ]; then + echo "PR number not specified" + usage + exit 1 +fi +if [ -z "$n" ]; then + echo "build number not specified" + usage + exit 1 +fi +pr_num=`echo "$pr_num"|sed "s/PR-//"` +container_count=`docker ps -a -f name=$container_name|wc -l` +if [ $container_count -gt 1 ]; then + docker ps -a -f name=$container_name + echo "container $container_name exists" + exit 1 +fi +cd $(dirname $0) +info=`grep -n "^[0-9]\{8\}-[0-9]\{6\}" jenkins.log | grep -A 1 "PR-${pr_num}:${n}:"` +# 22131:20220625-113105 NewTest/PR-14228:PR-14228:1:3.0 +# 22270:20220625-121154 NewTest/PR-14221:PR-14221:2:3.0 +ci_hosts="\ +192.168.0.212 \ +192.168.0.215 \ +192.168.0.217 \ +192.168.0.219 \ +" +if [ -z "$info" ]; then + echo "PR-${pr_num}:${n} not found" + for host in $ci_hosts; do + ssh root@$host "sh -c \"grep -n \\\"^[0-9]\\\\\{8\\\\\}-[0-9]\\\\\{6\\\\\}\\\" /var/lib/jenkins/workspace/jenkins.log | grep \\\"PR-${pr_num}:${n}:\\\"\"" + if [ $? -eq 0 ]; then + echo "this PR is possibly on host $host" + break + fi + done + exit 1 +fi +line_num=`echo "$info"|wc -l` +curr=`echo "$info"|head -n1` +if [ "$line_num" == "2" ]; then + next=`echo "$info"|tail -n1` +fi + +# check if it is TDinternal CI +internal=0 +commit_prefix=community +echo "$curr"|grep -q TDinternalCI +if [ $? -eq 0 ]; then + internal=1 + commit_prefix=tdinternal +fi + +curr_line=`echo "$curr"|cut -d: -f1` +next_line='$' +if [ ! -z "$next" ]; then + next_line=`echo "$next"|cut -d: -f1` + next_line=$(( next_line - 1 )) +fi +# echo "$curr_line, $next_line" + +details=`sed -n "${curr_line},${next_line}p" $JENKINS_LOG` +merge_line=`echo "$details"|grep -A 10 "$commit_prefix log merged: "|grep "Merge .* into"|head -n1` +if [ -z "$merge_line" ]; then + echo "merge commit not found" + exit 1 +fi +echo "$merge_line" +branch=`echo "$merge_line"|awk '{print $2}'` +commit_id=`echo "$merge_line"|awk '{print $4}'` +# echo "$details" +community_id=`echo "$details"|grep "community log: commit"|awk '{print $NF}'` +internal_id=`echo "$details"|grep "tdinternal log: commit"|awk '{print $NF}'` +python_connector_id=`echo "$details"|grep "python connector log: commit"|awk '{print $NF}'` +# change_branch=`echo "$details"|grep "CHANGE_BRANCH"|sed "s/CHANGE_BRANCH://"` + +# if [ -z "${branch}" ]; then +# branch="$change_branch" +# fi + +PWD=`pwd` +log_dir=`ls log|grep "PR-${pr_num}_${n}_"` +if [ -z "$log_dir" ]; then + echo "no log dir found" +else + mount_dir="-v ${PWD}/log/$log_dir:/home/log" + build_dir=`ls log/$log_dir | grep "build_"` + if [ ! -z "$build_dir" ]; then + mount_dir="$mount_dir -v ${PWD}/log/$log_dir/$build_dir:/home/TDinternal/debug/build" + fi +fi + +docker run -d --privileged -it --name $container_name \ + $mount_dir \ + taos_test:v1.0 bash + +if [ $internal -eq 0 ]; then + docker exec $container_name /home/setup.sh -c $commit_id -m $branch -n + echo "TDinternal checkout: $internal_id" + docker exec $container_name sh -c "cd /home/TDinternal; git checkout $internal_id" +else + docker exec $container_name /home/setup.sh -e -c $commit_id -m $branch -n + echo "community checkout: $community_id" + docker exec $container_name sh -c "cd /home/TDinternal/community; git checkout $community_id" +fi +echo +echo "* run the following command to enter the container:" +echo " docker exec -it $container_name bash" +if [ -z "$log_dir" ]; then + echo "* no log dir found" +else + echo "* log and coredump files are located in /home/log" +fi +if [ -z "$build_dir" ]; then + echo "* no build dir found" +else + echo "* build files are located in /home/TDinternal/debug/build" +fi +echo "* source files are located in /home/TDinternal" + diff --git a/tests/ci/start_http.sh b/tests/ci/start_http.sh new file mode 100644 index 0000000000..f73ffc85c8 --- /dev/null +++ b/tests/ci/start_http.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +script_dir=$(dirname $0) +cd $script_dir || exit +script_name=$(basename $0) +ps -ef|grep -v grep|grep -v $$|grep -q "$script_name" +if [ $? -eq 0 ]; then + exit 0 +fi +while [ 1 ]; do + ps -ef|grep python|grep -q 8081 + if [ $? -ne 0 ]; then + python3 -m http.server 8081 + fi + sleep 60 +done \ No newline at end of file diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 3d3069f05f..85e3d2ab73 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -61,14 +61,28 @@ docker run \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cos-local.1:/root/.cos-local.2 \ - -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ - -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ + -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ + -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ + -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ + -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ - -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ + -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ + -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ + -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ + -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ + -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ + -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ @@ -96,15 +110,29 @@ docker run \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cos-local.1:/root/.cos-local.2 \ - -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ + -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ + -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ + -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ + -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ - -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ + -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ + -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ + -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ + -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ + -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ + -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ + -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan