xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/TD-29367-8

This commit is contained in:
Hongze Cheng 2024-07-23 17:45:55 +08:00
parent 83746f52e2 0740eb88b6
commit 799d89a3be
30 changed files with 1029 additions and 882 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -44,11 +44,16 @@ static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter);
static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash);
static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSubActionEncode(pSub);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
int32_t mndInitSubscribe(SMnode *pMnode) {
@ -75,6 +80,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
}
static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) {
int32_t code = 0;
SMqSubscribeObj *pSub = tNewSubscribeObj(subKey);
if (pSub == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -86,7 +92,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
pSub->subType = pTopic->subType;
pSub->withMeta = pTopic->withMeta;
if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
if ((terrno = mndSchedInitSubEp(pMnode, pTopic, pSub)) < 0) {
tDeleteSubscribeObj(pSub);
taosMemoryFree(pSub);
return NULL;
@ -97,6 +103,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg,
SSubplan *pPlan) {
int32_t code = 0;
SMqRebVgReq req = {0};
req.oldConsumerId = pRebVg->oldConsumerId;
req.newConsumerId = pRebVg->newConsumerId;
@ -106,8 +113,8 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
pPlan->execNode.nodeId = pRebVg->pVgEp->vgId;
int32_t msgLen;
if (qSubPlanToString(pPlan, &req.qmsg, &msgLen) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
} else {
req.qmsg = taosStrdup("");
@ -122,7 +129,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret);
if (ret < 0) {
taosMemoryFree(req.qmsg);
return -1;
TAOS_RETURN(ret);
}
tlen += sizeof(SMsgHead);
@ -130,7 +137,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(req.qmsg);
return -1;
TAOS_RETURN(ret);
}
SMsgHead *pMsgHead = (SMsgHead *)buf;
@ -140,40 +147,41 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
SEncoder encoder = {0};
tEncoderInit(&encoder, POINTER_SHIFT(buf, sizeof(SMsgHead)), tlen);
if (tEncodeSMqRebVgReq(&encoder, &req) < 0) {
if ((code = tEncodeSMqRebVgReq(&encoder, &req)) < 0) {
taosMemoryFreeClear(buf);
tEncoderClear(&encoder);
taosMemoryFree(req.qmsg);
return -1;
TAOS_RETURN(code);
}
tEncoderClear(&encoder);
*pBuf = buf;
*pLen = tlen;
taosMemoryFree(req.qmsg);
return 0;
TAOS_RETURN(code);
}
static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub,
const SMqRebOutputVg *pRebVg, SSubplan *pPlan) {
int32_t code = 0;
if (pRebVg->oldConsumerId == pRebVg->newConsumerId) {
if (pRebVg->oldConsumerId == -1) return 0; // drop stream, no consumer, while split vnode,all consumerId is -1
terrno = TSDB_CODE_MND_INVALID_SUB_OPTION;
return -1;
code = TSDB_CODE_MND_INVALID_SUB_OPTION;
TAOS_RETURN(code);
}
void *buf;
int32_t tlen;
if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan) < 0) {
return -1;
if ((code = mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan)) < 0) {
TAOS_RETURN(code);
}
int32_t vgId = pRebVg->pVgEp->vgId;
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
if (pVgObj == NULL) {
taosMemoryFree(buf);
terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST;
return -1;
code = TSDB_CODE_MND_VGROUP_NOT_EXIST;
TAOS_RETURN(code);
}
STransAction action = {0};
@ -183,11 +191,11 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubsc
action.msgType = TDMT_VND_TMQ_SUBSCRIBE;
mndReleaseVgroup(pMnode, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(buf);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, bool fullName) {
@ -209,6 +217,7 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup,
}
static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
terrno = 0;
SMqRebInfo *pRebInfo = taosHashGet(pHash, key, strlen(key) + 1);
if (pRebInfo == NULL) {
pRebInfo = tNewSMqRebSubscribe(key);
@ -612,7 +621,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
if (strcmp(pOutput->pSub->qmsg, "") != 0) {
code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan);
if (code != 0) {
terrno = code;
goto END;
}
}
@ -623,7 +631,8 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
@ -665,7 +674,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
END:
nodesDestroyNode((SNode *)pPlan);
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static void freeRebalanceItem(void *param) {
@ -827,6 +836,7 @@ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) {
}
static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) {
int32_t code = 0;
const char *key = rebInput->pRebInfo->key;
SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key);
@ -838,8 +848,10 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
if (pTopic == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("[rebalance] mq rebalance %s ignored since topic %s doesn't exist", key, topic);
return -1;
TAOS_RETURN(code);
}
taosRLockLatch(&pTopic->lock);
@ -848,10 +860,12 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu
rebOutput->pSub = mndCreateSubscription(pMnode, pTopic, key);
if (rebOutput->pSub == NULL) {
mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, terrstr());
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, tstrerror(code));
taosRUnLockLatch(&pTopic->lock);
mndReleaseTopic(pMnode, pTopic);
return -1;
TAOS_RETURN(code);
}
memcpy(rebOutput->pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN);
@ -869,7 +883,7 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu
mInfo("[rebalance] sub topic:%s has %d consumers sub till now", key, rebInput->oldConsumerNum);
mndReleaseSubscribe(pMnode, pSub);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
@ -885,6 +899,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
if (rebSubHash == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
if (terrno != 0) code = terrno;
goto END;
}
taosHashSetFreeFp(rebSubHash, freeRebalanceItem);
@ -915,8 +930,8 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
mndDoRebalance(pMnode, &rebInput, &rebOutput);
if (mndPersistRebResult(pMnode, pMsg, &rebOutput) != 0) {
mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr())
if ((code = mndPersistRebResult(pMnode, pMsg, &rebOutput)) != 0) {
mError("mq rebalance persist output error, possibly vnode splitted or dropped,msg:%s", tstrerror(code))
}
clearRebOutput(&rebOutput);
@ -931,7 +946,7 @@ END:
taosHashCleanup(rebSubHash);
mndRebCntDec();
return code;
TAOS_RETURN(code);
}
static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) {
@ -950,8 +965,7 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran
}
SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
ret = -1;
ret = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
pReq->head.vgId = htonl(pVgObj->vgId);
@ -968,15 +982,14 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran
action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST;
sdbRelease(pMnode->pSdb, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
ret = -1;
if ((ret = mndTransAppendRedoAction(pTrans, &action)) != 0) {
goto END;
}
}
END:
sdbRelease(pMnode->pSdb, pVgObj);
sdbCancelFetch(pMnode->pSdb, pIter);
return ret;
TAOS_RETURN(ret);
}
static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) {
@ -1012,7 +1025,7 @@ static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgro
END:
sdbRelease(pMnode->pSdb, pConsumer);
sdbCancelFetch(pMnode->pSdb, pIter);
return ret;
TAOS_RETURN(ret);
}
static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
@ -1022,8 +1035,8 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
int32_t code = TSDB_CODE_ACTION_IN_PROGRESS;
if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
TAOS_RETURN(code);
}
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, dropReq.cgroup, dropReq.topic);
@ -1032,24 +1045,24 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
mInfo("cgroup:%s on topic:%s, not exist, ignore not exist is set", dropReq.cgroup, dropReq.topic);
return 0;
} else {
terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, terrstr());
return -1;
code = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, tstrerror(code));
TAOS_RETURN(code);
}
}
taosWLockLatch(&pSub->lock);
if (taosHashGetSize(pSub->consumerHash) != 0) {
terrno = TSDB_CODE_MND_CGROUP_USED;
mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr());
code = -1;
code = TSDB_CODE_MND_CGROUP_USED;
mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code));
goto end;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup");
if (pTrans == NULL) {
mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr());
code = -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code));
goto end;
}
@ -1088,9 +1101,9 @@ end:
if (code != 0) {
mError("cgroup %s on topic:%s, failed to drop", dropReq.cgroup, dropReq.topic);
return code;
TAOS_RETURN(code);
}
return TSDB_CODE_ACTION_IN_PROGRESS;
TAOS_RETURN(TSDB_CODE_ACTION_IN_PROGRESS);
}
void mndCleanupSubscribe(SMnode *pMnode) {}
@ -1340,7 +1353,7 @@ END:
sdbRelease(pSdb, pSub);
sdbCancelFetch(pSdb, pIter);
return code;
TAOS_RETURN(code);
}
static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic,

51
source/dnode/mnode/impl/src/mndSync.c
View File

@ -123,15 +123,16 @@ _OUT:
}
static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (pTrans->stage == TRN_STAGE_PREPARE) {
if (mndTransCheckConflict(pMnode, pTrans) < 0) {
if ((code = mndTransCheckConflict(pMnode, pTrans)) < 0) {
mError("trans:%d, failed to validate trans conflicts.", pTrans->id);
return -1;
TAOS_RETURN(code);
}
return mndTransValidatePrepareStage(pMnode, pTrans);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) {
@ -139,10 +140,18 @@ static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) {
int32_t code = -1;
SSdbRow *pRow = mndTransDecode(pRaw);
if (pRow == NULL) goto _OUT;
if (pRow == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OUT;
}
pTrans = sdbGetRowObj(pRow);
if (pTrans == NULL) goto _OUT;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OUT;
}
code = mndTransValidateImp(pMnode, pTrans);
@ -150,11 +159,10 @@ _OUT:
if (pTrans) mndTransDropData(pTrans);
if (pRow) taosMemoryFreeClear(pRow);
if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT);
return code;
TAOS_RETURN(code);
}
int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
terrno = TSDB_CODE_SUCCESS;
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SSdbRaw *pRaw = pMsg->pCont;
STrans *pTrans = NULL;
@ -163,7 +171,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (transId <= 0) {
mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq);
terrno = TSDB_CODE_INVALID_MSG;
code = TSDB_CODE_INVALID_MSG;
goto _OUT;
}
@ -176,7 +184,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (code != 0) {
mError("trans:%d, failed to validate requested trans since %s", transId, terrstr());
code = 0;
pMeta->code = terrno;
pMeta->code = code;
goto _OUT;
}
@ -184,13 +192,15 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (code != 0) {
mError("trans:%d, failed to write to sdb since %s", transId, terrstr());
code = 0;
pMeta->code = terrno;
pMeta->code = code;
goto _OUT;
}
pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans == NULL) {
mError("trans:%d, not found while execute in mnode since %s", transId, terrstr());
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("trans:%d, not found while execute in mnode since %s", transId, tstrerror(code));
goto _OUT;
}
@ -207,7 +217,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
_OUT:
if (pTrans) mndReleaseTrans(pMnode, pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) {
@ -258,7 +268,7 @@ _OUT:
mndPostMgmtCode(pMnode, code ? code : pMeta->code);
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
return code;
TAOS_RETURN(code);
}
SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) {
@ -466,16 +476,18 @@ int32_t mndInitSync(SMnode *pMnode) {
pNode->clusterId);
}
int32_t code = 0;
tsem_init(&pMgmt->syncSem, 0, 0);
pMgmt->sync = syncOpen(&syncInfo, true);
if (pMgmt->sync <= 0) {
mError("failed to open sync since %s", terrstr());
return -1;
if (terrno != 0) code = terrno;
mError("failed to open sync since %s", tstrerror(code));
TAOS_RETURN(code);
}
pMnode->pSdb->sync = pMgmt->sync;
mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
return 0;
TAOS_RETURN(code);
}
void mndCleanupSync(SMnode *pMnode) {
@ -518,10 +530,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)};
if (req.contLen <= 0) return -1;
if (req.contLen <= 0) return TSDB_CODE_OUT_OF_MEMORY;
req.pCont = rpcMallocCont(req.contLen);
if (req.pCont == NULL) return -1;
if (req.pCont == NULL) return TSDB_CODE_OUT_OF_MEMORY;
memcpy(req.pCont, pRaw, req.contLen);
taosThreadMutexLock(&pMgmt->lock);
@ -531,8 +543,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId);
taosThreadMutexUnlock(&pMgmt->lock);
rpcFreeCont(req.pCont);
terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED;
return terrno;
TAOS_RETURN(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED);
}
mInfo("trans:%d, will be proposed", transId);

1
source/dnode/vnode/CMakeLists.txt
View File

@ -22,7 +22,6 @@ set(
# meta
"src/meta/metaOpen.c"
"src/meta/metaIdx.c"
"src/meta/metaTable.c"
"src/meta/metaSma.c"
"src/meta/metaQuery.c"

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -318,7 +318,7 @@ void* tdUidStoreFree(STbUidStore* pStore);
// SMetaSnapReader ========================================
int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
int32_t metaSnapReaderClose(SMetaSnapReader** ppReader);
void metaSnapReaderClose(SMetaSnapReader** ppReader);
int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
// SMetaSnapWriter ========================================
int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter);

139
source/dnode/vnode/src/meta/metaCache.c
View File

@ -117,91 +117,81 @@ static void freeCacheEntryFp(void* param) {
}
int32_t metaCacheOpen(SMeta* pMeta) {
int32_t code = 0;
SMetaCache* pCache = NULL;
int32_t code = 0;
int32_t lino;
pCache = (SMetaCache*)taosMemoryMalloc(sizeof(SMetaCache));
if (pCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
pMeta->pCache = (SMetaCache*)taosMemoryCalloc(1, sizeof(SMetaCache));
if (pMeta->pCache == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
// open entry cache
pCache->sEntryCache.nEntry = 0;
pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET;
pCache->sEntryCache.aBucket =
(SMetaCacheEntry**)taosMemoryCalloc(pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*));
if (pCache->sEntryCache.aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
pMeta->pCache->sEntryCache.nEntry = 0;
pMeta->pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET;
pMeta->pCache->sEntryCache.aBucket =
(SMetaCacheEntry**)taosMemoryCalloc(pMeta->pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*));
if (pMeta->pCache->sEntryCache.aBucket == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
// open stats cache
pCache->sStbStatsCache.nEntry = 0;
pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET;
pCache->sStbStatsCache.aBucket =
(SMetaStbStatsEntry**)taosMemoryCalloc(pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*));
if (pCache->sStbStatsCache.aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
pMeta->pCache->sStbStatsCache.nEntry = 0;
pMeta->pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET;
pMeta->pCache->sStbStatsCache.aBucket =
(SMetaStbStatsEntry**)taosMemoryCalloc(pMeta->pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*));
if (pMeta->pCache->sStbStatsCache.aBucket == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pCache->sTagFilterResCache.pUidResCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
pMeta->pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pMeta->pCache->sTagFilterResCache.pUidResCache == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
pCache->sTagFilterResCache.accTimes = 0;
pCache->sTagFilterResCache.pTableEntry =
pMeta->pCache->sTagFilterResCache.accTimes = 0;
pMeta->pCache->sTagFilterResCache.pTableEntry =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->sTagFilterResCache.pTableEntry == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
if (pMeta->pCache->sTagFilterResCache.pTableEntry == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->sTagFilterResCache.lock, NULL);
taosHashSetFreeFp(pMeta->pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pMeta->pCache->sTagFilterResCache.lock, NULL);
pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pCache->STbGroupResCache.pResCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
pMeta->pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pMeta->pCache->STbGroupResCache.pResCache == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
pCache->STbGroupResCache.accTimes = 0;
pCache->STbGroupResCache.pTableEntry =
pMeta->pCache->STbGroupResCache.accTimes = 0;
pMeta->pCache->STbGroupResCache.pTableEntry =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbGroupResCache.pTableEntry == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
if (pMeta->pCache->STbGroupResCache.pTableEntry == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL);
taosHashSetFreeFp(pMeta->pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pMeta->pCache->STbGroupResCache.lock, NULL);
pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pStb == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
pMeta->pCache->STbFilterCache.pStb =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pMeta->pCache->STbFilterCache.pStb == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
pCache->STbFilterCache.pStbName =
pMeta->pCache->STbFilterCache.pStbName =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pStbName == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
if (pMeta->pCache->STbFilterCache.pStbName == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
pMeta->pCache = pCache;
return code;
_err2:
entryCacheClose(pMeta);
_err:
taosMemoryFree(pCache);
metaError("vgId:%d, meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
_exit:
if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
metaCacheClose(pMeta);
} else {
metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
}
return code;
}
@ -289,8 +279,7 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) {
}
} else { // insert
if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) {
code = metaRehashCache(pCache, 1);
if (code) goto _exit;
TAOS_UNUSED(metaRehashCache(pCache, 1));
iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket;
}
@ -328,8 +317,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) {
pCache->sEntryCache.nEntry--;
if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 &&
pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) {
code = metaRehashCache(pCache, 0);
if (code) goto _exit;
TAOS_UNUSED(metaRehashCache(pCache, 0));
}
} else {
code = TSDB_CODE_NOT_FOUND;
@ -351,7 +339,9 @@ int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo) {
}
if (pEntry) {
*pInfo = pEntry->info;
if (pInfo) {
*pInfo = pEntry->info;
}
} else {
code = TSDB_CODE_NOT_FOUND;
}
@ -415,9 +405,7 @@ int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) {
(*ppEntry)->info.ctbNum = pInfo->ctbNum;
} else { // insert
if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) {
code = metaRehashStatsCache(pCache, 1);
if (code) goto _exit;
TAOS_UNUSED(metaRehashStatsCache(pCache, 1));
iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket;
}
@ -454,8 +442,7 @@ int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid) {
pCache->sStbStatsCache.nEntry--;
if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 &&
pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) {
code = metaRehashStatsCache(pCache, 0);
if (code) goto _exit;
TAOS_UNUSED(metaRehashStatsCache(pCache, 0));
}
} else {
code = TSDB_CODE_NOT_FOUND;
@ -477,7 +464,9 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) {
}
if (pEntry) {
*pInfo = pEntry->info;
if (pInfo) {
*pInfo = pEntry->info;
}
} else {
code = TSDB_CODE_NOT_FOUND;
}
@ -502,7 +491,9 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
// check whether it is existed in LRU cache, and remove it from linked list if not.
LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len);
if (pRes == NULL) { // remove the item in the linked list
taosArrayPush(pInvalidRes, &pNode);
if (taosArrayPush(pInvalidRes, &pNode) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
taosLRUCacheRelease(pCache, pRes, false);
}
@ -626,7 +617,7 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL
p->hitTimes = 0;
tdListInit(&p->list, keyLen);
taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES);
TAOS_CHECK_RETURN(taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES));
tdListAppend(&p->list, pKey);
return 0;
}
@ -956,9 +947,7 @@ int32_t metaInitTbFilterCache(SMeta* pMeta) {
}
if (tbNum && pTbArr) {
for (int32_t i = 0; i < tbNum; ++i) {
if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) {
return terrno ? terrno : -1;
}
TAOS_CHECK_RETURN(metaPutTbToFilterCache(pMeta, pTbArr[i], 1));
}
}
#else

48
source/dnode/vnode/src/meta/metaCommit.c
View File

@ -21,7 +21,10 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) {
static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }
// begin a meta txn
int metaBegin(SMeta *pMeta, int8_t heap) {
int32_t metaBegin(SMeta *pMeta, int8_t heap) {
int32_t code = 0;
int32_t lino;
void *(*xMalloc)(void *, size_t) = NULL;
void (*xFree)(void *, void *) = NULL;
void *xArg = NULL;
@ -36,12 +39,19 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
xArg = pMeta->pVnode->inUse;
}
if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
return -1;
code = tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
TSDB_CHECK_CODE(code, lino, _exit);
code = tdbCommit(pMeta->pEnv, pMeta->txn);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__,
tstrerror(terrno));
} else {
metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__);
}
tdbCommit(pMeta->pEnv, pMeta->txn);
return 0;
}
@ -49,20 +59,36 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; }
int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); }
int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); }
int metaPrepareAsyncCommit(SMeta *pMeta) {
// return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
int code = 0;
int metaPrepareAsyncCommit(SMeta *pMeta) {
// return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
int code = 0;
int32_t lino;
metaWLock(pMeta);
code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
TAOS_UNUSED(ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn));
metaULock(pMeta);
code = tdbCommit(pMeta->pEnv, pMeta->txn);
TSDB_CHECK_CODE(code, lino, _exit);
pMeta->changed = false;
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__,
tstrerror(terrno));
} else {
metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__);
}
return code;
}
// abort the meta txn
int metaAbort(SMeta *pMeta) {
if (!pMeta->txn) return 0;
if (!pMeta->txn) {
return 0;
}
int code = tdbAbort(pMeta->pEnv, pMeta->txn);
if (code) {
metaError("vgId:%d, failed to abort meta since %s", TD_VID(pMeta->pVnode), tstrerror(terrno));

145
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -17,159 +17,166 @@
int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) {
const SColCmprWrapper *pw = &pME->colCmpr;
if (tEncodeI32v(pCoder, pw->nCols) < 0) return -1;
if (tEncodeI32v(pCoder, pw->version) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols));
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->version));
uDebug("encode cols:%d", pw->nCols);
for (int32_t i = 0; i < pw->nCols; i++) {
SColCmpr *p = &pw->pColCmpr[i];
if (tEncodeI16v(pCoder, p->id) < 0) return -1;
if (tEncodeU32(pCoder, p->alg) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI16v(pCoder, p->id));
TAOS_CHECK_RETURN(tEncodeU32(pCoder, p->alg));
}
return 0;
}
int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) {
SColCmprWrapper *pWrapper = &pME->colCmpr;
if (tDecodeI32v(pDecoder, &pWrapper->nCols) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->nCols));
if (pWrapper->nCols == 0) {
return 0;
}
if (tDecodeI32v(pDecoder, &pWrapper->version) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->version));
uDebug("dencode cols:%d", pWrapper->nCols);
pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr));
if (pWrapper->pColCmpr == NULL) return -1;
if (pWrapper->pColCmpr == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 0; i < pWrapper->nCols; i++) {
SColCmpr *p = &pWrapper->pColCmpr[i];
if (tDecodeI16v(pDecoder, &p->id) < 0) return -1;
if (tDecodeU32(pDecoder, &p->alg) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &p->id));
TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &p->alg));
}
return 0;
}
static FORCE_INLINE void metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr,
SSchemaWrapper *pSchema) {
static FORCE_INLINE int32_t metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr,
SSchemaWrapper *pSchema) {
pCmpr->nCols = pSchema->nCols;
pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr));
if ((pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr))) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pCmpr->nCols; i++) {
SColCmpr *pColCmpr = &pCmpr->pColCmpr[i];
SSchema *pColSchema = &pSchema->pSchema[i];
pColCmpr->id = pColSchema->colId;
pColCmpr->alg = createDefaultColCmprByType(pColSchema->type);
}
return 0;
}
int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tStartEncode(pCoder) < 0) return -1;
TAOS_CHECK_RETURN(tStartEncode(pCoder));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->version));
TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->type));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->uid));
if (tEncodeI64(pCoder, pME->version) < 0) return -1;
if (tEncodeI8(pCoder, pME->type) < 0) return -1;
if (tEncodeI64(pCoder, pME->uid) < 0) return -1;
if (pME->name == NULL || tEncodeCStr(pCoder, pME->name) < 0) return -1;
if (pME->name == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->name));
if (pME->type == TSDB_SUPER_TABLE) {
if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor?
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->flags));
TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow));
TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag));
if (TABLE_IS_ROLLUP(pME->flags)) {
if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam));
}
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.btime));
TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ctbEntry.ttlDays));
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ctbEntry.commentLen));
if (pME->ctbEntry.commentLen > 0) {
if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ctbEntry.comment));
}
if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.suid));
TAOS_CHECK_RETURN(tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags));
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ntbEntry.btime));
TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ntbEntry.ttlDays));
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.commentLen));
if (pME->ntbEntry.commentLen > 0) {
if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ntbEntry.comment));
}
if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.ncid));
TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow));
} else if (pME->type == TSDB_TSMA_TABLE) {
if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeTSma(pCoder, pME->smaEntry.tsma));
} else {
metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type);
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (meteEncodeColCmprEntry(pCoder, pME) < 0) return -1;
TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME));
tEndEncode(pCoder);
return 0;
}
int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI64(pCoder, &pME->version) < 0) return -1;
if (tDecodeI8(pCoder, &pME->type) < 0) return -1;
if (tDecodeI64(pCoder, &pME->uid) < 0) return -1;
if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
TAOS_CHECK_RETURN(tStartDecode(pCoder));
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->version));
TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->type));
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->uid));
TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name));
if (pME->type == TSDB_SUPER_TABLE) {
if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor?
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->flags));
TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow));
TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag));
if (TABLE_IS_ROLLUP(pME->flags)) {
if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam));
}
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.btime));
TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ctbEntry.ttlDays));
TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ctbEntry.commentLen));
if (pME->ctbEntry.commentLen > 0) {
if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ctbEntry.comment));
}
if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO)
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.suid));
TAOS_CHECK_RETURN(tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags));
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ntbEntry.btime));
TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ntbEntry.ttlDays));
TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.commentLen));
if (pME->ntbEntry.commentLen > 0) {
if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ntbEntry.comment));
}
if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1;
if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.ncid));
TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow));
} else if (pME->type == TSDB_TSMA_TABLE) {
pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma));
if (!pME->smaEntry.tsma) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeTSma(pCoder, pME->smaEntry.tsma, true));
} else {
metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type);
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (pME->type == TSDB_SUPER_TABLE) {
if (TABLE_IS_COL_COMPRESSED(pME->flags)) {
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1;
TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME));
if (pME->colCmpr.nCols == 0) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow);
TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow));
}
} else {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow);
TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow));
TABLE_SET_COL_COMPRESSED(pME->flags);
}
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (!tDecodeIsEnd(pCoder)) {
uDebug("set type: %d, tableName:%s", pME->type, pME->name);
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1;
TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME));
if (pME->colCmpr.nCols == 0) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow);
TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow));
}
} else {
uDebug("set default type: %d, tableName:%s", pME->type, pME->name);
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow);
TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow));
}
TABLE_SET_COL_COMPRESSED(pME->flags);
}

118
source/dnode/vnode/src/meta/metaIdx.c
View File

@ -1,118 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef USE_INVERTED_INDEX
#include "index.h"
#endif
#include "meta.h"
struct SMetaIdx {
#ifdef USE_INVERTED_INDEX
SIndex *pIdx;
#endif
/* data */
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
};
int metaOpenIdx(SMeta *pMeta) {
#if 0
char idxDir[128]; // TODO
char * err = NULL;
rocksdb_options_t *options = rocksdb_options_create();
// TODO
sprintf(idxDir, "%s/index", pMeta->path);
if (pMeta->pCache) {
rocksdb_options_set_row_cache(options, pMeta->pCache);
}
rocksdb_options_set_create_if_missing(options, 1);
pMeta->pIdx = rocksdb_open(options, idxDir, &err);
if (pMeta->pIdx == NULL) {
// TODO: handle error
rocksdb_options_destroy(options);
return -1;
}
rocksdb_options_destroy(options);
#endif
#ifdef USE_INVERTED_INDEX
// SIndexOpts opts;
// if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) {
// return -1;
//}
#endif
return 0;
}
#ifdef BUILD_NO_CALL
void metaCloseIdx(SMeta *pMeta) { /* TODO */
#if 0
if (pMeta->pIdx) {
rocksdb_close(pMeta->pIdx);
pMeta->pIdx = NULL;
}
#endif
#ifdef USE_INVERTED_INDEX
// SIndexOpts opts;
// if (indexClose(pMeta->pIdx->pIdx) != 0) {
// return -1;
//}
// return 0;
#endif
}
int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) {
#ifdef USE_INVERTED_INDEX
// if (pTbCfgs->type == META_CHILD_TABLE) {
// char buf[8] = {0};
// int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId;
// sprintf(buf, "%d", colId); // colname
// char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId);
// tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id
// tb_uid_t tuid = 0; // child table uid
// SIndexMultiTerm *terms = indexMultiTermCreate();
// SIndexTerm *term =
// indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid);
// indexMultiTermAdd(terms, term);
// int ret = indexPut(pMeta->pIdx->pIdx, terms);
// indexMultiTermDestroy(terms);
// return ret;
//} else {
// return DB_DONOTINDEX;
//}
#endif
// TODO
return 0;
}
int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
#ifdef USE_INVERTED_INDEX
#endif
// TODO
return 0;
}
#endif

221
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -39,13 +39,13 @@ static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&p
static void metaCleanup(SMeta **ppMeta);
int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
SMeta *pMeta = NULL;
int ret;
int offset;
char path[TSDB_FILENAME_LEN] = {0};
*ppMeta = NULL;
int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
SMeta *pMeta = NULL;
int32_t code = 0;
int32_t lino;
int32_t offset;
char path[TSDB_FILENAME_LEN] = {0};
char indexFullPath[128] = {0};
// create handle
vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN);
@ -53,8 +53,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR);
if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
metaInitLock(pMeta);
@ -69,163 +68,104 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
taosMkDir(pMeta->path);
// open env
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey);
if (ret < 0) {
metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// open pTbDb
ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pSkmDb
ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pUidIdx
ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pNameIdx
ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pCtbIdx
ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pSuidIdx
ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
char indexFullPath[128] = {0};
sprintf(indexFullPath, "%s/%s", pMeta->path, "invert");
taosMkDir(indexFullPath);
TAOS_UNUSED(taosMkDir(indexFullPath));
SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024};
ret = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx);
if (ret < 0) {
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx);
TSDB_CHECK_CODE(code, lino, _exit);
ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pTtlMgr ("ttlv1.idx")
char logPrefix[128] = {0};
sprintf(logPrefix, "vgId:%d", TD_VID(pVnode));
ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold);
if (ret < 0) {
metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold);
TSDB_CHECK_CODE(code, lino, _exit);
// open pSmaIdx
ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// idx table create time
ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// idx num of col, normal table only
ret = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta ncol index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open index
if (metaOpenIdx(pMeta) < 0) {
metaError("vgId:%d, failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = metaCacheOpen(pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t code = metaCacheOpen(pMeta);
code = metaInitTbFilterCache(pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
terrno = code;
metaError("vgId:%d, failed to open meta cache since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
metaCleanup(&pMeta);
*ppMeta = NULL;
} else {
metaDebug("vgId:%d %s success", TD_VID(pVnode), __func__);
*ppMeta = pMeta;
}
if (metaInitTbFilterCache(pMeta) != 0) {
goto _err;
}
metaDebug("vgId:%d, meta is opened", TD_VID(pVnode));
*ppMeta = pMeta;
return 0;
_err:
metaCleanup(&pMeta);
return -1;
return code;
}
int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) {
int code = TSDB_CODE_SUCCESS;
SMeta *pMeta = *ppMeta;
int32_t metaUpgrade(SVnode *pVnode, SMeta **ppMeta) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino;
SMeta *pMeta = *ppMeta;
if (ttlMgrNeedUpgrade(pMeta->pEnv)) {
code = metaBegin(pMeta, META_BEGIN_HEAP_OS);
if (code < 0) {
metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta);
if (code < 0) {
metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
code = metaCommit(pMeta, pMeta->txn);
if (code < 0) {
metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
return TSDB_CODE_SUCCESS;
_err:
metaCleanup(ppMeta);
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
metaCleanup(ppMeta);
}
return code;
}
@ -235,33 +175,42 @@ int metaClose(SMeta **ppMeta) {
}
int metaAlterCache(SMeta *pMeta, int32_t nPage) {
int32_t code = 0;
metaWLock(pMeta);
if (tdbAlter(pMeta->pEnv, nPage) < 0) {
metaULock(pMeta);
return -1;
}
code = tdbAlter(pMeta->pEnv, nPage);
metaULock(pMeta);
return 0;
if (code) {
metaError("vgId:%d %s failed since %s", TD_VID(pMeta->pVnode), __func__, tstrerror(code));
}
return code;
}
int32_t metaRLock(SMeta *pMeta) {
metaTrace("meta rlock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockRdlock(&pMeta->lock);
return ret;
int32_t code = taosThreadRwlockRdlock(&pMeta->lock);
if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
}
int32_t metaWLock(SMeta *pMeta) {
metaTrace("meta wlock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockWrlock(&pMeta->lock);
return ret;
int32_t code = taosThreadRwlockWrlock(&pMeta->lock);
if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
}
int32_t metaULock(SMeta *pMeta) {
metaTrace("meta ulock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockUnlock(&pMeta->lock);
return ret;
int32_t code = taosThreadRwlockUnlock(&pMeta->lock);
if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
}
static void metaCleanup(SMeta **ppMeta) {

102
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -25,14 +25,14 @@ struct SMetaSnapReader {
int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
int32_t code = 0;
int32_t lino;
int32_t c = 0;
SMetaSnapReader* pReader = NULL;
// alloc
pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pReader->pMeta = pMeta;
pReader->sver = sver;
@ -40,36 +40,29 @@ int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapRe
// impl
code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL);
if (code) {
taosMemoryFree(pReader);
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
taosMemoryFree(pReader);
goto _err;
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
metaSnapReaderClose(&pReader);
*ppReader = NULL;
} else {
metaInfo("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
*ppReader = pReader;
}
metaInfo("vgId:%d, vnode snapshot meta reader opened", TD_VID(pMeta->pVnode));
*ppReader = pReader;
return code;
_err:
metaError("vgId:%d, vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
*ppReader = NULL;
return code;
}
int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) {
int32_t code = 0;
tdbTbcClose((*ppReader)->pTbc);
taosMemoryFree(*ppReader);
*ppReader = NULL;
return code;
void metaSnapReaderClose(SMetaSnapReader** ppReader) {
if (ppReader && *ppReader) {
tdbTbcClose((*ppReader)->pTbc);
taosMemoryFree(*ppReader);
*ppReader = NULL;
}
}
int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
@ -106,7 +99,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
goto _exit;
}
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
@ -122,10 +115,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
}
_exit:
return code;
_err:
metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code));
if (code) {
metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode),
tstrerror(code));
}
return code;
}
@ -138,26 +131,30 @@ struct SMetaSnapWriter {
int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
int32_t code = 0;
int32_t lino;
SMetaSnapWriter* pWriter;
// alloc
pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pWriter->pMeta = pMeta;
pWriter->sver = sver;
pWriter->ever = ever;
metaBegin(pMeta, META_BEGIN_HEAP_NIL);
code = metaBegin(pMeta, META_BEGIN_HEAP_NIL);
TSDB_CHECK_CODE(code, lino, _exit);
*ppWriter = pWriter;
return code;
_err:
metaError("vgId:%d, meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
*ppWriter = NULL;
_exit:
if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
taosMemoryFree(pWriter);
*ppWriter = NULL;
} else {
metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
*ppWriter = pWriter;
}
return code;
}
@ -189,25 +186,24 @@ _err:
int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
int32_t line = 0;
int32_t lino = 0;
SMeta* pMeta = pWriter->pMeta;
SMetaEntry metaEntry = {0};
SDecoder* pDecoder = &(SDecoder){0};
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
code = metaDecodeEntry(pDecoder, &metaEntry);
VND_CHECK_CODE(code, line, _err);
TSDB_CHECK_CODE(code, lino, _exit);
code = metaHandleEntry(pMeta, &metaEntry);
VND_CHECK_CODE(code, line, _err);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
}
tDecoderClear(pDecoder);
return code;
_err:
tDecoderClear(pDecoder);
metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line);
return code;
}
typedef struct STableInfoForChildTable {
@ -468,19 +464,17 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
return c;
}
void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid){
bool ret = false;
SSchemaWrapper *schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1);
if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){
void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) {
bool ret = false;
SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1);
if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true;
}
tDeleteSchemaWrapper(schema);
ctx->hasPrimaryKey = ret;
}
bool taosXGetTablePrimaryKey(SSnapContext* ctx){
return ctx->hasPrimaryKey;
}
bool taosXGetTablePrimaryKey(SSnapContext* ctx) { return ctx->hasPrimaryKey; }
int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) {
int32_t ret = 0;

3
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -312,8 +312,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
goto _exit;
} else {
pReader->metaDone = 1;
code = metaSnapReaderClose(&pReader->pMetaReader);
TSDB_CHECK_CODE(code, lino, _exit);
metaSnapReaderClose(&pReader->pMetaReader);
}
}

1
source/libs/tdb/CMakeLists.txt
View File

@ -10,7 +10,6 @@ target_sources(tdb
"src/db/tdbTable.c"
"src/db/tdbTxn.c"
"src/db/tdbPage.c"
"src/db/tdbOs.c"
)
target_include_directories(

4
source/libs/tdb/src/db/tdbBtree.c
View File

@ -76,14 +76,14 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
if (keyLen == 0) {
tdbError("tdb/btree-open: key len cannot be zero.");
return -1;
return TSDB_CODE_INVALID_PARA;
}
*ppBt = NULL;
pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt));
if (pBt == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
// pBt->keyLen

37
source/libs/tdb/src/db/tdbDb.c
View File

@ -15,8 +15,8 @@
#include "tdbInt.h"
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm,
char *encryptKey) {
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback,
int32_t encryptAlgorithm, char *encryptKey) {
TDB *pDb;
int dsize;
int zsize;
@ -31,7 +31,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
pDb = (TDB *)pPtr;
@ -51,38 +51,38 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pDb->jfd = -1;
pDb->encryptAlgorithm = encryptAlgorithm;
if(encryptKey != NULL){
strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN);
if (encryptKey != NULL) {
strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN);
}
ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
if (ret < 0) {
return -1;
return ret;
}
pDb->nPgrHash = 8;
tsize = sizeof(SPager *) * pDb->nPgrHash;
pDb->pgrHash = tdbOsMalloc(tsize);
if (pDb->pgrHash == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
memset(pDb->pgrHash, 0, tsize);
ret = taosMulModeMkDir(dbname, 0755, false);
if (ret < 0) {
return -1;
return TAOS_SYSTEM_ERROR(ret);
}
#ifdef USE_MAINDB
// open main db
ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback);
if (ret < 0) {
return -1;
return ret;
}
ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback);
if (ret < 0) {
return -1;
return ret;
}
#endif
@ -125,12 +125,13 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (
TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn));
if (!pTxn) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags) < 0) {
ret = tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags);
if (ret < 0) {
tdbOsFree(pTxn);
return -1;
return ret;
}
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
@ -139,7 +140,7 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (
tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId);
tdbTxnClose(pTxn);
return -1;
return ret;
}
}
@ -157,7 +158,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId);
return -1;
return ret;
}
}
@ -173,7 +174,7 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId);
return -1;
return ret;
}
}
@ -191,7 +192,7 @@ int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId);
return -1;
return ret;
}
}
@ -207,7 +208,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) {
if (ret < 0) {
tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId);
return -1;
return ret;
}
}

98
source/libs/tdb/src/db/tdbOs.c
View File

@ -1,98 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbOs.h"
#ifndef TDB_FOR_TDENGINE
// tdbOsRead
i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes) {
i64 nRead = 0;
i64 iRead = 0;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iRead = read(fd, pBuf, nBytes);
if (iRead < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (iRead == 0) {
break;
}
nRead += iRead;
pBuf += iRead;
nBytes -= iRead;
}
return nRead;
}
// tdbOsPRead
i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset) {
i64 nRead = 0;
i64 iRead = 0;
i64 iOffset = offset;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iRead = pread(fd, pBuf, nBytes, iOffset);
if (iRead < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (iRead == 0) {
break;
}
nRead += iRead;
pBuf += iRead;
iOffset += iRead;
nBytes -= iRead;
}
return nRead;
}
// tdbOsWrite
i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes) {
i64 nWrite = 0;
i64 iWrite = 0;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iWrite = write(fd, pBuf, nBytes);
if (iWrite < 0) {
if (errno == EINTR) {
continue;
}
return -1;
}
nWrite += iWrite;
pBuf += iWrite;
nBytes -= iWrite;
}
return nWrite;
}
#endif

57
source/libs/tdb/src/db/tdbPCache.c
View File

@ -50,30 +50,36 @@ static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); }
static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); }
int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
int32_t code = 0;
int32_t lino;
SPCache *pCache;
void *pPtr;
SPage *pPgHdr;
pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize);
if (pCache == NULL) {
return -1;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pCache->szPage = pageSize;
pCache->nPages = cacheSize;
pCache->aPage = (SPage **)tdbOsCalloc(cacheSize, sizeof(SPage *));
if (pCache->aPage == NULL) {
tdbOsFree(pCache);
return -1;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
if (tdbPCacheOpenImpl(pCache) < 0) {
tdbOsFree(pCache);
return -1;
}
code = tdbPCacheOpenImpl(pCache);
TSDB_CHECK_CODE(code, lino, _exit);
*ppCache = pCache;
return 0;
_exit:
if (code) {
tdbError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(code));
tdbPCacheClose(pCache);
*ppCache = NULL;
} else {
*ppCache = pCache;
}
return code;
}
int tdbPCacheClose(SPCache *pCache) {
@ -99,14 +105,14 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) {
} else if (pCache->nPages < nPage) {
SPage **aPage = tdbOsCalloc(nPage, sizeof(SPage *));
if (aPage == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) {
if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) {
// TODO: handle error
int32_t code = tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL);
if (code) {
tdbOsFree(aPage);
return -1;
return code;
}
// pPage->pgid = 0;
@ -156,15 +162,11 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) {
}
int tdbPCacheAlter(SPCache *pCache, int32_t nPage) {
int ret = 0;
int code;
tdbPCacheLock(pCache);
ret = tdbPCacheAlterImpl(pCache, nPage);
code = tdbPCacheAlterImpl(pCache, nPage);
tdbPCacheUnlock(pCache);
return ret;
return code;
}
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
@ -180,9 +182,6 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
tdbPCacheUnlock(pCache);
// printf("thread %" PRId64 " fetch page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id,
// TDB_PAGE_PGNO(pPage), pPage, nRef);
if (pPage) {
tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef);
} else {
@ -285,6 +284,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
if (!pTxn) {
tdbError("tdb/pcache: null ptr pTxn, fetch impl failed.");
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
@ -327,7 +327,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg);
if (ret < 0 || pPage == NULL) {
tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage);
// TODO: recycle other backup pages
terrno = ret;
return NULL;
}
@ -475,10 +475,8 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
pCache->nFree = 0;
pCache->pFree = NULL;
for (int i = 0; i < pCache->nPages; i++) {
if (tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL) < 0) {
// TODO: handle error
return -1;
}
ret = tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL);
if (ret) return ret;
// pPage->pgid = 0;
pPage->isAnchor = 0;
@ -504,8 +502,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages;
pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *));
if (pCache->pgHash == NULL) {
// TODO
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
// Open LRU list

60
source/libs/tdb/src/db/tdbPage.c
View File

@ -45,12 +45,12 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
if (!xMalloc) {
tdbError("tdb/page-create: null xMalloc.");
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (!TDB_IS_PGSIZE_VLD(pageSize)) {
tdbError("tdb/page-create: invalid pageSize: %d.", pageSize);
return -1;
return TSDB_CODE_INVALID_PARA;
}
*ppPage = NULL;
@ -58,7 +58,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
ptr = (u8 *)(xMalloc(arg, size));
if (ptr == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
memset(ptr, 0, size);
@ -86,12 +86,12 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg)
if (pPage->isDirty) {
tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty);
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (!xFree) {
tdbError("tdb/page-destroy: null xFree.");
return -1;
return TSDB_CODE_INVALID_PARA;
}
for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) {
@ -129,7 +129,8 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize);
pPage->pPageHdr = pPage->pData + szAmHdr;
if (TDB_PAGE_NCELLS(pPage) == 0) {
return tdbPageZero(pPage, szAmHdr, xCellSize);
tdbPageZero(pPage, szAmHdr, xCellSize);
return;
}
pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage);
pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage);
@ -159,7 +160,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) {
tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell,
TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData));
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
nFree = TDB_PAGE_NFREE(pPage);
@ -207,7 +208,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) {
tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart,
pPage->pCellIdx, nCells);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
}
@ -234,7 +235,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
if (idx < 0 || idx >= nCells + pPage->nOverflow) {
tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow);
return -1;
return TSDB_CODE_INVALID_PARA;
}
iOvfl = 0;
@ -265,14 +266,14 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
pPage->aiOvfl[iOvfl]--;
if (pPage->aiOvfl[iOvfl] <= 0) {
tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
}
return 0;
}
void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
int delta, nFree;
pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr);
@ -280,7 +281,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (pToPage->pFreeEnd < pToPage->pFreeStart) {
tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd);
return;
return TSDB_CODE_INVALID_DATA_FMT;
}
memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr);
@ -289,7 +290,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) {
tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage),
pToPage->pFreeEnd - pToPage->pData);
return;
return TSDB_CODE_INVALID_DATA_FMT;
}
delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData);
@ -304,6 +305,10 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (deepCopyOvfl) {
int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL);
pNewCell = (SCell *)tdbOsMalloc(szCell);
if (pNewCell == NULL) {
tdbError("tdb/page-copy: out of memory, size: %d", szCell);
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell);
tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage);
}
@ -312,6 +317,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl];
}
pToPage->nOverflow = pFromPage->nOverflow;
return 0;
}
int tdbPageCapacity(int pageSize, int amHdrSize) {
@ -343,12 +349,12 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) {
tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell,
TDB_PAGE_OFFSET_SIZE(pPage));
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) {
tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage),
pPage->pFreeEnd - pPage->pData);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
// 1. Try to allocate from the free space block area
@ -363,7 +369,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
cellFree = TDB_PAGE_FCELL(pPage);
if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) {
tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) {
SCell *pPrevFreeCell = NULL;
@ -408,19 +414,19 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
}
// 3. Try to dfragment and allocate again
tdbPageDefragment(pPage);
TAOS_CHECK_RETURN(tdbPageDefragment(pPage));
if (pPage->pFreeEnd - pPage->pFreeStart != nFree) {
tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
if (TDB_PAGE_NFREE(pPage) != nFree) {
tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage));
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) {
tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart,
pPage->pData);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
pPage->pFreeEnd -= szCell;
@ -430,7 +436,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
_alloc_finish:
if (NULL == pCell) {
tdbError("tdb/page-allocate: null ptr pCell.");
return -1;
return TSDB_CODE_OUT_OF_BUFFER;
}
pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage);
@ -447,15 +453,15 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) {
if (pCell < pPage->pFreeEnd) {
tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd);
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (pCell + szCell > (u8 *)(pPage->pPageFtr)) {
tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd);
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) {
tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx));
return -1;
return TSDB_CODE_INVALID_PARA;
}
nFree = TDB_PAGE_NFREE(pPage);
@ -470,7 +476,7 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) {
TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData);
} else {
tdbError("tdb/page-free: invalid cell size: %d", szCell);
return -1;
return TSDB_CODE_INVALID_PARA;
}
}
@ -502,7 +508,9 @@ static int tdbPageDefragment(SPage *pPage) {
int32_t nCell = TDB_PAGE_NCELLS(pPage);
SCellIdx *aCellIdx = (SCellIdx *)tdbOsMalloc(sizeof(SCellIdx) * nCell);
if (aCellIdx == NULL) return -1;
if (aCellIdx == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t iCell = 0; iCell < nCell; iCell++) {
aCellIdx[iCell].iCell = iCell;
aCellIdx[iCell].offset = TDB_PAGE_CELL_OFFSET_AT(pPage, iCell);

244
source/libs/tdb/src/db/tdbPager.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
#include "crypt.h"
#include "tdbInt.h"
#include "tglobal.h"
/*
#pragma pack(push, 1)
@ -41,9 +41,10 @@ struct hashset_st {
static const unsigned int prime = 39;
static const unsigned int prime2 = 5009;
hashset_t hashset_create(void) {
static hashset_t hashset_create(void) {
hashset_t set = tdbOsCalloc(1, sizeof(struct hashset_st));
if (!set) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -52,6 +53,7 @@ hashset_t hashset_create(void) {
set->items = tdbOsCalloc(set->capacity, sizeof(size_t));
if (!set->items) {
tdbOsFree(set);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
set->mask = set->capacity - 1;
@ -69,7 +71,7 @@ void hashset_destroy(hashset_t set) {
}
}
int hashset_add_member(hashset_t set, void *item) {
static int hashset_add_member(hashset_t set, void *item) {
size_t value = (size_t)item;
size_t h;
@ -88,7 +90,7 @@ int hashset_add_member(hashset_t set, void *item) {
return 1;
}
int hashset_add(hashset_t set, void *item) {
static int hashset_add(hashset_t set, void *item) {
int ret = hashset_add_member(set, item);
size_t old_capacity = set->capacity;
@ -113,7 +115,7 @@ int hashset_add(hashset_t set, void *item) {
return ret;
}
int hashset_remove(hashset_t set, void *item) {
static int hashset_remove(hashset_t set, void *item) {
size_t value = (size_t)item;
for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) {
@ -127,7 +129,7 @@ int hashset_remove(hashset_t set, void *item) {
return 0;
}
int hashset_contains(hashset_t set, void *item) {
static int hashset_contains(hashset_t set, void *item) {
size_t value = (size_t)item;
for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) {
@ -177,7 +179,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
+ fsize + 8 + 1; /* jFileName */
pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
pPager = (SPager *)pPtr;
@ -198,12 +200,12 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (TDB_FD_INVALID(pPager->fd)) {
// if (pPager->fd < 0) {
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
ret = tdbGnrtFileID(pPager->fd, pPager->fid, false);
if (ret < 0) {
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
// pPager->jfd = -1;
@ -221,11 +223,6 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
int tdbPagerClose(SPager *pPager) {
if (pPager) {
/*
if (pPager->inTran) {
tdbOsClose(pPager->jfd);
}
*/
tdbOsClose(pPager->fd);
tdbOsFree(pPager);
}
@ -254,8 +251,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
!hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) {
ret = tdbPagerWritePageToJournal(pPager, pPage);
if (ret < 0) {
tdbError("failed to write page to journal since %s", tstrerror(terrno));
return -1;
tdbError("failed to write page to journal since %s", tstrerror(ret));
return ret;
}
if (pPager->pActiveTxn->jPageSet) {
@ -278,11 +275,13 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (TDB_FD_INVALID(pTxn->jfd)) {
tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
pTxn->jPageSet = hashset_create();
if (pTxn->jPageSet == NULL) {
return terrno;
}
pPager->pActiveTxn = pTxn;
@ -319,8 +318,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) {
tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
// loop to write the dirty pages to file
@ -331,13 +329,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
if (pPage->nOverflow != 0) {
tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1;
return ret;
}
}
@ -368,8 +366,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
// sync the db file
if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
return 0;
@ -382,14 +379,12 @@ int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) {
// remove the journal file
if (tdbOsClose(pTxn->jfd) < 0) {
tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) {
tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
// pPager->inTran = 0;
@ -408,8 +403,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) {
tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
// loop to write the dirty pages to file
@ -426,7 +420,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1;
return ret;
}
}
@ -445,36 +439,28 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
}
/*
tdbTrace("reset dirty tree: %p", &pPager->rbt);
tRBTreeCreate(&pPager->rbt, pageCmpFn);
// sync the db file
if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
*/
return 0;
}
static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, const char* function,
int64_t offset){
static char *tdbEncryptPage(SPager *pPager, char *pPageData, int32_t pageSize, const char *function, int64_t offset) {
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey;
char *encryptKey = pPager->pEnv->encryptKey;
char* buf = pPageData;
char *buf = pPageData;
if(encryptAlgorithm == DND_CA_SM4){
//tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0);
if (encryptAlgorithm == DND_CA_SM4) {
// tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
// ASSERT(strlen(encryptKey) > 0);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]);
// tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]);
buf = taosMemoryMalloc(pageSize);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
unsigned char packetData[128];
unsigned char packetData[128];
int32_t count = 0;
while (count < pageSize) {
@ -488,19 +474,19 @@ static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, c
int32_t newLen = CBC_Encrypt(&opts);
memcpy(buf + count, packetData, newLen);
count += newLen;
count += newLen;
}
//tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function);
// tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]);
// tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]);
}
return buf;
}
void tdbFreeEncryptBuf(SPager *pPager, char* buf){
void tdbFreeEncryptBuf(SPager *pPager, char *buf) {
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf);
if (encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf);
}
// recovery dirty pages
int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
@ -518,26 +504,24 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) {
tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
tdb_fd_t jfd = pTxn->jfd;
ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
if (ret < 0) {
return -1;
return ret;
}
if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) {
tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize);
if (pageBuf == NULL) {
return -1;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
tdbDebug("pager/abort: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId);
@ -549,7 +533,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
if (ret < 0) {
tdbOsFree(pageBuf);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
tdbTrace("pager/abort: restore pgno:%d,", pgno);
@ -559,27 +543,28 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
if (ret < 0) {
tdbOsFree(pageBuf);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
if (buf == NULL) {
return terrno;
}
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
tdbFreeEncryptBuf(pPager, buf);
@ -587,9 +572,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
tdbOsFree(pageBuf);
@ -617,8 +601,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
// 4, remove the journal file
if (tdbOsClose(pTxn->jfd) < 0) {
tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
char jTxnFileName[TDB_FILENAME_LEN];
@ -626,8 +609,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) {
tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
// pPager->inTran = 0;
@ -658,7 +640,7 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) {
ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1;
return ret;
}
tdbTrace("tdb/flush:%p, pgno:%d, %d/%d/%d", pPager, pgno, pPager->dbOrigSize, pPager->dbFileSize, maxPgno);
@ -717,13 +699,13 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
ret = tdbPagerAllocPage(pPager, &pgno, pTxn);
if (ret < 0) {
tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno);
return -1;
return ret;
}
}
if (pgno == 0) {
tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
// fetch a page container
@ -739,7 +721,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage);
if (ret < 0) {
tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage);
return -1;
return ret;
}
}
@ -748,11 +730,11 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
if (!TDB_PAGE_INITIALIZED(pPage)) {
tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
if (pPage->pPager != pPager) {
tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager);
return -1;
return TSDB_CODE_INVALID_DATA_FMT;
}
*ppgno = pgno;
@ -771,12 +753,17 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
SPgno pgno = TDB_PAGE_PGNO(pPage);
if (pPager->frps) {
taosArrayPush(pPager->frps, &pgno);
if (taosArrayPush(pPager->frps, &pgno) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pPage->pPager = NULL;
return code;
}
pPager->frps = taosArrayInit(8, sizeof(SPgno));
if (pPager->frps == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// memset(pPage->pData, 0, pPage->pageSize);
tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno);
// printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno);
@ -785,7 +772,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
taosArrayDestroy(pPager->frps);
pPager->frps = NULL;
return -1;
return code;
}
while (TARRAY_SIZE(pPager->frps) > 0) {
@ -796,7 +783,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
taosArrayDestroy(pPager->frps);
pPager->frps = NULL;
return -1;
return code;
}
}
@ -822,7 +809,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn);
if (code < 0) {
return 0;
return code;
}
code = tdbTbcMoveToFirst(pCur);
@ -924,20 +911,19 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
}
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey;
char *encryptKey = pPager->pEnv->encryptKey;
if(encryptAlgorithm == DND_CA_SM4){
//tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0);
if (encryptAlgorithm == DND_CA_SM4) {
// tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
// ASSERT(strlen(encryptKey) > 0);
//uint8_t flags = pPage->pData[0];
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags);
// uint8_t flags = pPage->pData[0];
// tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags);
unsigned char packetData[128];
unsigned char packetData[128];
int32_t count = 0;
while(count < pPage->pageSize)
{
while (count < pPage->pageSize) {
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pPage->pData + count;
@ -945,20 +931,23 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
int newLen = CBC_Decrypt(&opts);
int newLen = CBC_Decrypt(&opts);
memcpy(pPage->pData + count, packetData, newLen);
count += newLen;
}
//tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, __FUNCTION__);
// tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count,
// __FUNCTION__);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__);
// tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1),
// pPage->pData[0], __FUNCTION__);
}
} else {
init = 0;
}
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__);
// tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0],
// __FUNCTION__);
ret = (*initPage)(pPage, arg, init);
if (ret < 0) {
@ -1001,16 +990,14 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
if (ret < 0) {
tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName,
pgno, pPager->pActiveTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize);
if (ret < 0) {
tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno),
pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
return 0;
@ -1044,15 +1031,14 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) {
offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
char* buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset);
char *buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset);
ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset);
if (ret < 0) {
tdbFreeEncryptBuf(pPager, buf);
tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName,
pPage->pageSize);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
tdbFreeEncryptBuf(pPager, buf);
@ -1072,18 +1058,17 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
if (ret < 0) {
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) {
tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
pageBuf = tdbOsCalloc(1, pPager->pageSize);
if (pageBuf == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName);
@ -1095,7 +1080,7 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
if (ret < 0) {
tdbOsFree(pageBuf);
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
tdbTrace("pager/restore: restore pgno:%d,", pgno);
@ -1103,27 +1088,28 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
if (ret < 0) {
tdbOsFree(pageBuf);
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
if (buf == NULL) {
return terrno;
}
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
tdbFreeEncryptBuf(pPager, buf);
@ -1131,9 +1117,8 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
tdbOsFree(pageBuf);
@ -1160,21 +1145,27 @@ static int32_t txnIdCompareDesc(const void *pLeft, const void *pRight) {
}
int tdbPagerRestoreJournals(SPager *pPager) {
int32_t code = 0;
tdbDirEntryPtr pDirEntry;
tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName);
if (pDir == NULL) {
tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno));
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
SArray *pTxnList = taosArrayInit(16, sizeof(int64_t));
if (pTxnList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry));
if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) {
int64_t txnId = -1;
sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId);
taosArrayPush(pTxnList, &txnId);
if (taosArrayPush(pTxnList, &txnId) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
}
taosArraySort(pTxnList, txnIdCompareDesc);
@ -1185,12 +1176,12 @@ int tdbPagerRestoreJournals(SPager *pPager) {
memcpy(jname, pPager->pEnv->dbName, dirLen);
jname[dirLen] = '/';
sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId);
if (tdbPagerRestore(pPager, jname) < 0) {
code = tdbPagerRestore(pPager, jname);
if (code) {
taosArrayDestroy(pTxnList);
tdbCloseDir(&pDir);
tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname);
return -1;
tdbError("failed to restore file due to %s. jFileName:%s", strerror(code), jname);
return code;
}
}
@ -1205,7 +1196,7 @@ int tdbPagerRollback(SPager *pPager) {
tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName);
if (pDir == NULL) {
tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno));
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
@ -1221,8 +1212,7 @@ int tdbPagerRollback(SPager *pPager) {
tdbCloseDir(&pDir);
tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
}
}

10
source/libs/tdb/src/db/tdbTable.c
View File

@ -41,7 +41,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb));
if (pTb == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
// pTb->pEnv
@ -54,7 +54,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
pPager = tdbEnvGetPager(pEnv, fFullName);
if (!pPager) {
tdbOsFree(pTb);
return -1;
return terrno;
}
ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData);
@ -74,7 +74,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) {
tdbOsFree(pTb);
return -1;
return ret;
}
tdbEnvAddPager(pEnv, pPager);
@ -109,7 +109,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbPagerRestoreJournals(pPager);
if (ret < 0) {
tdbOsFree(pTb);
return -1;
return ret;
}
} else {
tdbPagerRollback(pPager);
@ -119,7 +119,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt));
if (ret < 0) {
tdbOsFree(pTb);
return -1;
return ret;
}
*ppTb = pTb;

4
source/libs/tdb/src/db/tdbTxn.c
View File

@ -20,7 +20,7 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void
// not support read-committed version at the moment
if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) {
tdbError("tdb/txn: invalid txn flags: %" PRId32, flags);
return -1;
return TSDB_CODE_INVALID_PARA;
}
pTxn->flags = flags;
@ -39,7 +39,7 @@ int tdbTxnCloseImpl(TXN *pTxn) {
}
if (pTxn->jfd) {
tdbOsClose(pTxn->jfd);
TAOS_UNUSED(tdbOsClose(pTxn->jfd));
ASSERT(pTxn->jfd == NULL);
}

7
source/libs/tdb/src/db/tdbUtil.c
View File

@ -38,9 +38,8 @@ void tdbFree(void *p) {
int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) {
int64_t stDev = 0, stIno = 0;
if (taosDevInoFile(fd, &stDev, &stIno) < 0) {
return -1;
}
int32_t code = taosDevInoFile(fd, &stDev, &stIno);
return code;
memset(fileid, 0, TDB_FILE_ID_LEN);
@ -59,7 +58,7 @@ int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) {
ret = tdbOsFileSize(fd, &szBytes);
if (ret < 0) {
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
*size = szBytes / szPage;

19
source/libs/tdb/src/inc/tdbInt.h
View File

@ -17,6 +17,7 @@
#define _TD_TDB_INTERNAL_H_
#include "tdb.h"
#include "tutil.h"
#include "tdef.h"
#include "tlog.h"
@ -338,15 +339,15 @@ static inline int tdbTryLockPage(tdb_spinlock_t *pLock) {
((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset)
#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg);
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg);
void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt);
int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt);
void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int copyOvflCells);
int tdbPageCapacity(int pageSize, int amHdrSize);
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg);
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg);
void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt);
int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt);
int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl);
int tdbPageCapacity(int pageSize, int amHdrSize);
static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
SCell *pCell;

30
source/os/src/osFile.c
View File

@ -67,7 +67,7 @@ typedef struct TdFile {
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) {
#ifdef WINDOWS
char tmpPath[PATH_MAX];
char tmpPath[PATH_MAX];
int32_t len = (int32_t)strlen(inputTmpDir);
memcpy(tmpPath, inputTmpDir, len);
@ -269,13 +269,13 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
#else
if (pFile == NULL || pFile->fd < 0) {
return -1;
return TSDB_CODE_INVALID_PARA;
}
struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat);
if (code < 0) {
printf("taosFStatFile run fstat fail.");
return code;
return TAOS_SYSTEM_ERROR(errno);
}
if (stDev != NULL) {
@ -1239,7 +1239,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
#ifdef WINDOWS
size_t bufferSize = 512;
*ptrBuf = taosMemoryMalloc(bufferSize);
if (*ptrBuf == NULL) goto END;
if (*ptrBuf == NULL) goto END;
size_t bytesRead = 0;
size_t totalBytesRead = 0;
@ -1274,7 +1274,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
ret = getline(ptrBuf, &len, pFile->fp);
#endif
END:
END:
#if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock));
#endif
@ -1413,34 +1413,30 @@ int32_t taosLinkFile(char *src, char *dst) {
return 0;
}
FILE* taosOpenCFile(const char* filename, const char* mode) {
return fopen(filename, mode);
}
FILE *taosOpenCFile(const char *filename, const char *mode) { return fopen(filename, mode); }
int taosSeekCFile(FILE* file, int64_t offset, int whence) {
int taosSeekCFile(FILE *file, int64_t offset, int whence) {
#ifdef WINDOWS
return _fseeki64(file, offset, whence);
#else
return fseeko(file, offset, whence);
#endif
#endif
}
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ) {
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) {
return fread(buffer, size, count, stream);
}
size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream) {
size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) {
return fwrite(ptr, size, nitems, stream);
}
int taosCloseCFile(FILE *f) {
return fclose(f);
}
int taosCloseCFile(FILE *f) { return fclose(f); }
int taosSetAutoDelFile(char* path) {
int taosSetAutoDelFile(char *path) {
#ifdef WINDOWS
return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
#else
return unlink(path);
#endif
#endif
}

8
tests/ci/ci_deploy_dependency_file_list.txt Normal file
View File

@ -0,0 +1,8 @@
/home/m.json
/home/log_server.json
/var/lib/jenkins/workspace/restore.sh
/var/lib/jenkins/workspace/start_http.sh
/var/lib/jenkins/workspace/TDinternal
/var/lib/jenkins/workspace/remove_corefile.sh
/var/lib/jenkins/workspace/CI_disk_Monitor.py
/var/lib/jenkins/workspace/start_CI_Monitor.sh

88
tests/ci/container_build_newmachine.sh Executable file
View File

@ -0,0 +1,88 @@
#!/bin/bash
set -e
function usage() {
echo "$0"
echo -e "\t -w work dir"
echo -e "\t -e enterprise edition"
echo -e "\t -t make thread count"
echo -e "\t -h help"
}
ent=0
while getopts "w:t:eh" opt; do
case $opt in
w)
WORKDIR=$OPTARG
;;
e)
ent=1
;;
t)
THREAD_COUNT=$OPTARG
;;
h)
usage
exit 0
;;
\?)
echo "Invalid option: -$OPTARG"
usage
exit 0
;;
esac
done
if [ -z "$WORKDIR" ]; then
usage
exit 1
fi
# if [ -z "$THREAD_COUNT" ]; then
# THREAD_COUNT=1
# fi
ulimit -c unlimited
if [ $ent -eq 0 ]; then
REP_DIR=/home/TDengine
REP_REAL_PATH=$WORKDIR/TDengine
REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDengine
else
REP_DIR=/home/TDinternal
REP_REAL_PATH=$WORKDIR/TDinternal
REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal
fi
date
docker run \
-v $REP_MOUNT_PARAM \
-v /root/.cargo/registry:/root/.cargo/registry \
-v /root/.cargo/git:/root/.cargo/git \
-v /root/go/pkg/mod:/root/go/pkg/mod \
-v /root/.cache/go-build:/root/.cache/go-build \
-v /root/.cos-local.1:/root/.cos-local.2 \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 "
# -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
if [[ -d ${WORKDIR}/debugNoSan ]] ;then
echo "delete ${WORKDIR}/debugNoSan"
rm -rf ${WORKDIR}/debugNoSan
fi
if [[ -d ${WORKDIR}/debugSan ]] ;then
echo "delete ${WORKDIR}/debugSan"
rm -rf ${WORKDIR}/debugSan
fi
if [ "$(uname -m)" = "aarch64" ] ;then
CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Debug"
else
CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Release"
fi
mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan
date
ret=$?
exit $ret

69
tests/ci/deploy_ci_machines.sh Normal file
View File

@ -0,0 +1,69 @@
#!/bin/bash
function scp_file_from_host {
# check at least three parameters
if [ "$#" -lt 3 ]; then
echo "Usage: $0 host passwd source_filename [dest_filename]"
exit 1
fi
host=$1
passwd=$2
source_filename=$3
# If the fourth parameter is not provided, use the third parameter as the default value
dest_filename=${4:-$3}
# use sshpass and scp for secure file transfer
sshpass -p "$passwd" scp -o StrictHostKeyChecking=no -r "$host":"$source_filename" "$dest_filename"
}
# install docker and sshpass
curl -fsSL https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository "deb [arch=amd64] https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
sudo apt-get update
sudo apt-get install -y docker-ce sshpass jq
sudo systemctl enable docker
sudo systemctl start docker
# create a log directory
mkdir -p /var/lib/jenkins/workspace/log
# Assuming you have a file called 'file_list.txt' with one filename per line
file_list="ci_deploy_dependency_file_list.txt"
monitorip="192.168.1.59"
passwd_all="abcdefg"
# Read the file list and call scp_file_from_host for each file
while IFS= read -r source_filename; do
scp_file_from_host "$monitorip" "$passwd_all" "$source_filename"
done < "$file_list"
# modify the configuration file
ip=$(ifconfig |grep inet|grep 192 |awk '{print $2}')
sed -i "s/${monitorip}/$ip/" /home/log_server.json
sed -i "s/${monitorip}/$ip/" /home/m.json
#mkdir corefile dir and configure the system to automatically set corefile dir at startup
mkdir -p /home/coredump/ && echo "echo '/home/coredump/core_%e-%p' | sudo tee /proc/sys/kernel/core_pattern " >> /root/.bashrc
# get image from 0.212
image_ip="192.168.0.212"
scp_file_from_host $image_ip $passwd_all "/home/tang/work/image/taos_image.tar " "/home/taos_image.tar"
docker load -i /home/taos_image.tar
#start http server
nohup /var/lib/jenkins/workspace/log/start_http.sh &
# start CI monitor and remove corefile in crontable
(crontab -l;echo "0 1 * * * /usr/bin/bash /var/lib/jenkins/workspace/remove_corefile.sh") | crontab
(crontab -l;echo "@reboot /usr/bin/bash /var/lib/jenkins/workspace/start_CI_Monitor.sh") | crontab
# generate cache dir
cd /var/lib/jenkins/workspace/TDinternal/community/tests/parallel_test || exit
time ./container_build_newmachine.sh -w /var/lib/jenkins/workspace -e
# test if the CI machine compilation is successful
time ./container_build.sh -w /var/lib/jenkins/workspace -e

17
tests/ci/remove_corefile.sh Normal file
View File

@ -0,0 +1,17 @@
#!/bin/bash
date_str=$(date -d "4 day ago" +%Y%m%d)
if [ ! -z "$1" ]; then
date_str="$1"
fi
script_dir=$(dirname $0)
cd "${script_dir}"/log || exit
# date >>core.list
# find . -name "core.*" | grep "$date_str" >>core.list
# find . -name "core.*" | grep "$date_str" | xargs rm -rf
# find . -name "build_*" | grep "$date_str" | xargs rm -rf
for file in *; do
if [[ $file == *"$date_str"* ]]; then
rm -rf "$file"
fi
done

159
tests/ci/restore.sh Normal file
View File

@ -0,0 +1,159 @@
#!/bin/bash
set -x
JENKINS_LOG=jenkins.log
# pr_num=14228
# n=1
function usage() {
echo "$0"
echo -e "\t -p PR number"
echo -e "\t -n build number"
echo -e "\t -c container name"
echo -e "\t -h help"
}
while getopts "p:n:c:h" opt; do
case $opt in
p)
pr_num=$OPTARG
;;
n)
n=$OPTARG
;;
c)
container_name=$OPTARG
;;
h)
usage
exit 0
;;
\?)
echo "Invalid option: -$OPTARG"
usage
exit 1
;;
esac
done
if [ -z "$container_name" ]; then
echo "container name not specified"
usage
exit 1
fi
if [ -z "$pr_num" ]; then
echo "PR number not specified"
usage
exit 1
fi
if [ -z "$n" ]; then
echo "build number not specified"
usage
exit 1
fi
pr_num=`echo "$pr_num"|sed "s/PR-//"`
container_count=`docker ps -a -f name=$container_name|wc -l`
if [ $container_count -gt 1 ]; then
docker ps -a -f name=$container_name
echo "container $container_name exists"
exit 1
fi
cd $(dirname $0)
info=`grep -n "^[0-9]\{8\}-[0-9]\{6\}" jenkins.log | grep -A 1 "PR-${pr_num}:${n}:"`
# 22131:20220625-113105 NewTest/PR-14228:PR-14228:1:3.0
# 22270:20220625-121154 NewTest/PR-14221:PR-14221:2:3.0
ci_hosts="\
192.168.0.212 \
192.168.0.215 \
192.168.0.217 \
192.168.0.219 \
"
if [ -z "$info" ]; then
echo "PR-${pr_num}:${n} not found"
for host in $ci_hosts; do
ssh root@$host "sh -c \"grep -n \\\"^[0-9]\\\\\{8\\\\\}-[0-9]\\\\\{6\\\\\}\\\" /var/lib/jenkins/workspace/jenkins.log | grep \\\"PR-${pr_num}:${n}:\\\"\""
if [ $? -eq 0 ]; then
echo "this PR is possibly on host $host"
break
fi
done
exit 1
fi
line_num=`echo "$info"|wc -l`
curr=`echo "$info"|head -n1`
if [ "$line_num" == "2" ]; then
next=`echo "$info"|tail -n1`
fi
# check if it is TDinternal CI
internal=0
commit_prefix=community
echo "$curr"|grep -q TDinternalCI
if [ $? -eq 0 ]; then
internal=1
commit_prefix=tdinternal
fi
curr_line=`echo "$curr"|cut -d: -f1`
next_line='$'
if [ ! -z "$next" ]; then
next_line=`echo "$next"|cut -d: -f1`
next_line=$(( next_line - 1 ))
fi
# echo "$curr_line, $next_line"
details=`sed -n "${curr_line},${next_line}p" $JENKINS_LOG`
merge_line=`echo "$details"|grep -A 10 "$commit_prefix log merged: "|grep "Merge .* into"|head -n1`
if [ -z "$merge_line" ]; then
echo "merge commit not found"
exit 1
fi
echo "$merge_line"
branch=`echo "$merge_line"|awk '{print $2}'`
commit_id=`echo "$merge_line"|awk '{print $4}'`
# echo "$details"
community_id=`echo "$details"|grep "community log: commit"|awk '{print $NF}'`
internal_id=`echo "$details"|grep "tdinternal log: commit"|awk '{print $NF}'`
python_connector_id=`echo "$details"|grep "python connector log: commit"|awk '{print $NF}'`
# change_branch=`echo "$details"|grep "CHANGE_BRANCH"|sed "s/CHANGE_BRANCH://"`
# if [ -z "${branch}" ]; then
# branch="$change_branch"
# fi
PWD=`pwd`
log_dir=`ls log|grep "PR-${pr_num}_${n}_"`
if [ -z "$log_dir" ]; then
echo "no log dir found"
else
mount_dir="-v ${PWD}/log/$log_dir:/home/log"
build_dir=`ls log/$log_dir | grep "build_"`
if [ ! -z "$build_dir" ]; then
mount_dir="$mount_dir -v ${PWD}/log/$log_dir/$build_dir:/home/TDinternal/debug/build"
fi
fi
docker run -d --privileged -it --name $container_name \
$mount_dir \
taos_test:v1.0 bash
if [ $internal -eq 0 ]; then
docker exec $container_name /home/setup.sh -c $commit_id -m $branch -n
echo "TDinternal checkout: $internal_id"
docker exec $container_name sh -c "cd /home/TDinternal; git checkout $internal_id"
else
docker exec $container_name /home/setup.sh -e -c $commit_id -m $branch -n
echo "community checkout: $community_id"
docker exec $container_name sh -c "cd /home/TDinternal/community; git checkout $community_id"
fi
echo
echo "* run the following command to enter the container:"
echo " docker exec -it $container_name bash"
if [ -z "$log_dir" ]; then
echo "* no log dir found"
else
echo "* log and coredump files are located in /home/log"
fi
if [ -z "$build_dir" ]; then
echo "* no build dir found"
else
echo "* build files are located in /home/TDinternal/debug/build"
fi
echo "* source files are located in /home/TDinternal"

16
tests/ci/start_http.sh Normal file
View File

@ -0,0 +1,16 @@
#!/bin/bash
script_dir=$(dirname $0)
cd $script_dir || exit
script_name=$(basename $0)
ps -ef|grep -v grep|grep -v $$|grep -q "$script_name"
if [ $? -eq 0 ]; then
exit 0
fi
while [ 1 ]; do
ps -ef|grep python|grep -q 8081
if [ $? -ne 0 ]; then
python3 -m http.server 8081
fi
sleep 60
done

40
tests/parallel_test/container_build.sh
View File

@ -61,14 +61,28 @@ docker run \
-v /root/go/pkg/mod:/root/go/pkg/mod \
-v /root/.cache/go-build:/root/.cache/go-build \
-v /root/.cos-local.1:/root/.cos-local.2 \
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
-v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \
-v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \
-v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \
-v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \
-v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \
-v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
-v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
-v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
-v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \
-v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \
-v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \
-v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
-v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \
-v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
-v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \
-v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \
-v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \
-v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \
-v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1"
# -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
@ -96,15 +110,29 @@ docker run \
-v /root/go/pkg/mod:/root/go/pkg/mod \
-v /root/.cache/go-build:/root/.cache/go-build \
-v /root/.cos-local.1:/root/.cos-local.2 \
-v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
-v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \
-v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \
-v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \
-v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \
-v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
-v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \
-v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \
-v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
-v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
-v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
-v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \
-v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \
-v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \
-v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
-v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \
-v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
-v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \
-v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \
-v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \
-v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \
-v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
-v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 "
mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan