xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #13386 from taosdata/feature/TD-14481-3.0 

feat: tsma days refactor
This commit is contained in:
Cary Xu 2022-06-02 15:31:00 +08:00 committed by GitHub
parent 0ac78291ae bc4cb4cb56
commit d7288bfa45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1152 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/common/tmsg.h
View File

@ -2354,19 +2354,19 @@ typedef struct {
STSma* tSma; STSma* tSma;
} STSmaWrapper; } STSmaWrapper;
static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { static FORCE_INLINE void tDestroyTSma(STSma* pSma) {
if (pSma) { if (pSma) {
taosMemoryFreeClear(pSma->expr); taosMemoryFreeClear(pSma->expr);
taosMemoryFreeClear(pSma->tagsFilter); taosMemoryFreeClear(pSma->tagsFilter);
} }
} }
static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { static FORCE_INLINE void tDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) {
if (pSW) { if (pSW) {
if (pSW->tSma) { if (pSW->tSma) {
if (deepCopy) { if (deepCopy) {
for (uint32_t i = 0; i < pSW->number; ++i) { for (uint32_t i = 0; i < pSW->number; ++i) {
tdDestroyTSma(pSW->tSma + i); tDestroyTSma(pSW->tSma + i);
} }
} }
taosMemoryFreeClear(pSW->tSma); taosMemoryFreeClear(pSW->tSma);
@ -2374,8 +2374,8 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy)
} }
} }
static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { static FORCE_INLINE void* tFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) {
tdDestroyTSmaWrapper(pSW, deepCopy); tDestroyTSmaWrapper(pSW, deepCopy);
taosMemoryFreeClear(pSW); taosMemoryFreeClear(pSW);
return NULL; return NULL;
} }

44
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -180,6 +180,24 @@ static void vmGenerateWrapperCfg(SVnodeMgmt *pMgmt, SCreateVnodeReq *pCreate, SW
snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCreate->vgId); snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCreate->vgId);
} }
static int32_t vmTsmaAdjustDays(SVnodeCfg *pCfg, SCreateVnodeReq *pReq) {
if (pReq->isTsma) {
SMsgHead *smaMsg = pReq->pTsma;
uint32_t contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead));
return smaGetTSmaDays(pCfg, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen, &pCfg->tsdbCfg.days);
}
return 0;
}
static int32_t vmTsmaProcessCreate(SVnode *pVnode, SCreateVnodeReq *pReq) {
if (pReq->isTsma) {
SMsgHead *smaMsg = pReq->pTsma;
uint32_t contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead));
return vnodeProcessCreateTSma(pVnode, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen);
}
return 0;
}
int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SCreateVnodeReq createReq = {0}; SCreateVnodeReq createReq = {0};
SVnodeCfg vnodeCfg = {0}; SVnodeCfg vnodeCfg = {0};
@ -195,6 +213,13 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dDebug("vgId:%d, create vnode req is received, tsma:%d standby:%d", createReq.vgId, createReq.isTsma, dDebug("vgId:%d, create vnode req is received, tsma:%d standby:%d", createReq.vgId, createReq.isTsma,
createReq.standby); createReq.standby);
vmGenerateVnodeCfg(&createReq, &vnodeCfg); vmGenerateVnodeCfg(&createReq, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", createReq.vgId, terrstr());
code = terrno;
goto _OVER;
}
vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
@ -203,14 +228,16 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
tFreeSCreateVnodeReq(&createReq); tFreeSCreateVnodeReq(&createReq);
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
return -1; code = terrno;
goto _OVER;
} }
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId);
if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) { if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) {
tFreeSCreateVnodeReq(&createReq); tFreeSCreateVnodeReq(&createReq);
dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
return -1; code = terrno;
goto _OVER;
} }
SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb);
@ -227,14 +254,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _OVER; goto _OVER;
} }
if (createReq.isTsma) { code = vmTsmaProcessCreate(pImpl, &createReq);
SMsgHead *smaMsg = createReq.pTsma; if (code != 0) {
uint32_t contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead)); dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr());
if (vnodeProcessCreateTSma(pImpl, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen) < 0) { code = terrno;
dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr()); goto _OVER;
code = terrno;
goto _OVER;
};
} }
code = vnodeStart(pImpl); code = vnodeStart(pImpl);

2
source/dnode/vnode/CMakeLists.txt
View File

@ -32,7 +32,7 @@ target_sources(
"src/sma/smaEnv.c" "src/sma/smaEnv.c"
"src/sma/smaOpen.c" "src/sma/smaOpen.c"
"src/sma/smaRollup.c" "src/sma/smaRollup.c"
"src/sma/smaTimeRange.c" "src/sma/smaTimeRange2.c"
# tsdb # tsdb
"src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit.c"

3
source/dnode/vnode/inc/vnode.h
View File

@ -147,6 +147,9 @@ bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids
int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
int32_t *pNumOfRows, int16_t *pNumOfCols); int32_t *pNumOfRows, int16_t *pNumOfCols);
// sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
// need to reposition // need to reposition
// structs // structs

2
source/dnode/vnode/src/inc/sma.h
View File

@ -223,6 +223,8 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t ve
// TODO: This is the basic params, and should wrap the params to a queryHandle. // TODO: This is the basic params, and should wrap the params to a queryHandle.
int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -464,7 +464,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
_err: _err:
metaReaderClear(&mr); metaReaderClear(&mr);
taosArrayDestroy(pSmaIds); taosArrayDestroy(pSmaIds);
tdFreeTSmaWrapper(pSW, deepCopy); tFreeTSmaWrapper(pSW, deepCopy);
return NULL; return NULL;
} }

8
source/dnode/vnode/src/sma/sma.c
View File

@ -51,3 +51,11 @@ int32_t tdGetTSmaData(SSma* pSma, char* pData, int64_t indexUid, TSKEY querySKey
} }
return code; return code;
} }
int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t *days) {
int32_t code = TSDB_CODE_SUCCESS;
if ((code = tdGetTSmaDaysImpl(pCfg, pCont, contLen, days)) < 0) {
smaWarn("vgId:%d get tSma days failed since %s", pCfg->vgId, tstrerror(terrno));
}
return code;
}

2
source/dnode/vnode/src/sma/smaEnv.c
View File

@ -278,7 +278,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) {
void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) {
if (pSmaStatItem) { if (pSmaStatItem) {
tdDestroyTSma(pSmaStatItem->pTSma); tDestroyTSma(pSmaStatItem->pTSma);
taosMemoryFreeClear(pSmaStatItem->pTSma); taosMemoryFreeClear(pSmaStatItem->pTSma);
taosHashCleanup(pSmaStatItem->expiredWindows); taosHashCleanup(pSmaStatItem->expiredWindows);
taosMemoryFreeClear(pSmaStatItem); taosMemoryFreeClear(pSmaStatItem);

12
source/dnode/vnode/src/sma/smaTimeRange.c
View File

@ -828,7 +828,7 @@ int32_t tdDropTSma(SSma *pSma, char *pMsg) {
// TODO: send msg to stream computing to drop tSma // TODO: send msg to stream computing to drop tSma
// if ((send msg to stream computing) < 0) { // if ((send msg to stream computing) < 0) {
// tdDestroyTSma(&vCreateSmaReq); // tDestroyTSma(&vCreateSmaReq);
// return -1; // return -1;
// } // }
// //
@ -982,25 +982,25 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t ve
SSubmitBlkIter blkIter = {0}; SSubmitBlkIter blkIter = {0};
if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) {
pSW = tdFreeTSmaWrapper(pSW, false); pSW = tFreeTSmaWrapper(pSW, false);
break; break;
} }
while (true) { while (true) {
STSRow *row = tGetSubmitBlkNext(&blkIter); STSRow *row = tGetSubmitBlkNext(&blkIter);
if (!row) { if (!row) {
pSW = tdFreeTSmaWrapper(pSW, false); pSW = tFreeTSmaWrapper(pSW, false);
break; break;
} }
if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) {
if (pSW) { if (pSW) {
pSW = tdFreeTSmaWrapper(pSW, false); pSW = tFreeTSmaWrapper(pSW, false);
} }
if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) {
break; break;
} }
if ((pSW->number) <= 0 || !pSW->tSma) { if ((pSW->number) <= 0 || !pSW->tSma) {
pSW = tdFreeTSmaWrapper(pSW, false); pSW = tFreeTSmaWrapper(pSW, false);
break; break;
} }
@ -1020,7 +1020,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t ve
if (lastWinSKey != winSKey) { if (lastWinSKey != winSKey) {
lastWinSKey = winSKey; lastWinSKey = winSKey;
if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) {
pSW = tdFreeTSmaWrapper(pSW, false); pSW = tFreeTSmaWrapper(pSW, false);
tdUnRefSmaStat(pSma, pStat); tdUnRefSmaStat(pSma, pStat);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }

1084
source/dnode/vnode/src/sma/smaTimeRange2.c Normal file
View File

File diff suppressed because it is too large Load Diff

14
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -147,8 +147,8 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
// resource release // resource release
taosMemoryFreeClear(pSW); taosMemoryFreeClear(pSW);
tdDestroyTSma(&tSma); tDestroyTSma(&tSma);
tdDestroyTSmaWrapper(&dstTSmaWrapper); tDestroyTSmaWrapper(&dstTSmaWrapper);
} }
#endif #endif
@ -218,7 +218,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : "");
EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1);
EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid);
tdDestroyTSma(qSmaCfg); tDestroyTSma(qSmaCfg);
taosMemoryFreeClear(qSmaCfg); taosMemoryFreeClear(qSmaCfg);
qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2, true); qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2, true);
@ -229,7 +229,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : "");
EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2);
EXPECT_EQ(qSmaCfg->interval, tSma.interval); EXPECT_EQ(qSmaCfg->interval, tSma.interval);
tdDestroyTSma(qSmaCfg); tDestroyTSma(qSmaCfg);
taosMemoryFreeClear(qSmaCfg); taosMemoryFreeClear(qSmaCfg);
// get index name by table uid // get index name by table uid
@ -265,7 +265,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
EXPECT_EQ((pSW->tSma + 1)->indexUid, indexUid2); EXPECT_EQ((pSW->tSma + 1)->indexUid, indexUid2);
EXPECT_EQ((pSW->tSma + 1)->tableUid, tbUid); EXPECT_EQ((pSW->tSma + 1)->tableUid, tbUid);
tdDestroyTSmaWrapper(pSW); tDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW); taosMemoryFreeClear(pSW);
// get all sma table uids // get all sma table uids
@ -282,7 +282,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
metaRemoveSmaFromDb(pMeta, indexUid1); metaRemoveSmaFromDb(pMeta, indexUid1);
metaRemoveSmaFromDb(pMeta, indexUid2); metaRemoveSmaFromDb(pMeta, indexUid2);
tdDestroyTSma(&tSma); tDestroyTSma(&tSma);
metaClose(pMeta); metaClose(pMeta);
} }
#endif #endif
@ -576,7 +576,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
taosArrayDestroy(pDataBlocks); taosArrayDestroy(pDataBlocks);
// release meta // release meta
tdDestroyTSma(&tSma); tDestroyTSma(&tSma);
tfsClose(pTsdb->pTfs); tfsClose(pTsdb->pTfs);
tsdbClose(pTsdb); tsdbClose(pTsdb);
metaClose(pMeta); metaClose(pMeta);