xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

more work

This commit is contained in:
Hongze Cheng 2022-06-16 12:53:25 +00:00
parent 338383182d
commit 75816b7e7a
3 changed files with 183 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -346,19 +346,31 @@ _err:
return code; return code;
} }
static FORCE_INLINE bool tsdbCommitIterEnd(SCommitter *pCommitter, STbDataIter *pIter) { #define ROW_END(pRow, maxKey) (((pRow) == NULL) || ((pRow)->pTSRow->ts > (maxKey)))
TSDBROW *pRow = tsdbTbDataIterGet(pIter);
return ((pRow == NULL) || (pRow->pTSRow->ts <= pCommitter->maxKey)); static int32_t tsdbMergeCommit(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) {
int32_t code = 0;
// TODO
return code;
} }
static int32_t tsdbCommitTableDataImpl(SCommitter *pCommitter, STbDataIter *pIter, SBlockIdx *pBlockIdx) { static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBlockIdx *pBlockIdx) {
int32_t code = 0; int32_t code = 0;
int32_t c; STbDataIter iter;
int32_t iBlock; STbDataIter *pIter = &iter;
int32_t nBlock; TSDBROW *pRow;
SBlock *pBlock; SBlockIdx blockIdx; // TODO
SBlock block;
SBlockIdx blockIdx; // todo // create iter
if (pTbData) {
tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = 0}, 0, pIter);
} else {
pIter = NULL;
}
// check
pRow = tsdbTbDataIterGet(pIter);
if (ROW_END(pRow, pCommitter->maxKey) && pBlockIdx == NULL) goto _exit;
// start ================================ // start ================================
tMapDataReset(&pCommitter->oBlock); tMapDataReset(&pCommitter->oBlock);
@ -369,48 +381,39 @@ static int32_t tsdbCommitTableDataImpl(SCommitter *pCommitter, STbDataIter *pIte
} }
// impl =============================== // impl ===============================
iBlock = 0; SBlock block;
nBlock = pCommitter->oBlock.nItem; SBlock *pBlock = &block;
int32_t iBlock = 0;
int32_t nBlock = pCommitter->oBlock.nItem;
if (iBlock < nBlock) { // merge
pBlock = &block; pRow = tsdbTbDataIterGet(pIter);
tMapDataGetItemByIdx(&pCommitter->nBlock, iBlock, pBlock, tGetBlock); while (!ROW_END(pRow, pCommitter->maxKey) && iBlock < nBlock) {
} else { tMapDataGetItemByIdx(&pCommitter->oBlock, iBlock, pBlock, tGetBlock);
pBlock = NULL; code = tsdbMergeCommit(pCommitter, pIter, pBlock);
if (code) goto _err;
pRow = tsdbTbDataIterGet(pIter);
iBlock++;
} }
while (true) {
TSDBROW *pRow = tsdbTbDataIterGet(pIter);
bool iterEnd = ((pRow == NULL) || (pRow->pTSRow->ts > pCommitter->maxKey));
bool blockEnd = (pBlock == NULL);
if (iterEnd && blockEnd) break; // mem
pRow = tsdbTbDataIterGet(pIter);
while (!ROW_END(pRow, pCommitter->maxKey)) {
code = tsdbMergeCommit(pCommitter, pIter, NULL);
if (code) goto _err;
if (!iterEnd && !blockEnd) { pRow = tsdbTbDataIterGet(pIter);
c = tBlockCmprFn(&(SBlock){.maxKey.ts = pRow->pTSRow->ts}, pBlock);
if (c == 0) {
// merge until pBlock->maxKey
// if (pBlock->last), merge until pCommitter->maxKey
// else merge until pBlock->maxKey
} else if (c < 0) {
// tsdbCommitTableMemData(pIter, pBlock);
// if (pBlock->last), merge until pCommitter->maxKey
// else, commit until pBlock->minKey-1
} else {
// tsdbCommitTableDiskData(pBlock);
// if (pBlock->last), merge until pCommitter->maxKey
// else, move the block to new one
}
} else if (!iterEnd) {
// no block on disk, commit to last when there are no enough data
// commit memory data to pCommitter->maxKey
// tsdbCommitTableMemData(pIter, NULL);
} else {
// tsdbCommitTableDiskData(pBlock); // only left block
// if (last block ? ) else ?
// tMapDataPutItem(&pCommitter->nBlock, pBlock, tPutBlock);
iBlock++; // get new SBlock
} }
// disk
while (iBlock < nBlock) {
tMapDataGetItemByIdx(&pCommitter->oBlock, iBlock, pBlock, tGetBlock);
code = tsdbMergeCommit(pCommitter, NULL, pBlock);
if (code) goto _err;
iBlock++;
} }
// end =============================== // end ===============================
@ -420,32 +423,6 @@ static int32_t tsdbCommitTableDataImpl(SCommitter *pCommitter, STbDataIter *pIte
code = tMapDataPutItem(&pCommitter->nBlockIdx, &blockIdx, tPutBlockIdx); code = tMapDataPutItem(&pCommitter->nBlockIdx, &blockIdx, tPutBlockIdx);
if (code) goto _err; if (code) goto _err;
return code;
_err:
tsdbError("vgId:%d commit table data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
return code;
}
static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBlockIdx *pBlockIdx) {
int32_t code = 0;
STbDataIter *pIter = NULL;
STbDataIter iter;
TSDBROW *pRow;
// create iter if can
if (pTbData) {
pIter = &iter;
tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = 0}, 0, pIter);
}
// check
if (tsdbCommitIterEnd(pCommitter, pIter) && pBlockIdx == NULL) goto _exit;
// impl
code = tsdbCommitTableDataImpl(pCommitter, pIter, pBlockIdx);
if (code) goto _err;
_exit: _exit:
pRow = tsdbTbDataIterGet(pIter); pRow = tsdbTbDataIterGet(pIter);
if (pRow) { if (pRow) {

106
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -107,8 +107,8 @@ typedef struct SBlockLoadSuppInfo {
struct STsdbReader { struct STsdbReader {
STsdb* pTsdb; STsdb* pTsdb;
uint64_t suid; uint64_t suid;
SQueryFilePos cur; // current position
int16_t order; int16_t order;
SQueryFilePos cur; // current position
STimeWindow window; // the primary query time window that applies to all queries STimeWindow window; // the primary query time window that applies to all queries
// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time
// SColumnDataAgg** pstatis;// the ptr array list to return to caller // SColumnDataAgg** pstatis;// the ptr array list to return to caller
@ -292,66 +292,31 @@ struct STsdbReader {
// } // }
// } // }
// static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReader* pReadHandle, TSKEY winSKey, SRetention* retentions) { static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId,
// if (VND_IS_RSMA(pVnode)) { STsdbReader** ppReader) {
// int level = 0; int32_t code = 0;
// int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); STsdbReader* pReader = NULL;
// for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { // alloc
// SRetention* pRetention = retentions + level; pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader));
// if (pRetention->keep <= 0) { if (pReader == NULL) {
// if (level > 0) { code = TSDB_CODE_OUT_OF_MEMORY;
// --level; goto _err;
// } }
// break; pReader->pTsdb = pVnode->pTsdb; // TODO: pass in pTsdb directly
// } pReader->suid = pCond->suid;
// if ((now - pRetention->keep) <= winSKey) { pReader->order = pCond->order;
// break; pReader->loadType = pCond->type;
// } pReader->loadExternalRow = pCond->loadExternalRows;
// ++level; pReader->currentLoadExternalRows = pCond->loadExternalRows;
// } pReader->type = TSDB_QUERY_TYPE_ALL;
pReader->cur.fid = INT32_MIN;
// if (level == TSDB_RETENTION_L0) { pReader->cur.win = TSWINDOW_INITIALIZER;
// tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, pReader->checkFiles = true;
// TSDB_RETENTION_L0); pReader->activeIndex = 0; // current active table index
// return VND_RSMA0(pVnode); pReader->allocSize = 0;
// } else if (level == TSDB_RETENTION_L1) { pReader->locateStart = false;
// tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, pReader->outputCapacity = 4096; //((STsdb*)tsdb)->config.maxRowsPerFileBlock;
// TSDB_RETENTION_L1);
// return VND_RSMA1(pVnode);
// } else {
// tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle,
// TSDB_RETENTION_L2);
// return VND_RSMA2(pVnode);
// }
// }
// return VND_TSDB(pVnode);
// }
// static STsdbReader* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) {
// STsdbReader* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReader));
// if (pReadHandle == NULL) {
// goto _end;
// }
// STsdb* pTsdb = getTsdbByRetentions(pVnode, pReadHandle, pCond->twindows[0].skey,
// pVnode->config.tsdbCfg.retentions);
// pReadHandle->pTsdb = pTsdb;
// pReadHandle->suid = pCond->suid;
// pReadHandle->order = pCond->order;
// pReadHandle->loadType = pCond->type;
// pReadHandle->loadExternalRow = pCond->loadExternalRows;
// pReadHandle->currentLoadExternalRows = pCond->loadExternalRows;
// pReadHandle->type = TSDB_QUERY_TYPE_ALL;
// pReadHandle->cur.fid = INT32_MIN;
// pReadHandle->cur.win = TSWINDOW_INITIALIZER;
// pReadHandle->checkFiles = true;
// pReadHandle->activeIndex = 0; // current active table index
// pReadHandle->allocSize = 0;
// pReadHandle->locateStart = false;
// pReadHandle->outputCapacity = 4096; //((STsdb*)tsdb)->config.maxRowsPerFileBlock;
// char buf[128] = {0}; // char buf[128] = {0};
// snprintf(buf, tListLen(buf), "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, qId); // snprintf(buf, tListLen(buf), "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, qId);
@ -361,7 +326,6 @@ struct STsdbReader {
// // goto _end; // // goto _end;
// // } // // }
// assert(pCond != NULL);
// setQueryTimewindow(pReadHandle, pCond, 0); // setQueryTimewindow(pReadHandle, pCond, 0);
// if (pCond->numOfCols > 0) { // if (pCond->numOfCols > 0) {
@ -423,7 +387,15 @@ struct STsdbReader {
// tsdbReaderClose(pReadHandle); // tsdbReaderClose(pReadHandle);
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
// return NULL; // return NULL;
// }
*ppReader = pReader;
return code;
_err:
// tsdbError("");
*ppReader = NULL;
return code;
}
// static int32_t setCurrentSchema(SVnode* pVnode, STsdbReader* pTsdbReadHandle) { // static int32_t setCurrentSchema(SVnode* pVnode, STsdbReader* pTsdbReadHandle) {
// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); // STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0);
@ -2659,10 +2631,8 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInf
uint64_t taskId, STsdbReader** ppReader) { uint64_t taskId, STsdbReader** ppReader) {
int32_t code = 0; int32_t code = 0;
// STsdbReader* pReader = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId); code = tsdbReaderCreate(pVnode, pCond, qId, taskId, ppReader);
// if (pReader == NULL) { if (code) goto _err;
// return NULL;
// }
// if (emptyQueryTimewindow(pReader)) { // if (emptyQueryTimewindow(pReader)) {
// return (STsdbReader*)pReader; // return (STsdbReader*)pReader;
@ -2706,6 +2676,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInf
// taosArrayGetSize(pReader->pTableCheckInfo), taosArrayGetSize(tableList->pTableList), pReader->idStr); // taosArrayGetSize(pReader->pTableCheckInfo), taosArrayGetSize(tableList->pTableList), pReader->idStr);
return code; return code;
_err:
// tsdbError("");
return code;
} }
void tsdbReaderClose(STsdbReader* pReader) { void tsdbReaderClose(STsdbReader* pReader) {