xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

tsdb/cache: relayout cache row reader

This commit is contained in:
Minglei Jin 2023-07-17 14:16:54 +08:00
parent 8263c49c17
commit 9b0b4061e0
5 changed files with 161 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
source/dnode/vnode/src/inc/tsdb.h
View File

@ -833,9 +833,25 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el); void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el);
// tsdbCache ============================================================================================== // tsdbCache ==============================================================================================
typedef enum {
READ_MODE_COUNT_ONLY = 0x1,
READ_MODE_ALL,
} EReadMode;
typedef struct STsdbReaderInfo {
uint64_t suid;
STSchema *pSchema;
EReadMode readMode;
uint64_t rowsNum;
STimeWindow window;
SVersionRange verRange;
int16_t order;
} STsdbReaderInfo;
typedef struct SCacheRowsReader { typedef struct SCacheRowsReader {
STsdb *pTsdb; STsdb *pTsdb;
SVersionRange verRange; STsdbReaderInfo info;
// SVersionRange verRange;
TdThreadMutex readerMutex; TdThreadMutex readerMutex;
SVnode *pVnode; SVnode *pVnode;
STSchema *pSchema; STSchema *pSchema;

66
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1720,25 +1720,10 @@ typedef struct {
SMergeTree *pMergeTree; SMergeTree *pMergeTree;
} SFSLastIter; } SFSLastIter;
static int32_t lastIterClose(SFSLastIter **iter) { static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) {
int32_t code = 0; int32_t code = 0;
if ((*iter)->pMergeTree) {
tMergeTreeClose((*iter)->pMergeTree);
(*iter)->pMergeTree = NULL;
}
*iter = NULL;
return code;
}
static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, STFileSet *pFileSet, int16_t *aCols,
int nCols) {
int32_t code = 0;
if (!iter->pMergeTree) {
int64_t loadBlocks = 0; int64_t loadBlocks = 0;
double elapse = 0; double elapse = 0;
pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, &loadBlocks, &elapse); pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, &loadBlocks, &elapse);
@ -1767,9 +1752,26 @@ static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow, STsdb *pTsdb, ST
} }
iter->pMergeTree = &iter->mergeTree; iter->pMergeTree = &iter->mergeTree;
return code;
} }
// retrieve next row static int32_t lastIterClose(SFSLastIter **iter) {
int32_t code = 0;
if ((*iter)->pMergeTree) {
tMergeTreeClose((*iter)->pMergeTree);
(*iter)->pMergeTree = NULL;
}
*iter = NULL;
return code;
}
static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow) {
int32_t code = 0;
bool hasVal = tMergeTreeNext(iter->pMergeTree); bool hasVal = tMergeTreeNext(iter->pMergeTree);
if (!hasVal) { if (!hasVal) {
*ppRow = NULL; *ppRow = NULL;
@ -1823,6 +1825,17 @@ typedef struct SFSNextRowIter {
struct CacheNextRowIter *pRowIter; struct CacheNextRowIter *pRowIter;
} SFSNextRowIter; } SFSNextRowIter;
static void clearLastFileSet(SFSNextRowIter *state) {
if (state->pLastIter) {
lastIterClose(&state->pLastIter);
}
if (state->pBlockData) {
tBlockDataDestroy(state->pBlockData);
state->pBlockData = NULL;
}
}
static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) { int nCols) {
SFSNextRowIter *state = (SFSNextRowIter *)iter; SFSNextRowIter *state = (SFSNextRowIter *)iter;
@ -1837,8 +1850,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
if (SFSNEXTROW_FILESET == state->state) { if (SFSNEXTROW_FILESET == state->state) {
_next_fileset: _next_fileset:
if (--state->iFileSet < 0) { // no fileset left, cleanup and return NULL row if (--state->iFileSet < 0) {
clearLastFileSet(state);
*ppRow = NULL;
return code;
} else { } else {
state->pFileSet = TARRAY2_GET(state->aDFileSet, state->iFileSet); state->pFileSet = TARRAY2_GET(state->aDFileSet, state->iFileSet);
} }
@ -1872,9 +1888,16 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
goto _err; goto _err;
} }
// TODO: load tomb data from data and sttt code = lastIterOpen(&state->lastIter, state->pFileSet, state->pTsdb, state->pTSchema, state->suid, state->uid,
state->pr, state->lastTs, aCols, nCols);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
state->pLastIter = &state->lastIter; state->pLastIter = &state->lastIter;
// TODO: load tomb data from data and stt
if (!state->pIndexList) { if (!state->pIndexList) {
state->pIndexList = taosArrayInit(1, sizeof(SBrinBlk)); state->pIndexList = taosArrayInit(1, sizeof(SBrinBlk));
} else { } else {
@ -1990,8 +2013,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
if (!state->pLastRow) { if (!state->pLastRow) {
// get next row from fslast and process with fs row, --state->Row if select fs row // get next row from fslast and process with fs row, --state->Row if select fs row
code = lastIterNext(&state->lastIter, &state->pLastRow, state->pTsdb, state->pTSchema, state->suid, state->uid, code = lastIterNext(&state->lastIter, &state->pLastRow);
state->pr, state->lastTs, state->pFileSet, aCols, nCols);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _err; goto _err;
} }

2
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -143,7 +143,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
p->type = type; p->type = type;
p->pVnode = pVnode; p->pVnode = pVnode;
p->pTsdb = p->pVnode->pTsdb; p->pTsdb = p->pVnode->pTsdb;
p->verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; p->info.verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX};
p->numOfCols = numOfCols; p->numOfCols = numOfCols;
p->pCidList = pCidList; p->pCidList = pCidList;
p->pSlotIds = pSlotIds; p->pSlotIds = pSlotIds;

48
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -30,12 +30,12 @@ typedef enum {
EXTERNAL_ROWS_MAIN = 0x2, EXTERNAL_ROWS_MAIN = 0x2,
EXTERNAL_ROWS_NEXT = 0x3, EXTERNAL_ROWS_NEXT = 0x3,
} EContentData; } EContentData;
/*
typedef enum { typedef enum {
READ_MODE_COUNT_ONLY = 0x1, READ_MODE_COUNT_ONLY = 0x1,
READ_MODE_ALL, READ_MODE_ALL,
} EReadMode; } EReadMode;
*/
typedef struct { typedef struct {
STbDataIter* iter; STbDataIter* iter;
int32_t index; int32_t index;
@ -743,7 +743,8 @@ void tsdbReleaseDataBlock(STsdbReader* pReader) {
} }
} }
static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, SQueryTableDataCond* pCond) { static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock,
SQueryTableDataCond* pCond) {
pResBlockInfo->capacity = capacity; pResBlockInfo->capacity = capacity;
pResBlockInfo->pResBlock = pResBlock; pResBlockInfo->pResBlock = pResBlock;
terrno = 0; terrno = 0;
@ -937,7 +938,8 @@ static void cleanupTableScanInfo(SReaderStatus* pStatus) {
pStatus->mapDataCleaned = true; pStatus->mapDataCleaned = true;
} }
static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, SArray* pTableScanInfoList) { static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum,
SArray* pTableScanInfoList) {
size_t sizeInDisk = 0; size_t sizeInDisk = 0;
size_t numOfTables = taosArrayGetSize(pIndexList); size_t numOfTables = taosArrayGetSize(pIndexList);
@ -1562,7 +1564,8 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks,
SArray* pTableList) {
bool asc = ASCENDING_TRAVERSE(pReader->order); bool asc = ASCENDING_TRAVERSE(pReader->order);
SBlockOrderSupporter sup = {0}; SBlockOrderSupporter sup = {0};
@ -1973,7 +1976,8 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
pLastBlockReader->currentKey = key; pLastBlockReader->currentKey = key;
pScanInfo->lastKeyInStt = key; pScanInfo->lastKeyInStt = key;
if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order,
pVerRange)) {
return true; return true;
} }
} }
@ -2220,7 +2224,8 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
// create local variable to hold the row value // create local variable to hold the row value
TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData};
tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, pReader->idStr); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid,
pReader->idStr);
// only last block exists // only last block exists
if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
@ -2240,7 +2245,8 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
tsdbRowMergerAdd(pMerger, pRow1, NULL); tsdbRowMergerAdd(pMerger, pRow1, NULL);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange,
pReader->idStr);
code = tsdbRowMergerGetRow(pMerger, &pTSRow); code = tsdbRowMergerGetRow(pMerger, &pTSRow);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -2874,7 +2880,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
} }
} else { // file blocks not exist } else { // file blocks not exist
pBlockScanInfo = *pReader->status.pTableIter; pBlockScanInfo = *pReader->status.pTableIter;
if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { if (pReader->pIgnoreTables &&
taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) {
setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
return code; return code;
} }
@ -3261,7 +3268,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
return terrno; return terrno;
} }
initLastBlockReader(pLastBlockReader, pScanInfo, pReader); initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader);
@ -3493,7 +3499,8 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
} }
STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter;
if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { if (pReader->pIgnoreTables &&
taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) {
bool hasNexTable = moveToNextTable(pUidList, pStatus); bool hasNexTable = moveToNextTable(pUidList, pStatus);
if (!hasNexTable) { if (!hasNexTable) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -3737,13 +3744,14 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, const
endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion; endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion;
} }
tsdbDebug("queried verRange:%"PRId64"-%"PRId64", revised query verRange:%"PRId64"-%"PRId64", %s", pCond->startVersion, tsdbDebug("queried verRange:%" PRId64 "-%" PRId64 ", revised query verRange:%" PRId64 "-%" PRId64 ", %s",
pCond->endVersion, startVer, endVer, id); pCond->startVersion, pCond->endVersion, startVer, endVer, id);
return (SVersionRange){.minVer = startVer, .maxVer = endVer}; return (SVersionRange){.minVer = startVer, .maxVer = endVer};
} }
bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange) { bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order,
SVersionRange* pVerRange) {
if (pDelList == NULL) { if (pDelList == NULL) {
return false; return false;
} }
@ -3761,8 +3769,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
return false; return false;
} else if (key == last->ts) { } else if (key == last->ts) {
TSDBKEY* prev = taosArrayGet(pDelList, num - 2); TSDBKEY* prev = taosArrayGet(pDelList, num - 2);
return (prev->version >= ver && prev->version <= pVerRange->maxVer && return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer);
prev->version >= pVerRange->minVer);
} }
} else { } else {
TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pCurrent = taosArrayGet(pDelList, *index);
@ -4125,8 +4132,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p
} }
tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema);
code = code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader);
doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -4458,7 +4464,8 @@ static void freeSchemaFunc(void* param) {
// ====================================== EXPOSED APIs ====================================== // ====================================== EXPOSED APIs ======================================
int32_t tsdbReaderOpen(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, int32_t tsdbReaderOpen(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, SHashObj** pIgnoreTables) { SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly,
SHashObj** pIgnoreTables) {
STimeWindow window = pCond->twindows; STimeWindow window = pCond->twindows;
SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); SVnodeCfg* pConf = &(((SVnode*)pVnode)->config);
@ -4950,7 +4957,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) {
*hasNext = false; *hasNext = false;
if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT || pReader->code != TSDB_CODE_SUCCESS) { if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT ||
pReader->code != TSDB_CODE_SUCCESS) {
return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code;
} }

15
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -30,11 +30,6 @@ typedef enum {
READER_STATUS_NORMAL = 0x2, READER_STATUS_NORMAL = 0x2,
} EReaderStatus; } EReaderStatus;
typedef enum {
READ_MODE_COUNT_ONLY = 0x1,
READ_MODE_ALL,
} EReadMode;
typedef enum { typedef enum {
EXTERNAL_ROWS_PREV = 0x1, EXTERNAL_ROWS_PREV = 0x1,
EXTERNAL_ROWS_MAIN = 0x2, EXTERNAL_ROWS_MAIN = 0x2,
@ -69,16 +64,6 @@ typedef struct STableBlockScanInfo {
bool iterInit; // whether to initialize the in-memory skip list iterator or not bool iterInit; // whether to initialize the in-memory skip list iterator or not
} STableBlockScanInfo; } STableBlockScanInfo;
typedef struct STsdbReaderInfo {
uint64_t suid;
STSchema* pSchema;
EReadMode readMode;
uint64_t rowsNum;
STimeWindow window;
SVersionRange verRange;
int16_t order;
} STsdbReaderInfo;
typedef struct SResultBlockInfo { typedef struct SResultBlockInfo {
SSDataBlock* pResBlock; SSDataBlock* pResBlock;
bool freeBlock; bool freeBlock;