xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'enh/tsdb_optimize' of https://github.com/taosdata/TDengine into enh/tsdb_optimize

This commit is contained in:
Minglei Jin 2023-07-19 18:11:14 +08:00
parent 4eaecd11de 37ae80d135
commit ce71b8bbe7
4 changed files with 65 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
source/dnode/vnode/src/inc/tsdb.h
View File

@ -710,9 +710,8 @@ typedef struct {
typedef struct SSttBlockLoadInfo {
SBlockData blockData[2];
void *pBlockArray;
void *pSttStatisBlkArray;
SArray *aSttBlk;
SArray *pTombBlockArray; // tomb block array list
int32_t blockIndex[2]; // to denote the loaded block in the corresponding position.
int32_t currentLoadBlockIndex;
int32_t loadBlocks;
@ -720,10 +719,9 @@ typedef struct SSttBlockLoadInfo {
STSchema *pSchema;
int16_t *colIds;
int32_t numOfCols;
bool checkRemainingRow;
bool checkRemainingRow; // todo: no assign value?
bool isLast;
bool sttBlockLoaded;
int32_t numOfStt;
// keep the last access position, this position may be used to reduce the binary times for
// starting last block data for a new table

4
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1799,10 +1799,6 @@ static int32_t loadSttTomb(STsdbReader *pTsdbReader, SSttFileReader *pSttFileRea
SCacheRowsReader *pReader = (SCacheRowsReader *)pTsdbReader;
if (pLoadInfo->pTombBlockArray == NULL) {
pLoadInfo->pTombBlockArray = taosArrayInit(4, POINTER_BYTES);
}
const TTombBlkArray *pBlkArray = NULL;
code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray);
if (code != TSDB_CODE_SUCCESS) {

104
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -29,8 +29,6 @@ SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList,
return NULL;
}
pLoadInfo->numOfStt = numOfSttTrigger;
for (int32_t i = 0; i < numOfSttTrigger; ++i) {
pLoadInfo[i].blockIndex[0] = -1;
pLoadInfo[i].blockIndex[1] = -1;
@ -61,7 +59,6 @@ SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colLi
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pLoadInfo->numOfStt = 1;
pLoadInfo->blockIndex[0] = -1;
pLoadInfo->blockIndex[1] = -1;
@ -78,7 +75,6 @@ SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colLi
}
pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
pLoadInfo->pTombBlockArray = taosArrayInit(4, POINTER_BYTES);
pLoadInfo->pSchema = pSchema;
pLoadInfo->colIds = colList;
pLoadInfo->numOfCols = numOfCols;
@ -87,7 +83,7 @@ SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colLi
}
void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) {
for (int32_t i = 0; i < 1; ++i) {
pLoadInfo[i].currentLoadBlockIndex = 1;
pLoadInfo[i].blockIndex[0] = -1;
pLoadInfo[i].blockIndex[1] = -1;
@ -101,7 +97,7 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
}
void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el) {
for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) {
for (int32_t i = 0; i < 1; ++i) {
*el += pLoadInfo[i].elapsedTime;
*blocks += pLoadInfo[i].loadBlocks;
}
@ -118,7 +114,7 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
return NULL;
}
for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) {
for (int32_t i = 0; i < 1; ++i) {
pLoadInfo[i].currentLoadBlockIndex = 1;
pLoadInfo[i].blockIndex[0] = -1;
pLoadInfo[i].blockIndex[1] = -1;
@ -129,8 +125,6 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
taosArrayDestroy(pLoadInfo[i].aSttBlk);
}
taosArrayDestroyEx(pLoadInfo->pTombBlockArray, freeTombBlock);
taosMemoryFree(pLoadInfo);
return NULL;
}
@ -317,8 +311,8 @@ int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t
return 0;
}
static int32_t loadSttBlockInfo(SLDataIter *pIter, SSttBlockLoadInfo *pBlockLoadInfo, uint64_t suid) {
TSttBlkArray *pArray = pBlockLoadInfo->pBlockArray;
static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray, SSttBlockLoadInfo *pBlockLoadInfo,
uint64_t suid) {
if (TARRAY2_SIZE(pArray) <= 0) {
return TSDB_CODE_SUCCESS;
}
@ -358,40 +352,49 @@ static int32_t loadSttBlockInfo(SLDataIter *pIter, SSttBlockLoadInfo *pBlockLoad
return TSDB_CODE_SUCCESS;
}
static int32_t loadSttTombBlockData(SSttFileReader *pSttFileReader, uint64_t suid, SSttBlockLoadInfo *pLoadInfo) {
if (pLoadInfo->pTombBlockArray == NULL) {
pLoadInfo->pTombBlockArray = taosArrayInit(4, POINTER_BYTES);
static int32_t uidComparFn(const void* p1, const void* p2) {
const uint64_t* uid1 = p1;
const uint64_t* uid2 = p2;
return (*uid1) - (*uid2);
}
const TTombBlkArray *pBlkArray = NULL;
int32_t code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray);
if (code != TSDB_CODE_SUCCESS) {
return code;
static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint64_t suid, uint64_t uid,
SSttFileReader *pReader) {
if (TARRAY2_SIZE(pStatisBlkArray) <= 0) {
return true;
}
for (int32_t j = 0; j < pBlkArray->size; ++j) {
STombBlk *pTombBlk = &pBlkArray->data[j];
if (pTombBlk->maxTbid.suid < suid) {
continue; // todo use binary search instead
int32_t i = 0;
for (i = 0; i < TARRAY2_SIZE(pStatisBlkArray); ++i) {
SStatisBlk *p = &pStatisBlkArray->data[i];
if (p->minTbid.suid == suid) {
break;
}
}
if (pTombBlk->minTbid.suid > suid) {
for (; i < TARRAY2_SIZE(pStatisBlkArray); ++i) {
SStatisBlk *p = &pStatisBlkArray->data[i];
if (p->minTbid.uid <= uid && p->maxTbid.uid >= uid) {
break;
}
STombBlock *pTombBlock = taosMemoryCalloc(1, sizeof(STombBlock));
code = tsdbSttFileReadTombBlock(pSttFileReader, pTombBlk, pTombBlock);
if (code != TSDB_CODE_SUCCESS) {
// todo handle error
}
void *p = taosArrayPush(pLoadInfo->pTombBlockArray, &pTombBlock);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
if (p->maxTbid.uid < uid) {
break;
}
}
return TSDB_CODE_SUCCESS;
if (i >= TARRAY2_SIZE(pStatisBlkArray)) {
return false;
}
SStatisBlk *p = &pStatisBlkArray->data[i];
STbStatisBlock block = {0};
tsdbSttFileReadStatisBlock(pReader, p, &block);
int32_t index = tarray2SearchIdx(block.uid, &uid, sizeof(int64_t), uidComparFn, TD_EQ);
tStatisBlockDestroy(&block);
return (index != -1);
}
int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader, int32_t iStt, int8_t backward,
@ -412,27 +415,45 @@ int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader
if (!pBlockLoadInfo->sttBlockLoaded) {
int64_t st = taosGetTimestampUs();
const TSttBlkArray*pSttBlkArray = NULL;
pBlockLoadInfo->sttBlockLoaded = true;
code = tsdbSttFileReadSttBlk(pIter->pReader, (const TSttBlkArray **)&pBlockLoadInfo->pBlockArray);
// load the stt block info for each stt-block
code = tsdbSttFileReadSttBlk(pIter->pReader, &pSttBlkArray);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("load stt blk, code:%s, %s", tstrerror(code), idStr);
tsdbError("load stt blk failed, code:%s, %s", tstrerror(code), idStr);
return code;
}
code = loadSttBlockInfo(pIter, pBlockLoadInfo, suid);
code = extractSttBlockInfo(pIter, pSttBlkArray, pBlockLoadInfo, suid);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("load stt block info failed, code:%s, %s", tstrerror(code), idStr);
return code;
}
code = loadTombFn(pReader1, pIter->pReader, pBlockLoadInfo);
double el = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
// load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file
code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pBlockLoadInfo->pSttStatisBlkArray);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), idStr);
return code;
}
// find the start block
code = loadTombFn(pReader1, pIter->pReader, pIter->pBlockLoadInfo);
double el = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug("load the stt file info completed, elapsed time:%.2fms, %s", el, idStr);
}
bool exists = existsFromSttBlkStatis(pBlockLoadInfo->pSttStatisBlkArray, suid, uid, pIter->pReader);
if (!exists) {
pIter->iSttBlk = -1;
pIter->pSttBlk = NULL;
return TSDB_CODE_SUCCESS;
}
// find the start block, actually we could load the position to avoid repeatly searching for the start position when
// the skey is updated.
size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
if (pIter->iSttBlk != -1) {
@ -744,6 +765,7 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
goto _end;
}
// add the list/iter placeholder
while (taosArrayGetSize(pConf->pSttFileBlockIterArray) < size) {
SArray *pList = taosArrayInit(4, POINTER_BYTES);
taosArrayPush(pConf->pSttFileBlockIterArray, &pList);

4
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -590,10 +590,6 @@ int32_t loadDataFileTombDataForAll(STsdbReader* pReader) {
}
int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo) {
if (pLoadInfo->pTombBlockArray == NULL) {
pLoadInfo->pTombBlockArray = taosArrayInit(4, POINTER_BYTES);
}
const TTombBlkArray* pBlkArray = NULL;
int32_t code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray);
if (code != TSDB_CODE_SUCCESS) {