From c5e094efd0fd053914cbc8d1954c0e1d8728889a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 9 Oct 2022 14:34:10 +0800 Subject: [PATCH 1/2] enh(query): optimize the perf for ordinary table query with data in last file. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 23 ++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 7e6a0d04ff..ef1650a8dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -235,6 +235,10 @@ static int32_t binarySearchForStartRowIndex(uint64_t* uidList, int32_t num, uint } } +static bool queryChildTable(uint64_t suid) { + return suid != 0; +} + int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo) { int32_t code = 0; @@ -252,15 +256,20 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t (*pIter)->timeWindow = *pTimeWindow; (*pIter)->pBlockLoadInfo = pBlockLoadInfo; - if (taosArrayGetSize(pBlockLoadInfo->aSttBlk) == 0) { + + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + if (size == 0) { code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); if (code) { goto _exit; - } else { - size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); - SArray* pTmp = taosArrayInit(size, sizeof(SSttBlk)); - for(int32_t i = 0; i < size; ++i) { - SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + } + + // only apply to the child tables, ordinary tables will not incur this filter procedure. + if (queryChildTable(suid)) { + size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + SArray *pTmp = taosArrayInit(size, sizeof(SSttBlk)); + for (int32_t i = 0; i < size; ++i) { + SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); if (p->suid == suid) { taosArrayPush(pTmp, p); } @@ -271,7 +280,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t } } - size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); // find the start block (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); From 5402f6bc2663235e454fce5829dc9b0fe063bce3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 9 Oct 2022 15:25:42 +0800 Subject: [PATCH 2/2] enh(query): opt query perf when handling the query of last files. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index ef1650a8dc..cd78e95677 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -240,7 +240,8 @@ static bool queryChildTable(uint64_t suid) { } int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, - uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo) { + uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo, + const char* idStr) { int32_t code = 0; *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); if (*pIter == NULL) { @@ -259,6 +260,8 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); if (size == 0) { + int64_t st = taosGetTimestampUs(); + code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); if (code) { goto _exit; @@ -278,6 +281,9 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t taosArrayDestroy(pBlockLoadInfo->aSttBlk); pBlockLoadInfo->aSttBlk = pTmp; } + + double el = (taosGetTimestampUs() - st)/1000.0; + tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); } size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); @@ -502,7 +508,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter* pIter = NULL; - code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pMTree->pLoadInfo[i]); + code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pMTree->pLoadInfo[i], pMTree->idStr); if (code != TSDB_CODE_SUCCESS) { goto _end; }