From e7b97cc08654da001039ded31ca639c9dd6d1476 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 6 Feb 2023 15:59:47 +0800 Subject: [PATCH 1/3] fix(query): set the correct timestamp value in case that only imem has data. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 45 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 15432b01c8..6532a99746 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1552,8 +1552,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock // log the reason why load the datablock for profile if (loadDataBlock) { - tsdbDebug("%p uid:%" PRIu64 - " need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, " + tsdbDebug("%p uid:%" PRIu64 " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, " "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s", pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired, info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock, @@ -2595,21 +2594,37 @@ _err: } TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { - TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; + bool asc = ASCENDING_TRAVERSE(pReader->order); + TSKEY initialVal = asc? TSKEY_MIN:TSKEY_MAX; + + TSDBKEY key = {.ts = initialVal}, ikey = {.ts = initialVal}; + + bool hasKey = false, hasIKey = false; TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { + hasKey = true; key = TSDBROW_KEY(pRow); } - pRow = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader); - if (pRow != NULL) { - TSDBKEY k = TSDBROW_KEY(pRow); - if (key.ts > k.ts) { - key = k; - } + TSDBROW* pIRow = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader); + if (pIRow != NULL) { + hasIKey = true; + ikey = TSDBROW_KEY(pIRow); + } + + if (hasKey) { + if (hasIKey) { // has data in mem & imem + if (asc) { + return key.ts <= ikey.ts ? key : ikey; + } else { + return key.ts <= ikey.ts ? ikey: key; + } + } else { // no data in imem + return key; + } + } else { // no data in mem & imem, return the initial value + return hasIKey? ikey:key; } - - return key; } static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { @@ -3997,11 +4012,9 @@ void tsdbReaderClose(STsdbReader* pReader) { } tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, fileBlocks:%" PRId64 - ", fileBlocks-load-time:%.2f ms, " - "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 - ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 - ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, creatTime:%.2f ms, %s", + " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-load-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms, %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, pCost->buildComposedBlockTime, From 08b3ce7cc06def2e45d26e6ed3e680661a69ac90 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 6 Feb 2023 16:08:41 +0800 Subject: [PATCH 2/3] fix(query): remove timing code. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index aa5d8fa3a4..c3f5e0f3c5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -79,8 +79,8 @@ typedef struct SIOCostSummary { int64_t composedBlocks; double buildComposedBlockTime; double createScanInfoList; - double getTbFromMemTime; - double getTbFromIMemTime; +// double getTbFromMemTime; +// double getTbFromIMemTime; double initDelSkylineIterTime; } SIOCostSummary; @@ -2187,9 +2187,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STbData* d = NULL; if (pReader->pReadSnap->pMem != NULL) { - st = taosGetTimestampUs(); d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); - pReader->cost.getTbFromMemTime += (taosGetTimestampUs() - st) / 1000.0; if (d != NULL) { code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); if (code == TSDB_CODE_SUCCESS) { @@ -2210,9 +2208,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STbData* di = NULL; if (pReader->pReadSnap->pIMem != NULL) { - st = taosGetTimestampUs(); di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); - pReader->cost.getTbFromIMemTime += (taosGetTimestampUs() - st) / 1000.0; if (di != NULL) { code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); if (code == TSDB_CODE_SUCCESS) { @@ -4030,12 +4026,11 @@ void tsdbReaderClose(STsdbReader* pReader) { " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-load-time:%.2f ms, " "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 - ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, creatTime:%.2f ms," - "getTbFromMem-time:%.2f ms, getTbFromIMem-time:%.2f ms, initDelSkylineIterTime:%.2f ms, %s", + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,initDelSkylineIterTime:%.2f ms, %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList, - pCost->getTbFromMemTime, pCost->getTbFromIMemTime, pCost->initDelSkylineIterTime, pReader->idStr); + pCost->initDelSkylineIterTime, pReader->idStr); taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); From 5e2bb7527508f4f9d9ea7bef1a8e85176f599b55 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 6 Feb 2023 17:56:45 +0800 Subject: [PATCH 3/3] fix(query):set correct initial key value for mem/imem data. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c3f5e0f3c5..717b76046a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2604,9 +2604,9 @@ _err: TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pReader->order); - TSKEY initialVal = asc? TSKEY_MIN:TSKEY_MAX; +// TSKEY initialVal = asc? TSKEY_MIN:TSKEY_MAX; - TSDBKEY key = {.ts = initialVal}, ikey = {.ts = initialVal}; + TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL}; bool hasKey = false, hasIKey = false; TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); @@ -2631,8 +2631,10 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) } else { // no data in imem return key; } - } else { // no data in mem & imem, return the initial value - return hasIKey? ikey:key; + } else { + // no data in mem & imem, return the initial value + // only imem has data, return ikey + return ikey; } } @@ -2862,21 +2864,14 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { ASSERT(pBlockInfo != NULL); - // if (pBlockInfo != NULL) { pScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); - // } else { - // pScanInfo = *pReader->status.pTableIter; - // } - if (pScanInfo == NULL) { tsdbError("failed to get table scan-info, %s", pReader->idStr); code = TSDB_CODE_INVALID_PARA; return code; } - // if (pBlockInfo != NULL) { pBlock = getCurrentBlock(pBlockIter); - // } initLastBlockReader(pLastBlockReader, pScanInfo, pReader); TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader);