From 272a161a610caeddab6338c746c218084ed89547 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 22 Nov 2022 22:06:39 +0800 Subject: [PATCH 1/4] fix: build data from mem/imem first when large nchar column incur fileBlockShouldLoad since SDataBlockToLoadInfo.moreThanCapacity --- source/dnode/vnode/src/tsdb/tsdbRead.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bb2de2ec93..d4f87525a5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2805,6 +2805,12 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pBlockInfo == NULL) { // build data block from last data file ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); + } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { + // data in memory that are earlier than current file block + // rows in buffer should be less than the file block in asc, greater than file block in desc + // process mem/imem data first even if fileBlockShouldLoad + int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; + code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { @@ -2813,11 +2819,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); - } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { - // data in memory that are earlier than current file block - // rows in buffer should be less than the file block in asc, greater than file block in desc - int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; - code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { // only return the rows in last block From c1ba62b26f35538e736beba8249cc9811c5305f1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 23 Nov 2022 14:52:07 +0800 Subject: [PATCH 2/4] fix: build from buf first when buf data is in block gap and only capacity caused load --- source/dnode/vnode/src/tsdb/tsdbRead.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4a12185108..297429da58 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2800,20 +2800,26 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pBlockInfo == NULL) { // build data block from last data file ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); - } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { - // data in memory that are earlier than current file block - // rows in buffer should be less than the file block in asc, greater than file block in desc - // process mem/imem data first even if fileBlockShouldLoad - int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; - code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader) && + bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { + //load only caused by outputbuf is not enough to hold all rows and buf data is in file block gap + int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; + code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); + } else { + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); + } if (code != TSDB_CODE_SUCCESS) { return code; } // build composed data block code = buildComposedDataBlock(pReader); + } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { + // data in memory that are earlier than current file block + // rows in buffer should be less than the file block in asc, greater than file block in desc + int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; + code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { // only return the rows in last block From 9a85050ff3a431e15aa611e6948b635cd4613a63 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 23 Nov 2022 15:43:57 +0800 Subject: [PATCH 3/4] fix: direct return file block only rows <= capacity --- source/dnode/vnode/src/tsdb/tsdbRead.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 297429da58..534fe89818 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2432,7 +2432,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) { + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && + pBlock->nRow <= pReader->capacity) { if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { copyBlockDataToSDataBlock(pReader, pBlockScanInfo); @@ -2801,14 +2802,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader) && - bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { - //load only caused by outputbuf is not enough to hold all rows and buf data is in file block gap - int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; - code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); - } else { - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); - } + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; } From 8b9c34504dd45f34e11032dc95fe0d3191bc80c0 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 23 Nov 2022 16:38:59 +0800 Subject: [PATCH 4/4] fix: core dump when left data block id is equal to right data block id --- source/libs/executor/src/joinoperator.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 61dc7a1b76..a1b44307d4 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -54,22 +54,26 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown SColumnNode* col2 = (SColumnNode*)pNode->pRight; SColumnNode* leftTsCol = NULL; SColumnNode* rightTsCol = NULL; - if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { - ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); + if (col1->dataBlockId == col2->dataBlockId ) { leftTsCol = col1; rightTsCol = col2; } else { - ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); - ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); - leftTsCol = col2; - rightTsCol = col1; + if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { + ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); + leftTsCol = col1; + rightTsCol = col2; + } else { + ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); + ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); + leftTsCol = col2; + rightTsCol = col1; + } } setJoinColumnInfo(&pInfo->leftCol, leftTsCol); setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } else { ASSERT(false); - } -} + }} SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) {