From f3d7d6671b08f916e477907300f13024c70ba669 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Jul 2022 20:06:29 +0800 Subject: [PATCH 1/4] fix(query):filter invisible columns --- source/libs/executor/src/scanoperator.c | 41 ++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e60f6f8a5b..4796f0405e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1196,7 +1196,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock if (pResCol->info.colId == pColMatchInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); colDataAssign(pDst, pResCol, pBlock->info.rows, &pInfo->pRes->info); - // taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); colExists = true; break; } @@ -1225,6 +1224,40 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock return 0; } +// todo opt perf +static SSDataBlock* doDropInvisibleCol(SSDataBlock* pBlock, SArray* pColMatchInfo) { + size_t numOfMatchInfo = taosArrayGetSize(pColMatchInfo); + + bool ignoreCols = false; + for (int32_t j = 0; j < numOfMatchInfo; ++j) { + SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, j); + if (!pInfo->output) { + ignoreCols = true; + break; + } + } + + SSDataBlock* pRes = createOneDataBlock(pBlock, false); + if (ignoreCols) { + int32_t i = 0; + while(i < taosArrayGetSize(pRes->pDataBlock)) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); + for (int32_t j = 0; j < numOfMatchInfo; ++j) { + SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, j); + if (pInfo->colId == pCol->info.colId && !pInfo->output) { + taosArrayRemove(pBlock->pDataBlock, i); + i -= 1; + break; + } + } + + i += 1; + } + } + + return pRes; +} + static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1435,8 +1468,9 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } } } + qDebug("scan rows: %d", pBlockInfo->rows); - return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + return (pBlockInfo->rows == 0) ? NULL : doDropInvisibleCol(pInfo->pRes, pInfo->pColMatchInfo); } else { ASSERT(0); @@ -1507,8 +1541,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } - SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan; - + SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan; SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; pInfo->pTagCond = pTagCond; From 484d043b4418af5e2cc9443b3645dd116c2c8d4a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Jul 2022 20:58:52 +0800 Subject: [PATCH 2/4] fix(query):filter invisible columns --- source/libs/executor/src/scanoperator.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4796f0405e..4ff0e6b936 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1237,7 +1237,7 @@ static SSDataBlock* doDropInvisibleCol(SSDataBlock* pBlock, SArray* pColMatchInf } } - SSDataBlock* pRes = createOneDataBlock(pBlock, false); + SSDataBlock* pRes = createOneDataBlock(pBlock, true); if (ignoreCols) { int32_t i = 0; while(i < taosArrayGetSize(pRes->pDataBlock)) { @@ -1255,10 +1255,11 @@ static SSDataBlock* doDropInvisibleCol(SSDataBlock* pBlock, SArray* pColMatchInf } } + pRes->info.rows = pBlock->info.rows; return pRes; } -static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { +static SSDataBlock* doStreamScanImpl(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; @@ -1470,7 +1471,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } qDebug("scan rows: %d", pBlockInfo->rows); - return (pBlockInfo->rows == 0) ? NULL : doDropInvisibleCol(pInfo->pRes, pInfo->pColMatchInfo); + return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; } else { ASSERT(0); @@ -1478,6 +1479,16 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } } +static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { + SSDataBlock* pBlock = doStreamScanImpl(pOperator); + if (pBlock != NULL) { + SStreamScanInfo* pInfo = (SStreamScanInfo*) pOperator->info; + return doDropInvisibleCol(pInfo->pRes, pInfo->pColMatchInfo); + } else { + return NULL; + } +} + static SSDataBlock* doRawScan(SOperatorInfo* pInfo) { // return NULL; From 2f199bd80b7a112b4093abb56fcbeb0592a62f92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Jul 2022 10:28:29 +0800 Subject: [PATCH 3/4] fix(query):revoke remove operation --- source/libs/executor/src/scanoperator.c | 47 +------------------------ 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4ff0e6b936..946057e0d9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1224,42 +1224,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock return 0; } -// todo opt perf -static SSDataBlock* doDropInvisibleCol(SSDataBlock* pBlock, SArray* pColMatchInfo) { - size_t numOfMatchInfo = taosArrayGetSize(pColMatchInfo); - - bool ignoreCols = false; - for (int32_t j = 0; j < numOfMatchInfo; ++j) { - SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, j); - if (!pInfo->output) { - ignoreCols = true; - break; - } - } - - SSDataBlock* pRes = createOneDataBlock(pBlock, true); - if (ignoreCols) { - int32_t i = 0; - while(i < taosArrayGetSize(pRes->pDataBlock)) { - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); - for (int32_t j = 0; j < numOfMatchInfo; ++j) { - SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, j); - if (pInfo->colId == pCol->info.colId && !pInfo->output) { - taosArrayRemove(pBlock->pDataBlock, i); - i -= 1; - break; - } - } - - i += 1; - } - } - - pRes->info.rows = pBlock->info.rows; - return pRes; -} - -static SSDataBlock* doStreamScanImpl(SOperatorInfo* pOperator) { +static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; @@ -1479,16 +1444,6 @@ static SSDataBlock* doStreamScanImpl(SOperatorInfo* pOperator) { } } -static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { - SSDataBlock* pBlock = doStreamScanImpl(pOperator); - if (pBlock != NULL) { - SStreamScanInfo* pInfo = (SStreamScanInfo*) pOperator->info; - return doDropInvisibleCol(pInfo->pRes, pInfo->pColMatchInfo); - } else { - return NULL; - } -} - static SSDataBlock* doRawScan(SOperatorInfo* pInfo) { // return NULL; From 5e754a7662e8efa77ee806eafb9b8d808e800095 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Jul 2022 14:00:45 +0800 Subject: [PATCH 4/4] fix(query): add check for deleting record --- source/dnode/vnode/src/tsdb/tsdbRead.c | 69 +++++++++++++++++++------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0b05bb7224..2801e09f06 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1453,38 +1453,71 @@ static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVer (pBlock->minVersion <= pVerRange->maxVer); } -static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { - if (pBlockScanInfo->delSkyline == NULL) { - return false; - } - TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); - TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); - - // ts is not overlap - if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) { - return false; - } - - int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; - - // version is not overlap +static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); - for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += step) { + + for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) { if (p->version >= pBlock->minVersion) { return true; } - } else if (p->ts > pBlock->maxKey.ts) { + } else if (p->ts < pBlock->minKey.ts) { // p->ts < pBlock->minKey.ts + if (p->version >= pBlock->minVersion) { + if (i < num - 1) { + TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); + if (i + 1 == num - 1) { // pnext is the last point + if (pnext->ts >= pBlock->minKey.ts) { + return true; + } + } else { + if (pnext->ts >= pBlock->minKey.ts && pnext->version >= pBlock->minVersion) { + return true; + } + } + } else { // it must be the last point + ASSERT(p->version == 0); + } + } + } else { // (p->ts > pBlock->maxKey.ts) { return false; } } - ASSERT(0); return false; } +static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { + if (pBlockScanInfo->delSkyline == NULL) { + return false; + } + + // ts is not overlap + TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); + TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); + if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) { + return false; + } + + // version is not overlap + if (ASCENDING_TRAVERSE(order)) { + return doCheckforDatablockOverlap(pBlockScanInfo, pBlock); + } else { + int32_t index = pBlockScanInfo->fileDelIndex; + while(1) { + TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index); + if (p->ts > pBlock->minKey.ts && index > 0) { + index -= 1; + } else { // find the first point that is smaller than the minKey.ts of dataBlock. + break; + } + } + + return doCheckforDatablockOverlap(pBlockScanInfo, pBlock); + } +} + // 1. the version of all rows should be less than the endVersion // 2. current block should not overlap with next neighbor block // 3. current timestamp should not be overlap with each other