From a2150031461af90ba441db5f62a44e66cea28f35 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 18 Jun 2022 15:20:30 +0800 Subject: [PATCH 01/12] feature: eliminate projection optimization --- source/libs/function/src/tudf.c | 5 ++ source/libs/planner/src/planOptimizer.c | 59 ++++++++++++++++++- source/libs/planner/test/planOptimizeTest.cpp | 6 ++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 472d672607..f6ae027e48 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -972,6 +972,11 @@ void releaseUdfFuncHandle(char* udfName) { } int32_t cleanUpUdfs() { + int8_t initialized = atomic_load_8(&gUdfdProxy.initialized); + if (!initialized) { + return TSDB_CODE_SUCCESS; + } + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); int32_t i = 0; SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index a7c25162b7..0f2abc4568 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1082,13 +1082,70 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub return code; } +static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) { + return false; + } + + SProjectLogicNode* pProjectNode = (SProjectLogicNode*)pNode; + SNode* pProjection; + FOREACH(pProjection, pProjectNode->pProjections) { + SExprNode* pExprNode = (SExprNode*)pProjection; + if (QUERY_NODE_COLUMN != nodeType(pExprNode)) { + return false; + } + } + + return true; +} + +static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SProjectLogicNode* pProjectNode) { + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0); + SNodeList* pNewChildTargets = nodesMakeList(); + + SNode* pProjection = NULL; + FOREACH(pProjection, pProjectNode->pProjections) { + SColumnNode* projColumn = (SColumnNode*)pProjection; + SNode* pChildTarget = NULL; + FOREACH(pChildTarget, pChild->pTargets) { + SExprNode* childExpr = (SExprNode*)pChildTarget; + char* projColumnName = projColumn->colName; + if (QUERY_NODE_COLUMN == nodeType(childExpr) && strcmp(projColumnName, ((SColumnNode*)childExpr)->colName) == 0 || + strcmp(projColumnName, childExpr->aliasName) == 0) { + nodesListAppend(pNewChildTargets, pChildTarget); + } + } + } + + TSWAP(pChild->pTargets, pNewChildTargets); + int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pProjectNode, pChild); + if (TSDB_CODE_SUCCESS == code) { + NODES_CLEAR_LIST(pProjectNode->node.pChildren); + nodesDestroyNode((SNode*)pProjectNode); + } + NODES_CLEAR_LIST(pNewChildTargets); + return code; +} + +static int32_t eliminateProjOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SProjectLogicNode* pProjectNode = + (SProjectLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, eliminateProjOptMayBeOptimized); + + if (NULL == pProjectNode) { + return TSDB_CODE_SUCCESS; + } + + return eliminateProjOptimizeImpl(pCxt, pLogicSubplan, pProjectNode); +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - {.pName = "PartitionByTags", .optimizeFunc = partTagsOptimize} + {.pName = "PartitionByTags", .optimizeFunc = partTagsOptimize}, + {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 84ccea668d..b9e2be4e16 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -52,3 +52,9 @@ TEST_F(PlanOptimizeTest, orderByPrimaryKey) { run("SELECT COUNT(*) FROM t1 INTERVAL(10S) ORDER BY _WSTARTTS DESC"); } + +TEST_F(PlanOptimizeTest, eliminateProjection) { + useDb("root", "test"); + + run("SELECT c1, sum(c3) FROM t1 GROUP BY c1"); +} From 683cf876760c78a866c21ed0653f60ff9e7967e6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 18 Jun 2022 16:24:29 +0800 Subject: [PATCH 02/12] fix: set parent of new logic node to null --- source/libs/planner/src/planUtil.c | 1 + source/libs/planner/test/planOptimizeTest.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 7f650c7c9a..77e4e05530 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -107,6 +107,7 @@ int32_t createColumnByRewriteExpr(SNode* pExpr, SNodeList** pList) { int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew) { if (NULL == pOld->pParent) { pSubplan->pNode = (SLogicNode*)pNew; + pNew->pParent = NULL; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index b9e2be4e16..10bdd5b21d 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -57,4 +57,6 @@ TEST_F(PlanOptimizeTest, eliminateProjection) { useDb("root", "test"); run("SELECT c1, sum(c3) FROM t1 GROUP BY c1"); + run("SELECT c1 FROM t1"); + run("SELECT * FROM st1"); } From 53a07f0c3700cd64bdf6b6ba3d920ee76c604d38 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 18 Jun 2022 17:58:15 +0800 Subject: [PATCH 03/12] feat: add test case sql --- source/libs/planner/test/planOptimizeTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 10bdd5b21d..07b3adbc1f 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -59,4 +59,5 @@ TEST_F(PlanOptimizeTest, eliminateProjection) { run("SELECT c1, sum(c3) FROM t1 GROUP BY c1"); run("SELECT c1 FROM t1"); run("SELECT * FROM st1"); + run("SELECT c1 FROM st1s3"); } From 8c402d2aa3b9aa1b586b88309abccd4f60f223c3 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 19 Jun 2022 21:39:35 +0800 Subject: [PATCH 04/12] fix: column target/output desc match error --- source/libs/executor/src/dataDispatcher.c | 10 ++++++++-- source/libs/executor/src/executil.c | 17 +++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index b7c7102143..808dd78ac3 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -75,8 +75,14 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { - int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots); - + int32_t numOfCols = 0; + SNode* pNode; + FOREACH(pNode, pHandle->pSchema->pSlots) { + SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; + if (pSlotDesc->output) { + ++numOfCols; + } + } SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = (int8_t)needCompress(pInput->pData, numOfCols); pEntry->numOfRows = pInput->pData->info.rows; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 99139be409..b319d77c17 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -193,9 +193,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {{0}}; SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); - // if (!pDescNode->output) { // todo disable it temporarily - // continue; - // } + if (!pDescNode->output) { // todo disable it temporarily + continue; + } idata.info.type = pDescNode->dataType.type; idata.info.bytes = pDescNode->dataType.bytes; @@ -319,7 +319,16 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod continue; } - SColMatchInfo* info = taosArrayGet(pList, pNode->slotId); + bool foundSource = false; + SColMatchInfo* info = NULL; + for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) { + info = taosArrayGet(pList, j); + if (info->targetSlotId == pNode->slotId) { + foundSource = true; + break; + } + } + ASSERT(foundSource); if (pNode->output) { (*numOfOutputCols) += 1; } else { From 413d26ee30b8093c34d3bd93e78acf8c91f71471 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 19 Jun 2022 22:00:57 +0800 Subject: [PATCH 05/12] fix: add column to datablock even no output --- source/libs/executor/src/executil.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b319d77c17..18fc1ff477 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -193,9 +193,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {{0}}; SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); - if (!pDescNode->output) { // todo disable it temporarily - continue; - } +// if (!pDescNode->output) { // todo disable it temporarily +// continue; +// } idata.info.type = pDescNode->dataType.type; idata.info.bytes = pDescNode->dataType.bytes; @@ -594,10 +594,10 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) { SColumnInfoData* p = taosArrayGet(pCols, i); SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, j); - if (!pmInfo->output) { - j++; - continue; - } +// if (!pmInfo->output) { +// j++; +// continue; +// } if (p->info.colId == pmInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId); From 218d455dccb9b03e7319bc1161ea42dd9302dbcc Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 19 Jun 2022 22:32:26 +0800 Subject: [PATCH 06/12] fix: elimate projection error --- source/libs/executor/src/dataDispatcher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 808dd78ac3..802f9ea5b5 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -86,7 +86,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = (int8_t)needCompress(pInput->pData, numOfCols); pEntry->numOfRows = pInput->pData->info.rows; - pEntry->numOfCols = pInput->pData->info.numOfCols; + pEntry->numOfCols = numOfCols; pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); From 8c955ef017276cb46cd419b602df02f83d5eef14 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 20 Jun 2022 07:52:29 +0800 Subject: [PATCH 07/12] fix: limit/slimit would prevent optimization projection elimination --- source/libs/executor/src/executil.c | 3 ++- source/libs/planner/src/planOptimizer.c | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 18fc1ff477..77a18028b4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -328,10 +328,11 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod break; } } - ASSERT(foundSource); + if (pNode->output) { (*numOfOutputCols) += 1; } else { + ASSERT(foundSource); info->output = false; } } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 0f2abc4568..b8c6df4b54 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1088,6 +1088,10 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { } SProjectLogicNode* pProjectNode = (SProjectLogicNode*)pNode; + if (-1 != pProjectNode->limit || -1 != pProjectNode->slimit || -1 != pProjectNode->offset || -1 != pProjectNode->soffset) { + return false; + } + SNode* pProjection; FOREACH(pProjection, pProjectNode->pProjections) { SExprNode* pExprNode = (SExprNode*)pProjection; From 9bb21ebbc2e89afef67cf87c6cd7fc05113e1211 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 20 Jun 2022 08:45:57 +0800 Subject: [PATCH 08/12] fix: double free error during plan optimize --- source/libs/planner/src/planOptimizer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index b8c6df4b54..be6186f891 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1116,18 +1116,18 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* char* projColumnName = projColumn->colName; if (QUERY_NODE_COLUMN == nodeType(childExpr) && strcmp(projColumnName, ((SColumnNode*)childExpr)->colName) == 0 || strcmp(projColumnName, childExpr->aliasName) == 0) { - nodesListAppend(pNewChildTargets, pChildTarget); + nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); } } } - - TSWAP(pChild->pTargets, pNewChildTargets); + nodesDestroyList(pChild->pTargets); + pChild->pTargets = pNewChildTargets; + int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pProjectNode, pChild); if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pProjectNode->node.pChildren); nodesDestroyNode((SNode*)pProjectNode); } - NODES_CLEAR_LIST(pNewChildTargets); return code; } From 056301fb7bf38c7a248b553a4d0087d16b3c6cfd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 20 Jun 2022 14:22:50 +0800 Subject: [PATCH 09/12] fix: disable eliminate projection when repeat proj column name --- source/libs/executor/src/executil.c | 13 +++++-------- source/libs/planner/src/planOptimizer.c | 16 +++++++++++++++- source/libs/qworker/src/qworker.c | 2 +- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 77a18028b4..b493f3222e 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -235,7 +235,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo terrno = code; return code; } else { - qDebug("sucess to get tableIds, size: %d, suid: %" PRIu64 "", (int)taosArrayGetSize(res), tableUid); + qDebug("success to get tableIds, size: %d, suid: %" PRIu64 "", (int)taosArrayGetSize(res), tableUid); } for (int i = 0; i < taosArrayGetSize(res); i++) { @@ -319,12 +319,10 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod continue; } - bool foundSource = false; SColMatchInfo* info = NULL; for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) { info = taosArrayGet(pList, j); if (info->targetSlotId == pNode->slotId) { - foundSource = true; break; } } @@ -332,7 +330,6 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod if (pNode->output) { (*numOfOutputCols) += 1; } else { - ASSERT(foundSource); info->output = false; } } @@ -595,10 +592,10 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) { SColumnInfoData* p = taosArrayGet(pCols, i); SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, j); -// if (!pmInfo->output) { -// j++; -// continue; -// } + if (!pmInfo->output) { + j++; + continue; + } if (p->info.colId == pmInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index be6186f891..02c1779224 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1092,14 +1092,27 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return false; } + SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SNode* pProjection; FOREACH(pProjection, pProjectNode->pProjections) { SExprNode* pExprNode = (SExprNode*)pProjection; if (QUERY_NODE_COLUMN != nodeType(pExprNode)) { + taosHashCleanup(pProjColNameHash); return false; } + + char* projColumnName = ((SColumnNode*)pProjection)->colName; + int32_t* pExist = taosHashGet(pProjColNameHash, projColumnName, strlen(projColumnName)); + if (NULL != pExist) { + taosHashCleanup(pProjColNameHash); + return false; + } else { + int32_t exist = 1; + taosHashPut(pProjColNameHash, projColumnName, strlen(projColumnName), &exist, sizeof(exist)); + } } + taosHashCleanup(pProjColNameHash); return true; } @@ -1110,13 +1123,14 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* SNode* pProjection = NULL; FOREACH(pProjection, pProjectNode->pProjections) { SColumnNode* projColumn = (SColumnNode*)pProjection; + char* projColumnName = projColumn->colName; SNode* pChildTarget = NULL; FOREACH(pChildTarget, pChild->pTargets) { SExprNode* childExpr = (SExprNode*)pChildTarget; - char* projColumnName = projColumn->colName; if (QUERY_NODE_COLUMN == nodeType(childExpr) && strcmp(projColumnName, ((SColumnNode*)childExpr)->colName) == 0 || strcmp(projColumnName, childExpr->aliasName) == 0) { nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); + break; } } } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 800cc4c6e5..9161f4456a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -526,7 +526,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex atomic_store_8(&ctx->taskType, taskType); atomic_store_8(&ctx->explain, explain); - /*QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);*/ + QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { From a28a275dffce45b62eb3d057796ac403f174df85 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 20 Jun 2022 16:39:19 +0800 Subject: [PATCH 10/12] fix: fix eliminate projection bugs --- source/libs/executor/inc/executil.h | 2 +- source/libs/executor/src/executil.c | 5 +++-- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 8 ++++---- source/libs/planner/test/planOptimizeTest.cpp | 1 + 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 07686893db..1117be5db0 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -114,7 +114,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); -void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); +void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b493f3222e..aadab4f22a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -585,14 +585,15 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, } // NOTE: sources columns are more than the destination SSDatablock columns. -void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols) { +// doFilter in table scan needs every column even its output is false +void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn) { size_t numOfSrcCols = taosArrayGetSize(pCols); int32_t i = 0, j = 0; while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) { SColumnInfoData* p = taosArrayGet(pCols, i); SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, j); - if (!pmInfo->output) { + if (!outputEveryColumn && !pmInfo->output) { j++; continue; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 58918667f3..902dae754c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2084,7 +2084,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo // data from mnode pRes->info.rows = numOfRows; - relocateColumnData(pRes, pColList, pBlock->pDataBlock); + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); taosArrayDestroy(pBlock->pDataBlock); taosMemoryFree(pBlock); // blockDataDestroy(pBlock); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6f0187fa53..b9d90c4cf4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -258,7 +258,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return terrno; } - relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols); + relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true); // currently only the tbname pseudo column if (pTableScanInfo->pseudoSup.numOfExprs > 0) { @@ -1469,7 +1469,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { p->info.rows = numOfRows; pInfo->pRes->info.rows = numOfRows; - relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock); + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); doFilterResult(pInfo); blockDataDestroy(p); @@ -1561,7 +1561,7 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { getPerfDbMeta(&pSysDbTableMeta, &size); p->info.rows = buildDbTableInfoBlock(p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); - relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock); + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); pInfo->pRes->info.rows = p->info.rows; blockDataDestroy(p); @@ -2042,7 +2042,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc return terrno; } - relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols); + relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true); // currently only the tbname pseudo column if (pTableScanInfo->numOfPseudoExpr > 0) { diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 07b3adbc1f..6b514ef2c3 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -60,4 +60,5 @@ TEST_F(PlanOptimizeTest, eliminateProjection) { run("SELECT c1 FROM t1"); run("SELECT * FROM st1"); run("SELECT c1 FROM st1s3"); + //run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) order by 1 nulls first"); } From c31ec1988531825630955f4d39579cee08558ec8 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 20 Jun 2022 17:01:00 +0800 Subject: [PATCH 11/12] fix: fix eliminate project error --- source/libs/planner/src/planOptimizer.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 02c1779224..cb4a967761 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1111,8 +1111,8 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { taosHashPut(pProjColNameHash, projColumnName, strlen(projColumnName), &exist, sizeof(exist)); } } - taosHashCleanup(pProjColNameHash); + return true; } @@ -1123,12 +1123,9 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* SNode* pProjection = NULL; FOREACH(pProjection, pProjectNode->pProjections) { SColumnNode* projColumn = (SColumnNode*)pProjection; - char* projColumnName = projColumn->colName; SNode* pChildTarget = NULL; FOREACH(pChildTarget, pChild->pTargets) { - SExprNode* childExpr = (SExprNode*)pChildTarget; - if (QUERY_NODE_COLUMN == nodeType(childExpr) && strcmp(projColumnName, ((SColumnNode*)childExpr)->colName) == 0 || - strcmp(projColumnName, childExpr->aliasName) == 0) { + if (strcmp(projColumn->colName, ((SColumnNode*)pChildTarget)->colName) == 0) { nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); break; } From 56b5ff03e64a912c0e444bab87dee115c1c4e9d6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 20 Jun 2022 17:21:12 +0800 Subject: [PATCH 12/12] fix: disable project elimination when the project node has parent --- source/libs/planner/src/planOptimizer.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index cb4a967761..5ed49f6aae 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1083,6 +1083,11 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { + //TODO: enable this optimization after new mechanising that map projection and targets of project node + if (NULL != pNode->pParent) { + return false; + } + if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) { return false; } @@ -1122,10 +1127,9 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* SNode* pProjection = NULL; FOREACH(pProjection, pProjectNode->pProjections) { - SColumnNode* projColumn = (SColumnNode*)pProjection; SNode* pChildTarget = NULL; FOREACH(pChildTarget, pChild->pTargets) { - if (strcmp(projColumn->colName, ((SColumnNode*)pChildTarget)->colName) == 0) { + if (strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName) == 0) { nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); break; }