From cf245fa52096edbc64037870968a7b5a4ee4672a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 23 Nov 2022 12:56:16 +0800 Subject: [PATCH] enh: last cache optimize --- source/libs/planner/src/planLogicCreater.c | 86 +++++++++++++++++----- source/libs/planner/src/planOptimizer.c | 26 +++++-- 2 files changed, 87 insertions(+), 25 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 89e8a85895..dc28b2ba31 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -36,6 +36,7 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi typedef struct SRewriteExprCxt { int32_t errCode; SNodeList* pExprs; + bool* pOutputs; } SRewriteExprCxt; static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { @@ -63,14 +64,30 @@ static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { } static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { + SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext; switch (nodeType(*pNode)) { + case QUERY_NODE_COLUMN: { + if (NULL != pCxt->pOutputs) { + SNode* pExpr; + int32_t index = 0; + FOREACH(pExpr, pCxt->pExprs) { + if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) { + pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0); + } + if (nodesEqualNode(pExpr, *pNode)) { + pCxt->pOutputs[index] = true; + break; + } + } + } + break; + } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: case QUERY_NODE_CASE_WHEN: { - SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext; - SNode* pExpr; - int32_t index = 0; + SNode* pExpr; + int32_t index = 0; FOREACH(pExpr, pCxt->pExprs) { if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) { pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0); @@ -89,6 +106,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; + if (NULL != pCxt->pOutputs) { + pCxt->pOutputs[index] = true; + } return DEAL_RES_IGNORE_CHILD; } ++index; @@ -121,7 +141,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExpr(pExpr, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL}; cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); if (TSDB_CODE_SUCCESS == cxt.errCode) { nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); @@ -130,23 +150,50 @@ static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClau return cxt.errCode; } -static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { +static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** pRewriteExpr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t index = 0; + SNode* pExpr = NULL; + FOREACH(pExpr, pExprs) { + if (pOutputs[index]) { + code = nodesListMakeStrictAppend(pRewriteExpr, nodesCloneNode(pExpr)); + if (TSDB_CODE_SUCCESS != code) { + NODES_DESTORY_LIST(*pRewriteExpr); + break; + } + } + } + return code; +} + +static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, + SNodeList** pRewriteExpr) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + if (NULL != pRewriteExpr) { + cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); + if (NULL == cxt.pOutputs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); + if (TSDB_CODE_SUCCESS == cxt.errCode && NULL != pRewriteExpr) { + cxt.errCode = cloneRewriteExprs(pExprs, cxt.pOutputs, pRewriteExpr); + } + taosMemoryFree(cxt.pOutputs); return cxt.errCode; } static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } @@ -311,7 +358,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM); + code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM, NULL); } pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType, pSelect->tagScan); @@ -509,7 +556,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { - code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY, NULL); } if (NULL != pSelect->pGroupByList) { @@ -524,8 +571,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } // rewrite the expression in subsequent clauses + SNodeList* pOutputGroupKeys = NULL; if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); + code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY, &pOutputGroupKeys); } if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { @@ -536,8 +584,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } // set the output - if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) { - code = createColumnByRewriteExprs(pAgg->pGroupKeys, &pAgg->node.pTargets); + if (TSDB_CODE_SUCCESS == code && NULL != pOutputGroupKeys) { + code = createColumnByRewriteExprs(pOutputGroupKeys, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets); @@ -574,7 +622,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt // indefinite rows functions and _select_values functions int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pIdfRowsFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); + code = rewriteExprsForSelect(pIdfRowsFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT, NULL); } // set the output @@ -612,7 +660,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p // interp functions and _group_key functions int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, isInterpFunc, &pInterpFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); + code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT, NULL); } if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pFill) { @@ -656,7 +704,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); + code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW, NULL); } if (TSDB_CODE_SUCCESS == code) { @@ -854,10 +902,10 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect int32_t code = partFillExprs(pSelect, &pFill->pFillExprs, &pFill->pNotFillExprs); if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pFill->pFillExprs, pSelect, SQL_CLAUSE_FILL); + code = rewriteExprsForSelect(pFill->pFillExprs, pSelect, SQL_CLAUSE_FILL, NULL); } if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pFill->pNotFillExprs, pSelect, SQL_CLAUSE_FILL); + code = rewriteExprsForSelect(pFill->pNotFillExprs, pSelect, SQL_CLAUSE_FILL, NULL); } if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExprs(pFill->pFillExprs, &pFill->node.pTargets); @@ -1066,7 +1114,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT); + code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT, NULL); } // set the output diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 45fa67faef..b8b6e44412 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1476,19 +1476,33 @@ static bool partTagsHasIndefRowsSelectFunc(SNodeList* pFuncs) { return false; } -static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, int32_t start, SNodeList* pAggFuncs) { - bool hasIndefRowsSelectFunc = partTagsHasIndefRowsSelectFunc(pAggFuncs); +static bool partTagsNeedOutput(SNode* pExpr, SNodeList* pTargets) { + SNode* pOutput = NULL; + FOREACH(pOutput, pTargets) { + if (QUERY_NODE_COLUMN == nodeType(pExpr)) { + if (nodesEqualNode(pExpr, pOutput)) { + return true; + } + } else if (0 == strcmp(((SExprNode*)pExpr)->aliasName, ((SColumnNode*)pOutput)->colName)) { + return true; + } + } + return false; +} + +static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, int32_t start, SAggLogicNode* pAgg) { + bool hasIndefRowsSelectFunc = partTagsHasIndefRowsSelectFunc(pAgg->pAggFuncs); int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; SNode* pNode = NULL; FOREACH(pNode, pGroupTags) { - if (index++ < start) { + if (index++ < start || !partTagsNeedOutput(pNode, pAgg->node.pTargets)) { continue; } if (hasIndefRowsSelectFunc) { - code = nodesListStrictAppend(pAggFuncs, partTagsCreateWrapperFunc("_select_value", pNode)); + code = nodesListStrictAppend(pAgg->pAggFuncs, partTagsCreateWrapperFunc("_select_value", pNode)); } else { - code = nodesListStrictAppend(pAggFuncs, partTagsCreateWrapperFunc("_group_key", pNode)); + code = nodesListStrictAppend(pAgg->pAggFuncs, partTagsCreateWrapperFunc("_group_key", pNode)); } if (TSDB_CODE_SUCCESS != code) { break; @@ -1541,7 +1555,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } NODES_DESTORY_LIST(pAgg->pGroupKeys); if (TSDB_CODE_SUCCESS == code && start >= 0) { - code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg->pAggFuncs); + code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg); } } if (TSDB_CODE_SUCCESS == code) {