From 927464113bcb6e54b3bdf2ee15d53212ee83dcc6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 18 Jun 2022 20:15:47 +0800 Subject: [PATCH] feat: tag condition split --- include/libs/nodes/plannodes.h | 32 +++++----- include/libs/nodes/querynodes.h | 13 ++-- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 11 +++- source/libs/nodes/src/nodesUtilFuncs.c | 60 ++++++++++++++++--- source/libs/parser/src/parTranslater.c | 4 +- source/libs/planner/src/planLogicCreater.c | 4 +- source/libs/planner/src/planOptimizer.c | 34 +---------- source/libs/planner/src/planPhysiCreater.c | 9 +++ source/libs/planner/test/planOptimizeTest.cpp | 2 + 10 files changed, 104 insertions(+), 66 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index a9002b5d19..ac37b0c44f 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -64,6 +64,7 @@ typedef struct SScanLogicNode { int8_t intervalUnit; int8_t slidingUnit; SNode* pTagCond; + SNode* pTagIndexCond; int8_t triggerType; int64_t watermark; int16_t tsColId; @@ -143,21 +144,21 @@ typedef enum EWindowAlgorithm { } EWindowAlgorithm; typedef struct SWindowLogicNode { - SLogicNode node; - EWindowType winType; - SNodeList* pFuncs; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - int64_t sessionGap; - SNode* pTspk; - SNode* pStateExpr; - int8_t triggerType; - int64_t watermark; - double filesFactor; - EWindowAlgorithm windowAlgo; + SLogicNode node; + EWindowType winType; + SNodeList* pFuncs; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + int64_t sessionGap; + SNode* pTspk; + SNode* pStateExpr; + int8_t triggerType; + int64_t watermark; + double filesFactor; + EWindowAlgorithm windowAlgo; } SWindowLogicNode; typedef struct SFillLogicNode { @@ -438,6 +439,7 @@ typedef struct SSubplan { SPhysiNode* pNode; // physical plan of current subplan SDataSinkNode* pDataSink; // data of the subplan flow into the datasink SNode* pTagCond; + SNode* pTagIndexCond; } SSubplan; typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 3a0d3178a3..78c6f3467e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -283,11 +283,11 @@ typedef enum ESqlClause { } ESqlClause; typedef struct SDeleteStmt { - ENodeType type; // QUERY_NODE_DELETE_STMT - SNode* pFromTable; // FROM clause - SNode* pWhere; // WHERE clause - SNode* pCountFunc; // count the number of rows affected - SNode* pTagIndexCond; // pWhere divided into pTagIndexCond and timeRange + ENodeType type; // QUERY_NODE_DELETE_STMT + SNode* pFromTable; // FROM clause + SNode* pWhere; // WHERE clause + SNode* pCountFunc; // count the number of rows affected + SNode* pTagCond; // pWhere divided into pTagCond and timeRange STimeWindow timeRange; uint8_t precision; bool deleteZeroRows; @@ -391,7 +391,8 @@ void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal); char* nodesGetFillModeString(EFillMode mode); int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc); -int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond); +int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond, + SNode** pOtherCond); #ifdef __cplusplus } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index d0eb1548c1..02c6c582e3 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -351,6 +351,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(intervalUnit); COPY_SCALAR_FIELD(slidingUnit); CLONE_NODE_FIELD(pTagCond); + CLONE_NODE_FIELD(pTagIndexCond); COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(tsColId); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 1b2eadd8e8..3d5e018dfa 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2193,6 +2193,7 @@ static const char* jkSubplanNodeAddr = "NodeAddr"; static const char* jkSubplanRootNode = "RootNode"; static const char* jkSubplanDataSink = "DataSink"; static const char* jkSubplanTagCond = "TagCond"; +static const char* jkSubplanTagIndexCond = "TagIndexCond"; static int32_t subplanToJson(const void* pObj, SJson* pJson) { const SSubplan* pNode = (const SSubplan*)pObj; @@ -2222,6 +2223,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSubplanTagCond, nodeToJson, pNode->pTagCond); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSubplanTagIndexCond, nodeToJson, pNode->pTagIndexCond); + } return code; } @@ -2255,6 +2259,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkSubplanTagCond, (SNode**)&pNode->pTagCond); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSubplanTagIndexCond, (SNode**)&pNode->pTagIndexCond); + } return code; } @@ -3821,7 +3828,7 @@ static int32_t deleteStmtToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkDeleteStmtCountFunc, nodeToJson, pNode->pCountFunc); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagIndexCond); + code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeStartKey, pNode->timeRange.skey); @@ -3850,7 +3857,7 @@ static int32_t jsonToDeleteStmt(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkDeleteStmtCountFunc, &pNode->pCountFunc); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagIndexCond); + code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeStartKey, &pNode->timeRange.skey); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 6ebd0f2d39..b0c2aa7407 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -646,7 +646,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pFromTable); nodesDestroyNode(pStmt->pWhere); nodesDestroyNode(pStmt->pCountFunc); - nodesDestroyNode(pStmt->pTagIndexCond); + nodesDestroyNode(pStmt->pTagCond); break; } case QUERY_NODE_QUERY: { @@ -665,7 +665,13 @@ void nodesDestroyNode(SNode* pNode) { SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pScanCols); + nodesDestroyList(pLogicNode->pScanPseudoCols); taosMemoryFreeClear(pLogicNode->pVgroupList); + nodesDestroyList(pLogicNode->pDynamicScanFuncs); + nodesDestroyNode(pLogicNode->pTagCond); + nodesDestroyNode(pLogicNode->pTagIndexCond); + taosArrayDestroy(pLogicNode->pSmaIndexes); + nodesDestroyList(pLogicNode->pPartTags); break; } case QUERY_NODE_LOGIC_PLAN_JOIN: { @@ -858,6 +864,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(pSubplan->pChildren); nodesDestroyNode((SNode*)pSubplan->pNode); nodesDestroyNode((SNode*)pSubplan->pDataSink); + nodesDestroyNode((SNode*)pSubplan->pTagCond); + nodesDestroyNode((SNode*)pSubplan->pTagIndexCond); nodesClearList(pSubplan->pParents); break; } @@ -1609,6 +1617,7 @@ int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc) { typedef struct SClassifyConditionCxt { bool hasPrimaryKey; bool hasTagIndexCol; + bool hasTagCol; bool hasOtherCol; } SClassifyConditionCxt; @@ -1620,6 +1629,9 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) { pCxt->hasPrimaryKey = true; } else if (pCol->hasIndex) { pCxt->hasTagIndexCol = true; + pCxt->hasTagCol = true; + } else if (COLUMN_TYPE_TAG == pCol->colType) { + pCxt->hasTagCol = true; } else { pCxt->hasOtherCol = true; } @@ -1628,23 +1640,31 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -typedef enum EConditionType { COND_TYPE_PRIMARY_KEY = 1, COND_TYPE_TAG_INDEX, COND_TYPE_NORMAL } EConditionType; +typedef enum EConditionType { + COND_TYPE_PRIMARY_KEY = 1, + COND_TYPE_TAG_INDEX, + COND_TYPE_TAG, + COND_TYPE_NORMAL +} EConditionType; static EConditionType classifyCondition(SNode* pNode) { SClassifyConditionCxt cxt = {.hasPrimaryKey = false, .hasTagIndexCol = false, .hasOtherCol = false}; nodesWalkExpr(pNode, classifyConditionImpl, &cxt); return cxt.hasOtherCol ? COND_TYPE_NORMAL - : (cxt.hasPrimaryKey && cxt.hasTagIndexCol + : (cxt.hasPrimaryKey && cxt.hasTagCol ? COND_TYPE_NORMAL - : (cxt.hasPrimaryKey ? COND_TYPE_PRIMARY_KEY : COND_TYPE_TAG_INDEX)); + : (cxt.hasPrimaryKey ? COND_TYPE_PRIMARY_KEY + : (cxt.hasTagIndexCol ? COND_TYPE_TAG_INDEX : COND_TYPE_TAG))); } -static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond) { +static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond, + SNode** pOtherCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(*pCondition); int32_t code = TSDB_CODE_SUCCESS; SNodeList* pPrimaryKeyConds = NULL; + SNodeList* pTagIndexConds = NULL; SNodeList* pTagConds = NULL; SNodeList* pOtherConds = NULL; SNode* pCond = NULL; @@ -1656,6 +1676,14 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S } break; case COND_TYPE_TAG_INDEX: + if (NULL != pTagIndexCond) { + code = nodesListMakeAppend(&pTagIndexConds, nodesCloneNode(pCond)); + } + if (NULL != pTagCond) { + code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond)); + } + break; + case COND_TYPE_TAG: if (NULL != pTagCond) { code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond)); } @@ -1673,11 +1701,15 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S } SNode* pTempPrimaryKeyCond = NULL; + SNode* pTempTagIndexCond = NULL; SNode* pTempTagCond = NULL; SNode* pTempOtherCond = NULL; if (TSDB_CODE_SUCCESS == code) { code = nodesMergeConds(&pTempPrimaryKeyCond, &pPrimaryKeyConds); } + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagIndexCond, &pTagIndexConds); + } if (TSDB_CODE_SUCCESS == code) { code = nodesMergeConds(&pTempTagCond, &pTagConds); } @@ -1689,6 +1721,9 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S if (NULL != pPrimaryKeyCond) { *pPrimaryKeyCond = pTempPrimaryKeyCond; } + if (NULL != pTagIndexCond) { + *pTagIndexCond = pTempTagIndexCond; + } if (NULL != pTagCond) { *pTagCond = pTempTagCond; } @@ -1699,9 +1734,11 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S *pCondition = NULL; } else { nodesDestroyList(pPrimaryKeyConds); + nodesDestroyList(pTagIndexConds); nodesDestroyList(pTagConds); nodesDestroyList(pOtherConds); nodesDestroyNode(pTempPrimaryKeyCond); + nodesDestroyNode(pTempTagIndexCond); nodesDestroyNode(pTempTagCond); nodesDestroyNode(pTempOtherCond); } @@ -1709,10 +1746,11 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S return code; } -int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond) { +int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond, + SNode** pOtherCond) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCondition) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pCondition)->condType) { - return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagCond, pOtherCond); + return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagIndexCond, pTagCond, pOtherCond); } switch (classifyCondition(*pCondition)) { @@ -1722,6 +1760,14 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** } break; case COND_TYPE_TAG_INDEX: + if (NULL != pTagIndexCond) { + *pTagIndexCond = *pCondition; + } + if (NULL != pTagCond) { + *pTagCond = *pCondition; + } + break; + case COND_TYPE_TAG: if (NULL != pTagCond) { *pTagCond = *pCondition; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3cb885c8e1..66990a8e41 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1917,7 +1917,7 @@ static int32_t getFillTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWin } SNode* pPrimaryKeyCond = NULL; - nodesPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL); + nodesPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL, NULL); int32_t code = TSDB_CODE_SUCCESS; if (NULL != pPrimaryKeyCond) { @@ -2503,7 +2503,7 @@ static int32_t partitionDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelet SNode* pPrimaryKeyCond = NULL; SNode* pOtherCond = NULL; - int32_t code = nodesPartitionCond(&pDelete->pWhere, &pPrimaryKeyCond, &pDelete->pTagIndexCond, &pOtherCond); + int32_t code = nodesPartitionCond(&pDelete->pWhere, &pPrimaryKeyCond, NULL, &pDelete->pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pOtherCond) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DELETE_WHERE); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 3cd5eeb655..7fad7416cd 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1033,8 +1033,8 @@ static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p } } - if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagIndexCond) { - pScan->pTagCond = nodesCloneNode(pDelete->pTagIndexCond); + if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagCond) { + pScan->pTagCond = nodesCloneNode(pDelete->pTagCond); if (NULL == pScan->pTagCond) { code = TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index a7c25162b7..3a2cbd090c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -15,7 +15,6 @@ #include "filter.h" #include "functionMgt.h" -#include "index.h" #include "planInt.h" #include "ttime.h" @@ -307,32 +306,6 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, return code; } -static int32_t cpdApplyTagIndex(SScanLogicNode* pScan, SNode** pTagCond, SNode** pOtherCond) { - int32_t code = TSDB_CODE_SUCCESS; - SIdxFltStatus idxStatus = idxGetFltStatus(*pTagCond); - switch (idxStatus) { - case SFLT_NOT_INDEX: - code = cpdCondAppend(pOtherCond, pTagCond); - break; - case SFLT_COARSE_INDEX: - pScan->pTagCond = nodesCloneNode(*pTagCond); - if (NULL == pScan->pTagCond) { - code = TSDB_CODE_OUT_OF_MEMORY; - break; - } - code = cpdCondAppend(pOtherCond, pTagCond); - break; - case SFLT_ACCURATE_INDEX: - pScan->pTagCond = *pTagCond; - *pTagCond = NULL; - break; - default: - code = TSDB_CODE_FAILED; - break; - } - return code; -} - static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || TSDB_SYSTEM_TABLE == pScan->tableType) { @@ -340,15 +313,12 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* } SNode* pPrimaryKeyCond = NULL; - SNode* pTagCond = NULL; SNode* pOtherCond = NULL; - int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pTagCond, &pOtherCond); + int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond, + &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) { code = cpdCalcTimeRange(pScan, &pPrimaryKeyCond, &pOtherCond); } - if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) { - code = cpdApplyTagIndex(pScan, &pTagCond, &pOtherCond); - } if (TSDB_CODE_SUCCESS == code) { pScan->node.pConditions = pOtherCond; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 656976b3ca..567df6c4d1 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -436,6 +436,15 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS } } + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pScanLogicNode->pTagIndexCond) { + pSubplan->pTagIndexCond = nodesCloneNode(pScanLogicNode->pTagIndexCond); + if (NULL == pSubplan->pTagIndexCond) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + } + if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pScanPhysiNode; } else { diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 84ccea668d..ead3956450 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -40,6 +40,8 @@ TEST_F(PlanOptimizeTest, ConditionPushDown) { run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 or tag1 < 2"); run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello'"); + + run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello' AND c1 > 10"); } TEST_F(PlanOptimizeTest, orderByPrimaryKey) {