From 1bda5a722348d4210bf0263435da1cb61af6ddcb Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 27 Oct 2023 14:47:56 +0800 Subject: [PATCH 01/16] enhance: use tbname cond to set vgroup list of table node --- source/libs/parser/src/parTranslater.c | 105 +++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 72293e2f8c..15d0a1f511 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3848,6 +3848,110 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec return code; } +typedef struct SEqCondTbNameTableInfo { + SRealTableNode* pRealTable; + char tbName[TSDB_TABLE_NAME_LEN]; +} SEqCondTbNameTableInfo; + +//[tableAlias.]tbname = tbNamVal +static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, char** ppTbNameVal) { + if (pOperator->opType != OP_TYPE_EQUAL) return false; + SFunctionNode* pTbnameFunc = NULL; + SValueNode* pValueNode = NULL; + if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION && + ((SFunctionNode*)(pOperator->pLeft))->funcType == FUNCTION_TYPE_TBNAME && + nodeType(pOperator->pRight) == QUERY_NODE_VALUE) { + pTbnameFunc = (SFunctionNode*)pOperator->pLeft; + pValueNode = (SValueNode*)pOperator->pRight; + } else if (nodeType(pOperator->pRight) == QUERY_NODE_FUNCTION && + ((SFunctionNode*)(pOperator->pRight))->funcType == FUNCTION_TYPE_TBNAME && + nodeType(pOperator->pLeft) == QUERY_NODE_VALUE) { + pTbnameFunc = (SFunctionNode*)pOperator->pRight; + pValueNode = (SValueNode*)pOperator->pLeft; + } else { + return false; + } + + if (LIST_LENGTH(pTbnameFunc->pParameterList) == 0) { + *ppTableAlias = NULL; + } else if (LIST_LENGTH(pTbnameFunc->pParameterList) == 1) { + SNode* pQualNode = nodesListGetNode(pTbnameFunc->pParameterList, 0); + if (nodeType(pQualNode) != QUERY_NODE_VALUE) return false; + SValueNode* pQualValNode = (SValueNode*)pQualNode; + *ppTableAlias = pQualValNode->literal; + } else { + return false; + } + *ppTbNameVal = pValueNode->literal; + return true; +} + +static int32_t findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SArray* aTables) { + char* pTableAlias = NULL; + char* pTbNameVal = NULL; + if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pTbNameVal)) { + STableNode* pTable; + if (pTableAlias == NULL) { + pTable = (STableNode*)((SSelectStmt*)(pCxt->pCurrStmt))->pFromTable; + } else { + findTable(pCxt, pTableAlias, &pTable); + } + if (nodeType(pTable) == QUERY_NODE_REAL_TABLE) { + SEqCondTbNameTableInfo info = {0}; + strcpy(info.tbName, pTbNameVal); + info.pRealTable = (SRealTableNode*)pTable; + taosArrayPush(aTables, &info); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTables) { + if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { + findEqCondTbNameInOperatorNode(pCxt, pWhere, aTables); + } else if (nodeType(pWhere) == QUERY_NODE_LOGIC_CONDITION && + ((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_AND) { + SNode* pTmpNode = NULL; + FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { + if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { + findEqCondTbNameInOperatorNode(pCxt, pTmpNode, aTables); + } + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* pSelect, SArray* aTables) { + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < taosArrayGetSize(aTables); ++i) { + SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i); + char* dbName = pInfo->pRealTable->table.dbName; + SName snameTb; + toName(pCxt->pParseCxt->acctId, dbName, pInfo->tbName, &snameTb); + SVgroupInfo vgInfo; + bool bExists; + code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); + if (code == TSDB_CODE_SUCCESS && bExists) { + taosMemoryFree(pInfo->pRealTable->pVgroupList); + pInfo->pRealTable->pVgroupList = taosMemoryMalloc(sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + pInfo->pRealTable->pVgroupList->numOfVgroups = 1; + pInfo->pRealTable->pVgroupList->vgroups[0] = vgInfo; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setTableVgroupsFromEqualTbnameCond(STranslateContext* pCxt, SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* aTables = taosArrayInit(1, sizeof(SEqCondTbNameTableInfo)); + code = findEqualCondTbname(pCxt, pSelect->pWhere, aTables); + if (code == TSDB_CODE_SUCCESS) { + code = setEqualTbnameTableVgroups(pCxt, pSelect, aTables); + } + taosArrayDestroy(aTables); + return code; +} + static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_WHERE; int32_t code = translateExpr(pCxt, &pSelect->pWhere); @@ -3857,6 +3961,7 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code && pSelect->timeRange.skey > pSelect->timeRange.ekey) { pSelect->isEmptyResult = true; } + setTableVgroupsFromEqualTbnameCond(pCxt, pSelect); return code; } From 15aedb8d8cd593518a2a91a2505c9197bf437ba4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 27 Oct 2023 15:16:40 +0800 Subject: [PATCH 02/16] fix: fix bugs of pWhere is null --- source/libs/parser/src/parTranslater.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 15d0a1f511..171d9757c1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3961,7 +3961,9 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code && pSelect->timeRange.skey > pSelect->timeRange.ekey) { pSelect->isEmptyResult = true; } - setTableVgroupsFromEqualTbnameCond(pCxt, pSelect); + if (pSelect->pWhere != NULL) { + setTableVgroupsFromEqualTbnameCond(pCxt, pSelect); + } return code; } From bab886b22984ae3ffd1ae529645783a72e7fb61a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 3 Nov 2023 14:12:04 +0800 Subject: [PATCH 03/16] enhance: refact code --- source/libs/parser/src/parTranslater.c | 41 ++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 15d0a1f511..59239c08b5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3886,35 +3886,46 @@ static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOper return true; } -static int32_t findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SArray* aTables) { +static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; char* pTableAlias = NULL; char* pTbNameVal = NULL; + if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pTbNameVal)) { STableNode* pTable; if (pTableAlias == NULL) { pTable = (STableNode*)((SSelectStmt*)(pCxt->pCurrStmt))->pFromTable; } else { - findTable(pCxt, pTableAlias, &pTable); + code = findTable(pCxt, pTableAlias, &pTable); } - if (nodeType(pTable) == QUERY_NODE_REAL_TABLE) { - SEqCondTbNameTableInfo info = {0}; - strcpy(info.tbName, pTbNameVal); - info.pRealTable = (SRealTableNode*)pTable; - taosArrayPush(aTables, &info); + if (code == TSDB_CODE_SUCCESS && nodeType(pTable) == QUERY_NODE_REAL_TABLE) { + strcpy(pInfo->tbName, pTbNameVal); + pInfo->pRealTable = (SRealTableNode*)pTable; + return true; } } - return TSDB_CODE_SUCCESS; + return false; } static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTables) { if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { - findEqCondTbNameInOperatorNode(pCxt, pWhere, aTables); - } else if (nodeType(pWhere) == QUERY_NODE_LOGIC_CONDITION && - ((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_AND) { - SNode* pTmpNode = NULL; - FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { - if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { - findEqCondTbNameInOperatorNode(pCxt, pTmpNode, aTables); + SEqCondTbNameTableInfo info = {0}; + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info); + if (bIsEqTbnameCond) { + taosArrayPush(aTables, &info); + } + } else if (nodeType(pWhere) == QUERY_NODE_LOGIC_CONDITION) { + if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_AND) { + SNode* pTmpNode = NULL; + FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { + if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { + SEqCondTbNameTableInfo info = {0}; + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + if (bIsEqTbnameCond) { + taosArrayPush(aTables, &info); + break; + } + } } } } From 893f7cc080060d124d5e518c0f4a7199d480f867 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 3 Nov 2023 16:35:43 +0800 Subject: [PATCH 04/16] enhance: prepare for or processing --- source/libs/parser/src/parTranslater.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 75581c80af..f1808ff911 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3907,12 +3907,22 @@ static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWher return false; } -static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTables) { +static bool isTableExistInTableTbnames(SArray* aTableTbNames, SRealTableNode* pTable) { + for (int i = 0; i < taosArrayGetSize(aTableTbNames); ++i) { + SEqCondTbNameTableInfo* info = taosArrayGet(aTableTbNames, i); + if (info->pRealTable == pTable) { + return true; + } + } + return false; +} + +static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info); if (bIsEqTbnameCond) { - taosArrayPush(aTables, &info); + taosArrayPush(aTableTbnames, &info); } } else if (nodeType(pWhere) == QUERY_NODE_LOGIC_CONDITION) { if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_AND) { @@ -3921,12 +3931,14 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); - if (bIsEqTbnameCond) { - taosArrayPush(aTables, &info); + if (bIsEqTbnameCond && !isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { + taosArrayPush(aTableTbnames, &info); break; } } } + } else if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_OR) { + // deal with or condition } } return TSDB_CODE_SUCCESS; From 88c8463e18be24103fae595b02dfbd9f144ce257 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 6 Nov 2023 11:06:22 +0800 Subject: [PATCH 05/16] fix: process logic or conditions --- source/libs/parser/src/parTranslater.c | 116 +++++++++++++++++++------ 1 file changed, 91 insertions(+), 25 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f1808ff911..139dd75ed2 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3851,6 +3851,7 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec typedef struct SEqCondTbNameTableInfo { SRealTableNode* pRealTable; char tbName[TSDB_TABLE_NAME_LEN]; + bool done; } SEqCondTbNameTableInfo; //[tableAlias.]tbname = tbNamVal @@ -3898,7 +3899,8 @@ static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWher } else { code = findTable(pCxt, pTableAlias, &pTable); } - if (code == TSDB_CODE_SUCCESS && nodeType(pTable) == QUERY_NODE_REAL_TABLE) { + if (code == TSDB_CODE_SUCCESS && nodeType(pTable) == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pTable)->pMeta && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { strcpy(pInfo->tbName, pTbNameVal); pInfo->pRealTable = (SRealTableNode*)pTable; return true; @@ -3917,6 +3919,43 @@ static bool isTableExistInTableTbnames(SArray* aTableTbNames, SRealTableNode* pT return false; } +static void findEqualCondTbnameInLogicCondAnd(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { + SNode* pTmpNode = NULL; + FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { + if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { + SEqCondTbNameTableInfo info = {0}; + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + if (bIsEqTbnameCond && !isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { + taosArrayPush(aTableTbnames, &info); + break; + } + } + } +} + +static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { + bool bAllTbName = true; + SNode* pTmpNode = NULL; + FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { + if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { + SEqCondTbNameTableInfo info = {0}; + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + if (!bIsEqTbnameCond) { + bAllTbName = false; + break; + } else if (isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { + taosArrayPush(aTableTbnames, &info); + } + } else { + bAllTbName = false; + break; + } + } + if (!bAllTbName) { + taosArrayClear(aTableTbnames); + } +} + static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; @@ -3926,19 +3965,45 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra } } else if (nodeType(pWhere) == QUERY_NODE_LOGIC_CONDITION) { if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_AND) { - SNode* pTmpNode = NULL; - FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { - if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { - SEqCondTbNameTableInfo info = {0}; - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); - if (bIsEqTbnameCond && !isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { - taosArrayPush(aTableTbnames, &info); - break; - } + findEqualCondTbnameInLogicCondAnd(pCxt, pWhere, aTableTbnames); + } else if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_OR) { + findEqualCondTbnameInLogicCondOr(pCxt, pWhere, aTableTbnames); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTables, int32_t start, SVgroupsInfo* vgsInfo) { + int32_t nVgroups = 0; + int32_t nTbls = taosArrayGetSize(aTables); + SEqCondTbNameTableInfo* pInfo1 = taosArrayGet(aTables, start); + + for (int j = start; j < nTbls; ++j) { + SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, j); + if (pInfo->done || pInfo->pRealTable != pInfo1->pRealTable) { + continue; + } + char* dbName = pInfo->pRealTable->table.dbName; + SName snameTb; + toName(pCxt->pParseCxt->acctId, dbName, pInfo->tbName, &snameTb); + SVgroupInfo vgInfo; + bool bExists; + int32_t code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); + if (code == TSDB_CODE_SUCCESS && bExists) { + bool bFoundVg = false; + for (int32_t k = 0; k < nVgroups; ++k) { + if (vgsInfo->vgroups[k].vgId == vgInfo.vgId) { + bFoundVg = true; } } - } else if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_OR) { - // deal with or condition + if (!bFoundVg) { + vgsInfo->vgroups[nVgroups] = vgInfo; + ++nVgroups; + vgsInfo->numOfVgroups = nVgroups; + } + } else { + vgsInfo->numOfVgroups = 0; + break; } } return TSDB_CODE_SUCCESS; @@ -3946,20 +4011,21 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* pSelect, SArray* aTables) { int32_t code = TSDB_CODE_SUCCESS; - for (int i = 0; i < taosArrayGetSize(aTables); ++i) { - SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i); - char* dbName = pInfo->pRealTable->table.dbName; - SName snameTb; - toName(pCxt->pParseCxt->acctId, dbName, pInfo->tbName, &snameTb); - SVgroupInfo vgInfo; - bool bExists; - code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); - if (code == TSDB_CODE_SUCCESS && bExists) { - taosMemoryFree(pInfo->pRealTable->pVgroupList); - pInfo->pRealTable->pVgroupList = taosMemoryMalloc(sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); - pInfo->pRealTable->pVgroupList->numOfVgroups = 1; - pInfo->pRealTable->pVgroupList->vgroups[0] = vgInfo; + int32_t nTbls = taosArrayGetSize(aTables); + for (int i = 0; i < nTbls; ++i) { + SEqCondTbNameTableInfo* pInfo1 = taosArrayGet(aTables, i); + if (pInfo1->done) { + continue; } + SVgroupsInfo* vgsInfo = taosMemoryMalloc(sizeof(SVgroupsInfo) + nTbls * sizeof(SVgroupInfo)); + int32_t nVgroups = 0; + findVgroupsFromEqualTbname(pCxt, aTables, i, vgsInfo); + if (vgsInfo->numOfVgroups != 0) { + taosMemoryFree(pInfo1->pRealTable->pVgroupList); + pInfo1->pRealTable->pVgroupList = vgsInfo; + } else { + taosMemoryFree(vgsInfo); + } } return TSDB_CODE_SUCCESS; } From 6136b1d018b405c5fc9b629a57df2efa0efd3eef Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 6 Nov 2023 13:49:01 +0800 Subject: [PATCH 06/16] fix: fix tbname or error --- source/libs/parser/src/parTranslater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1aa0052007..398d31d750 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4020,7 +4020,7 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh if (!bIsEqTbnameCond) { bAllTbName = false; break; - } else if (isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { + } else { taosArrayPush(aTableTbnames, &info); } } else { @@ -4066,6 +4066,7 @@ static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTabl SVgroupInfo vgInfo; bool bExists; int32_t code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); + pInfo->done = true; if (code == TSDB_CODE_SUCCESS && bExists) { bool bFoundVg = false; for (int32_t k = 0; k < nVgroups; ++k) { From 41fa664916e2f642531fc4f6ea758940f162199c Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 6 Nov 2023 14:52:34 +0800 Subject: [PATCH 07/16] enhance: add test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/tbname_vgroup.py | 118 +++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 tests/system-test/2-query/tbname_vgroup.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 21dcd16441..e00ac4e089 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -21,6 +21,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3 diff --git a/tests/system-test/2-query/tbname_vgroup.py b/tests/system-test/2-query/tbname_vgroup.py new file mode 100644 index 0000000000..dedf473929 --- /dev/null +++ b/tests/system-test/2-query/tbname_vgroup.py @@ -0,0 +1,118 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-] + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tbname_vgroup") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tbname_vgroup") + tdSql.execute("create database if not exists tbname_vgroup") + tdSql.execute('use tbname_vgroup') + tdSql.execute('create database dbvg vgroups 8;') + + tdSql.execute('use dbvg;') + + tdSql.execute('create table st(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1)") + + tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 2)") + + tdSql.execute("insert into ct3 using st tags(3) values('2021-04-19 00:00:03', 3)") + + tdSql.execute("insert into ct4 using st tags(4) values('2021-04-19 00:00:04', 4)") + + tdSql.query("select * from st where tbname='ct1'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + + tdSql.query("select * from st where tbname='ct3'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 3)) + tdSql.checkData(0, 1, 3) + tdSql.checkData(0, 2, 3) + + tdSql.query("select * from st where tbname='ct3' and f=2") + tdSql.checkRows(0) + + tdSql.query("select * from st where tbname='ct1' and tbname='ct4'") + tdSql.checkRows(0) + + tdSql.query("select * from st where tbname='ct1' or tbname='ct4' order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 4)) + tdSql.checkData(1, 1, 4) + tdSql.checkData(1, 2, 4) + + tdSql.query("select * from st where tbname='ct2' or tbname='ct3' order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 2, 2) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 3)) + tdSql.checkData(1, 1, 3) + tdSql.checkData(1, 2, 3) + + tdSql.query("select * from st where tbname='ct1' or tbname='ct4' or tbname='ct3' or tbname='ct2' order by ts") + tdSql.checkRows(4) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 3)) + tdSql.checkData(2, 1, 3) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 4)) + tdSql.checkData(3, 1, 4) + tdSql.checkData(3, 2, 4) + + tdSql.query("select * from st where tbname='ct4' or 1=1 order by ts") + tdSql.checkRows(4) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 3)) + tdSql.checkData(2, 1, 3) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 4)) + tdSql.checkData(3, 1, 4) + tdSql.checkData(3, 2, 4) + + tdSql.execute('drop database dbvg;') + + tdSql.execute('drop database tbname_vgroup') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From edeeb490b12357fbbed7365a9ea6c68fa48f6baf Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 7 Nov 2023 16:40:05 +0800 Subject: [PATCH 08/16] enhance: add tbname in values --- source/libs/parser/src/parTranslater.c | 113 ++++++++++++++++++------- 1 file changed, 84 insertions(+), 29 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 398d31d750..83feb22cc3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3927,12 +3927,11 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec typedef struct SEqCondTbNameTableInfo { SRealTableNode* pRealTable; - char tbName[TSDB_TABLE_NAME_LEN]; - bool done; + SArray* aTbnames; } SEqCondTbNameTableInfo; //[tableAlias.]tbname = tbNamVal -static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, char** ppTbNameVal) { +static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray* aTabNames) { if (pOperator->opType != OP_TYPE_EQUAL) return false; SFunctionNode* pTbnameFunc = NULL; SValueNode* pValueNode = NULL; @@ -3960,16 +3959,50 @@ static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOper } else { return false; } - *ppTbNameVal = pValueNode->literal; + taosArrayPush(aTabNames, &(pValueNode->literal)); return true; } +//[tableAlias.]tbname in (value1, value2, ...) +static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray* aTbNames) { + if (pOperator->opType != OP_TYPE_IN) return false; + if (nodeType(pOperator->pLeft) != QUERY_NODE_FUNCTION || + ((SFunctionNode*)(pOperator->pLeft))->funcType != FUNCTION_TYPE_TBNAME || + nodeType(pOperator->pRight) == QUERY_NODE_NODE_LIST) { + return false; + } + + SFunctionNode* pTbnameFunc = (SFunctionNode*)pOperator->pLeft; + if (LIST_LENGTH(pTbnameFunc->pParameterList) == 0) { + *ppTableAlias = NULL; + } else if (LIST_LENGTH(pTbnameFunc->pParameterList) == 1) { + SNode* pQualNode = nodesListGetNode(pTbnameFunc->pParameterList, 0); + if (nodeType(pQualNode) != QUERY_NODE_VALUE) return false; + SValueNode* pQualValNode = (SValueNode*)pQualNode; + *ppTableAlias = pQualValNode->literal; + } else { + return false; + } + SNodeListNode* pValueListNode = (SNodeListNode*)pOperator->pRight; + SNodeList* pValueNodeList = pValueListNode->pNodeList; + SNode* pValNode = NULL; + FOREACH(pValNode, pValueNodeList) { + if (nodeType(pValNode) != QUERY_NODE_VALUE) { + return false; + } + taosArrayPush(aTbNames, &((SValueNode*)pValNode)->literal); + } + return true; + +} + static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; char* pTableAlias = NULL; char* pTbNameVal = NULL; - - if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pTbNameVal)) { + SArray* aTableNames = taosArrayInit(1, sizeof(void*)); + if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, aTableNames) || + isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, aTableNames)) { STableNode* pTable; if (pTableAlias == NULL) { pTable = (STableNode*)((SSelectStmt*)(pCxt->pCurrStmt))->pFromTable; @@ -3978,11 +4011,12 @@ static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWher } if (code == TSDB_CODE_SUCCESS && nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { - strcpy(pInfo->tbName, pTbNameVal); pInfo->pRealTable = (SRealTableNode*)pTable; + taosArrayAddAll(pInfo->aTbnames, aTableNames); return true; } } + taosArrayDestroy(aTableNames); return false; } @@ -4001,27 +4035,47 @@ static void findEqualCondTbnameInLogicCondAnd(STranslateContext* pCxt, SNode* pW FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; + info.aTbnames = taosArrayInit(1, sizeof(void*)); bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); if (bIsEqTbnameCond && !isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { + //TODO: intersect tbNames of same table? speed taosArrayPush(aTableTbnames, &info); - break; + } else { + taosArrayDestroy(info.aTbnames); } } } } +static void unionTbnamesOfTbNameCond(SArray* aTableTbnames, SEqCondTbNameTableInfo* pInfo) { + bool bFoundTable = false; + for (int i = 0; i < taosArrayGetSize(aTableTbnames); ++i) { + SEqCondTbNameTableInfo* info = taosArrayGet(aTableTbnames, i); + if (info->pRealTable == pInfo->pRealTable) { + taosArrayAddAll(info->aTbnames, pInfo->aTbnames); + taosArrayDestroy(pInfo->aTbnames); + bFoundTable = true; + break; + } + } + if (!bFoundTable) { + taosArrayPush(aTableTbnames, pInfo); + } +} + static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { bool bAllTbName = true; SNode* pTmpNode = NULL; FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + info.aTbnames = taosArrayInit(1, sizeof(void*)); + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); if (!bIsEqTbnameCond) { bAllTbName = false; break; } else { - taosArrayPush(aTableTbnames, &info); + unionTbnamesOfTbNameCond(aTableTbnames, &info); } } else { bAllTbName = false; @@ -4029,6 +4083,10 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh } } if (!bAllTbName) { + for (int i = 0; i < taosArrayGetSize(aTableTbnames); ++i) { + SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTableTbnames, i); + taosArrayDestroy(pInfo->aTbnames); + } taosArrayClear(aTableTbnames); } } @@ -4050,23 +4108,18 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra return TSDB_CODE_SUCCESS; } -static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTables, int32_t start, SVgroupsInfo* vgsInfo) { +static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo, SVgroupsInfo* vgsInfo) { int32_t nVgroups = 0; - int32_t nTbls = taosArrayGetSize(aTables); - SEqCondTbNameTableInfo* pInfo1 = taosArrayGet(aTables, start); + int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); - for (int j = start; j < nTbls; ++j) { - SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, j); - if (pInfo->done || pInfo->pRealTable != pInfo1->pRealTable) { - continue; - } + for (int j = 0; j < nTbls; ++j) { char* dbName = pInfo->pRealTable->table.dbName; SName snameTb; - toName(pCxt->pParseCxt->acctId, dbName, pInfo->tbName, &snameTb); + char* tbName = taosArrayGetP(pInfo->aTbnames, j); + toName(pCxt->pParseCxt->acctId, dbName, tbName, &snameTb); SVgroupInfo vgInfo; bool bExists; int32_t code = catalogGetCachedTableHashVgroup(pCxt->pParseCxt->pCatalog, &snameTb, &vgInfo, &bExists); - pInfo->done = true; if (code == TSDB_CODE_SUCCESS && bExists) { bool bFoundVg = false; for (int32_t k = 0; k < nVgroups; ++k) { @@ -4089,18 +4142,16 @@ static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SArray* aTabl static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* pSelect, SArray* aTables) { int32_t code = TSDB_CODE_SUCCESS; - int32_t nTbls = taosArrayGetSize(aTables); - for (int i = 0; i < nTbls; ++i) { - SEqCondTbNameTableInfo* pInfo1 = taosArrayGet(aTables, i); - if (pInfo1->done) { - continue; - } + for (int i = 0; i < taosArrayGetSize(aTables); ++i) { + SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i); + int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); + SVgroupsInfo* vgsInfo = taosMemoryMalloc(sizeof(SVgroupsInfo) + nTbls * sizeof(SVgroupInfo)); int32_t nVgroups = 0; - findVgroupsFromEqualTbname(pCxt, aTables, i, vgsInfo); + findVgroupsFromEqualTbname(pCxt, pInfo, vgsInfo); if (vgsInfo->numOfVgroups != 0) { - taosMemoryFree(pInfo1->pRealTable->pVgroupList); - pInfo1->pRealTable->pVgroupList = vgsInfo; + taosMemoryFree(pInfo->pRealTable->pVgroupList); + pInfo->pRealTable->pVgroupList = vgsInfo; } else { taosMemoryFree(vgsInfo); } @@ -4115,6 +4166,10 @@ static int32_t setTableVgroupsFromEqualTbnameCond(STranslateContext* pCxt, SSele if (code == TSDB_CODE_SUCCESS) { code = setEqualTbnameTableVgroups(pCxt, pSelect, aTables); } + for (int i = 0; i < taosArrayGetSize(aTables); ++i) { + SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i); + taosArrayDestroy(pInfo->aTbnames); + } taosArrayDestroy(aTables); return code; } From 4ff81b84b0018b8ffe21a45ce32f93b74a2ff0d4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 7 Nov 2023 20:47:14 +0800 Subject: [PATCH 09/16] fix: fix address sanitizer error --- source/libs/parser/src/parTranslater.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 83feb22cc3..9e26f64c5d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4013,6 +4013,7 @@ static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWher ((SRealTableNode*)pTable)->pMeta && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { pInfo->pRealTable = (SRealTableNode*)pTable; taosArrayAddAll(pInfo->aTbnames, aTableNames); + taosArrayDestroy(aTableNames); return true; } } @@ -4072,6 +4073,7 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh info.aTbnames = taosArrayInit(1, sizeof(void*)); bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); if (!bIsEqTbnameCond) { + taosArrayDestroy(info.aTbnames); bAllTbName = false; break; } else { @@ -4094,9 +4096,12 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; + info.aTbnames = taosArrayInit(1, sizeof(void*)); bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info); if (bIsEqTbnameCond) { taosArrayPush(aTableTbnames, &info); + } else { + taosArrayDestroy(info.aTbnames); } } else if (nodeType(pWhere) == QUERY_NODE_LOGIC_CONDITION) { if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_AND) { From fabf4e8cf003a63c67dae6399cd1a21982f04709 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 8 Nov 2023 09:09:03 +0800 Subject: [PATCH 10/16] fix: tbname in (value...) apply --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9e26f64c5d..4aa2e0263e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3968,7 +3968,7 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper if (pOperator->opType != OP_TYPE_IN) return false; if (nodeType(pOperator->pLeft) != QUERY_NODE_FUNCTION || ((SFunctionNode*)(pOperator->pLeft))->funcType != FUNCTION_TYPE_TBNAME || - nodeType(pOperator->pRight) == QUERY_NODE_NODE_LIST) { + nodeType(pOperator->pRight) != QUERY_NODE_NODE_LIST) { return false; } From 680afd41b0d6d3ea98f8220af15327a31f397398 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 8 Nov 2023 10:40:07 +0800 Subject: [PATCH 11/16] enhance: add test case --- tests/system-test/2-query/tbname_vgroup.py | 60 ++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/system-test/2-query/tbname_vgroup.py b/tests/system-test/2-query/tbname_vgroup.py index dedf473929..b3cbd207d4 100644 --- a/tests/system-test/2-query/tbname_vgroup.py +++ b/tests/system-test/2-query/tbname_vgroup.py @@ -107,6 +107,66 @@ class TDTestCase: tdSql.checkData(3, 1, 4) tdSql.checkData(3, 2, 4) + tdSql.query("select * from st where tbname in ('ct1') order by ts") + tdSql.checkRows(1) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + + tdSql.query("select * from st where tbname in ('ct1', 'ct2') order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + + tdSql.query("select * from st where tbname in ('ct1', 'ct2') or tbname in ('ct3', 'ct4') order by ts") + tdSql.checkRows(4) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 3)) + tdSql.checkData(2, 1, 3) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 4)) + tdSql.checkData(3, 1, 4) + tdSql.checkData(3, 2, 4) + + tdSql.query("select * from st where tbname in ('ct1', 'ct2') or tbname='ct3' order by ts") + tdSql.checkRows(3) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 3)) + tdSql.checkData(2, 1, 3) + tdSql.checkData(2, 2, 3) + + tdSql.query("select * from st where tbname in ('ct1', 'ct2') and tbname='ct3' order by ts") + tdSql.checkRows(0) + + tdSql.query("select * from st where tbname in ('ct1') or 1=1 order by ts") + tdSql.checkRows(4) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 3)) + tdSql.checkData(2, 1, 3) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 0, 0, 4)) + tdSql.checkData(3, 1, 4) + tdSql.checkData(3, 2, 4) + tdSql.execute('drop database dbvg;') tdSql.execute('drop database tbname_vgroup') From 144856893cceff2c349145ce903e29afacc818bc Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 8 Nov 2023 14:19:30 +0800 Subject: [PATCH 12/16] enhance: check no exchange operator --- tests/system-test/2-query/tbname_vgroup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/system-test/2-query/tbname_vgroup.py b/tests/system-test/2-query/tbname_vgroup.py index b3cbd207d4..1e489356e6 100644 --- a/tests/system-test/2-query/tbname_vgroup.py +++ b/tests/system-test/2-query/tbname_vgroup.py @@ -167,6 +167,9 @@ class TDTestCase: tdSql.checkData(3, 1, 4) tdSql.checkData(3, 2, 4) + tdSql.query("explain select * from st where tbname='ct1'") + tdSql.checkRows(2) + tdSql.execute('drop database dbvg;') tdSql.execute('drop database tbname_vgroup') From 30cbf9ae5c3b89d5f41e723a6e523f4d37efa269 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 9 Nov 2023 10:31:01 +0800 Subject: [PATCH 13/16] feature: if num of tbnames is greater than vgroups, do not update vgroupList of table --- source/libs/parser/src/parTranslater.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4aa2e0263e..d38122a6ad 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4116,6 +4116,10 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo, SVgroupsInfo* vgsInfo) { int32_t nVgroups = 0; int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); + + if (nTbls >= pInfo->pRealTable->pVgroupList->numOfVgroups) { + vgsInfo->numOfVgroups = 0; + } for (int j = 0; j < nTbls; ++j) { char* dbName = pInfo->pRealTable->table.dbName; From 97804695d214731c27c601cd7862dd0668c74864 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 9 Nov 2023 15:59:49 +0800 Subject: [PATCH 14/16] query: add test case --- tests/system-test/2-query/tbname_vgroup.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/system-test/2-query/tbname_vgroup.py b/tests/system-test/2-query/tbname_vgroup.py index 1e489356e6..90be1bf394 100644 --- a/tests/system-test/2-query/tbname_vgroup.py +++ b/tests/system-test/2-query/tbname_vgroup.py @@ -169,6 +169,15 @@ class TDTestCase: tdSql.query("explain select * from st where tbname='ct1'") tdSql.checkRows(2) + + tdSql.query("select table_name, vgroup_id from information_schema.ins_tables where db_name='dbvg' and type='CHILD_TABLE'"); + print(tdSql.queryResult); + + tdSql.query("explain select * from st where tbname in ('ct1', 'ct2')") + if tdSql.queryResult[0][0].count("Data Exchange 2:1") == 0: + tdLog.exit("failed, not two vgroups") + else: + tdLog.info("select * from st where tbname in ('ct1', 'ct2') involves two vgroups") tdSql.execute('drop database dbvg;') From 9d448c7c6c4fcf22506f4d18a924fca26070cac9 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 10 Nov 2023 16:30:16 +0800 Subject: [PATCH 15/16] fix: review comments --- source/libs/parser/src/parTranslater.c | 53 ++++++++++++---------- tests/system-test/2-query/tbname_vgroup.py | 17 ++++++- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d38122a6ad..a55bd5663f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3931,7 +3931,7 @@ typedef struct SEqCondTbNameTableInfo { } SEqCondTbNameTableInfo; //[tableAlias.]tbname = tbNamVal -static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray* aTabNames) { +static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray** ppTabNames) { if (pOperator->opType != OP_TYPE_EQUAL) return false; SFunctionNode* pTbnameFunc = NULL; SValueNode* pValueNode = NULL; @@ -3959,12 +3959,13 @@ static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOper } else { return false; } - taosArrayPush(aTabNames, &(pValueNode->literal)); + *ppTabNames = taosArrayInit(1, sizeof(void*)); + taosArrayPush(*ppTabNames, &(pValueNode->literal)); return true; } //[tableAlias.]tbname in (value1, value2, ...) -static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray* aTbNames) { +static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray** ppTbNames) { if (pOperator->opType != OP_TYPE_IN) return false; if (nodeType(pOperator->pLeft) != QUERY_NODE_FUNCTION || ((SFunctionNode*)(pOperator->pLeft))->funcType != FUNCTION_TYPE_TBNAME || @@ -3983,6 +3984,7 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper } else { return false; } + *ppTbNames = taosArrayInit(1, sizeof(void*)); SNodeListNode* pValueListNode = (SNodeListNode*)pOperator->pRight; SNodeList* pValueNodeList = pValueListNode->pNodeList; SNode* pValNode = NULL; @@ -3990,7 +3992,7 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper if (nodeType(pValNode) != QUERY_NODE_VALUE) { return false; } - taosArrayPush(aTbNames, &((SValueNode*)pValNode)->literal); + taosArrayPush(*ppTbNames, &((SValueNode*)pValNode)->literal); } return true; @@ -4000,9 +4002,8 @@ static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWher int32_t code = TSDB_CODE_SUCCESS; char* pTableAlias = NULL; char* pTbNameVal = NULL; - SArray* aTableNames = taosArrayInit(1, sizeof(void*)); - if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, aTableNames) || - isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, aTableNames)) { + if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames) || + isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames)) { STableNode* pTable; if (pTableAlias == NULL) { pTable = (STableNode*)((SSelectStmt*)(pCxt->pCurrStmt))->pFromTable; @@ -4012,12 +4013,11 @@ static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWher if (code == TSDB_CODE_SUCCESS && nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { pInfo->pRealTable = (SRealTableNode*)pTable; - taosArrayAddAll(pInfo->aTbnames, aTableNames); - taosArrayDestroy(aTableNames); return true; } + taosArrayDestroy(pInfo->aTbnames); + pInfo->aTbnames = NULL; } - taosArrayDestroy(aTableNames); return false; } @@ -4036,25 +4036,28 @@ static void findEqualCondTbnameInLogicCondAnd(STranslateContext* pCxt, SNode* pW FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - info.aTbnames = taosArrayInit(1, sizeof(void*)); - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); - if (bIsEqTbnameCond && !isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { - //TODO: intersect tbNames of same table? speed - taosArrayPush(aTableTbnames, &info); - } else { - taosArrayDestroy(info.aTbnames); + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + if (bIsEqTbnameCond) { + if (!isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { + //TODO: intersect tbNames of same table? speed + taosArrayPush(aTableTbnames, &info); + } else { + taosArrayDestroy(info.aTbnames); + } } } + //TODO: logic cond } } -static void unionTbnamesOfTbNameCond(SArray* aTableTbnames, SEqCondTbNameTableInfo* pInfo) { +static void unionEqualCondTbnamesOfSameTable(SArray* aTableTbnames, SEqCondTbNameTableInfo* pInfo) { bool bFoundTable = false; for (int i = 0; i < taosArrayGetSize(aTableTbnames); ++i) { SEqCondTbNameTableInfo* info = taosArrayGet(aTableTbnames, i); if (info->pRealTable == pInfo->pRealTable) { taosArrayAddAll(info->aTbnames, pInfo->aTbnames); taosArrayDestroy(pInfo->aTbnames); + pInfo->aTbnames = NULL; bFoundTable = true; break; } @@ -4068,16 +4071,15 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh bool bAllTbName = true; SNode* pTmpNode = NULL; FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { + //TODO: logic cond if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - info.aTbnames = taosArrayInit(1, sizeof(void*)); bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); if (!bIsEqTbnameCond) { - taosArrayDestroy(info.aTbnames); bAllTbName = false; break; } else { - unionTbnamesOfTbNameCond(aTableTbnames, &info); + unionEqualCondTbnamesOfSameTable(aTableTbnames, &info); } } else { bAllTbName = false; @@ -4088,20 +4090,19 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh for (int i = 0; i < taosArrayGetSize(aTableTbnames); ++i) { SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTableTbnames, i); taosArrayDestroy(pInfo->aTbnames); + pInfo->aTbnames = NULL; } taosArrayClear(aTableTbnames); } } static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { + //TODO: optimize nested and/or condition. now only the fist level is processed. if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - info.aTbnames = taosArrayInit(1, sizeof(void*)); bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info); if (bIsEqTbnameCond) { taosArrayPush(aTableTbnames, &info); - } else { - taosArrayDestroy(info.aTbnames); } } else if (nodeType(pWhere) == QUERY_NODE_LOGIC_CONDITION) { if (((SLogicConditionNode*)pWhere)->condType == LOGIC_COND_TYPE_AND) { @@ -4116,9 +4117,10 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo, SVgroupsInfo* vgsInfo) { int32_t nVgroups = 0; int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); - + if (nTbls >= pInfo->pRealTable->pVgroupList->numOfVgroups) { vgsInfo->numOfVgroups = 0; + return TSDB_CODE_SUCCESS; } for (int j = 0; j < nTbls; ++j) { @@ -4134,6 +4136,7 @@ static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbName for (int32_t k = 0; k < nVgroups; ++k) { if (vgsInfo->vgroups[k].vgId == vgInfo.vgId) { bFoundVg = true; + break; } } if (!bFoundVg) { diff --git a/tests/system-test/2-query/tbname_vgroup.py b/tests/system-test/2-query/tbname_vgroup.py index 90be1bf394..a4deea8089 100644 --- a/tests/system-test/2-query/tbname_vgroup.py +++ b/tests/system-test/2-query/tbname_vgroup.py @@ -27,6 +27,7 @@ class TDTestCase: tdSql.execute("drop database if exists tbname_vgroup") tdSql.execute("create database if not exists tbname_vgroup") tdSql.execute('use tbname_vgroup') + tdSql.execute('drop database if exists dbvg') tdSql.execute('create database dbvg vgroups 8;') tdSql.execute('use dbvg;') @@ -179,7 +180,21 @@ class TDTestCase: else: tdLog.info("select * from st where tbname in ('ct1', 'ct2') involves two vgroups") - tdSql.execute('drop database dbvg;') + tdSql.execute('create table st2(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct21 using st2 tags(1) values('2021-04-19 00:00:01', 1)") + + tdSql.execute("insert into ct22 using st2 tags(2) values('2021-04-19 00:00:02', 2)") + + tdSql.execute("insert into ct23 using st2 tags(3) values('2021-04-19 00:00:03', 3)") + + tdSql.execute("insert into ct24 using st2 tags(4) values('2021-04-19 00:00:04', 4)") + + tdSql.query("select * from st, st2 where st.ts=st2.ts and st.tbname in ('ct1', 'ct2') and st2.tbname in ('ct1', 'ct3')"); + tdSql.checkRows(0); + + + #tdSql.execute('drop database dbvg;') tdSql.execute('drop database tbname_vgroup') def stop(self): From 792e0db35091bd738cf9365aaf5f0c032485413d Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 10 Nov 2023 17:00:10 +0800 Subject: [PATCH 16/16] enhance: add join test case --- tests/system-test/2-query/tbname_vgroup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/tbname_vgroup.py b/tests/system-test/2-query/tbname_vgroup.py index a4deea8089..564c78b47a 100644 --- a/tests/system-test/2-query/tbname_vgroup.py +++ b/tests/system-test/2-query/tbname_vgroup.py @@ -190,9 +190,14 @@ class TDTestCase: tdSql.execute("insert into ct24 using st2 tags(4) values('2021-04-19 00:00:04', 4)") - tdSql.query("select * from st, st2 where st.ts=st2.ts and st.tbname in ('ct1', 'ct2') and st2.tbname in ('ct1', 'ct3')"); - tdSql.checkRows(0); - + tdSql.query("select * from st, st2 where st.ts=st2.ts and st.tbname in ('ct1', 'ct2') and st2.tbname in ('ct21', 'ct23')"); + tdSql.checkRows(1); + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(0, 4, 1) + tdSql.checkData(0, 5, 1) #tdSql.execute('drop database dbvg;')