diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 49370876d8..b1d8be2f0e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -61,6 +61,7 @@ typedef struct SExprNode { bool asAlias; bool asParam; bool asPosition; + int32_t projIdx; } SExprNode; typedef enum EColumnType { @@ -91,6 +92,8 @@ typedef struct SColumnNode { int16_t numOfPKs; bool tableHasPk; bool isPk; + int32_t projRefIdx; + int32_t resIdx; } SColumnNode; typedef struct SColumnRefNode { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 78cf78f6db..e2c5b42e39 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -103,6 +103,7 @@ static int32_t exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { COPY_CHAR_ARRAY_FIELD(aliasName); COPY_CHAR_ARRAY_FIELD(userAlias); COPY_SCALAR_FIELD(orderAlias); + COPY_SCALAR_FIELD(projIdx); return TSDB_CODE_SUCCESS; } @@ -124,6 +125,8 @@ static int32_t columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { COPY_SCALAR_FIELD(tableHasPk); COPY_SCALAR_FIELD(isPk); COPY_SCALAR_FIELD(numOfPKs); + COPY_SCALAR_FIELD(projRefIdx); + COPY_SCALAR_FIELD(resIdx); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 32981d2ed9..1d6c19346a 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2260,8 +2260,13 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN } else { len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); } - if (NULL == taosHashGet(pCxt->pColHash, name, len)) { - pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); + if (pCol->projRefIdx > 0) { + len = taosHashBinary(name, strlen(name)); + len += sprintf(name + len, "_%d", pCol->projRefIdx); + } + SNode** pNodeFound = taosHashGet(pCxt->pColHash, name, len); + if (pNodeFound == NULL) { + pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, &pNode, POINTER_BYTES); if (TSDB_CODE_SUCCESS == pCxt->errCode) { SNode* pNew = NULL; pCxt->errCode = nodesCloneNode(pNode, &pNew); @@ -2304,7 +2309,6 @@ static EDealRes collectColumnsExt(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } - int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, SNodeList** pCols) { if (NULL == pSelect || NULL == pCols) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c01e95531f..57a21af091 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1270,7 +1270,7 @@ static void setColumnPrimTs(STranslateContext* pCxt, SColumnNode* pCol, const ST } } -static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList) { +static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList, bool skipProjRef) { int32_t code = 0; if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; @@ -1305,7 +1305,11 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p SListCell* pCell = nodesListGetCell(pList, LIST_LENGTH(pList) - 1); code = setColumnInfoByExpr(pTempTable, (SExprNode*)pNode, (SColumnNode**)&pCell->pNode); } - if (TSDB_CODE_SUCCESS != code) break; + if (TSDB_CODE_SUCCESS == code) { + if (!skipProjRef) pCol->projRefIdx = ((SExprNode*)pNode)->projIdx; // only set proj ref when select * from (select ...) + } else { + break; + } } } return code; @@ -4770,7 +4774,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList* size_t nums = taosArrayGetSize(pTables); for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); - int32_t code = createColumnsByTable(pCxt, pTable, igTags, *pCols); + int32_t code = createColumnsByTable(pCxt, pTable, igTags, *pCols, nums > 1); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -4833,7 +4837,7 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, bo } } if (TSDB_CODE_SUCCESS == code) { - code = createColumnsByTable(pCxt, pTable, igTags, *pOutput); + code = createColumnsByTable(pCxt, pTable, igTags, *pOutput, false); } return code; } @@ -5175,6 +5179,11 @@ static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSe if (!pSelect->isSubquery) { return rewriteProjectAlias(pSelect->pProjectionList); } else { + SNode* pNode; + int32_t projIdx = 1; + FOREACH(pNode, pSelect->pProjectionList) { + ((SExprNode*)pNode)->projIdx = projIdx++; + } return TSDB_CODE_SUCCESS; } } diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 090071ee7f..beb277493c 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -68,6 +68,8 @@ int32_t tagScanSetExecutionMode(SScanLogicNode* pScan); int32_t cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat, bool* pCloned); int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, SSortLogicNode* pSort, bool* pNotOptimize, SNodeList** pSequencingNodes, bool* keepSort); +bool isColRefExpr(const SColumnNode* pCol, const SExprNode* pExpr); +void rewriteTargetsWithResId(SNodeList* pTargets); #ifdef __cplusplus diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index d5ee5c51df..3d13ad4c62 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -122,6 +122,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); + pCol->node.projIdx = ((SExprNode*)(*pNode))->projIdx; if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } @@ -637,6 +638,10 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } nodesDestroyList(pColList); } + + if (TSDB_CODE_SUCCESS == code) { + rewriteTargetsWithResId(pJoin->node.pTargets); + } if (NULL == pJoin->node.pTargets && NULL != pLeft) { code = nodesCloneList(pLeft->pTargets, &pJoin->node.pTargets); @@ -1342,6 +1347,9 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect ? (pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); + if (TSDB_CODE_SUCCESS == code) { + rewriteTargetsWithResId(pSort->node.pTargets); + } if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { SNode* pNew = NULL; code = nodesCloneNode(nodesListGetNode(pCxt->pCurrRoot->pTargets, 0), &pNew); @@ -1390,11 +1398,14 @@ static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pS } SNode* pNode; + int32_t projIdx = 1; FOREACH(pNode, pExprs) { - if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)createColumnByExpr(pStmtName, (SExprNode*)pNode)))) { + SColumnNode* pCol = createColumnByExpr(pStmtName, (SExprNode*)pNode); + if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)pCol))) { nodesDestroyList(pList); return code; } + pCol->resIdx = ((SExprNode*)pNode)->projIdx; } *pCols = pList; @@ -1459,6 +1470,9 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS code = nodesListMakeStrictAppend(&pPartition->node.pTargets, pNew); } } + if (TSDB_CODE_SUCCESS == code) { + rewriteTargetsWithResId(pPartition->node.pTargets); + } if (TSDB_CODE_SUCCESS == code) { // code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, NULL, fmIsAggFunc, &pPartition->pAggFuncs); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 3aab190eda..7da4f93dee 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3475,6 +3475,20 @@ static EDealRes eliminateProjOptRewriteScanTableAlias(SNode* pNode, void* pConte } +static void eliminateProjPushdownProjIdx(SNodeList* pParentProjects, SNodeList* pChildTargets) { + SNode* pChildTarget = NULL, *pParentProject = NULL; + FOREACH(pChildTarget, pChildTargets) { + SColumnNode* pTargetCol = (SColumnNode*)pChildTarget; + FOREACH(pParentProject, pParentProjects) { + SExprNode* pProject = (SExprNode*)pParentProject; + if (0 == strcmp(pTargetCol->colName, pProject->aliasName)) { + pTargetCol->resIdx = pProject->projIdx; + break; + } + } + } +} + static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SProjectLogicNode* pProjectNode) { SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0); @@ -3546,6 +3560,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* nodesWalkExprs(pScan->pScanPseudoCols, eliminateProjOptRewriteScanTableAlias, &cxt); nodesWalkExpr(pScan->node.pConditions, eliminateProjOptRewriteScanTableAlias, &cxt); nodesWalkExprs(pChild->pTargets, eliminateProjOptRewriteScanTableAlias, &cxt); + eliminateProjPushdownProjIdx(pProjectNode->pProjections, pChild->pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -4883,6 +4898,31 @@ typedef struct SMergeProjectionsContext { int32_t errCode; } SMergeProjectionsContext; +static EDealRes mergeProjectionsExpr2(SNode** pNode, void* pContext) { + SMergeProjectionsContext* pCxt = pContext; + SProjectLogicNode* pChildProj = pCxt->pChildProj; + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SColumnNode* pProjCol = (SColumnNode*)(*pNode); + SNode* pProjection; + int32_t projIdx = 1; + FOREACH(pProjection, pChildProj->pProjections) { + if (isColRefExpr(pProjCol, (SExprNode*)pProjection)) { + SNode* pExpr = NULL; + pCxt->errCode = nodesCloneNode(pProjection, &pExpr); + if (pExpr == NULL) { + return DEAL_RES_ERROR; + } + snprintf(((SExprNode*)pExpr)->aliasName, sizeof(((SExprNode*)pExpr)->aliasName), "%s", + ((SExprNode*)*pNode)->aliasName); + nodesDestroyNode(*pNode); + *pNode = pExpr; + return DEAL_RES_IGNORE_CHILD; + } + } + } + return DEAL_RES_CONTINUE; +} + static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) { SMergeProjectionsContext* pCxt = pContext; SProjectLogicNode* pChildProj = pCxt->pChildProj; @@ -4917,7 +4957,7 @@ static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* ((SProjectLogicNode*)pSelfNode)->inputIgnoreGroup = true; } SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS}; - nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt); + nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr2, &cxt); int32_t code = cxt.errCode; if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e50e574f01..2b0f449b77 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -35,17 +35,18 @@ typedef struct SPhysiPlanContext { int32_t errCode; int16_t nextDataBlockId; SArray* pLocationHelper; + SArray* pProjIdxLocHelper; bool hasScan; bool hasSysScan; } SPhysiPlanContext; -static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t *pLen) { +static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t *pLen, uint16_t extraBufLen) { int32_t code = 0; if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; if (NULL != pStmtName) { if ('\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -55,7 +56,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } else { - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -65,7 +66,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int } } if ('\0' == pCol->tableAlias[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -74,7 +75,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int return code; } - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -89,7 +90,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); if (pVal) { if (NULL != pStmtName && '\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -99,7 +100,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); return code; } - *ppKey = taosMemoryCalloc(1, strlen(pVal->literal) + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, strlen(pVal->literal) + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -113,7 +114,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int } if (NULL != pStmtName && '\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -124,7 +125,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int return code; } - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1); + *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); if (!*ppKey) { return terrno; } @@ -191,22 +192,34 @@ static int32_t putSlotToHash(const char* pName, int32_t len, int16_t dataBlockId } static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId, - SHashObj** pDescHash) { + SHashObj** pDescHash, SHashObj** ppProjIdxDescHash) { SHashObj* pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == pHash) { return TSDB_CODE_OUT_OF_MEMORY; } + SHashObj* pProjIdxHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (!pProjIdxHash) { + taosHashCleanup(pHash); + return TSDB_CODE_OUT_OF_MEMORY; + } if (NULL == taosArrayInsert(pCxt->pLocationHelper, dataBlockId, &pHash)) { taosHashCleanup(pHash); + taosHashCleanup(pProjIdxHash); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (NULL == taosArrayInsert(pCxt->pProjIdxLocHelper, dataBlockId, &pProjIdxHash)) { + taosHashCleanup(pHash); + taosHashCleanup(pProjIdxHash); return TSDB_CODE_OUT_OF_MEMORY; } *pDescHash = pHash; + *ppProjIdxDescHash = pProjIdxHash; return TSDB_CODE_SUCCESS; } static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, - SHashObj* pHash) { + SHashObj* pHash, SHashObj* pProjIdxDescHash) { pDataBlockDesc->pSlots = NULL; int32_t code = nodesMakeList(&pDataBlockDesc->pSlots); if (NULL == pDataBlockDesc->pSlots) { @@ -218,12 +231,16 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD FOREACH(pNode, pList) { char* name = NULL; int32_t len = 0; - code = getSlotKey(pNode, NULL, &name, &len); + code = getSlotKey(pNode, NULL, &name, &len, 16); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false)); } + code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); if (TSDB_CODE_SUCCESS == code) { - code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); + if (nodeType(pNode) == QUERY_NODE_COLUMN && ((SColumnNode*)pNode)->resIdx > 0) { + sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->resIdx); + code = putSlotToHash(name, strlen(name), pDataBlockDesc->dataBlockId, slotId, pNode, pProjIdxDescHash); + } } taosMemoryFree(name); if (TSDB_CODE_SUCCESS == code) { @@ -246,9 +263,10 @@ static int32_t createDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SD pDesc->dataBlockId = pCxt->nextDataBlockId++; SHashObj* pHash = NULL; - code = createDataBlockDescHash(pCxt, LIST_LENGTH(pList), pDesc->dataBlockId, &pHash); + SHashObj* pProjIdxHash = NULL; + code = createDataBlockDescHash(pCxt, LIST_LENGTH(pList), pDesc->dataBlockId, &pHash, &pProjIdxHash); if (TSDB_CODE_SUCCESS == code) { - code = buildDataBlockSlots(pCxt, pList, pDesc, pHash); + code = buildDataBlockSlots(pCxt, pList, pDesc, pHash, pProjIdxHash); } if (TSDB_CODE_SUCCESS == code) { @@ -286,7 +304,7 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode; char *name = NULL; int32_t len = 0; - code = getSlotKey(pExpr, pStmtName, &name, &len); + code = getSlotKey(pExpr, pStmtName, &name, &len, 0); if (TSDB_CODE_SUCCESS == code) { SSlotIndex* pIndex = taosHashGet(pHash, name, len); if (NULL == pIndex) { @@ -355,7 +373,9 @@ static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, typedef struct SSetSlotIdCxt { int32_t errCode; SHashObj* pLeftHash; + SHashObj* pLeftProjIdxHash; SHashObj* pRightHash; + SHashObj* pRightProdIdxHash; } SSetSlotIdCxt; static void dumpSlots(const char* pName, SHashObj* pHash) { @@ -379,13 +399,22 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; char *name = NULL; int32_t len = 0; - pCxt->errCode = getSlotKey(pNode, NULL, &name, &len); + pCxt->errCode = getSlotKey(pNode, NULL, &name, &len, 16); if (TSDB_CODE_SUCCESS != pCxt->errCode) { return DEAL_RES_ERROR; } - SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len); - if (NULL == pIndex) { - pIndex = taosHashGet(pCxt->pRightHash, name, len); + SSlotIndex *pIndex = NULL; + if (((SColumnNode*)pNode)->projRefIdx > 0) { + sprintf(name + strlen(name), "_%d", ((SColumnNode*)pNode)->projRefIdx); + pIndex = taosHashGet(pCxt->pLeftProjIdxHash, name, strlen(name)); + if (!pIndex) { + pIndex = taosHashGet(pCxt->pRightProdIdxHash, name, strlen(name)); + } + } else { + pIndex = taosHashGet(pCxt->pLeftHash, name, len); + if (NULL == pIndex) { + pIndex = taosHashGet(pCxt->pRightHash, name, len); + } } // pIndex is definitely not NULL, otherwise it is a bug if (NULL == pIndex) { @@ -396,9 +425,9 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { taosMemoryFree(name); return DEAL_RES_ERROR; } - taosMemoryFree(name); ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; ((SColumnNode*)pNode)->slotId = ((SSlotIdInfo*)taosArrayGet(pIndex->pSlotIdsInfo, 0))->slotId; + taosMemoryFree(name); return DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -419,7 +448,9 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), - .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))}; + .pLeftProjIdxHash = taosArrayGetP(pCxt->pProjIdxLocHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)), + .pRightProdIdxHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pProjIdxLocHelper, rightDataBlockId))}; nodesWalkExpr(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); @@ -445,7 +476,9 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), - .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))}; + .pLeftProjIdxHash = taosArrayGetP(pCxt->pProjIdxLocHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)), + .pRightProdIdxHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pProjIdxLocHelper, rightDataBlockId))}; nodesWalkExprs(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); @@ -1254,7 +1287,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys SColumnNode* pCol = (SColumnNode*)pNode; char *pName = NULL; int32_t len = 0; - code = getSlotKey(pNode, NULL, &pName, &len); + code = getSlotKey(pNode, NULL, &pName, &len, 0); if (TSDB_CODE_SUCCESS == code) { code = tSimpleHashPut(pHash, pName, len, &pCol, POINTER_BYTES); } @@ -1272,7 +1305,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys char* pName = NULL; SColumnNode* pCol = (SColumnNode*)pNode; int32_t len = 0; - code = getSlotKey(pNode, NULL, &pName, &len); + code = getSlotKey(pNode, NULL, &pName, &len, 0); if (TSDB_CODE_SUCCESS == code) { SNode** p = tSimpleHashGet(pHash, pName, len); if (p) { @@ -1293,7 +1326,7 @@ static int32_t sortHashJoinTargets(int16_t lBlkId, int16_t rBlkId, SHashJoinPhys char* pName = NULL; SColumnNode* pCol = (SColumnNode*)pNode; int32_t len = 0; - code = getSlotKey(pNode, NULL, &pName, &len); + code = getSlotKey(pNode, NULL, &pName, &len, 0); if (TSDB_CODE_SUCCESS == code) { SNode** p = tSimpleHashGet(pHash, pName, len); if (p) { @@ -2992,6 +3025,7 @@ static void destoryLocationHash(void* p) { static void destoryPhysiPlanContext(SPhysiPlanContext* pCxt) { taosArrayDestroyEx(pCxt->pLocationHelper, destoryLocationHash); + taosArrayDestroyEx(pCxt->pProjIdxLocHelper, destoryLocationHash); } static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) { @@ -3023,9 +3057,12 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP .errCode = TSDB_CODE_SUCCESS, .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), + .pProjIdxLocHelper = taosArrayInit(32, POINTER_BYTES), .hasScan = false, .hasSysScan = false}; - if (NULL == cxt.pLocationHelper) { + if (NULL == cxt.pLocationHelper || !cxt.pProjIdxLocHelper) { + taosArrayDestroy(cxt.pLocationHelper); + taosArrayDestroy(cxt.pProjIdxLocHelper); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index e1e98f221f..b5f0bc50e8 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -716,5 +716,16 @@ int32_t tagScanSetExecutionMode(SScanLogicNode* pScan) { return TSDB_CODE_SUCCESS; } +bool isColRefExpr(const SColumnNode* pCol, const SExprNode* pExpr) { + if (pCol->projRefIdx > 0) return pCol->projRefIdx == pExpr->projIdx; + return 0 == strcmp(pCol->colName, pExpr->aliasName); +} +void rewriteTargetsWithResId(SNodeList* pTargets) { + SNode* pNode; + FOREACH(pNode, pTargets) { + SColumnNode* pCol = (SColumnNode*)pNode; + pCol->resIdx = pCol->projRefIdx; + } +} diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a56e4b98c0..43193128fa 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -144,6 +144,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py diff --git a/tests/system-test/2-query/nestedQuery2.py b/tests/system-test/2-query/nestedQuery2.py new file mode 100644 index 0000000000..dce119b1f2 --- /dev/null +++ b/tests/system-test/2-query/nestedQuery2.py @@ -0,0 +1,167 @@ +from random import randrange +import time +import threading +import secrets +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4} + + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = '1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s" % (dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s" % ( + dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s" % (dbName)) + return + + def create_stable(self, tsql, paraDict): + colString = tdCom.gen_column_type_str( + colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str( + tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)" % ( + paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s" % (sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self, tsql=None, dbName='dbx', stbName='stb', ctbPrefix='ctb', ctbNum=1, ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % (dbName, ctbPrefix, i+ctbStartIdx, dbName, stbName, (i+ctbStartIdx) % 5, i+ctbStartIdx + random.randint( + 1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100)) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" % + (ctbNum, dbName, stbName)) + return + + def init_normal_tb(self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int): + sql = 'CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))' % ( + db_name, tb_name) + tsql.execute(sql) + sql = 'INSERT INTO %s.%s values' % (db_name, tb_name) + for j in range(rows): + sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % (start_ts + j * ts_step + randrange(500), j % + 10 + randrange(200), j % 10, j % 10, j % 10 + randrange(100)) + tsql.execute(sql) + + def insert_data(self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" % dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s.%s%d values " % (dbName, ctbPrefix, i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % (startTs + j*tsStep + randrange( + 500), j % 10 + randrange(100), j % 10 + randrange(200), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " % ( + startTs + j*tsStep + randrange(500), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s.%s%d values " % (dbName, ctbPrefix, i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def init_data(self, db: str = 'test', ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500): + tdLog.printNoPrefix( + "======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': db, + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'FLOAT', 'count': 1}, {'type': 'DOUBLE', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'tinyint', 'count': 1}, {'type': 'bool', 'count': 1}, {'type': 'binary', 'len': 10, 'count': 1}, {'type': 'nchar', 'len': 10, 'count': 1}], + 'tagSchema': [{'type': 'INT', 'count': 1}, {'type': 'nchar', 'len': 20, 'count': 1}, {'type': 'binary', 'len': 20, 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'DOUBLE', 'count': 1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': ctb_num, + 'rowsPerTbl': rows_per_ctb, + 'batchNum': 3000, + 'startTs': start_ts, + 'tsStep': ts_step} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = ctb_num + paraDict['rowsPerTbl'] = rows_per_ctb + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], + vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], + stbName=paraDict["stbName"], ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"], ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"], + ctbPrefix=paraDict["ctbPrefix"], ctbNum=paraDict["ctbNum"], + rowsPerTbl=paraDict["rowsPerTbl"], batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"], tsStep=paraDict["tsStep"]) + self.init_normal_tb(tdSql, paraDict['dbName'], 'norm_tb', + paraDict['rowsPerTbl'], paraDict['startTs'], paraDict['tsStep']) + + def test_select_asterisk_from_subquery_with_duplicate_aliasname(self): + sql = "select * from (select c8 as a, c9 as a from t1 order by ts desc limit 10)t;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 0, "binary9") + tdSql.checkData(0, 1, "nchar9") + sql = "select * from (select c8 as a, c9 as a, ts from t1 order by ts desc limit 10)t order by ts desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 0, "binary9") + tdSql.checkData(0, 1, "nchar9") + sql = "select * from (select c8 as a, c9 as a, ts, t1 from t1 order by ts desc limit 10)t partition by t1 order by ts desc;" + tdSql.query(sql, queryTimes=1) + tdSql.checkData(0, 0, "binary9") + tdSql.checkData(0, 1, "nchar9") + sql = " select * from (select a.c8, b.c8, a.ts, a.t1,b.t1 from t1 a, t3 b where a.ts = b.ts order by a.ts)ttt" + tdSql.query(sql, queryTimes=1) + + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 3) + + def run(self): + self.init_data() + self.test_select_asterisk_from_subquery_with_duplicate_aliasname() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())