From 7964e5e6b7a72a9696ae3bb2962e242cd4e47eea Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 9 Dec 2024 16:21:14 +0800 Subject: [PATCH 01/45] cols function --- include/libs/function/functionMgt.h | 5 + include/libs/scalar/scalar.h | 1 + include/util/taoserror.h | 1 + source/libs/function/inc/builtinsimpl.h | 5 + source/libs/function/inc/functionMgtInt.h | 1 + source/libs/function/src/builtins.c | 50 ++++- source/libs/function/src/builtinsimpl.c | 74 +++++++ source/libs/function/src/functionMgt.c | 2 + source/libs/parser/inc/parAst.h | 2 + source/libs/parser/inc/sql.y | 18 ++ source/libs/parser/src/parAstCreater.c | 45 ++++ source/libs/parser/src/parTokenizer.c | 1 + source/libs/parser/src/parTranslater.c | 239 ++++++++++++++++++++- source/libs/scalar/src/sclfunc.c | 4 + source/util/src/terror.c | 3 +- tests/parallel_test/cases.task | 4 + tests/system-test/2-query/cols_function.py | 119 ++++++++++ 17 files changed, 564 insertions(+), 10 deletions(-) create mode 100644 tests/system-test/2-query/cols_function.py diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 853f70e755..286068cebb 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -155,6 +155,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_FORECAST_LOW, FUNCTION_TYPE_FORECAST_HIGH, FUNCTION_TYPE_FORECAST_ROWTS, + FUNCTION_TYPE_COLS, FUNCTION_TYPE_IROWTS_ORIGIN, // internal function @@ -208,6 +209,9 @@ typedef enum EFunctionType { FUNCTION_TYPE_HYPERLOGLOG_STATE, FUNCTION_TYPE_HYPERLOGLOG_STATE_MERGE, + FUNCTION_TYPE_COLS_PARTIAL, + FUNCTION_TYPE_COLS_MERGE, + // geometry functions FUNCTION_TYPE_GEOM_FROM_TEXT = 4250, FUNCTION_TYPE_AS_TEXT, @@ -295,6 +299,7 @@ bool fmisSelectGroupConstValueFunc(int32_t funcId); bool fmIsElapsedFunc(int32_t funcId); bool fmIsDBUsageFunc(int32_t funcId); bool fmIsRowTsOriginFunc(int32_t funcId); +bool fmIsSelectColsFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType, int32_t pkBytes); int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index fd936dd087..60aefa1491 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -149,6 +149,7 @@ int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t colsScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 64ef0b3829..366ffc6ecc 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -906,6 +906,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT TAOS_DEF_ERROR_CODE(0, 0x2684) #define TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x2685) #define TSDB_CODE_PAR_INVALID_VGID_LIST TAOS_DEF_ERROR_CODE(0, 0x2686) +#define TSDB_CODE_PAR_INVALID_COLS_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2687) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index d548ae956d..883d80d1cd 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -251,6 +251,11 @@ int32_t blockDBUsageSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo int32_t blockDBUsageFunction(SqlFunctionCtx* pCtx); int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +bool getColsFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); +int32_t colsFunction(SqlFunctionCtx* pCtx); +int32_t colsPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t colsCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index e10581beb6..d676d4b728 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -59,6 +59,7 @@ extern "C" { #define FUNC_MGT_COUNT_LIKE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(30) // funcs that should also return 0 when no rows found #define FUNC_MGT_PROCESS_BY_ROW FUNC_MGT_FUNC_CLASSIFICATION_MASK(31) #define FUNC_MGT_FORECAST_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(32) +#define FUNC_MGT_SELECT_COLS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(33) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 95a332ac05..e91b80aa4a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1233,6 +1233,17 @@ static int32_t translateForecastConf(SFunctionNode* pFunc, char* pErrBuf, int32_ return TSDB_CODE_SUCCESS; } +static int32_t translateCols(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_FLOAT].bytes, .type = TSDB_DATA_TYPE_FLOAT}; + return TSDB_CODE_SUCCESS; +} + +bool getColFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); + + return true; +} + static EFuncReturnRows forecastEstReturnRows(SFunctionNode* pFunc) { return FUNC_RETURN_ROWS_N; } static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { @@ -5641,7 +5652,44 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .paramInfoPattern = 0, .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}}, .translateFunc = translateOutVarchar, - } + }, + { + .name = "cols", + .type = FUNCTION_TYPE_COLS, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_SELECT_COLS_FUNC, + .translateFunc = translateCols, + .dynDataRequiredFunc = NULL, + .getEnvFunc = getColsFuncEnv, + .initFunc = functionSetup, + .processFunc = colsFunction, + .sprocessFunc = colsScalarFunction, + .pPartialFunc = "_cols_partial", + .pMergeFunc = "_cols_merge", + .finalizeFunc = NULL + }, + { + .name = "_cols_partial", + .type = FUNCTION_TYPE_COLS_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_SELECT_COLS_FUNC, + .translateFunc = translateCols, + .dynDataRequiredFunc = NULL, + .getEnvFunc = getColsFuncEnv, + .initFunc = functionSetup, + .processFunc = colsFunction, + .finalizeFunc = colsPartialFinalize, + .combineFunc = colsCombine, + }, + { + .name = "_cols_merge", + .type = FUNCTION_TYPE_COLS_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_SELECT_COLS_FUNC, + .translateFunc = translateOutFirstIn, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunctionMerge, + .finalizeFunc = colsPartialFinalize, + .combineFunc = colsCombine, + }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1c77d12fb2..43e40bee80 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -7194,3 +7194,77 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } + +bool getColsFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); + pEnv->calcMemSize = pNode->node.resType.bytes; + return true; +} + +int32_t colsFunction(SqlFunctionCtx* pCtx) { + int32_t code = TSDB_CODE_SUCCESS; + SFunctionNode* pSelectFunc = ((SFunctionNode*)(pCtx[0].pExpr->pExpr->_function.pFunctNode->pParameterList->pHead->pNode)); + + SFuncExecFuncs selectExecFuncs; + code = fmGetFuncExecFuncs(pCtx->functionId, &selectExecFuncs); + if(TSDB_CODE_SUCCESS != code) { + return code; + } + if(selectExecFuncs.process == NULL) { + return TSDB_CODE_SUCCESS; + } + return selectExecFuncs.process(pCtx); +} + +int32_t colsPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); + + int32_t resultBytes = getFirstLastInfoSize(pRes->bytes, pRes->pkBytes); + + // todo check for failure + char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + if (NULL == res) { + return terrno; + } + (void)memcpy(varDataVal(res), pRes, resultBytes); + + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + if (NULL == pCol) { + taosMemoryFree(res); + return TSDB_CODE_OUT_OF_RANGE; + } + + if (pEntryInfo->numOfRes == 0) { + colDataSetNULL(pCol, pBlock->info.rows); + code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, pBlock->info.rows); + } else { + code = colDataSetVal(pCol, pBlock->info.rows, res, false); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(res); + return code; + } + code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); + } + taosMemoryFree(res); + return code; +} + +int32_t colsCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { + SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); + SFirstLastRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + int32_t bytes = pDBuf->bytes; + + SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); + SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + + pDBuf->hasResult = firstLastTransferInfoImpl(pSBuf, pDBuf, false); + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); + pDResInfo->isNullRes &= pSResInfo->isNullRes; + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 4fb3aab49d..97d6a5851b 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -173,6 +173,8 @@ bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC bool fmIsVectorFunc(int32_t funcId) { return !fmIsScalarFunc(funcId) && !fmIsPseudoColumnFunc(funcId); } +bool fmIsSelectColsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_COLS_FUNC); } + bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); } bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index e69a3da4a9..8b4e6d56ee 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -170,6 +170,7 @@ SNode* createInterpTimeAround(SAstCreateContext* pCxt, SNode* pTimepoint, SN SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); SNode* createAlterSingleTagColumnNode(SAstCreateContext* pCtx, SToken* token, SNode* pVal); +SNode* createColsFunctionNode(SAstCreateContext* pCxt, SNode* pFunc, SNodeList* pList); SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); @@ -329,6 +330,7 @@ SNode* createDropTSMAStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* SNode* createShowCreateTSMAStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createShowTSMASStmt(SAstCreateContext* pCxt, SNode* dbName); SNode* createShowDiskUsageStmt(SAstCreateContext* pCxt, SNode* dbName, ENodeType type); +SNodeList* createColsFuncParamNodeList(SAstCreateContext* pCxt, SNode* pFuncNode, SNodeList* pNodeList, SToken* pAlias); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 63eb09d509..1d74816461 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1249,6 +1249,7 @@ pseudo_column(A) ::= IROWTS_ORIGIN(B). function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } +function_expression(A) ::= cols_func(B) NK_LP cols_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= CAST(B) NK_LP expr_or_subquery(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } function_expression(A) ::= @@ -1311,6 +1312,23 @@ star_func(A) ::= FIRST(B). star_func(A) ::= LAST(B). { A = B; } star_func(A) ::= LAST_ROW(B). { A = B; } +%type cols_func { SToken } +%destructor cols_func { } +cols_func(A) ::= COLS(B). { A = B; } + +%type cols_func_para_list { SNodeList* } +%destructor cols_func_para_list { nodesDestroyList($$); } +cols_func_para_list(A) ::= function_expression(B) NK_COMMA cols_func_expression_list(C). { A = createColsFuncParamNodeList(pCxt, B, C, NULL); } + +cols_func_expression(A) ::= expr_or_subquery(B). { A = releaseRawExprNode(pCxt, B); } +cols_func_expression(A) ::= expr_or_subquery(B) column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C);} +cols_func_expression(A) ::= expr_or_subquery(B) AS column_alias(C). { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C);} + +%type cols_func_expression_list { SNodeList* } +%destructor cols_func_expression_list { nodesDestroyList($$); } +cols_func_expression_list(A) ::= cols_func_expression(B). { A = createNodeList(pCxt, B); } +cols_func_expression_list(A) ::= cols_func_expression_list(B) NK_COMMA cols_func_expression(C). { A = addNodeToList(pCxt, B, C); } + %type star_func_para_list { SNodeList* } %destructor star_func_para_list { nodesDestroyList($$); } star_func_para_list(A) ::= NK_STAR(B). { A = createNodeList(pCxt, createColumnNode(pCxt, NULL, &B)); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a13472620b..54aa94a352 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -16,6 +16,7 @@ #include #include +#include "nodes.h" #include "parAst.h" #include "parUtil.h" #include "tglobal.h" @@ -356,6 +357,32 @@ SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { return t; } +SNodeList* createColsFuncParamNodeList(SAstCreateContext* pCxt, SNode* pNode, SNodeList* pNodeList, SToken* pAlias) { + CHECK_PARSER_STATUS(pCxt); + if (NULL == pNode || QUERY_NODE_RAW_EXPR != nodeType(pNode)) { + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; + } + CHECK_PARSER_STATUS(pCxt); + SRawExprNode* pRawExpr = (SRawExprNode*)pNode; + SNode* pFuncNode = pRawExpr->pNode; + if(pFuncNode->type != QUERY_NODE_FUNCTION) { + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; + } + CHECK_PARSER_STATUS(pCxt); + SNodeList* list = NULL; + pCxt->errCode = nodesMakeList(&list); + CHECK_MAKE_NODE(list); + pCxt->errCode = nodesListAppend(list, pFuncNode); + CHECK_PARSER_STATUS(pCxt); + pCxt->errCode = nodesListAppendList(list, pNodeList); + return list; + + _err: + nodesDestroyNode(pFuncNode); + nodesDestroyList(pNodeList); + return NULL; +} + SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { CHECK_PARSER_STATUS(pCxt); SNodeList* list = NULL; @@ -1150,6 +1177,24 @@ _err: return NULL; } +SNode* createColsFunctionNode(SAstCreateContext* pCxt, SNode* pColFunc, SNodeList* pExpr){ + SFunctionNode* func = NULL; + CHECK_PARSER_STATUS(pCxt); + pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); + CHECK_MAKE_NODE(func); + strcpy(func->functionName, "cols"); + pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pColFunc); + CHECK_PARSER_STATUS(pCxt); + pCxt->errCode = nodesListMakeStrictAppendList(&func->pParameterList, pExpr); + CHECK_PARSER_STATUS(pCxt); + return (SNode*)func; +_err: + nodesDestroyNode((SNode*)func); + nodesDestroyNode(pColFunc); + nodesDestroyList(pExpr); + return NULL; +} + SNode* createSubstrFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* pExpr2, SNode* pExpr3) { SFunctionNode* func = NULL; CHECK_PARSER_STATUS(pCxt); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index ea2e9d712f..4bb4bb9ced 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -355,6 +355,7 @@ static SKeyword keywordTable[] = { {"FORCE_WINDOW_CLOSE", TK_FORCE_WINDOW_CLOSE}, {"DISK_INFO", TK_DISK_INFO}, {"AUTO", TK_AUTO}, + {"COLS", TK_COLS}, }; // clang-format on diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f191080512..d6225ce2b6 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1097,6 +1097,10 @@ static bool isVectorFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsVectorFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isColsFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectColsFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isDistinctOrderBy(STranslateContext* pCxt) { return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && isSelectStmt(pCxt->pCurrStmt) && ((SSelectStmt*)pCxt->pCurrStmt)->isDistinct); @@ -2450,9 +2454,10 @@ static int32_t rewriteCountTbname(STranslateContext* pCxt, SFunctionNode* pCount return code; } -static bool hasInvalidFuncNesting(SNodeList* pParameterList) { +static bool hasInvalidFuncNesting(SFunctionNode* pFunc) { + if(pFunc->funcType == FUNCTION_TYPE_COLS) return false; bool hasInvalidFunc = false; - nodesWalkExprs(pParameterList, haveVectorFunction, &hasInvalidFunc); + nodesWalkExprs(pFunc->pParameterList, haveVectorFunction, &hasInvalidFunc); return hasInvalidFunc; } @@ -2474,7 +2479,7 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (beforeHaving(pCxt->currClause)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); } - if (hasInvalidFuncNesting(pFunc->pParameterList)) { + if (hasInvalidFuncNesting(pFunc)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } // The auto-generated COUNT function in the DELETE statement is legal @@ -2518,7 +2523,7 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function is not supported in window query or group query", pFunc->functionName); } - if (hasInvalidFuncNesting(pFunc->pParameterList)) { + if (hasInvalidFuncNesting(pFunc)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } return TSDB_CODE_SUCCESS; @@ -2533,7 +2538,7 @@ static int32_t translateMultiRowsFunc(STranslateContext* pCxt, SFunctionNode* pF ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } - if (hasInvalidFuncNesting(pFunc->pParameterList)) { + if (hasInvalidFuncNesting(pFunc)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } return TSDB_CODE_SUCCESS; @@ -2564,7 +2569,7 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function is not supported in window query or group query", pFunc->functionName); } - if (hasInvalidFuncNesting(pFunc->pParameterList)) { + if (hasInvalidFuncNesting(pFunc)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } return TSDB_CODE_SUCCESS; @@ -2630,7 +2635,7 @@ static int32_t translateForecastFunc(STranslateContext* pCxt, SFunctionNode* pFu return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function is not supported in window query or group query", pFunc->functionName); } - if (hasInvalidFuncNesting(pFunc->pParameterList)) { + if (hasInvalidFuncNesting(pFunc)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } return TSDB_CODE_SUCCESS; @@ -2860,6 +2865,9 @@ static int32_t calcSelectFuncNum(SFunctionNode* pFunc, int32_t currSelectFuncNum if (fmIsCumulativeFunc(pFunc->funcId)) { return currSelectFuncNum > 0 ? currSelectFuncNum : 1; } + if(fmIsSelectColsFunc(pFunc->funcId)) { + return currSelectFuncNum; + } return currSelectFuncNum + ((fmIsMultiResFunc(pFunc->funcId) && !fmIsLastRowFunc(pFunc->funcId)) ? getMultiResFuncNum(pFunc->pParameterList) : 1); @@ -3798,6 +3806,9 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (isVectorFunc(*pNode) && !isDistinctOrderBy(pCxt)) { return DEAL_RES_IGNORE_CHILD; } + if (isColsFunc(*pNode)) { + return DEAL_RES_IGNORE_CHILD; + } SNode* pGroupNode = NULL; FOREACH(pGroupNode, getGroupByList(pCxt)) { SNode* pActualNode = getGroupByNode(pGroupNode); @@ -3891,6 +3902,7 @@ static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; bool existCol; + bool hasColFunc; SNodeList* pColList; } CheckAggColCoexistCxt; @@ -3899,6 +3911,10 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { if (isVectorFunc(*pNode)) { return DEAL_RES_IGNORE_CHILD; } + if(isColsFunc(*pNode)) { + pCxt->hasColFunc = true; + return DEAL_RES_IGNORE_CHILD; + } SNode* pPartKey = NULL; bool partionByTbname = false; if (fromSingleTable(((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pFromTable) || @@ -3977,7 +3993,7 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (!pSelect->onlyHasKeepOrderFunc) { pSelect->timeLineResMode = TIME_LINE_NONE; } - CheckAggColCoexistCxt cxt = {.pTranslateCxt = pCxt, .existCol = false}; + CheckAggColCoexistCxt cxt = {.pTranslateCxt = pCxt, .existCol = false, .hasColFunc = false}; nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { nodesRewriteExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); @@ -3989,6 +4005,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); } + if (cxt.hasColFunc) { + return rewriteColsToSelectValFunc(pCxt, pSelect); + } return TSDB_CODE_SUCCESS; } @@ -5169,6 +5188,54 @@ static int32_t createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr, SNo return code; } +static int32_t createMultiResColsFunc(SFunctionNode* pSrcFunc, SFunctionNode* pSelectFunc, SExprNode* pExpr, SNode** ppNodeOut) { + SFunctionNode* pFunc = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = nodesMakeList(&pFunc->pParameterList); + SNode* pClonedFuncNode = NULL; + if (TSDB_CODE_SUCCESS != (code = nodesCloneNode((SNode*)pSelectFunc, &pClonedFuncNode)) || + TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pFunc->pParameterList, pClonedFuncNode))) { + nodesDestroyNode((SNode*)pFunc); + return code; + } + SNode* pClonedExprNode = NULL; + if (TSDB_CODE_SUCCESS != (code = nodesCloneNode((SNode*)pExpr, &pClonedExprNode)) || + TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pFunc->pParameterList, pClonedExprNode))) { + nodesDestroyNode((SNode*)pFunc); + return code; + } + + pFunc->node.resType = pExpr->resType; + pFunc->funcId = pSrcFunc->funcId; + pFunc->funcType = pSrcFunc->funcType; + strcpy(pFunc->functionName, pSrcFunc->functionName); + char buf[TSDB_FUNC_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + TSDB_NAME_DELIMITER_LEN + 3] = {0}; + int32_t len = 0; + if(pExpr->asAlias) { + strncpy(pFunc->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN - 1); + strncpy(pFunc->node.userAlias, pExpr->userAlias, TSDB_COL_NAME_LEN - 1); + pFunc->node.asAlias = true; + } else if (QUERY_NODE_COLUMN == nodeType(pExpr)) { + SColumnNode* pCol = (SColumnNode*)pExpr; + len = tsnprintf(buf, sizeof(buf) - 1, "%s.%s", pCol->tableAlias, pCol->colName); + (void)taosHashBinary(buf, len); + strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); + len = tsnprintf(buf, sizeof(buf) - 1, "%s", pCol->colName); + strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); + } else { + len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); + (void)taosHashBinary(buf, len); + strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); + len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); + strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); + } + *ppNodeOut = (SNode*)pFunc; + return code; +} + static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, bool igTags, SNodeList** pOutput) { STableNode* pTable = NULL; int32_t code = findTable(pCxt, pCol->tableAlias, &pTable); @@ -7272,10 +7339,166 @@ static int32_t translateSelectWithoutFrom(STranslateContext* pCxt, SSelectStmt* return translateExprList(pCxt, pSelect->pProjectionList); } +typedef struct SHasMultiColsFuncCxt { + bool hasMultiColsFunc; +} SHasMultiColsFuncCxt; + +static bool isMultiColsFunc(SFunctionNode* pFunc) { + if (strcasecmp(pFunc->functionName, "cols") != 0) { + return false; + } + return pFunc->pParameterList->length > 2; +} + +static EDealRes isMultiColsFuncNode(SNode** pNode, void* pContext) { + SHasMultiColsFuncCxt* pCxt = pContext; + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)*pNode; + if (isMultiColsFunc(pFunc)) { + pCxt->hasMultiColsFunc = true; + return DEAL_RES_END; + } + } + return DEAL_RES_CONTINUE; +} + +static bool hasMultiColsFuncInList(SNodeList* nodeList) { + SHasMultiColsFuncCxt pCxt = {false}; + + nodesRewriteExprs(nodeList, isMultiColsFuncNode, &pCxt); + + return pCxt.hasMultiColsFunc; +} + +static bool hasMultiColsFunc(SNode** pNode) { + SHasMultiColsFuncCxt pCxt = {false}; + + nodesRewriteExpr(pNode, isMultiColsFuncNode, &pCxt); + + return pCxt.hasMultiColsFunc; +} + +static int32_t hasInvalidColsFunction(STranslateContext* pCxt, SNodeList* nodeList) { + SNode* pTmpNode = NULL; + FOREACH(pTmpNode, nodeList) { + if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; + if (pFunc->funcType == FUNCTION_TYPE_COLS) { + // cols function at here is valid. + } else { + if (hasMultiColsFuncInList(pFunc->pParameterList)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, + "Invalid cols function in function %s", pFunc->functionName); + } + } + } else { + if (hasMultiColsFunc(&pTmpNode)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, + "Invalid cols function, can't be used at here"); + } + } + } + return TSDB_CODE_SUCCESS; +} + +static bool invalidColsAlias(SFunctionNode* pFunc) { + if (strcasecmp(pFunc->functionName, "cols") != 0) { + return false; + } + if (pFunc->node.asAlias) { + if (pFunc->pParameterList->length > 2) { + return true; + } else { + SNode* pNode; + FOREACH(pNode, pFunc->pParameterList) { + SExprNode* pExpr = (SExprNode*)pNode; + if (pExpr->asAlias) { + return true; + } + } + } + } + + return false; +} + +static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList) { + int32_t code = hasInvalidColsFunction(pCxt, *nodeList); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + bool needRewrite = false; + SNode* pTmpNode = NULL; + FOREACH(pTmpNode, *nodeList) { + if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; + if(invalidColsAlias(pFunc)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, + "Invalid using alias for cols function"); + } + if (isMultiColsFunc(pFunc)) { + needRewrite = true; + break; + } + } + } + + SNodeList* pNewNodeList = NULL; + if (needRewrite) { + code = nodesMakeList(&pNewNodeList); + if (NULL == pNewNodeList) { + return code; + } + FOREACH(pTmpNode, *nodeList) { + if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; + if (isMultiColsFunc(pFunc)) { + // start from index 1, because the first parameter is select function which needn't to output. + SFunctionNode* pSelectFunc = (SFunctionNode*)nodesListGetNode(pFunc->pParameterList, 0); + for (int i = 1; i < pFunc->pParameterList->length; ++i) { + SNode* pExpr = nodesListGetNode(pFunc->pParameterList, i); + SNode* pNewFunc = NULL; + code = createMultiResColsFunc(pFunc, pSelectFunc, (SExprNode*)pExpr, &pNewFunc); + if (TSDB_CODE_SUCCESS != code) goto _end; + code = nodesListMakeStrictAppend(&pNewNodeList, pNewFunc); + if (TSDB_CODE_SUCCESS != code) goto _end; + } + continue; + } + } + SNode* pNewNode = NULL; + code = nodesCloneNode(pTmpNode, &pNewNode); + if (TSDB_CODE_SUCCESS != code) goto _end; + code = nodesListMakeStrictAppend(&pNewNodeList, pNewNode); + if (TSDB_CODE_SUCCESS != code) goto _end; + } + nodesDestroyList(*nodeList); + *nodeList = pNewNodeList; + } + return TSDB_CODE_SUCCESS; + +_end: + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pNewNodeList); + } + return code; +} + +static int32_t translateColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { + int32_t code = rewriteColsFunction(pCxt, &pSelect->pProjectionList); + if (code == TSDB_CODE_SUCCESS) { + code = rewriteColsFunction(pCxt, &pSelect->pOrderByList); + } + return code; +} + static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = (SNode*)pSelect; pCxt->dual = false; int32_t code = translateFrom(pCxt, &pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = translateColsFunction(pCxt, pSelect); + } if (TSDB_CODE_SUCCESS == code) { pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision; code = translateWhere(pCxt, pSelect); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1da9a8f123..68b6b8de68 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -4588,3 +4588,7 @@ int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return selectScalarFunction(pInput, inputNum, pOutput); } + +int32_t colsScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 195cb21618..2c32339efc 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -748,8 +748,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ANOMALY_WIN_TYPE, "ANOMALY_WINDOW only TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ANOMALY_WIN_COL, "ANOMALY_WINDOW not support on tag column") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT, "ANOMALY_WINDOW option should include algo field") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE, "Invalid forecast clause") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR, "Syntax error in regular expression") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR, "Syntax error in regular expression") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VGID_LIST, "Invalid vgid list") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLS_FUNCTION, "Invalid cols function") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4aedc0991e..0800ff62d3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1084,6 +1084,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/fill_with_group.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/state_window.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cols_function.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cols_function.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cols_function.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cols_function.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py new file mode 100644 index 0000000000..9267fffa40 --- /dev/null +++ b/tests/system-test/2-query/cols_function.py @@ -0,0 +1,119 @@ +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.dbname = 'test' + + def create_test_data(self): + tdLog.info("create test data") + tdLog.info("taosBenchmark -y -t 10 -n 100 -b INT,FLOAT,NCHAR,BOOL") + os.system("taosBenchmark -y -t 10 -n 100 -b INT,FLOAT,NCHAR,BOOL") + + tdSql.execute('use test') + tdSql.execute(f'Create table {self.dbname}.normal_table (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool)') + tdSql.execute(f'insert into {self.dbname}.normal_table (select * from {self.dbname}.d0)') + + def one_cols_1output_test(self): + tdLog.info("one_cols_1output_test") + tdSql.query(f'select cols(last(ts), ts) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), ts) as t1 from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), ts as t1) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), c0) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), c1) from {self.dbname}.meters group by tbname') + + + tdSql.query(f'select cols(last(ts+1), ts) as t1 from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts+1), ts+2 as t1) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts+1), c0+10) from {self.dbname}.meters') + + + def one_cols_multi_output_test(self): + tdLog.info("one_cols_1output_test") + tdSql.query(f'select cols(last(ts), ts, c0) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), ts, c0) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), ts as time, c0 cc) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), c0, c1, c2, c3) from {self.dbname}.meters') + tdSql.query(f'select cols(last(c1), ts) from {self.dbname}.meters group by tbname') + + + tdSql.query(f'select cols(max(c0), ts) from {self.dbname}.meters') + tdSql.query(f'select cols(min(c1), ts, c0) from {self.dbname}.meters') + + tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.meters') + tdSql.query(f'select count(1), cols(last(ts), ts, c0), min(c0) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), ts as time, c0 cc), count(1) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), c0, c1, c2, c3), count(1) from {self.dbname}.meters') + tdSql.query(f'select cols(last(c1), ts), count(1) from {self.dbname}.meters group by tbname') + + + tdSql.query(f'select cols(max(c0), ts), count(1) from {self.dbname}.meters') + tdSql.query(f'select cols(min(c1), ts, c0), count(1) from {self.dbname}.meters') + tdSql.query(f'select count(1), cols(max(c0), ts) from {self.dbname}.meters') + tdSql.query(f'select max(c0), cols(max(c0), ts) from {self.dbname}.meters') + tdSql.query(f'select max(c1), cols(max(c0), ts) from {self.dbname}.meters') + + + + def multi_cols_output_test(self): + tdLog.info("multi_cols_output_test") + tdSql.query(f'select cols(last(c0), ts, c1), cols(first(c0), ts, c1), count(1) from {self.dbname}.meters') + tdSql.query(f'select cols(last(c0), ts as t1, c1 as c11), cols(first(c0), ts as c2, c1 c21), count(1) from {self.dbname}.meters') + + def parse_test(self): + tdLog.info("parse test") + + + #** error sql **# + tdSql.error(f'select cols(ts) from {self.dbname}.meters group by tbname') + tdSql.error(f'select cols(ts) from {self.dbname}.meters') + tdSql.error(f'select last(cols(ts)) from {self.dbname}.meters') + tdSql.error(f'select last(cols(ts, ts)) from {self.dbname}.meters') + tdSql.error(f'select last(cols(ts, ts), ts) from {self.dbname}.meters') + tdSql.error(f'select last(cols(last(ts), ts), ts) from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), ts as t1) as t1 from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), ts, c0) t1 from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), ts t1) tt from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), c0 cc0, c1 cc1) cc from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), c0 as cc0) as cc from {self.dbname}.meters') + tdSql.error(f'select cols(ts) + 1 from {self.dbname}.meters group by tbname') + tdSql.error(f'select last(cols(ts)+1) from {self.dbname}.meters') + tdSql.error(f'select last(cols(ts+1, ts)) from {self.dbname}.meters') + tdSql.error(f'select last(cols(ts, ts), ts+1) from {self.dbname}.meters') + tdSql.error(f'select last(cols(last(ts+1), ts+1), ts) from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), ts+1 as t1) as t1 from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts+1), ts, c0) t1 from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), ts t1) tt from {self.dbname}.meters') + tdSql.error(f'select cols(first(ts+1), c0+2 cc0, c1 cc1) cc from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts)+1, c0+2 as cc0) as cc from {self.dbname}.meters') + + tdSql.error(f'select cols(last(ts)+1, ts) from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts)+10, c1+10) from {self.dbname}.meters group by tbname') + + + def run(self): + self.create_test_data() + self.parse_test() + self.one_cols_1output_test() + self.one_cols_multi_output_test() + self.multi_cols_output_test() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 20f8d17c1688c0367f412d99951ae1cd4131cafa Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 19 Dec 2024 10:48:40 +0800 Subject: [PATCH 02/45] cols func --- include/libs/function/function.h | 2 + include/libs/nodes/querynodes.h | 3 + source/libs/executor/src/executil.c | 82 +++++++++++++++----- source/libs/function/src/functionMgt.c | 2 + source/libs/nodes/src/nodesCloneFuncs.c | 2 + source/libs/nodes/src/nodesEqualFuncs.c | 6 ++ source/libs/nodes/src/nodesMsgFuncs.c | 17 +++- source/libs/parser/src/parTranslater.c | 90 +++++++++++++++++----- tests/system-test/2-query/cols_function.py | 60 ++++++++++++++- 9 files changed, 223 insertions(+), 41 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 0ca1962b4e..f61960edd4 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -282,6 +282,8 @@ typedef struct tExprNode { struct SNode *pRootNode; } _optrRoot; }; + int32_t bindTupleFuncIdx; + int32_t tupleFuncIdx; } tExprNode; struct SScalarParam { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5e4e8b6292..e14a43b1c2 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -16,6 +16,7 @@ #ifndef _TD_QUERY_NODES_H_ #define _TD_QUERY_NODES_H_ +#include #ifdef __cplusplus extern "C" { #endif @@ -61,6 +62,8 @@ typedef struct SExprNode { bool asParam; bool asPosition; int32_t projIdx; + int32_t bindTupleFuncIdx; + int32_t tupleFuncIdx; } SExprNode; typedef enum EColumnType { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index cce754a8c8..13316a09ec 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -18,6 +18,8 @@ #include "index.h" #include "os.h" #include "query.h" +#include "querynodes.h" +#include "tarray.h" #include "tdatablock.h" #include "thash.h" #include "tmsg.h" @@ -1990,7 +1992,8 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; QUERY_CHECK_CODE(code, lino, _end); } - + pExp->pExpr->bindTupleFuncIdx = ((SExprNode*)pNode)->bindTupleFuncIdx; + pExp->pExpr->tupleFuncIdx = ((SExprNode*)pNode)->tupleFuncIdx; _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); @@ -2071,42 +2074,81 @@ int32_t createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, SExprInfo** return code; } +static void deleteSubsidiareCtx(void* pData) { + SSubsidiaryResInfo* pCtx = (SSubsidiaryResInfo*)pData; + if (pCtx->pCtx) { + taosMemoryFreeClear(pCtx->pCtx); + } +} + // set the output buffer for the selectivity + tag query static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { int32_t num = 0; int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; - SqlFunctionCtx* p = NULL; - SqlFunctionCtx** pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); - if (pValCtx == NULL) { - return terrno; + SArray* pValCtxArray = NULL; + for (int32_t i = numOfOutput - 1; i > 0; --i) { // select Func is at the end of the list + int32_t funcIdx = pCtx[i].pExpr->pExpr->tupleFuncIdx; + if (funcIdx > 0) { + if (pValCtxArray == NULL) { + // the end of the list is the select function of biggest index + pValCtxArray = taosArrayInit_s(sizeof(SSubsidiaryResInfo*), funcIdx); + if (pValCtxArray == NULL) { + return terrno; + } + } + if (funcIdx > pValCtxArray->size) { + qError("funcIdx:%d is out of range", funcIdx); + taosArrayDestroyP(pValCtxArray, deleteSubsidiareCtx); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } + SSubsidiaryResInfo* pSubsidiary = &pCtx[i].subsidiaries; + pSubsidiary->pCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); + if (pSubsidiary->pCtx == NULL) { + taosArrayDestroyP(pValCtxArray, deleteSubsidiareCtx); + return terrno; + } + pSubsidiary->num = 0; + taosArraySet(pValCtxArray, funcIdx - 1, &pSubsidiary); + } else { + break; + } } - SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - QUERY_CHECK_NULL(pSelectFuncs, code, lino, _end, terrno); + SqlFunctionCtx* p = NULL; + SqlFunctionCtx** pValCtx = NULL; + if (pValCtxArray == NULL) { + pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); + if (pValCtx == NULL) { + QUERY_CHECK_CODE(terrno, lino, _end); + } + } for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0) || (strcmp(pName, "_group_const_value") == 0)) { - pValCtx[num++] = &pCtx[i]; - } else if (fmIsSelectFunc(pCtx[i].functionId)) { - void* data = taosHashGet(pSelectFuncs, pName, strlen(pName)); - if (taosHashGetSize(pSelectFuncs) != 0 && data == NULL) { - p = NULL; - break; + if (pValCtxArray == NULL) { + pValCtx[num++] = &pCtx[i]; } else { - int32_t tempRes = taosHashPut(pSelectFuncs, pName, strlen(pName), &num, sizeof(num)); - if (tempRes != TSDB_CODE_SUCCESS && tempRes != TSDB_CODE_DUP_KEY) { - code = tempRes; - QUERY_CHECK_CODE(code, lino, _end); + int32_t bindFuncIndex = pCtx[i].pExpr->pExpr->bindTupleFuncIdx; // start from index 1; + if (bindFuncIndex > 0) { // 0 is default index related to the select function + bindFuncIndex -= 1; } - p = &pCtx[i]; + SSubsidiaryResInfo** pSubsidiary = taosArrayGet(pValCtxArray, bindFuncIndex); + if(pSubsidiary == NULL) { + QUERY_CHECK_CODE(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR, lino, _end); + } + (*pSubsidiary)->pCtx[(*pSubsidiary)->num] = &pCtx[i]; + (*pSubsidiary)->num++; } + } else if (fmIsSelectFunc(pCtx[i].functionId)) { + if (pValCtxArray == NULL) { + p = &pCtx[i]; + } } } - taosHashCleanup(pSelectFuncs); if (p != NULL) { p->subsidiaries.pCtx = pValCtx; @@ -2117,8 +2159,8 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu _end: if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroyP(pValCtxArray, deleteSubsidiareCtx); taosMemoryFreeClear(pValCtx); - taosHashCleanup(pSelectFuncs); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return code; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 97d6a5851b..a88bee48fa 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -443,6 +443,8 @@ int32_t createFunctionWithSrcFunc(const char* pName, const SFunctionNode* pSrcFu return code; } resetOutputChangedFunc(*ppFunc, pSrcFunc); + (*ppFunc)->node.bindTupleFuncIdx = pSrcFunc->node.bindTupleFuncIdx; + (*ppFunc)->node.tupleFuncIdx = pSrcFunc->node.tupleFuncIdx; return code; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 22f6dc7418..084b5001bb 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -103,6 +103,8 @@ static int32_t exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { COPY_CHAR_ARRAY_FIELD(aliasName); COPY_CHAR_ARRAY_FIELD(userAlias); COPY_SCALAR_FIELD(projIdx); + COPY_SCALAR_FIELD(bindTupleFuncIdx); + COPY_SCALAR_FIELD(tupleFuncIdx); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 891843761a..91bd6c6973 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "functionMgt.h" #include "querynodes.h" #define COMPARE_SCALAR_FIELD(fldname) \ @@ -137,6 +138,11 @@ static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_SCALAR_FIELD(funcId); COMPARE_STRING_FIELD(functionName); COMPARE_NODE_LIST_FIELD(pParameterList); + if (a->funcType == FUNCTION_TYPE_SELECT_VALUE) { + if ((a->node.bindTupleFuncIdx != b->node.bindTupleFuncIdx) && + (a->node.bindTupleFuncIdx != 0 != b->node.bindTupleFuncIdx != 0)) + return false; + } return true; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 930a88aea0..b122c3fffd 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -664,11 +664,18 @@ static int32_t msgToDataType(STlvDecoder* pDecoder, void* pObj) { return code; } -enum { EXPR_CODE_RES_TYPE = 1 }; +enum { EXPR_CODE_RES_TYPE = 1, EXPR_CODE_BIND_TUPLE_FUNC_IDX, EXPR_CODE_TUPLE_FUNC_IDX }; static int32_t exprNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { const SExprNode* pNode = (const SExprNode*)pObj; - return tlvEncodeObj(pEncoder, EXPR_CODE_RES_TYPE, dataTypeToMsg, &pNode->resType); + int32_t code = tlvEncodeObj(pEncoder, EXPR_CODE_RES_TYPE, dataTypeToMsg, &pNode->resType); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, EXPR_CODE_BIND_TUPLE_FUNC_IDX, pNode->bindTupleFuncIdx); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, EXPR_CODE_TUPLE_FUNC_IDX, pNode->tupleFuncIdx); + } + return code; } static int32_t msgToExprNode(STlvDecoder* pDecoder, void* pObj) { @@ -681,6 +688,12 @@ static int32_t msgToExprNode(STlvDecoder* pDecoder, void* pObj) { case EXPR_CODE_RES_TYPE: code = tlvDecodeObjFromTlv(pTlv, msgToDataType, &pNode->resType); break; + case EXPR_CODE_BIND_TUPLE_FUNC_IDX: + code = tlvDecodeI32(pTlv, &pNode->bindTupleFuncIdx); + break; + case EXPR_CODE_TUPLE_FUNC_IDX: + code = tlvDecodeI32(pTlv, &pNode->tupleFuncIdx); + break; default: break; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d6225ce2b6..5abc142584 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -13,8 +13,12 @@ * along with this program. If not, see . */ +#include "nodes.h" #include "parInt.h" #include "parTranslater.h" +#include +#include "query.h" +#include "querynodes.h" #include "tdatablock.h" #include "catalog.h" @@ -3534,6 +3538,8 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode tstrncpy(pFunc->functionName, "_select_value", TSDB_FUNC_NAME_LEN); tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); + pFunc->node.bindTupleFuncIdx = ((SExprNode*)*pNode)->bindTupleFuncIdx; + pFunc->node.tupleFuncIdx = ((SExprNode*)*pNode)->tupleFuncIdx; pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = getFuncInfo(pCxt, pFunc); @@ -3934,7 +3940,8 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { ((QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG))) { return rewriteExprToSelectTagFunc(pCxt->pTranslateCxt, pNode); } - if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { + if ((isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) && + ((!nodesIsExprNode(*pNode) || ((SExprNode*)*pNode)->bindTupleFuncIdx == 0))) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -7362,6 +7369,19 @@ static EDealRes isMultiColsFuncNode(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } +typedef struct SBindTupleFuncCxt { + int32_t bindTupleFuncIdx; +} SBindTupleFuncCxt; + +static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { + SBindTupleFuncCxt* pCxt = pContext; + if (nodesIsExprNode(*pNode)) { + ((SExprNode*)*pNode)->bindTupleFuncIdx = pCxt->bindTupleFuncIdx; + SFunctionNode* pFunc = (SFunctionNode*)*pNode; + } + return DEAL_RES_CONTINUE; +} + static bool hasMultiColsFuncInList(SNodeList* nodeList) { SHasMultiColsFuncCxt pCxt = {false}; @@ -7383,13 +7403,9 @@ static int32_t hasInvalidColsFunction(STranslateContext* pCxt, SNodeList* nodeLi FOREACH(pTmpNode, nodeList) { if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; - if (pFunc->funcType == FUNCTION_TYPE_COLS) { - // cols function at here is valid. - } else { - if (hasMultiColsFuncInList(pFunc->pParameterList)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, - "Invalid cols function in function %s", pFunc->functionName); - } + if (hasMultiColsFuncInList(pFunc->pParameterList)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, + "Invalid cols function in function %s", pFunc->functionName); } } else { if (hasMultiColsFunc(&pTmpNode)) { @@ -7436,43 +7452,80 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, "Invalid using alias for cols function"); } - if (isMultiColsFunc(pFunc)) { - needRewrite = true; - break; - } + needRewrite = true; } } SNodeList* pNewNodeList = NULL; + SNodeList* tmpFuncNodeList = NULL; if (needRewrite) { code = nodesMakeList(&pNewNodeList); if (NULL == pNewNodeList) { return code; } + code = nodesMakeList(&tmpFuncNodeList); + if (NULL == tmpFuncNodeList) { + return code; + } + SNode* pNewNode = NULL; + int32_t nums = 0; + int32_t selectFuncNum = 0; FOREACH(pTmpNode, *nodeList) { if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; - if (isMultiColsFunc(pFunc)) { + if (strcasecmp(pFunc->functionName, "cols") == 0) { + ++selectFuncNum; + SNode* pSelectFunc = nodesListGetNode(pFunc->pParameterList, 0); + if(nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { + code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + parserError("Invalid cols function, the first parameter must be a select function"); + goto _end; + } + nodesListMakeStrictAppend(&tmpFuncNodeList, pSelectFunc); // start from index 1, because the first parameter is select function which needn't to output. - SFunctionNode* pSelectFunc = (SFunctionNode*)nodesListGetNode(pFunc->pParameterList, 0); for (int i = 1; i < pFunc->pParameterList->length; ++i) { SNode* pExpr = nodesListGetNode(pFunc->pParameterList, i); - SNode* pNewFunc = NULL; - code = createMultiResColsFunc(pFunc, pSelectFunc, (SExprNode*)pExpr, &pNewFunc); + + code = nodesCloneNode(pExpr, &pNewNode); + if (nodesIsExprNode(pNewNode)) { + SBindTupleFuncCxt pCxt = {selectFuncNum}; + nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); + } else { + code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + parserError("Invalid cols function, the first parameter must be a select function"); + goto _end; + } if (TSDB_CODE_SUCCESS != code) goto _end; - code = nodesListMakeStrictAppend(&pNewNodeList, pNewFunc); + code = nodesListMakeStrictAppend(&pNewNodeList, pNewNode); if (TSDB_CODE_SUCCESS != code) goto _end; } continue; } } - SNode* pNewNode = NULL; code = nodesCloneNode(pTmpNode, &pNewNode); if (TSDB_CODE_SUCCESS != code) goto _end; code = nodesListMakeStrictAppend(&pNewNodeList, pNewNode); if (TSDB_CODE_SUCCESS != code) goto _end; } + SNode* pNode = NULL; + int32_t pNewSelectFuncIds = 0; + FOREACH(pNode, tmpFuncNodeList) { + ++pNewSelectFuncIds; + code = nodesCloneNode(pNode, &pNewNode); + if (TSDB_CODE_SUCCESS != code) goto _end; + if (nodesIsExprNode(pNewNode)) { + SExprNode* pExprNode = (SExprNode*)pNewNode; + pExprNode->tupleFuncIdx = pNewSelectFuncIds; + } else { + code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + parserError("Invalid cols function, the first parameter must be a select function"); + goto _end; + } + code = nodesListMakeStrictAppend(&pNewNodeList, pNewNode); + if (TSDB_CODE_SUCCESS != code) goto _end; + } nodesDestroyList(*nodeList); + nodesDestroyList(tmpFuncNodeList); *nodeList = pNewNodeList; } return TSDB_CODE_SUCCESS; @@ -7480,6 +7533,7 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList _end: if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pNewNodeList); + nodesDestroyList(tmpFuncNodeList); } return code; } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 9267fffa40..26e2e4a0cb 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -71,10 +71,64 @@ class TDTestCase: tdSql.query(f'select cols(last(c0), ts, c1), cols(first(c0), ts, c1), count(1) from {self.dbname}.meters') tdSql.query(f'select cols(last(c0), ts as t1, c1 as c11), cols(first(c0), ts as c2, c1 c21), count(1) from {self.dbname}.meters') + + def funcNestTest(self): + tdSql.execute('create database db;') + tdSql.execute('use db') + tdSql.execute(f'drop table if exists db.d1') + + tdSql.execute('create table db.d1 (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool)') + tdSql.execute('insert into db.d1 values(1734574929000, 1, 1.1, "a", true)') + tdSql.execute('insert into db.d1 values(1734574930000, 2, 2.2, "bbbbbbbbb", false)') + + tdSql.query(f'select cols(last(c0), ts, c2), cols(first(c0), ts, c2) from db.d1') + tdSql.checkRows(1) + #tdSql.checkCols(4) + tdSql.checkData(0, 0, 1734574930000) + tdSql.checkData(0, 1, 2.2) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1.1) + tdSql.query(f'select cols(last(c0), ts, c1, c2, c3), cols(first(c0), ts, c1, c2, c3) from db.d1') + tdSql.checkRows(1) + #tdSql.checkCols(6) + tdSql.checkData(0, 0, 1734574930000) + tdSql.checkData(0, 1, 2.2) + tdSql.checkData(0, 2, 'bbbbbbbbb') + tdSql.checkData(0, 3, False) + tdSql.checkData(0, 4, 1734574929000) + tdSql.checkData(0, 5, 1.1) + tdSql.checkData(0, 6, 'a') + tdSql.checkData(0, 7, True) + + tdSql.query(f'select cols(last(ts), c1), cols(first(ts), c1) from db.d1') + tdSql.checkRows(1) + #tdSql.checkCols(6) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 1) + + tdSql.query(f'select cols(first(ts), c1), cols(first(ts), c1) from db.d1') + tdSql.checkRows(1) + #tdSql.checkCols(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + + tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2)) from db.d1') + tdSql.checkRows(1) + #tdSql.checkCols(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2)) from db.d1') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + + + def parse_test(self): tdLog.info("parse test") - #** error sql **# tdSql.error(f'select cols(ts) from {self.dbname}.meters group by tbname') tdSql.error(f'select cols(ts) from {self.dbname}.meters') @@ -101,8 +155,12 @@ class TDTestCase: tdSql.error(f'select cols(last(ts)+1, ts) from {self.dbname}.meters') tdSql.error(f'select cols(last(ts)+10, c1+10) from {self.dbname}.meters group by tbname') + tdSql.error(f'select cols(cols(last(ts), c0), c0) as cc from {self.dbname}.meters') + def run(self): + self.funcNestTest() + return self.create_test_data() self.parse_test() self.one_cols_1output_test() From e26a3567bc0c941f71e656ddf92bd6a874558f67 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 25 Dec 2024 08:58:07 +0800 Subject: [PATCH 03/45] cols function --- source/libs/parser/src/parTranslater.c | 26 ++++++++++++++---- tests/system-test/2-query/cols_function.py | 32 +++++++++++++++++----- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5abc142584..6535c70c82 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7438,6 +7438,18 @@ static bool invalidColsAlias(SFunctionNode* pFunc) { return false; } +static int32_t getSelectFuncIndex(SNodeList* FuncNodeList, SNode* pSelectFunc) { + SNode* pNode = NULL; + int32_t selectFuncIndex = 0; + FOREACH(pNode, FuncNodeList) { + ++selectFuncIndex; + if (nodesEqualNode(pNode, pSelectFunc)) { + return selectFuncIndex; + } + } + return 0; +} + static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList) { int32_t code = hasInvalidColsFunction(pCxt, *nodeList); if (TSDB_CODE_SUCCESS != code) { @@ -7469,26 +7481,30 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList } SNode* pNewNode = NULL; int32_t nums = 0; - int32_t selectFuncNum = 0; + int32_t selectFuncCount = 0; FOREACH(pTmpNode, *nodeList) { if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; if (strcasecmp(pFunc->functionName, "cols") == 0) { - ++selectFuncNum; SNode* pSelectFunc = nodesListGetNode(pFunc->pParameterList, 0); - if(nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { + if (nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; parserError("Invalid cols function, the first parameter must be a select function"); goto _end; } - nodesListMakeStrictAppend(&tmpFuncNodeList, pSelectFunc); + int32_t selectFuncIndex = getSelectFuncIndex(tmpFuncNodeList, pSelectFunc); + if (selectFuncIndex == 0) { + ++selectFuncCount; + selectFuncIndex = selectFuncCount; + nodesListMakeStrictAppend(&tmpFuncNodeList, pSelectFunc); + } // start from index 1, because the first parameter is select function which needn't to output. for (int i = 1; i < pFunc->pParameterList->length; ++i) { SNode* pExpr = nodesListGetNode(pFunc->pParameterList, i); code = nodesCloneNode(pExpr, &pNewNode); if (nodesIsExprNode(pNewNode)) { - SBindTupleFuncCxt pCxt = {selectFuncNum}; + SBindTupleFuncCxt pCxt = {selectFuncIndex}; nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); } else { code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 26e2e4a0cb..dba7e346b1 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -85,9 +85,9 @@ class TDTestCase: tdSql.checkRows(1) #tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574930000) - tdSql.checkData(0, 1, 2.2) + tdSql.checkData(0, 1, 'bbbbbbbbb') tdSql.checkData(0, 2, 1734574929000) - tdSql.checkData(0, 3, 1.1) + tdSql.checkData(0, 3, 'a') tdSql.query(f'select cols(last(c0), ts, c1, c2, c3), cols(first(c0), ts, c1, c2, c3) from db.d1') tdSql.checkRows(1) #tdSql.checkCols(6) @@ -103,19 +103,37 @@ class TDTestCase: tdSql.query(f'select cols(last(ts), c1), cols(first(ts), c1) from db.d1') tdSql.checkRows(1) #tdSql.checkCols(6) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 0, 2.2) + tdSql.checkData(0, 1, 1.1) - tdSql.query(f'select cols(first(ts), c1), cols(first(ts), c1) from db.d1') + tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts), c0, c1) from db.d1') tdSql.checkRows(1) #tdSql.checkCols(6) tdSql.checkData(0, 0, 1) - tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 1, 1.1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, 1.1) + + tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts+1), c0, c1) from db.d1') + tdSql.checkRows(1) + #tdSql.checkCols(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1.1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, 1.1) + + tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts), c0+1, c1+2) from db.d1') + tdSql.checkRows(1) + #tdSql.checkCols(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1.1) + tdSql.checkData(0, 2, 2) + tdSql.checkData(0, 3, 3.1) tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2)) from db.d1') tdSql.checkRows(1) #tdSql.checkCols(6) - tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 0, 1734574929000) tdSql.checkData(0, 1, 1) tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2)) from db.d1') tdSql.checkRows(1) From 504a8af0cda197bec3ed3ea80c039ed2dd4f7efb Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 25 Dec 2024 17:40:09 +0800 Subject: [PATCH 04/45] fix: select_value function slotid --- include/libs/nodes/querynodes.h | 2 +- include/util/tdef.h | 1 + source/libs/parser/src/parTranslater.c | 2 ++ source/libs/planner/src/planLogicCreater.c | 2 ++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index e14a43b1c2..161d713ecc 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -55,7 +55,7 @@ typedef struct SDataType { typedef struct SExprNode { ENodeType type; SDataType resType; - char aliasName[TSDB_COL_NAME_LEN]; + char aliasName[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; char userAlias[TSDB_COL_NAME_LEN]; SArray* pAssociation; bool asAlias; diff --git a/include/util/tdef.h b/include/util/tdef.h index 2ee84b42bd..5deec72176 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -271,6 +271,7 @@ typedef enum ELogicConditionType { #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) #define TSDB_COL_NAME_LEN 65 +#define TSDB_COL_NAME_EXLEN 2 #define TSDB_COL_FNAME_LEN (TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6535c70c82..8da19736a4 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7377,6 +7377,8 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { SBindTupleFuncCxt* pCxt = pContext; if (nodesIsExprNode(*pNode)) { ((SExprNode*)*pNode)->bindTupleFuncIdx = pCxt->bindTupleFuncIdx; + int32_t len = strlen(((SExprNode*)*pNode)->aliasName); + tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, "%d", pCxt->bindTupleFuncIdx); SFunctionNode* pFunc = (SFunctionNode*)*pNode; } return DEAL_RES_CONTINUE; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 363aa71479..519194ef8a 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -710,6 +710,8 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) if (NULL != pStmtName) { snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName); } + pCol->node.bindTupleFuncIdx = pExpr->bindTupleFuncIdx; + pCol->node.tupleFuncIdx = pExpr->tupleFuncIdx; return pCol; } From c371fb200daa680ae55fbb2e74632bd8bb708c27 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 25 Dec 2024 18:27:50 +0800 Subject: [PATCH 05/45] fix: bindTupleFuncindex --- include/util/tdef.h | 2 +- source/libs/parser/src/parTranslater.c | 2 +- source/libs/planner/src/planLogicCreater.c | 2 ++ tests/system-test/2-query/cols_function.py | 32 ++++++++++++++++++---- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 5deec72176..f090512b1c 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -271,7 +271,7 @@ typedef enum ELogicConditionType { #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) #define TSDB_COL_NAME_LEN 65 -#define TSDB_COL_NAME_EXLEN 2 +#define TSDB_COL_NAME_EXLEN 4 #define TSDB_COL_FNAME_LEN (TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8da19736a4..bb547baee2 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7378,7 +7378,7 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { if (nodesIsExprNode(*pNode)) { ((SExprNode*)*pNode)->bindTupleFuncIdx = pCxt->bindTupleFuncIdx; int32_t len = strlen(((SExprNode*)*pNode)->aliasName); - tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, "%d", pCxt->bindTupleFuncIdx); + tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pCxt->bindTupleFuncIdx); SFunctionNode* pFunc = (SFunctionNode*)*pNode; } return DEAL_RES_CONTINUE; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 519194ef8a..4bb2d7be71 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -123,6 +123,8 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { tstrncpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias, TSDB_COL_NAME_LEN); tstrncpy(pCol->colName, ((SExprNode*)pExpr)->aliasName, TSDB_COL_NAME_LEN); pCol->node.projIdx = ((SExprNode*)(*pNode))->projIdx; + pCol->node.bindTupleFuncIdx = ((SExprNode*)(*pNode))->bindTupleFuncIdx; + pCol->node.tupleFuncIdx = ((SExprNode*)(*pNode))->tupleFuncIdx; if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index dba7e346b1..28e5de5d2e 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -134,14 +134,36 @@ class TDTestCase: tdSql.checkRows(1) #tdSql.checkCols(6) tdSql.checkData(0, 0, 1734574929000) - tdSql.checkData(0, 1, 1) - tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2)) from db.d1') + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1734574930000) + tdSql.checkData(0, 3, 36) + tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2) + 2) from db.d1') tdSql.checkRows(1) tdSql.checkData(0, 0, 1734574929000) - tdSql.checkData(0, 1, 1) - tdSql.checkData(0, 2, 1734574929000) - tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1734574930000) + tdSql.checkData(0, 3, 38) + + tdSql.query(f'select cols(first(c0), ts, c2), cols(last(c0), ts, length(c2) + 2) from db.d1') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 'a') + tdSql.checkData(0, 2, 1734574930000) + tdSql.checkData(0, 3, 38) + + tdSql.query(f'select cols(min(c0), ts, c2), cols(last(c0), ts, length(c2) + 2) from db.d1') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 'a') + tdSql.checkData(0, 2, 1734574930000) + tdSql.checkData(0, 3, 38) + tdSql.query(f'select cols(min(c0), ts, c2), cols(first(c0), ts, length(c2) + 2) from db.d1') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 'a') + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 6) def parse_test(self): From 983431bab6ea236f9ecb0d579209de53e4700041 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 26 Dec 2024 11:13:51 +0800 Subject: [PATCH 06/45] cols --- include/libs/scalar/scalar.h | 1 - source/libs/function/inc/builtinsimpl.h | 5 - source/libs/function/src/builtins.c | 37 ----- source/libs/function/src/builtinsimpl.c | 74 ---------- source/libs/parser/src/parTranslater.c | 163 ++++++++------------- source/libs/scalar/src/sclfunc.c | 3 - tests/system-test/2-query/cols_function.py | 21 +++ 7 files changed, 82 insertions(+), 222 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 60aefa1491..fd936dd087 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -149,7 +149,6 @@ int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); -int32_t colsScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 883d80d1cd..d548ae956d 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -251,11 +251,6 @@ int32_t blockDBUsageSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo int32_t blockDBUsageFunction(SqlFunctionCtx* pCtx); int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); -bool getColsFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); -int32_t colsFunction(SqlFunctionCtx* pCtx); -int32_t colsPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); -int32_t colsCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); - #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e91b80aa4a..ec7f3220dc 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -5653,43 +5653,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}}, .translateFunc = translateOutVarchar, }, - { - .name = "cols", - .type = FUNCTION_TYPE_COLS, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_SELECT_COLS_FUNC, - .translateFunc = translateCols, - .dynDataRequiredFunc = NULL, - .getEnvFunc = getColsFuncEnv, - .initFunc = functionSetup, - .processFunc = colsFunction, - .sprocessFunc = colsScalarFunction, - .pPartialFunc = "_cols_partial", - .pMergeFunc = "_cols_merge", - .finalizeFunc = NULL - }, - { - .name = "_cols_partial", - .type = FUNCTION_TYPE_COLS_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_SELECT_COLS_FUNC, - .translateFunc = translateCols, - .dynDataRequiredFunc = NULL, - .getEnvFunc = getColsFuncEnv, - .initFunc = functionSetup, - .processFunc = colsFunction, - .finalizeFunc = colsPartialFinalize, - .combineFunc = colsCombine, - }, - { - .name = "_cols_merge", - .type = FUNCTION_TYPE_COLS_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_SELECT_COLS_FUNC, - .translateFunc = translateOutFirstIn, - .getEnvFunc = getFirstLastFuncEnv, - .initFunc = functionSetup, - .processFunc = lastFunctionMerge, - .finalizeFunc = colsPartialFinalize, - .combineFunc = colsCombine, - }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 43e40bee80..1c77d12fb2 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -7194,77 +7194,3 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } - -bool getColsFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); - pEnv->calcMemSize = pNode->node.resType.bytes; - return true; -} - -int32_t colsFunction(SqlFunctionCtx* pCtx) { - int32_t code = TSDB_CODE_SUCCESS; - SFunctionNode* pSelectFunc = ((SFunctionNode*)(pCtx[0].pExpr->pExpr->_function.pFunctNode->pParameterList->pHead->pNode)); - - SFuncExecFuncs selectExecFuncs; - code = fmGetFuncExecFuncs(pCtx->functionId, &selectExecFuncs); - if(TSDB_CODE_SUCCESS != code) { - return code; - } - if(selectExecFuncs.process == NULL) { - return TSDB_CODE_SUCCESS; - } - return selectExecFuncs.process(pCtx); -} - -int32_t colsPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - int32_t code = TSDB_CODE_SUCCESS; - - SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); - SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); - - int32_t resultBytes = getFirstLastInfoSize(pRes->bytes, pRes->pkBytes); - - // todo check for failure - char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); - if (NULL == res) { - return terrno; - } - (void)memcpy(varDataVal(res), pRes, resultBytes); - - varDataSetLen(res, resultBytes); - - int32_t slotId = pCtx->pExpr->base.resSchema.slotId; - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - if (NULL == pCol) { - taosMemoryFree(res); - return TSDB_CODE_OUT_OF_RANGE; - } - - if (pEntryInfo->numOfRes == 0) { - colDataSetNULL(pCol, pBlock->info.rows); - code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, pBlock->info.rows); - } else { - code = colDataSetVal(pCol, pBlock->info.rows, res, false); - if (TSDB_CODE_SUCCESS != code) { - taosMemoryFree(res); - return code; - } - code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); - } - taosMemoryFree(res); - return code; -} - -int32_t colsCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { - SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); - SFirstLastRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); - int32_t bytes = pDBuf->bytes; - - SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); - SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - - pDBuf->hasResult = firstLastTransferInfoImpl(pSBuf, pDBuf, false); - pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); - pDResInfo->isNullRes &= pSResInfo->isNullRes; - return TSDB_CODE_SUCCESS; -} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bb547baee2..a1c723e3f2 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -19,6 +19,7 @@ #include #include "query.h" #include "querynodes.h" +#include "taoserror.h" #include "tdatablock.h" #include "catalog.h" @@ -5195,54 +5196,6 @@ static int32_t createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr, SNo return code; } -static int32_t createMultiResColsFunc(SFunctionNode* pSrcFunc, SFunctionNode* pSelectFunc, SExprNode* pExpr, SNode** ppNodeOut) { - SFunctionNode* pFunc = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - code = nodesMakeList(&pFunc->pParameterList); - SNode* pClonedFuncNode = NULL; - if (TSDB_CODE_SUCCESS != (code = nodesCloneNode((SNode*)pSelectFunc, &pClonedFuncNode)) || - TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pFunc->pParameterList, pClonedFuncNode))) { - nodesDestroyNode((SNode*)pFunc); - return code; - } - SNode* pClonedExprNode = NULL; - if (TSDB_CODE_SUCCESS != (code = nodesCloneNode((SNode*)pExpr, &pClonedExprNode)) || - TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pFunc->pParameterList, pClonedExprNode))) { - nodesDestroyNode((SNode*)pFunc); - return code; - } - - pFunc->node.resType = pExpr->resType; - pFunc->funcId = pSrcFunc->funcId; - pFunc->funcType = pSrcFunc->funcType; - strcpy(pFunc->functionName, pSrcFunc->functionName); - char buf[TSDB_FUNC_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + TSDB_NAME_DELIMITER_LEN + 3] = {0}; - int32_t len = 0; - if(pExpr->asAlias) { - strncpy(pFunc->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN - 1); - strncpy(pFunc->node.userAlias, pExpr->userAlias, TSDB_COL_NAME_LEN - 1); - pFunc->node.asAlias = true; - } else if (QUERY_NODE_COLUMN == nodeType(pExpr)) { - SColumnNode* pCol = (SColumnNode*)pExpr; - len = tsnprintf(buf, sizeof(buf) - 1, "%s.%s", pCol->tableAlias, pCol->colName); - (void)taosHashBinary(buf, len); - strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); - len = tsnprintf(buf, sizeof(buf) - 1, "%s", pCol->colName); - strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); - } else { - len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); - (void)taosHashBinary(buf, len); - strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1); - len = tsnprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); - strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1); - } - *ppNodeOut = (SNode*)pFunc; - return code; -} - static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, bool igTags, SNodeList** pOutput) { STableNode* pTable = NULL; int32_t code = findTable(pCxt, pCol->tableAlias, &pTable); @@ -7345,10 +7298,17 @@ static int32_t translateSelectWithoutFrom(STranslateContext* pCxt, SSelectStmt* pCxt->dual = true; return translateExprList(pCxt, pSelect->pProjectionList); } +typedef struct SCheckColsFuncCxt { + bool hasColsFunc; + bool hasMultiColsFunc; +} SCheckColsFuncCxt; -typedef struct SHasMultiColsFuncCxt { - bool hasMultiColsFunc; -} SHasMultiColsFuncCxt; +static bool isColsFuncByName(SFunctionNode* pFunc) { + if (strcasecmp(pFunc->functionName, "cols") != 0) { + return false; + } + return true; +} static bool isMultiColsFunc(SFunctionNode* pFunc) { if (strcasecmp(pFunc->functionName, "cols") != 0) { @@ -7358,12 +7318,15 @@ static bool isMultiColsFunc(SFunctionNode* pFunc) { } static EDealRes isMultiColsFuncNode(SNode** pNode, void* pContext) { - SHasMultiColsFuncCxt* pCxt = pContext; + SCheckColsFuncCxt* pCxt = pContext; if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { SFunctionNode* pFunc = (SFunctionNode*)*pNode; - if (isMultiColsFunc(pFunc)) { - pCxt->hasMultiColsFunc = true; - return DEAL_RES_END; + if (isColsFuncByName(pFunc)) { + pCxt->hasColsFunc = true; + if (pFunc->pParameterList->length > 2) { + pCxt->hasMultiColsFunc = true; + return DEAL_RES_END; + } } } return DEAL_RES_CONTINUE; @@ -7384,45 +7347,17 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool hasMultiColsFuncInList(SNodeList* nodeList) { - SHasMultiColsFuncCxt pCxt = {false}; - - nodesRewriteExprs(nodeList, isMultiColsFuncNode, &pCxt); - - return pCxt.hasMultiColsFunc; +static void checkColsFuncInList(SNodeList* nodeList, SCheckColsFuncCxt* pCheckouColsFuncCxt) { + nodesRewriteExprs(nodeList, isMultiColsFuncNode, pCheckouColsFuncCxt); + return; } -static bool hasMultiColsFunc(SNode** pNode) { - SHasMultiColsFuncCxt pCxt = {false}; - - nodesRewriteExpr(pNode, isMultiColsFuncNode, &pCxt); - - return pCxt.hasMultiColsFunc; -} - -static int32_t hasInvalidColsFunction(STranslateContext* pCxt, SNodeList* nodeList) { - SNode* pTmpNode = NULL; - FOREACH(pTmpNode, nodeList) { - if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { - SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; - if (hasMultiColsFuncInList(pFunc->pParameterList)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, - "Invalid cols function in function %s", pFunc->functionName); - } - } else { - if (hasMultiColsFunc(&pTmpNode)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, - "Invalid cols function, can't be used at here"); - } - } - } - return TSDB_CODE_SUCCESS; +static void checkColsFunc(SNode** pNode, SCheckColsFuncCxt* pCheckouColsFuncCxt) { + nodesRewriteExpr(pNode, isMultiColsFuncNode, pCheckouColsFuncCxt); + return; } static bool invalidColsAlias(SFunctionNode* pFunc) { - if (strcasecmp(pFunc->functionName, "cols") != 0) { - return false; - } if (pFunc->node.asAlias) { if (pFunc->pParameterList->length > 2) { return true; @@ -7436,10 +7371,40 @@ static bool invalidColsAlias(SFunctionNode* pFunc) { } } } - return false; } +static int32_t hasInvalidColsFunction(STranslateContext* pCxt, SNodeList* nodeList, + SCheckColsFuncCxt* pCheckouColsFuncCxt) { + SNode* pTmpNode = NULL; + FOREACH(pTmpNode, nodeList) { + if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; + if (isColsFuncByName(pFunc)) { + pCheckouColsFuncCxt->hasColsFunc = true; + if (invalidColsAlias(pFunc)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, + "Invalid using alias for cols function"); + } + } + checkColsFuncInList(pFunc->pParameterList, pCheckouColsFuncCxt); + if (pCheckouColsFuncCxt->hasMultiColsFunc) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, + "Invalid cols function in function %s", pFunc->functionName); + } + } else { + checkColsFunc(&pTmpNode, pCheckouColsFuncCxt); + if (pCheckouColsFuncCxt->hasMultiColsFunc) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, + "Invalid cols function, can't be used at here"); + } + } + } + return TSDB_CODE_SUCCESS; +} + + + static int32_t getSelectFuncIndex(SNodeList* FuncNodeList, SNode* pSelectFunc) { SNode* pNode = NULL; int32_t selectFuncIndex = 0; @@ -7453,21 +7418,14 @@ static int32_t getSelectFuncIndex(SNodeList* FuncNodeList, SNode* pSelectFunc) { } static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList) { - int32_t code = hasInvalidColsFunction(pCxt, *nodeList); + SCheckColsFuncCxt pCheckouColsFuncCxt = {false, false}; + int32_t code = hasInvalidColsFunction(pCxt, *nodeList, &pCheckouColsFuncCxt); if (TSDB_CODE_SUCCESS != code) { return code; } - bool needRewrite = false; - SNode* pTmpNode = NULL; - FOREACH(pTmpNode, *nodeList) { - if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { - SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; - if(invalidColsAlias(pFunc)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, - "Invalid using alias for cols function"); - } - needRewrite = true; - } + bool needRewrite = false; + if (pCheckouColsFuncCxt.hasColsFunc) { + needRewrite = true; } SNodeList* pNewNodeList = NULL; @@ -7484,6 +7442,7 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList SNode* pNewNode = NULL; int32_t nums = 0; int32_t selectFuncCount = 0; + SNode* pTmpNode = NULL; FOREACH(pTmpNode, *nodeList) { if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 68b6b8de68..8ab6134053 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -4589,6 +4589,3 @@ int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam return selectScalarFunction(pInput, inputNum, pOutput); } -int32_t colsScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - return selectScalarFunction(pInput, inputNum, pOutput); -} diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 28e5de5d2e..43244e095f 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -71,6 +71,25 @@ class TDTestCase: tdSql.query(f'select cols(last(c0), ts, c1), cols(first(c0), ts, c1), count(1) from {self.dbname}.meters') tdSql.query(f'select cols(last(c0), ts as t1, c1 as c11), cols(first(c0), ts as c2, c1 c21), count(1) from {self.dbname}.meters') + def funcSupperTableTest(self): + tdSql.execute('create database if not exists db;') + tdSql.execute('use db') + tdSql.execute(f'drop table if exists db.st') + + tdSql.execute('create table db.st (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool) tags (t1 nchar(30))') + tdSql.execute('create table db.st_1 using db.st tags("st1")') + tdSql.execute('create table db.st_2 using db.st tags("st1")') + tdSql.execute('insert into db.st_1 values(1734574929000, 1, 1, "a1", true)') + tdSql.execute('insert into db.st_1 values(1734574929001, 2, 2, "bbbbbbbbb1", false)') + tdSql.execute('insert into db.st_1 values(1734574929002, 3, 3, "a2", true)') + tdSql.execute('insert into db.st_1 values(1734574929003, 4, 4, "bbbbbbbbb2", false)') + + tdSql.query(f'select cols(last(c0), ts, c1, c2, c3), cols(first(c0), ts, c1, c2, c3) from db.st') + tdSql.checkRows(1) + tdSql.checkData(0, 1, 'bbbbbbbbb') + + #tdSql.execute(f'drop table if exists db.st') + def funcNestTest(self): tdSql.execute('create database db;') @@ -196,10 +215,12 @@ class TDTestCase: tdSql.error(f'select cols(last(ts)+10, c1+10) from {self.dbname}.meters group by tbname') tdSql.error(f'select cols(cols(last(ts), c0), c0) as cc from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), cols(last(ts), c0), c0) as cc from {self.dbname}.meters') def run(self): self.funcNestTest() + self.funcSupperTableTest() return self.create_test_data() self.parse_test() From 465f19567ec0f47f504bad07400420739b93dbdd Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 27 Dec 2024 08:56:28 +0800 Subject: [PATCH 07/45] fix: cols func --- include/libs/function/functionMgt.h | 2 -- source/libs/function/src/builtins.c | 13 +------------ source/libs/function/src/builtinsimpl.c | 2 +- source/libs/nodes/src/nodesEqualFuncs.c | 2 +- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 286068cebb..41b5d76371 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -209,8 +209,6 @@ typedef enum EFunctionType { FUNCTION_TYPE_HYPERLOGLOG_STATE, FUNCTION_TYPE_HYPERLOGLOG_STATE_MERGE, - FUNCTION_TYPE_COLS_PARTIAL, - FUNCTION_TYPE_COLS_MERGE, // geometry functions FUNCTION_TYPE_GEOM_FROM_TEXT = 4250, diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index ec7f3220dc..6fe01c5fc3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1233,17 +1233,6 @@ static int32_t translateForecastConf(SFunctionNode* pFunc, char* pErrBuf, int32_ return TSDB_CODE_SUCCESS; } -static int32_t translateCols(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_FLOAT].bytes, .type = TSDB_DATA_TYPE_FLOAT}; - return TSDB_CODE_SUCCESS; -} - -bool getColFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); - - return true; -} - static EFuncReturnRows forecastEstReturnRows(SFunctionNode* pFunc) { return FUNC_RETURN_ROWS_N; } static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { @@ -2742,7 +2731,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateOutFirstIn, .dynDataRequiredFunc = firstDynDataReq, .getEnvFunc = getFirstLastFuncEnv, - .initFunc = functionSetup, + .initFunc = firstLastFunctionSetup, .processFunc = firstFunction, .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1c77d12fb2..096201c3bb 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -946,7 +946,7 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu if (nullList[j]) { colDataSetNULL(pDstCol, rowIndex); } else { - code = colDataSetVal(pDstCol, rowIndex, pStart, false); + code = colDataSetValOrCover(pDstCol, rowIndex, pStart, false); if (TSDB_CODE_SUCCESS != code) { return code; } diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 91bd6c6973..501a9fe17d 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -140,7 +140,7 @@ static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_NODE_LIST_FIELD(pParameterList); if (a->funcType == FUNCTION_TYPE_SELECT_VALUE) { if ((a->node.bindTupleFuncIdx != b->node.bindTupleFuncIdx) && - (a->node.bindTupleFuncIdx != 0 != b->node.bindTupleFuncIdx != 0)) + (a->node.bindTupleFuncIdx != 0 || b->node.bindTupleFuncIdx != 0)) return false; } return true; From de801080e10f9dd579e79570ad2109b24f489c36 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 27 Dec 2024 20:08:21 +0800 Subject: [PATCH 08/45] enh: cols func --- source/libs/function/src/builtins.c | 3 +++ source/libs/function/src/builtinsimpl.c | 2 +- source/libs/parser/src/parTranslater.c | 4 ++++ source/libs/planner/src/planLogicCreater.c | 3 +++ source/libs/planner/src/planOptimizer.c | 7 ++++++- tests/system-test/2-query/cols_function.py | 9 ++++++++- 6 files changed, 25 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6fe01c5fc3..6d4e2c5bdb 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -5642,6 +5642,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}}, .translateFunc = translateOutVarchar, }, + { + .name = "cols", + }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 096201c3bb..3d1b04f223 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2740,7 +2740,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { char* data = colDataGetData(pInputCol, chosen); - int32_t code = doSaveCurrentVal(pCtx, i, cts, NULL, type, data); + int32_t code = doSaveCurrentVal(pCtx, chosen, cts, NULL, type, data); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a1c723e3f2..1680808c71 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -13726,6 +13726,10 @@ static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* int32_t index = 0; FOREACH(pNode, pProjections) { SExprNode* pExpr = (SExprNode*)pNode; + if(pExpr->tupleFuncIdx != 0) { + *numOfCols -= 1; + continue; + } if (TSDB_DATA_TYPE_NULL == pExpr->resType.type) { (*pSchema)[index].type = TSDB_DATA_TYPE_VARCHAR; (*pSchema)[index].bytes = VARSTR_HEADER_SIZE; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 4bb2d7be71..60228725db 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1600,6 +1600,9 @@ static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pS int32_t projIdx = 1; FOREACH(pNode, pExprs) { SColumnNode* pCol = createColumnByExpr(pStmtName, (SExprNode*)pNode); + if (pCol->node.tupleFuncIdx != 0) { + continue; + } if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)pCol))) { nodesDestroyList(pList); return code; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 11cf926081..63554e3a4e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3471,6 +3471,10 @@ static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild, nodesWalkExpr(pJoinLogicNode->pFullOnCond, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); if (!cxt.canUse) return false; } + if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pChild) && + ((SAggLogicNode*)pChild)->node.pTargets->length != pNewChildTargets->length) { + return false; + } return true; } @@ -3599,7 +3603,8 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* } else { FOREACH(pProjection, pProjectNode->pProjections) { FOREACH(pChildTarget, pChild->pTargets) { - if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { + if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName) + && ((SColumnNode*)pProjection)->node.bindTupleFuncIdx == 0) { SNode* pNew = NULL; code = nodesCloneNode(pChildTarget, &pNew); if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 43244e095f..0c1916b4dd 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -86,7 +86,14 @@ class TDTestCase: tdSql.query(f'select cols(last(c0), ts, c1, c2, c3), cols(first(c0), ts, c1, c2, c3) from db.st') tdSql.checkRows(1) - tdSql.checkData(0, 1, 'bbbbbbbbb') + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, '4.0') + tdSql.checkData(0, 2, 'bbbbbbbbb2') + tdSql.checkData(0, 3, False) + tdSql.checkData(0, 4, 1734574929000) + tdSql.checkData(0, 5, '1.0') + tdSql.checkData(0, 6, 'a1') + tdSql.checkData(0, 7, True) #tdSql.execute(f'drop table if exists db.st') From 2611eef0e279a0bf9c713cde7bd7b66dbf8e21d7 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 29 Dec 2024 10:02:20 +0800 Subject: [PATCH 09/45] fix: mem leak --- include/libs/nodes/querynodes.h | 2 +- source/libs/executor/src/executil.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 161d713ecc..f3c84c11bc 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -56,7 +56,7 @@ typedef struct SExprNode { ENodeType type; SDataType resType; char aliasName[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; - char userAlias[TSDB_COL_NAME_LEN]; + char userAlias[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; SArray* pAssociation; bool asAlias; bool asParam; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 13316a09ec..c57accfa66 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2162,6 +2162,8 @@ _end: taosArrayDestroyP(pValCtxArray, deleteSubsidiareCtx); taosMemoryFreeClear(pValCtx); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } else { + taosArrayDestroy(pValCtxArray); } return code; } From d3fa16b998cd88fd180dad4051e78ab38579e6d1 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 30 Dec 2024 07:56:02 +0800 Subject: [PATCH 10/45] test --- tests/system-test/2-query/cols_function.py | 43 +++++++++++++++------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 0c1916b4dd..5f48c401db 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -15,12 +15,24 @@ class TDTestCase: self.dbname = 'test' - def create_test_data(self): - tdLog.info("create test data") - tdLog.info("taosBenchmark -y -t 10 -n 100 -b INT,FLOAT,NCHAR,BOOL") - os.system("taosBenchmark -y -t 10 -n 100 -b INT,FLOAT,NCHAR,BOOL") + def create_test_data(self): + tdSql.execute(f'create database if not exists {self.dbname};') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(f'drop table if exists {self.dbname}.meters') - tdSql.execute('use test') + # tdLog.info("create test data") + # tdLog.info("taosBenchmark -y -t 10 -n 100 -b INT,FLOAT,NCHAR,BOOL") + # os.system("taosBenchmark -y -t 10 -n 100 -b INT,FLOAT,NCHAR,BOOL") + + tdSql.execute(f'create table {self.dbname}.meters (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool) tags (t1 nchar(30))') + tdSql.execute(f'create table {self.dbname}.d0 using {self.dbname}.meters tags("st1")') + tdSql.execute(f'create table {self.dbname}.d1 using {self.dbname}.meters tags("st1")') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929000, 1, 1, "a1", true)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929001, 2, 2, "bbbbbbbbb1", false)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929002, 3, 3, "a2", true)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929003, 4, 4, "bbbbbbbbb2", false)') + + tdSql.execute(f'use {self.dbname}') tdSql.execute(f'Create table {self.dbname}.normal_table (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool)') tdSql.execute(f'insert into {self.dbname}.normal_table (select * from {self.dbname}.d0)') @@ -109,14 +121,14 @@ class TDTestCase: tdSql.query(f'select cols(last(c0), ts, c2), cols(first(c0), ts, c2) from db.d1') tdSql.checkRows(1) - #tdSql.checkCols(4) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574930000) tdSql.checkData(0, 1, 'bbbbbbbbb') tdSql.checkData(0, 2, 1734574929000) tdSql.checkData(0, 3, 'a') tdSql.query(f'select cols(last(c0), ts, c1, c2, c3), cols(first(c0), ts, c1, c2, c3) from db.d1') tdSql.checkRows(1) - #tdSql.checkCols(6) + tdSql.checkCols(8) tdSql.checkData(0, 0, 1734574930000) tdSql.checkData(0, 1, 2.2) tdSql.checkData(0, 2, 'bbbbbbbbb') @@ -128,13 +140,13 @@ class TDTestCase: tdSql.query(f'select cols(last(ts), c1), cols(first(ts), c1) from db.d1') tdSql.checkRows(1) - #tdSql.checkCols(6) + tdSql.checkCols(2) tdSql.checkData(0, 0, 2.2) tdSql.checkData(0, 1, 1.1) tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts), c0, c1) from db.d1') tdSql.checkRows(1) - #tdSql.checkCols(6) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1) tdSql.checkData(0, 1, 1.1) tdSql.checkData(0, 2, 1) @@ -142,7 +154,7 @@ class TDTestCase: tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts+1), c0, c1) from db.d1') tdSql.checkRows(1) - #tdSql.checkCols(6) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1) tdSql.checkData(0, 1, 1.1) tdSql.checkData(0, 2, 1) @@ -150,7 +162,7 @@ class TDTestCase: tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts), c0+1, c1+2) from db.d1') tdSql.checkRows(1) - #tdSql.checkCols(6) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1) tdSql.checkData(0, 1, 1.1) tdSql.checkData(0, 2, 2) @@ -158,13 +170,14 @@ class TDTestCase: tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2)) from db.d1') tdSql.checkRows(1) - #tdSql.checkCols(6) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) tdSql.checkData(0, 1, 4) tdSql.checkData(0, 2, 1734574930000) tdSql.checkData(0, 3, 36) tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2) + 2) from db.d1') tdSql.checkRows(1) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) tdSql.checkData(0, 1, 4) tdSql.checkData(0, 2, 1734574930000) @@ -172,6 +185,7 @@ class TDTestCase: tdSql.query(f'select cols(first(c0), ts, c2), cols(last(c0), ts, length(c2) + 2) from db.d1') tdSql.checkRows(1) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) tdSql.checkData(0, 1, 'a') tdSql.checkData(0, 2, 1734574930000) @@ -179,6 +193,7 @@ class TDTestCase: tdSql.query(f'select cols(min(c0), ts, c2), cols(last(c0), ts, length(c2) + 2) from db.d1') tdSql.checkRows(1) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) tdSql.checkData(0, 1, 'a') tdSql.checkData(0, 2, 1734574930000) @@ -186,6 +201,7 @@ class TDTestCase: tdSql.query(f'select cols(min(c0), ts, c2), cols(first(c0), ts, length(c2) + 2) from db.d1') tdSql.checkRows(1) + tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) tdSql.checkData(0, 1, 'a') tdSql.checkData(0, 2, 1734574929000) @@ -201,7 +217,7 @@ class TDTestCase: tdSql.error(f'select last(cols(ts)) from {self.dbname}.meters') tdSql.error(f'select last(cols(ts, ts)) from {self.dbname}.meters') tdSql.error(f'select last(cols(ts, ts), ts) from {self.dbname}.meters') - tdSql.error(f'select last(cols(last(ts), ts), ts) from {self.dbname}.meters') + tdSql.error(f'd{self.dbname}.meters') tdSql.error(f'select cols(last(ts), ts as t1) as t1 from {self.dbname}.meters') tdSql.error(f'select cols(last(ts), ts, c0) t1 from {self.dbname}.meters') tdSql.error(f'select cols(last(ts), ts t1) tt from {self.dbname}.meters') @@ -228,7 +244,6 @@ class TDTestCase: def run(self): self.funcNestTest() self.funcSupperTableTest() - return self.create_test_data() self.parse_test() self.one_cols_1output_test() From d3e86d9854af18e17fa2ae6dfc85b4b5131d7f00 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 30 Dec 2024 18:21:26 +0800 Subject: [PATCH 11/45] cols --- include/util/taoserror.h | 3 + source/libs/function/src/builtins.c | 5 + source/libs/parser/src/parTranslater.c | 352 +++++++++++++-------- source/util/src/terror.c | 4 + tests/system-test/2-query/cols_function.py | 1 + 5 files changed, 233 insertions(+), 132 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 366ffc6ecc..2410016485 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -907,6 +907,9 @@ int32_t taosGetErrSize(); #define TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x2685) #define TSDB_CODE_PAR_INVALID_VGID_LIST TAOS_DEF_ERROR_CODE(0, 0x2686) #define TSDB_CODE_PAR_INVALID_COLS_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2687) +#define TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC TAOS_DEF_ERROR_CODE(0, 0x2688) +#define TSDB_CODE_INVALID_MULITI_COLS_FUNC TAOS_DEF_ERROR_CODE(0, 0x2688) +#define TSDB_CODE_INVALID_COLS_ALIAS TAOS_DEF_ERROR_CODE(0, 0x2689) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6d4e2c5bdb..16eb418ce5 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1696,6 +1696,10 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t return TSDB_CODE_SUCCESS; } +static int32_t translateColsFunction(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; +} + static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len)); int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); @@ -5644,6 +5648,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "cols", + .translateFunc = translateColsFunction, }, }; // clang-format on diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1680808c71..1ccfce3e0a 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1076,6 +1076,14 @@ static bool isForecastPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsForecastPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isColsFunctionResult(const SNode* pNode) { + return ((nodesIsExprNode(pNode)) && ((SExprNode*)pNode)->bindTupleFuncIdx > 0); +} + +static bool isInvalidColsBindFunction(const SFunctionNode* pFunc) { + return (!fmIsSelectFunc(pFunc->funcId) && pFunc->node.tupleFuncIdx != 0); +} + #ifdef BUILD_NO_CALL static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); @@ -2311,7 +2319,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { static EDealRes haveVectorFunction(SNode* pNode, void* pContext) { if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || - isInterpPseudoColumnFunc(pNode) || isForecastPseudoColumnFunc(pNode)) { + isInterpPseudoColumnFunc(pNode) || isForecastPseudoColumnFunc(pNode) || isColsFunctionResult(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -3298,6 +3306,10 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc pCxt->errCode = TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION; } } + if (isInvalidColsBindFunction(*pFunc)) { + pCxt->errCode = TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC; + return DEAL_RES_ERROR; + } if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = translateFunctionImpl(pCxt, pFunc); } @@ -3813,7 +3825,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (isVectorFunc(*pNode) && !isDistinctOrderBy(pCxt)) { return DEAL_RES_IGNORE_CHILD; } - if (isColsFunc(*pNode)) { + if (isColsFunctionResult(*pNode)) { return DEAL_RES_IGNORE_CHILD; } SNode* pGroupNode = NULL; @@ -3918,10 +3930,10 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { if (isVectorFunc(*pNode)) { return DEAL_RES_IGNORE_CHILD; } - if(isColsFunc(*pNode)) { + if(isColsFunctionResult(*pNode)) { pCxt->hasColFunc = true; - return DEAL_RES_IGNORE_CHILD; } + SNode* pPartKey = NULL; bool partionByTbname = false; if (fromSingleTable(((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pFromTable) || @@ -7299,8 +7311,9 @@ static int32_t translateSelectWithoutFrom(STranslateContext* pCxt, SSelectStmt* return translateExprList(pCxt, pSelect->pProjectionList); } typedef struct SCheckColsFuncCxt { - bool hasColsFunc; - bool hasMultiColsFunc; + bool hasColsFunc; + SNodeList** selectFuncList; + int32_t status; } SCheckColsFuncCxt; static bool isColsFuncByName(SFunctionNode* pFunc) { @@ -7310,26 +7323,16 @@ static bool isColsFuncByName(SFunctionNode* pFunc) { return true; } -static bool isMultiColsFunc(SFunctionNode* pFunc) { - if (strcasecmp(pFunc->functionName, "cols") != 0) { - return false; - } - return pFunc->pParameterList->length > 2; -} - -static EDealRes isMultiColsFuncNode(SNode** pNode, void* pContext) { - SCheckColsFuncCxt* pCxt = pContext; - if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { - SFunctionNode* pFunc = (SFunctionNode*)*pNode; +static bool isMultiColsFuncNode(SNode* pNode) { + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; if (isColsFuncByName(pFunc)) { - pCxt->hasColsFunc = true; if (pFunc->pParameterList->length > 2) { - pCxt->hasMultiColsFunc = true; - return DEAL_RES_END; + return true; } } } - return DEAL_RES_CONTINUE; + return false; } typedef struct SBindTupleFuncCxt { @@ -7347,16 +7350,6 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static void checkColsFuncInList(SNodeList* nodeList, SCheckColsFuncCxt* pCheckouColsFuncCxt) { - nodesRewriteExprs(nodeList, isMultiColsFuncNode, pCheckouColsFuncCxt); - return; -} - -static void checkColsFunc(SNode** pNode, SCheckColsFuncCxt* pCheckouColsFuncCxt) { - nodesRewriteExpr(pNode, isMultiColsFuncNode, pCheckouColsFuncCxt); - return; -} - static bool invalidColsAlias(SFunctionNode* pFunc) { if (pFunc->node.asAlias) { if (pFunc->pParameterList->length > 2) { @@ -7374,37 +7367,6 @@ static bool invalidColsAlias(SFunctionNode* pFunc) { return false; } -static int32_t hasInvalidColsFunction(STranslateContext* pCxt, SNodeList* nodeList, - SCheckColsFuncCxt* pCheckouColsFuncCxt) { - SNode* pTmpNode = NULL; - FOREACH(pTmpNode, nodeList) { - if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { - SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; - if (isColsFuncByName(pFunc)) { - pCheckouColsFuncCxt->hasColsFunc = true; - if (invalidColsAlias(pFunc)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, - "Invalid using alias for cols function"); - } - } - checkColsFuncInList(pFunc->pParameterList, pCheckouColsFuncCxt); - if (pCheckouColsFuncCxt->hasMultiColsFunc) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, - "Invalid cols function in function %s", pFunc->functionName); - } - } else { - checkColsFunc(&pTmpNode, pCheckouColsFuncCxt); - if (pCheckouColsFuncCxt->hasMultiColsFunc) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLS_FUNCTION, - "Invalid cols function, can't be used at here"); - } - } - } - return TSDB_CODE_SUCCESS; -} - - - static int32_t getSelectFuncIndex(SNodeList* FuncNodeList, SNode* pSelectFunc) { SNode* pNode = NULL; int32_t selectFuncIndex = 0; @@ -7417,109 +7379,235 @@ static int32_t getSelectFuncIndex(SNodeList* FuncNodeList, SNode* pSelectFunc) { return 0; } -static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList) { - SCheckColsFuncCxt pCheckouColsFuncCxt = {false, false}; - int32_t code = hasInvalidColsFunction(pCxt, *nodeList, &pCheckouColsFuncCxt); - if (TSDB_CODE_SUCCESS != code) { - return code; +static EDealRes checkHasColsFunc(SNode** pNode, void* pContext){ + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)*pNode; + if (isColsFuncByName(pFunc)) { + *(bool*)pContext = true; + return DEAL_RES_END; + } } - bool needRewrite = false; - if (pCheckouColsFuncCxt.hasColsFunc) { - needRewrite = true; + return DEAL_RES_CONTINUE; +} + +static int32_t checkMultColsFuncParam(SNodeList* pParameterList) { + if (!pParameterList || pParameterList->length < 2) { + return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + } + int32_t index = 0; + SNode* pNode = NULL; + FOREACH(pNode, pParameterList) { + if (index == 0) { // the first parameter is select function + if (QUERY_NODE_FUNCTION != nodeType(pNode)) { + return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + } + SFunctionNode* pFunc = (SFunctionNode*)pNode; + // pFunc->funcId is zero at here, so need to check at * step + // if(!fmIsSelectFunc(pFunc->funcId)) { + // return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + // } + SNode* pTmpNode = NULL; + FOREACH(pTmpNode, pFunc->pParameterList) { + bool hasColsFunc = false; + nodesRewriteExpr(&pTmpNode, checkHasColsFunc, (void*)&hasColsFunc); + if (hasColsFunc) { + return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + } + } + } else { + bool hasColsFunc = false; + nodesRewriteExpr(&pNode, checkHasColsFunc, &hasColsFunc); + if (hasColsFunc) { + return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + } + } + ++index; + } + return TSDB_CODE_SUCCESS; +} + +static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { + int32_t code = TSDB_CODE_SUCCESS; + if (QUERY_NODE_FUNCTION != nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + SCheckColsFuncCxt* pCxt = pContext; + SFunctionNode* pFunc = (SFunctionNode*)*pNode; + if (isColsFuncByName(pFunc)) { + if(pFunc->pParameterList->length > 2) { + pCxt->status = TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC; + return DEAL_RES_ERROR; + } + SNode* pSelectFunc = nodesListGetNode(pFunc->pParameterList, 0); + SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 1); + if (nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { + pCxt->status = TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC; + parserError("Invalid cols function, the first parameter must be a select function"); + return DEAL_RES_ERROR; + } + if (pFunc->node.asAlias) { + if (((SExprNode*)pExpr)->asAlias) { + pCxt->status = TSDB_CODE_INVALID_COLS_ALIAS; + parserError("Invalid using alias for cols function"); + return DEAL_RES_ERROR; + } else { + ((SExprNode*)pExpr)->asAlias = true; + tstrncpy(((SExprNode*)pExpr)->userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN); + } + } + if(*pCxt->selectFuncList == NULL) { + nodesMakeList(pCxt->selectFuncList); + if (NULL == *pCxt->selectFuncList) { + pCxt->status = terrno; + return DEAL_RES_ERROR; + } + } + int32_t selectFuncCount = (*pCxt->selectFuncList)->length; + int32_t selectFuncIndex = getSelectFuncIndex(*pCxt->selectFuncList, pSelectFunc); + if (selectFuncIndex == 0) { + ++selectFuncCount; + selectFuncIndex = selectFuncCount; + SNode* pNewNode = NULL; + code = nodesCloneNode(pSelectFunc, &pNewNode); + ((SExprNode*)pNewNode)->tupleFuncIdx = selectFuncIndex; + nodesListMakeStrictAppend(pCxt->selectFuncList, pNewNode); + } + + SNode* pNewNode = NULL; + code = nodesCloneNode(pExpr, &pNewNode); + if (nodesIsExprNode(pNewNode)) { + SBindTupleFuncCxt pCxt = {selectFuncIndex}; + nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); + } else { + pCxt->status = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + parserError("Invalid cols function, the first parameter must be a select function"); + return DEAL_RES_ERROR; + } + nodesDestroyNode(*pNode); + *pNode = pNewNode; + } + return DEAL_RES_CONTINUE; +_end: + return code; +} + +static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList, SNodeList** selectFuncList) { + int32_t code = TSDB_CODE_SUCCESS; + bool needRewrite = false; + SNode** pNode = NULL; + FOREACH_FOR_REWRITE(pNode, *nodeList) { + if (isMultiColsFuncNode(*pNode)) { + code = checkMultColsFuncParam(((SFunctionNode*)*pNode)->pParameterList); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + needRewrite = true; + } else { + SCheckColsFuncCxt pSelectFuncCxt = {false, selectFuncList, TSDB_CODE_SUCCESS}; + nodesRewriteExpr(pNode, rewriteSingleColsFunc, &pSelectFuncCxt); + if (pSelectFuncCxt.status != TSDB_CODE_SUCCESS) { + return pSelectFuncCxt.status; + } + } } SNodeList* pNewNodeList = NULL; - SNodeList* tmpFuncNodeList = NULL; if (needRewrite) { code = nodesMakeList(&pNewNodeList); if (NULL == pNewNodeList) { return code; } - code = nodesMakeList(&tmpFuncNodeList); - if (NULL == tmpFuncNodeList) { - return code; + if (*selectFuncList == NULL) { + code = nodesMakeList(selectFuncList); + if (NULL == *selectFuncList) { + nodesDestroyList(pNewNodeList); + return code; + } } - SNode* pNewNode = NULL; + + SNode* pNewNode = NULL; int32_t nums = 0; - int32_t selectFuncCount = 0; + int32_t selectFuncCount = (*selectFuncList)->length; SNode* pTmpNode = NULL; FOREACH(pTmpNode, *nodeList) { - if (QUERY_NODE_FUNCTION == nodeType(pTmpNode)) { + if (isMultiColsFuncNode(pTmpNode)) { SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; - if (strcasecmp(pFunc->functionName, "cols") == 0) { - SNode* pSelectFunc = nodesListGetNode(pFunc->pParameterList, 0); - if (nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { + if(pFunc->node.asAlias) { + code = TSDB_CODE_INVALID_COLS_ALIAS; + parserError("Invalid using alias for cols function"); + goto _end; + } + + SNode* pSelectFunc = nodesListGetNode(pFunc->pParameterList, 0); + if (nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { + code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + parserError("Invalid cols function, the first parameter must be a select function"); + goto _end; + } + int32_t selectFuncIndex = getSelectFuncIndex(*selectFuncList, pSelectFunc); + if (selectFuncIndex == 0) { + ++selectFuncCount; + selectFuncIndex = selectFuncCount; + code = nodesCloneNode(pSelectFunc, &pNewNode); + ((SExprNode*)pNewNode)->tupleFuncIdx = selectFuncIndex; + nodesListMakeStrictAppend(selectFuncList, pNewNode); + } + // start from index 1, because the first parameter is select function which needn't to output. + for (int i = 1; i < pFunc->pParameterList->length; ++i) { + SNode* pExpr = nodesListGetNode(pFunc->pParameterList, i); + + code = nodesCloneNode(pExpr, &pNewNode); + if (nodesIsExprNode(pNewNode)) { + SBindTupleFuncCxt pCxt = {selectFuncIndex}; + nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); + } else { code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; parserError("Invalid cols function, the first parameter must be a select function"); goto _end; } - int32_t selectFuncIndex = getSelectFuncIndex(tmpFuncNodeList, pSelectFunc); - if (selectFuncIndex == 0) { - ++selectFuncCount; - selectFuncIndex = selectFuncCount; - nodesListMakeStrictAppend(&tmpFuncNodeList, pSelectFunc); - } - // start from index 1, because the first parameter is select function which needn't to output. - for (int i = 1; i < pFunc->pParameterList->length; ++i) { - SNode* pExpr = nodesListGetNode(pFunc->pParameterList, i); - - code = nodesCloneNode(pExpr, &pNewNode); - if (nodesIsExprNode(pNewNode)) { - SBindTupleFuncCxt pCxt = {selectFuncIndex}; - nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); - } else { - code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; - parserError("Invalid cols function, the first parameter must be a select function"); - goto _end; - } - if (TSDB_CODE_SUCCESS != code) goto _end; - code = nodesListMakeStrictAppend(&pNewNodeList, pNewNode); - if (TSDB_CODE_SUCCESS != code) goto _end; - } - continue; + if (TSDB_CODE_SUCCESS != code) goto _end; + code = nodesListMakeStrictAppend(&pNewNodeList, pNewNode); + if (TSDB_CODE_SUCCESS != code) goto _end; } + continue; } + code = nodesCloneNode(pTmpNode, &pNewNode); if (TSDB_CODE_SUCCESS != code) goto _end; code = nodesListMakeStrictAppend(&pNewNodeList, pNewNode); if (TSDB_CODE_SUCCESS != code) goto _end; } - SNode* pNode = NULL; - int32_t pNewSelectFuncIds = 0; - FOREACH(pNode, tmpFuncNodeList) { - ++pNewSelectFuncIds; - code = nodesCloneNode(pNode, &pNewNode); - if (TSDB_CODE_SUCCESS != code) goto _end; - if (nodesIsExprNode(pNewNode)) { - SExprNode* pExprNode = (SExprNode*)pNewNode; - pExprNode->tupleFuncIdx = pNewSelectFuncIds; - } else { - code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; - parserError("Invalid cols function, the first parameter must be a select function"); - goto _end; - } - code = nodesListMakeStrictAppend(&pNewNodeList, pNewNode); - if (TSDB_CODE_SUCCESS != code) goto _end; - } nodesDestroyList(*nodeList); - nodesDestroyList(tmpFuncNodeList); *nodeList = pNewNodeList; + return TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; - -_end: - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(pNewNodeList); - nodesDestroyList(tmpFuncNodeList); - } - return code; + _end: + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pNewNodeList); + } + return code; } static int32_t translateColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { - int32_t code = rewriteColsFunction(pCxt, &pSelect->pProjectionList); + SNodeList* selectFuncList = NULL; + int32_t code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &selectFuncList); if (code == TSDB_CODE_SUCCESS) { - code = rewriteColsFunction(pCxt, &pSelect->pOrderByList); + code = rewriteColsFunction(pCxt, &pSelect->pOrderByList, &selectFuncList); } + if (TSDB_CODE_SUCCESS != code) { + goto _end; + } + + if (selectFuncList != NULL) { + nodesListAppendList(pSelect->pProjectionList, selectFuncList); + selectFuncList = NULL; + } + +_end: + if (selectFuncList) { + nodesDestroyList(selectFuncList); + } + return code; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2c32339efc..387d153b5a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -751,6 +751,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE, "Invalid forecast cl TAOS_DEFINE_ERROR(TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR, "Syntax error in regular expression") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VGID_LIST, "Invalid vgid list") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLS_FUNCTION, "Invalid cols function") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC, "cols function's first param must be a select function") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MULITI_COLS_FUNC, "Improper use of cols function with multiple output columns") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_COLS_ALIAS, "Invalid using alias for cols function") + TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 5f48c401db..f049f52480 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -233,6 +233,7 @@ class TDTestCase: tdSql.error(f'select cols(last(ts), ts t1) tt from {self.dbname}.meters') tdSql.error(f'select cols(first(ts+1), c0+2 cc0, c1 cc1) cc from {self.dbname}.meters') tdSql.error(f'select cols(last(ts)+1, c0+2 as cc0) as cc from {self.dbname}.meters') + tdSql.error(f'select cols(ABS(c0), c1) from {self.dbname}.meters group by tbname') tdSql.error(f'select cols(last(ts)+1, ts) from {self.dbname}.meters') tdSql.error(f'select cols(last(ts)+10, c1+10) from {self.dbname}.meters group by tbname') From 772c37b4d61d28c08703ebe75b1a5ce3c54f3c5b Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 1 Jan 2025 19:17:12 +0800 Subject: [PATCH 12/45] fix: empty block --- source/libs/executor/src/aggregateoperator.c | 6 + source/libs/function/src/builtinsimpl.c | 1 - source/libs/nodes/src/nodesMsgFuncs.c | 12 ++ source/libs/parser/src/parTranslater.c | 8 +- source/libs/planner/src/planOptimizer.c | 2 +- tests/system-test/2-query/cols_function.py | 175 ++++++++++++++++++- 6 files changed, 193 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 5713726501..bec8bf7f6f 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -395,6 +395,12 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc return TSDB_CODE_SUCCESS; } + // if the last expression is a tuple function, we don't need to create a empty data block + int32_t lastExprIndex = pOperator->exprSupp.numOfExprs - 1; + if(pOperator->exprSupp.pExprInfo[lastExprIndex].pExpr->tupleFuncIdx > 0) { + return TSDB_CODE_SUCCESS; + } + code = createDataBlock(&pBlock); if (code) { return code; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3d1b04f223..91751c41ef 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2403,7 +2403,6 @@ int32_t firstLastFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResIn } SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - SInputColumnInfoData* pInput = &pCtx->input; pRes->nullTupleSaved = false; pRes->nullTuplePos.pageId = -1; diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index b122c3fffd..5bd773796f 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -708,6 +708,12 @@ static int32_t columnNodeInlineToMsg(const void* pObj, STlvEncoder* pEncoder) { const SColumnNode* pNode = (const SColumnNode*)pObj; int32_t code = dataTypeInlineToMsg(&pNode->node.resType, pEncoder); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI32(pEncoder, pNode->node.bindTupleFuncIdx); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI32(pEncoder, pNode->node.tupleFuncIdx); + } if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueU64(pEncoder, pNode->tableId); } @@ -758,6 +764,12 @@ static int32_t msgToColumnNodeInline(STlvDecoder* pDecoder, void* pObj) { SColumnNode* pNode = (SColumnNode*)pObj; int32_t code = msgToDataTypeInline(pDecoder, &pNode->node.resType); + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueI32(pDecoder, &pNode->node.bindTupleFuncIdx); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueI32(pDecoder, &pNode->node.tupleFuncIdx); + } if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueU64(pDecoder, &pNode->tableId); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1ccfce3e0a..4a5258c5fc 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3825,9 +3825,6 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (isVectorFunc(*pNode) && !isDistinctOrderBy(pCxt)) { return DEAL_RES_IGNORE_CHILD; } - if (isColsFunctionResult(*pNode)) { - return DEAL_RES_IGNORE_CHILD; - } SNode* pGroupNode = NULL; FOREACH(pGroupNode, getGroupByList(pCxt)) { SNode* pActualNode = getGroupByNode(pGroupNode); @@ -3865,7 +3862,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { - if (pSelect->selectFuncNum > 1 || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) { + if ((pSelect->selectFuncNum > 1 || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) && + ((SExprNode*)*pNode)->bindTupleFuncIdx == 0) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } if (isWindowJoinStmt(pSelect) && @@ -3874,7 +3872,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } - if (pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { + if ((pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) && ((SExprNode*)*pNode)->bindTupleFuncIdx == 0) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 63554e3a4e..761b6d43b7 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3604,7 +3604,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* FOREACH(pProjection, pProjectNode->pProjections) { FOREACH(pChildTarget, pChild->pTargets) { if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName) - && ((SColumnNode*)pProjection)->node.bindTupleFuncIdx == 0) { + && ((SColumnNode*)pProjection)->node.tupleFuncIdx == 0) { SNode* pNew = NULL; code = nodesCloneNode(pChildTarget, &pNew); if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index f049f52480..a9b0d7349e 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -26,12 +26,14 @@ class TDTestCase: tdSql.execute(f'create table {self.dbname}.meters (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool) tags (t1 nchar(30))') tdSql.execute(f'create table {self.dbname}.d0 using {self.dbname}.meters tags("st1")') - tdSql.execute(f'create table {self.dbname}.d1 using {self.dbname}.meters tags("st1")') + tdSql.execute(f'create table {self.dbname}.d1 using {self.dbname}.meters tags("st2")') tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929000, 1, 1, "a1", true)') tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929001, 2, 2, "bbbbbbbbb1", false)') tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929002, 3, 3, "a2", true)') tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929003, 4, 4, "bbbbbbbbb2", false)') + tdSql.execute(f'insert into {self.dbname}.d1 values(1734574929000, 1, 1, "a1", true)') + tdSql.execute(f'use {self.dbname}') tdSql.execute(f'Create table {self.dbname}.normal_table (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool)') tdSql.execute(f'insert into {self.dbname}.normal_table (select * from {self.dbname}.d0)') @@ -53,16 +55,173 @@ class TDTestCase: def one_cols_multi_output_test(self): tdLog.info("one_cols_1output_test") tdSql.query(f'select cols(last(ts), ts, c0) from {self.dbname}.meters') - tdSql.query(f'select cols(last(ts), ts, c0) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) tdSql.query(f'select cols(last(ts), ts as time, c0 cc) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) tdSql.query(f'select cols(last(ts), c0, c1, c2, c3) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 4) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 'bbbbbbbbb2') + tdSql.checkData(0, 3, False) + tdSql.query(f'select cols(last(ts), c0, t1) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 4) + tdSql.checkData(0, 1, 'st1') tdSql.query(f'select cols(last(c1), ts) from {self.dbname}.meters group by tbname') - - + tdSql.checkRows(2) + tdSql.checkCols(1) + tdSql.query(f'select cols(last(c1), ts) from {self.dbname}.meters group by tbname order by tbname') + tdSql.checkCols(1) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(1, 0, 1734574929000) + tdSql.query(f'select cols(last(c1), ts) from {self.dbname}.meters group by tbname order by ts') + tdSql.checkCols(1) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(1, 0, 1734574929003) + tdSql.query(f'select cols(last(c1), ts), tbname from {self.dbname}.meters group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 'd0') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 'd1') + tdSql.query(f'select cols(last(c1), ts), tbname, t1 from {self.dbname}.meters group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 'd0') + tdSql.checkData(0, 2, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 'd1') + tdSql.checkData(1, 2, 'st2') + tdSql.query(f'select cols(last(c1), ts), tbname, t1 from {self.dbname}.meters group by tbname order by t1') + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 'd0') + tdSql.checkData(0, 2, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 'd1') + tdSql.checkData(1, 2, 'st2') tdSql.query(f'select cols(max(c0), ts) from {self.dbname}.meters') + tdSql.checkCols(1) + tdSql.checkData(0, 0, 1734574929003) tdSql.query(f'select cols(min(c1), ts, c0) from {self.dbname}.meters') + tdSql.checkCols(2) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.meters group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 4) + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.query(f'select cols(last(ts), ts, c0), count(1), tbname from {self.dbname}.meters group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 'd0') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(1, 3, 'd1') + tdSql.query(f'select cols(last(ts), ts, c0), count(1), tbname, t1 from {self.dbname}.meters group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 'd0') + tdSql.checkData(0, 4, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(1, 3, 'd1') + tdSql.checkData(1, 4, 'st2') + tdSql.query(f'select cols(last(ts), ts, c0), count(1), t1 from {self.dbname}.meters group by t1 order by t1') + tdSql.checkRows(2) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(1, 3, 'st2') + tdSql.error(f'select cols(last(ts), ts, c0), count(1), t1 from {self.dbname}.meters group by t1 order by tbname') + tdSql.query(f'select cols(last(ts), ts, c0), sum(c0), t1 from {self.dbname}.meters group by t1 order by t1') + tdSql.checkRows(2) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 10) + tdSql.checkData(0, 3, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(1, 3, 'st2') + + tdSql.query(f'select cols(max(c0), ts, c0), count(1) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 5) + tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.d0') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 4) + tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.d1') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.normal_table') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 4) + + tdSql.query(f'select cols(last(ts), ts, c0), avg(c0) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 2.2) tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 5) + + tdSql.query(f'select cols(last(ts), ts, c0), sum(c0) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 11) + + tdSql.query(f'select count(1), cols(last(ts), ts, c0), min(c0) from {self.dbname}.meters') tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.meters') tdSql.query(f'select cols(last(ts), ts as time, c0 cc), count(1) from {self.dbname}.meters') @@ -207,7 +366,14 @@ class TDTestCase: tdSql.checkData(0, 2, 1734574929000) tdSql.checkData(0, 3, 6) + def subquery_test(self): + tdSql.query(f'select count(1), cols(last(c0),c0) from (select * from test.d0)') + tdSql.query(f'select count(1), cols(last(c0),c0) from (select *, tbname from test.meters) group by tbname') + def window_test(self): + tdSql.query(f'select tbname, _wstart,_wend, max(c0), max(c1), cols( max(c0), c1) from test.meters partition \ + by tbname count_window(2) order by tbname') + def parse_test(self): tdLog.info("parse test") @@ -250,6 +416,7 @@ class TDTestCase: self.one_cols_1output_test() self.one_cols_multi_output_test() self.multi_cols_output_test() + self.subquery_test() def stop(self): From cb1ed298728678eed074fca64e597cfe30f82276 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 2 Jan 2025 15:31:02 +0800 Subject: [PATCH 13/45] error desc in doc --- docs/en/14-reference/09-error-code.md | 4 + docs/zh/14-reference/09-error-code.md | 6 +- include/util/taoserror.h | 4 +- tests/system-test/2-query/cols_function.py | 676 ++++++++++++++++----- 4 files changed, 527 insertions(+), 163 deletions(-) diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 1d3ea3f9a1..7a171f47be 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -458,6 +458,10 @@ This document details the server error codes that may be encountered when using | 0x80002665 | The _TAGS pseudocolumn can only be used for subtable and supertable queries | Illegal tag column query | Check and correct the SQL statement | | 0x80002666 | Subquery does not output primary timestamp column | Check and correct the SQL statement | | | 0x80002667 | Invalid usage of expr: %s | Illegal expression | Check and correct the SQL statement | +| 0x80002687 | Invalid using cols function | Illegal using cols function | Check and correct the SQL statement | +| 0x80002688 | Cols function's first param must be a select function | The first parameter of the cols function should be a selection function | Check and correct the SQL statement | +| 0x80002689 | Invalid using cols function with multiple output columns | Illegal using the cols function for multiple column output | Check and correct the SQL statement | +| 0x80002690 | Invalid using alias for cols function | Illegal cols function alias | Check and correct the SQL statement | | 0x800026FF | Parser internal error | Internal error in parser | Preserve the scene and logs, report issue on GitHub | | 0x80002700 | Planner internal error | Internal error in planner | Preserve the scene and logs, report issue on GitHub | | 0x80002701 | Expect ts equal | JOIN condition validation failed | Preserve the scene and logs, report issue on GitHub | diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index 2bebe2406b..3022a7ec75 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -473,8 +473,12 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80002663 | Not unique table/alias | 表名（别名）冲突 | 检查并修正SQL语句 | | 0x80002664 | Join requires valid time series input | 不支持子查询不含主键时间戳列输出的JOIN查询 | 检查并修正SQL语句 | | 0x80002665 | The _TAGS pseudo column can only be used for subtable and supertable queries | 非法TAG列查询 | 检查并修正SQL语句 | -| 0x80002666 | 子查询不含主键时间戳列输出 | 检查并修正SQL语句 | +| 0x80002666 | 子查询不含主键时间戳列输出 | 检查并修正SQL语句 | | 0x80002667 | Invalid usage of expr: %s | 非法表达式 | 检查并修正SQL语句 | +| 0x80002687 | Invalid using cols function | cols函数使用错误 | 检查并修正SQL语句 | +| 0x80002688 | Cols function's first param must be a select function | cols函数第一个参数应该为选择函数 | 检查并修正SQL语句 | +| 0x80002689 | Invalid using cols function with multiple output columns | 多列输出的 cols 函数使用错误 | 检查并修正SQL语句 | +| 0x80002690 | Invalid using alias for cols function | cols 函数输出列重命名错误 | 检查并修正SQL语句 | | 0x800026FF | Parser internal error | 解析器内部错误 | 保留现场和日志，github上报issue | | 0x80002700 | Planner internal error | 计划期内部错误 | 保留现场和日志，github上报issue | | 0x80002701 | Expect ts equal | JOIN条件校验失败 | 保留现场和日志，github上报issue | diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2410016485..58557dac1a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -908,8 +908,8 @@ int32_t taosGetErrSize(); #define TSDB_CODE_PAR_INVALID_VGID_LIST TAOS_DEF_ERROR_CODE(0, 0x2686) #define TSDB_CODE_PAR_INVALID_COLS_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2687) #define TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC TAOS_DEF_ERROR_CODE(0, 0x2688) -#define TSDB_CODE_INVALID_MULITI_COLS_FUNC TAOS_DEF_ERROR_CODE(0, 0x2688) -#define TSDB_CODE_INVALID_COLS_ALIAS TAOS_DEF_ERROR_CODE(0, 0x2689) +#define TSDB_CODE_INVALID_MULITI_COLS_FUNC TAOS_DEF_ERROR_CODE(0, 0x2689) +#define TSDB_CODE_INVALID_COLS_ALIAS TAOS_DEF_ERROR_CODE(0, 0x2690) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index a9b0d7349e..fd41c3a1a1 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -14,6 +14,10 @@ class TDTestCase: tdSql.init(conn.cursor()) self.dbname = 'test' + + def condition_check(self, condition, row, col, expected_value): + if condition: + tdSql.checkData(row, col, expected_value) def create_test_data(self): tdSql.execute(f'create database if not exists {self.dbname};') @@ -27,12 +31,13 @@ class TDTestCase: tdSql.execute(f'create table {self.dbname}.meters (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool) tags (t1 nchar(30))') tdSql.execute(f'create table {self.dbname}.d0 using {self.dbname}.meters tags("st1")') tdSql.execute(f'create table {self.dbname}.d1 using {self.dbname}.meters tags("st2")') - tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929000, 1, 1, "a1", true)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929000, 1, 1, "c2", true)') tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929001, 2, 2, "bbbbbbbbb1", false)') - tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929002, 3, 3, "a2", true)') - tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929003, 4, 4, "bbbbbbbbb2", false)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929002, 2, 2, "bbbbbbbbb1", false)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929003, 3, 3, "a2", true)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929004, 4, 4, "bbbbbbbbb2", false)') - tdSql.execute(f'insert into {self.dbname}.d1 values(1734574929000, 1, 1, "a1", true)') + tdSql.execute(f'insert into {self.dbname}.d1 values(1734574929000, 1, 1, "c2", true)') tdSql.execute(f'use {self.dbname}') tdSql.execute(f'Create table {self.dbname}.normal_table (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool)') @@ -51,142 +56,161 @@ class TDTestCase: tdSql.query(f'select cols(last(ts+1), ts+2 as t1) from {self.dbname}.meters') tdSql.query(f'select cols(last(ts+1), c0+10) from {self.dbname}.meters') + def one_cols_multi_output_with_group_test(self, from_table = 'test.meters', isTmpTable = False): + select_t1 = ["", ", t1", ", t1 as tag1"] + for t1 in select_t1: + tags_count = 0 if t1 == "" else 1 + tdLog.info("one_cols_1output_test_with_group") + tdSql.query(f'select cols(last(c1), ts) {t1} from {from_table} group by tbname') + tdSql.checkRows(2) + tdSql.checkCols(1 + tags_count) + tdSql.query(f'select cols(last(c1), ts) {t1} from {from_table} group by tbname order by tbname') + tdSql.checkCols(1 + tags_count) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(1, 0, 1734574929000) + tdSql.query(f'select cols(last(c1), ts) {t1} from {from_table} group by tbname order by ts') + tdSql.checkCols(1 + tags_count) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(1, 0, 1734574929004) + tdSql.query(f'select cols(last(c1), ts), tbname {t1} from {from_table} group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 'd0') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 'd1') + tdSql.query(f'select cols(last(c1), ts), tbname, t1 from {from_table} group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 'd0') + tdSql.checkData(0, 2, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 'd1') + tdSql.checkData(1, 2, 'st2') + tdSql.query(f'select cols(last(c1), ts), tbname, t1 from {from_table} group by tbname order by t1') + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 'd0') + tdSql.checkData(0, 2, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 'd1') + tdSql.checkData(1, 2, 'st2') + tdSql.query(f'select cols(last(ts), ts, c0), count(1) {t1} from {from_table} group by t1 order by t1') + tdSql.checkRows(2) + tdSql.checkCols(3 + tags_count) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 5) + self.condition_check(t1!="", 0, 3, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + self.condition_check(t1!="", 1, 3, 'st2') + + tdSql.query(f'select cols(last(ts), ts, c0), sum(c0) {t1} from {from_table} group by t1 order by t1') + tdSql.checkRows(2) + tdSql.checkCols(3 + tags_count) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 12) + self.condition_check(t1!="", 0, 3, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + self.condition_check(t1!="", 1, 3, 'st2') - def one_cols_multi_output_test(self): - tdLog.info("one_cols_1output_test") - tdSql.query(f'select cols(last(ts), ts, c0) from {self.dbname}.meters') + tdSql.error(f'select cols(last(ts), ts, c0), count(1), t1 from {from_table} group by t1 order by tbname') + + if t1 != "" and isTmpTable: + # Not a GROUP BY expression + tdSql.error(f'select cols(last(ts), ts, c0), count(1) {t1} from {from_table} group by tbname order by tbname') + tdSql.error(f'select cols(last(ts), ts, c0), count(1), tbname {t1} from {from_table} group by tbname order by tbname') + tdSql.error(f'select cols(max(c0), ts, c0), count(1), tbname {t1} from {from_table} group by tbname order by tbname') + tdSql.error(f'select cols(last(c1), ts), count(1) {t1} from {from_table} group by tbname') + continue + tdSql.query(f'select cols(last(ts), ts, c0), count(1) {t1} from {from_table} group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(3 + tags_count) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 5) + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.query(f'select cols(last(ts), ts, c0), count(1), tbname {t1} from {from_table} group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(4 + tags_count) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 5) + tdSql.checkData(0, 3, 'd0') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(1, 3, 'd1') + tdSql.query(f'select cols(max(c0), ts, c0), count(1), tbname {t1} from {from_table} group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(4 + tags_count) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 5) + tdSql.checkData(0, 3, 'd0') + self.condition_check(t1!="", 0, 4, 'st1') + tdSql.checkData(1, 0, 1734574929000) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(1, 3, 'd1') + self.condition_check(t1!="", 1, 4, 'st2') + + tdSql.query(f'select cols(last(c1), ts), count(1) {t1} from {from_table} group by tbname') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + + def one_cols_multi_output_test(self, from_table = 'test.meters'): + tdLog.info(f"one_cols_1output_test {from_table}") + tdSql.query(f'select cols(last(ts), ts, c0) from {from_table}') tdSql.checkRows(1) tdSql.checkCols(2) - tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) - tdSql.query(f'select cols(last(ts), ts as time, c0 cc) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), ts as time, c0 cc) from {from_table}') tdSql.checkRows(1) tdSql.checkCols(2) - tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) - tdSql.query(f'select cols(last(ts), c0, c1, c2, c3) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), c0, c1, c2, c3) from {from_table}') tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 4) tdSql.checkData(0, 1, 4) tdSql.checkData(0, 2, 'bbbbbbbbb2') tdSql.checkData(0, 3, False) - tdSql.query(f'select cols(last(ts), c0, t1) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), c0, t1) from {from_table}') tdSql.checkRows(1) tdSql.checkData(0, 0, 4) tdSql.checkData(0, 1, 'st1') - tdSql.query(f'select cols(last(c1), ts) from {self.dbname}.meters group by tbname') - tdSql.checkRows(2) + tdSql.query(f'select cols(max(c0), ts) from {from_table}') tdSql.checkCols(1) - tdSql.query(f'select cols(last(c1), ts) from {self.dbname}.meters group by tbname order by tbname') - tdSql.checkCols(1) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(1, 0, 1734574929000) - tdSql.query(f'select cols(last(c1), ts) from {self.dbname}.meters group by tbname order by ts') - tdSql.checkCols(1) - tdSql.checkData(0, 0, 1734574929000) - tdSql.checkData(1, 0, 1734574929003) - tdSql.query(f'select cols(last(c1), ts), tbname from {self.dbname}.meters group by tbname order by tbname') - tdSql.checkRows(2) - tdSql.checkCols(2) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 'd0') - tdSql.checkData(1, 0, 1734574929000) - tdSql.checkData(1, 1, 'd1') - tdSql.query(f'select cols(last(c1), ts), tbname, t1 from {self.dbname}.meters group by tbname order by tbname') - tdSql.checkRows(2) - tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 'd0') - tdSql.checkData(0, 2, 'st1') - tdSql.checkData(1, 0, 1734574929000) - tdSql.checkData(1, 1, 'd1') - tdSql.checkData(1, 2, 'st2') - tdSql.query(f'select cols(last(c1), ts), tbname, t1 from {self.dbname}.meters group by tbname order by t1') - tdSql.checkRows(2) - tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 'd0') - tdSql.checkData(0, 2, 'st1') - tdSql.checkData(1, 0, 1734574929000) - tdSql.checkData(1, 1, 'd1') - tdSql.checkData(1, 2, 'st2') - tdSql.query(f'select cols(max(c0), ts) from {self.dbname}.meters') - tdSql.checkCols(1) - tdSql.checkData(0, 0, 1734574929003) - tdSql.query(f'select cols(min(c1), ts, c0) from {self.dbname}.meters') + tdSql.checkData(0, 0, 1734574929004) + tdSql.query(f'select cols(min(c1), ts, c0) from {from_table}') tdSql.checkCols(2) tdSql.checkData(0, 0, 1734574929000) tdSql.checkData(0, 1, 1) - tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.meters group by tbname order by tbname') - tdSql.checkRows(2) - tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 4) - tdSql.checkData(1, 0, 1734574929000) - tdSql.checkData(1, 1, 1) - tdSql.checkData(1, 2, 1) - tdSql.query(f'select cols(last(ts), ts, c0), count(1), tbname from {self.dbname}.meters group by tbname order by tbname') - tdSql.checkRows(2) - tdSql.checkCols(4) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 4) - tdSql.checkData(0, 3, 'd0') - tdSql.checkData(1, 0, 1734574929000) - tdSql.checkData(1, 1, 1) - tdSql.checkData(1, 2, 1) - tdSql.checkData(1, 3, 'd1') - tdSql.query(f'select cols(last(ts), ts, c0), count(1), tbname, t1 from {self.dbname}.meters group by tbname order by tbname') - tdSql.checkRows(2) - tdSql.checkCols(5) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 4) - tdSql.checkData(0, 3, 'd0') - tdSql.checkData(0, 4, 'st1') - tdSql.checkData(1, 0, 1734574929000) - tdSql.checkData(1, 1, 1) - tdSql.checkData(1, 2, 1) - tdSql.checkData(1, 3, 'd1') - tdSql.checkData(1, 4, 'st2') - tdSql.query(f'select cols(last(ts), ts, c0), count(1), t1 from {self.dbname}.meters group by t1 order by t1') - tdSql.checkRows(2) - tdSql.checkCols(4) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 4) - tdSql.checkData(0, 3, 'st1') - tdSql.checkData(1, 0, 1734574929000) - tdSql.checkData(1, 1, 1) - tdSql.checkData(1, 2, 1) - tdSql.checkData(1, 3, 'st2') - tdSql.error(f'select cols(last(ts), ts, c0), count(1), t1 from {self.dbname}.meters group by t1 order by tbname') - tdSql.query(f'select cols(last(ts), ts, c0), sum(c0), t1 from {self.dbname}.meters group by t1 order by t1') - tdSql.checkRows(2) - tdSql.checkCols(4) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 10) - tdSql.checkData(0, 3, 'st1') - tdSql.checkData(1, 0, 1734574929000) - tdSql.checkData(1, 1, 1) - tdSql.checkData(1, 2, 1) - tdSql.checkData(1, 3, 'st2') - tdSql.query(f'select cols(max(c0), ts, c0), count(1) from {self.dbname}.meters') + tdSql.query(f'select cols(max(c0), ts, c0), count(1) from {from_table}') tdSql.checkRows(1) tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 5) + tdSql.checkData(0, 2, 6) tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.d0') tdSql.checkRows(1) tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 2, 5) tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.d1') tdSql.checkRows(1) tdSql.checkCols(3) @@ -196,51 +220,205 @@ class TDTestCase: tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.normal_table') tdSql.checkRows(1) tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 4) - - tdSql.query(f'select cols(last(ts), ts, c0), avg(c0) from {self.dbname}.meters') - tdSql.checkRows(1) - tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) - tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 2.2) - - tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.meters') - tdSql.checkRows(1) - tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) tdSql.checkData(0, 2, 5) - - tdSql.query(f'select cols(last(ts), ts, c0), sum(c0) from {self.dbname}.meters') + tdSql.query(f'select cols(first(ts), ts, c0), count(1) from {self.dbname}.normal_table') tdSql.checkRows(1) tdSql.checkCols(3) - tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 5) + tdSql.query(f'select cols(min(c0), ts, c0), count(1) from {self.dbname}.normal_table') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 5) + + tdSql.query(f'select cols(last(ts), ts, c0), avg(c0) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) - tdSql.checkData(0, 2, 11) - - - tdSql.query(f'select count(1), cols(last(ts), ts, c0), min(c0) from {self.dbname}.meters') - tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {self.dbname}.meters') - tdSql.query(f'select cols(last(ts), ts as time, c0 cc), count(1) from {self.dbname}.meters') - tdSql.query(f'select cols(last(ts), c0, c1, c2, c3), count(1) from {self.dbname}.meters') - tdSql.query(f'select cols(last(c1), ts), count(1) from {self.dbname}.meters group by tbname') + tdSql.checkData(0, 2, 2.1666666666666665) + + tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 6) + + tdSql.query(f'select cols(last(ts), ts, c0), sum(c0) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 13) + + tdSql.query(f'select count(1), cols(last(ts), ts, c0), min(c0) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 6) + tdSql.checkData(0, 1, 1734574929004) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 1) + + # there's some error on last_row func when using sub query. + tdSql.query(f'select count(1), cols(last_row(ts), ts, c0), min(c0) from test.meters') + tdSql.checkRows(1) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 6) + tdSql.checkData(0, 1, 1734574929004) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 1) + + # there's some error on last_row func when using sub query. + tdSql.query(f'select count(1), cols(last_row(ts), ts, c0), last_row(c1), last_row(c3) from test.meters') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 6) + tdSql.checkData(0, 1, 1734574929004) + tdSql.checkData(0, 2, 4) + tdSql.checkData(0, 3, 4) + tdSql.checkData(0, 4, False) + + + tdSql.query(f'select cols(last(ts), ts, c0), count(1) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 6) + tdSql.query(f'select cols(max(c0), ts, c0), count(1) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 6) - tdSql.query(f'select cols(max(c0), ts), count(1) from {self.dbname}.meters') - tdSql.query(f'select cols(min(c1), ts, c0), count(1) from {self.dbname}.meters') - tdSql.query(f'select count(1), cols(max(c0), ts) from {self.dbname}.meters') - tdSql.query(f'select max(c0), cols(max(c0), ts) from {self.dbname}.meters') - tdSql.query(f'select max(c1), cols(max(c0), ts) from {self.dbname}.meters') + tdSql.query(f'select cols(last(ts), ts as time, c0 cc), count(1) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 6) + tdSql.query(f'select cols(max(c1), ts as time, c0 cc), count(1) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 6) + tdSql.query(f'select cols(last(ts), c0, c1, c2, c3), count(1) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 4) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 'bbbbbbbbb2') + tdSql.checkData(0, 3, False) + tdSql.checkData(0, 4, 6) - def multi_cols_output_test(self): + tdSql.query(f'select cols(max(c0), ts), count(1) from {from_table}') + tdSql.query(f'select cols(min(c1), ts, c0), count(1) from {from_table}') + tdSql.query(f'select count(1), cols(max(c0), ts) from {from_table}') + tdSql.query(f'select max(c0), cols(max(c0), ts) from {from_table}') + tdSql.query(f'select max(c1), cols(max(c0), ts) from {from_table}') + + def multi_cols_output_test(self, from_table = 'test.meters', isTmpTable = False): tdLog.info("multi_cols_output_test") tdSql.query(f'select cols(last(c0), ts, c1), cols(first(c0), ts, c1), count(1) from {self.dbname}.meters') - tdSql.query(f'select cols(last(c0), ts as t1, c1 as c11), cols(first(c0), ts as c2, c1 c21), count(1) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 6) + tdSql.query(f'select cols(last(c0),ts lts, c1 lc1), cols(first(c0), ts fts, c1 as fc1), count(1) from test.meters') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 6) + tdSql.query(f'select cols(max(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 6) + tdSql.query(f'select cols(max(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from {self.dbname}.meters where c0 < 4') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 3) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 5) + tdSql.query(f'select cols(max(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkRows(2) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) + tdSql.checkData(1, 0, 1734574929003) + tdSql.checkData(1, 1, 3) + tdSql.checkData(1, 2, 1734574929000) + tdSql.checkData(1, 3, 1) + tdSql.checkData(1, 4, 4) + + tdSql.query(f'select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkRows(2) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) + tdSql.checkData(1, 0, 1734574929003) + tdSql.checkData(1, 1, 3) + tdSql.checkData(1, 2, 1734574929000) + tdSql.checkData(1, 3, 1) + tdSql.checkData(1, 4, 4) + + tdSql.query(f'select cols(last_row(c0), ts as t1, c1 as c11), cols(min(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkRows(2) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) + tdSql.checkData(1, 0, 1734574929003) + tdSql.checkData(1, 1, 3) + tdSql.checkData(1, 2, 1734574929000) + tdSql.checkData(1, 3, 1) + tdSql.checkData(1, 4, 4) + + tdSql.query(f'select cols(last_row(c0), ts as t1, c1 as c11), cols(mode(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkRows(2) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) + tdSql.checkData(1, 0, 1734574929003) + tdSql.checkData(1, 1, 3) + #tdSql.checkData(1, 2, 1734574929000) # mode(c0) is return a random ts of same c0 + tdSql.checkData(1, 3, 2) + tdSql.checkData(1, 4, 4) + + #select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4; def funcSupperTableTest(self): tdSql.execute('create database if not exists db;') @@ -250,20 +428,20 @@ class TDTestCase: tdSql.execute('create table db.st (ts timestamp, c0 int, c1 float, c2 nchar(30), c3 bool) tags (t1 nchar(30))') tdSql.execute('create table db.st_1 using db.st tags("st1")') tdSql.execute('create table db.st_2 using db.st tags("st1")') - tdSql.execute('insert into db.st_1 values(1734574929000, 1, 1, "a1", true)') + tdSql.execute('insert into db.st_1 values(1734574929000, 1, 1, "c2", true)') tdSql.execute('insert into db.st_1 values(1734574929001, 2, 2, "bbbbbbbbb1", false)') tdSql.execute('insert into db.st_1 values(1734574929002, 3, 3, "a2", true)') - tdSql.execute('insert into db.st_1 values(1734574929003, 4, 4, "bbbbbbbbb2", false)') + tdSql.execute('insert into db.st_1 values(1734574929004, 4, 4, "bbbbbbbbb2", false)') tdSql.query(f'select cols(last(c0), ts, c1, c2, c3), cols(first(c0), ts, c1, c2, c3) from db.st') tdSql.checkRows(1) - tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, '4.0') tdSql.checkData(0, 2, 'bbbbbbbbb2') tdSql.checkData(0, 3, False) tdSql.checkData(0, 4, 1734574929000) tdSql.checkData(0, 5, '1.0') - tdSql.checkData(0, 6, 'a1') + tdSql.checkData(0, 6, 'c2') tdSql.checkData(0, 7, True) #tdSql.execute(f'drop table if exists db.st') @@ -278,6 +456,8 @@ class TDTestCase: tdSql.execute('insert into db.d1 values(1734574929000, 1, 1.1, "a", true)') tdSql.execute('insert into db.d1 values(1734574930000, 2, 2.2, "bbbbbbbbb", false)') + groupby = ["", "group by tbname order ts", "group by tbname order t1", "group by tbname order ts"] + tdSql.query(f'select cols(last(c0), ts, c2), cols(first(c0), ts, c2) from db.d1') tdSql.checkRows(1) tdSql.checkCols(4) @@ -365,15 +545,167 @@ class TDTestCase: tdSql.checkData(0, 1, 'a') tdSql.checkData(0, 2, 1734574929000) tdSql.checkData(0, 3, 6) - - def subquery_test(self): - tdSql.query(f'select count(1), cols(last(c0),c0) from (select * from test.d0)') - tdSql.query(f'select count(1), cols(last(c0),c0) from (select *, tbname from test.meters) group by tbname') - - def window_test(self): - tdSql.query(f'select tbname, _wstart,_wend, max(c0), max(c1), cols( max(c0), c1) from test.meters partition \ - by tbname count_window(2) order by tbname') + def orderby_test(self, from_table = 'test.meters', isTmpTable = False): + select_t1 = ["", ", t1", ", t1 as tag1"] + for t1 in select_t1: + if t1 != "" and isTmpTable: + # Not a GROUP BY expression + tdSql.error(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.error(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.error(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.error(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + continue + tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.checkRows(2) + tags_count = 0 if t1 == "" else 1 + tdLog.debug(f'tags_count: {tags_count}') + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 0, 2, 'st1') + tdSql.checkData(1, 0, 1) + tdSql.checkData(1, 1, 'c2') + self.condition_check(t1 != "", 1, 2, 'st2') + + tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by 1') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by 2') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 0, 2, 'st1') + tdSql.checkData(1, 0, 1) + tdSql.checkData(1, 1, 'c2') + self.condition_check(t1 != "", 1, 2, 'st2') + + tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 0, 2, 'st1') + tdSql.checkData(1, 0, 1) + tdSql.checkData(1, 1, 'c2') + self.condition_check(t1 != "", 1, 2, 'st2') + + tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'bbbbbbbbb2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st1') + tdSql.checkData(1, 0, 1) + tdSql.checkData(1, 1, 'c2') + self.condition_check(t1 != "", 1, 2, 'st2') + + tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'c2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'c2') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'c2') + self.condition_check(t1 != "", 1, 2, 'st1') + def parse_test(self): tdLog.info("parse test") @@ -383,7 +715,6 @@ class TDTestCase: tdSql.error(f'select last(cols(ts)) from {self.dbname}.meters') tdSql.error(f'select last(cols(ts, ts)) from {self.dbname}.meters') tdSql.error(f'select last(cols(ts, ts), ts) from {self.dbname}.meters') - tdSql.error(f'd{self.dbname}.meters') tdSql.error(f'select cols(last(ts), ts as t1) as t1 from {self.dbname}.meters') tdSql.error(f'select cols(last(ts), ts, c0) t1 from {self.dbname}.meters') tdSql.error(f'select cols(last(ts), ts t1) tt from {self.dbname}.meters') @@ -406,7 +737,33 @@ class TDTestCase: tdSql.error(f'select cols(cols(last(ts), c0), c0) as cc from {self.dbname}.meters') tdSql.error(f'select cols(last(ts), cols(last(ts), c0), c0) as cc from {self.dbname}.meters') + + # Aggregate functions do not support nesting + tdSql.error(f'select count(1), cols(last_row(ts), ts, first(c0)), last_row(c1) from {self.dbname}.meters') + # Not a GROUP BY expression + tdSql.error(f'select count(1), cols(last(c0),c0) from test.meters group by tbname order by c3 desc') + + def subquery_test(self): + tdSql.query(f'select count(1), cols(last(c0),c0) from (select * from test.d0)') + tdSql.query(f'select count(1), cols(last(c0),c0) from (select *, tbname from test.meters) group by tbname') + + tdLog.info("subquery_test: orderby_test from meters") + self.orderby_test("test.meters", False) + tdLog.info("subquery_test: orderby_test from (select *, tbname from meters)") + self.orderby_test("(select *, tbname from test.meters)", True) + tdLog.info("subquery_test: one_cols_multi_output_with_group_test from meters") + self.one_cols_multi_output_with_group_test("test.meters", False) + tdLog.info("subquery_test: one_cols_multi_output_with_group_test from (select *, tbname from meters)") + self.one_cols_multi_output_with_group_test("(select *, tbname from test.meters)", True) + + self.one_cols_multi_output_test("test.meters") + self.one_cols_multi_output_test("(select *, tbname from test.meters)") + + + def window_test(self): + tdSql.query(f'select tbname, _wstart,_wend, max(c0), max(c1), cols( max(c0), c1) from test.meters partition \ + by tbname count_window(2) order by tbname') def run(self): self.funcNestTest() @@ -414,10 +771,9 @@ class TDTestCase: self.create_test_data() self.parse_test() self.one_cols_1output_test() - self.one_cols_multi_output_test() self.multi_cols_output_test() self.subquery_test() - + self.window_test() def stop(self): tdSql.close() From 9a34af9e9516b1f3d2aa5ffdf35991cd0ffe16d1 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 2 Jan 2025 18:04:45 +0800 Subject: [PATCH 14/45] build on win --- source/libs/parser/src/parTranslater.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4a5258c5fc..73f5569a7c 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7440,13 +7440,13 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 1); if (nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { pCxt->status = TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC; - parserError("Invalid cols function, the first parameter must be a select function"); + parserError("%s Invalid cols function, the first parameter must be a select function", __func__); return DEAL_RES_ERROR; } if (pFunc->node.asAlias) { if (((SExprNode*)pExpr)->asAlias) { pCxt->status = TSDB_CODE_INVALID_COLS_ALIAS; - parserError("Invalid using alias for cols function"); + parserError("%s Invalid using alias for cols function", __func__); return DEAL_RES_ERROR; } else { ((SExprNode*)pExpr)->asAlias = true; @@ -7478,7 +7478,7 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); } else { pCxt->status = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; - parserError("Invalid cols function, the first parameter must be a select function"); + parserError("%s Invalid cols function, the first parameter must be a select function", __func__); return DEAL_RES_ERROR; } nodesDestroyNode(*pNode); @@ -7532,14 +7532,14 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList SFunctionNode* pFunc = (SFunctionNode*)pTmpNode; if(pFunc->node.asAlias) { code = TSDB_CODE_INVALID_COLS_ALIAS; - parserError("Invalid using alias for cols function"); + parserError("%s Invalid using alias for cols function", __func__); goto _end; } SNode* pSelectFunc = nodesListGetNode(pFunc->pParameterList, 0); if (nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; - parserError("Invalid cols function, the first parameter must be a select function"); + parserError("%s Invalid cols function, the first parameter must be a select function", __func__); goto _end; } int32_t selectFuncIndex = getSelectFuncIndex(*selectFuncList, pSelectFunc); @@ -7560,7 +7560,7 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); } else { code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; - parserError("Invalid cols function, the first parameter must be a select function"); + parserError("%s Invalid cols function, the first parameter must be a select function", __func__); goto _end; } if (TSDB_CODE_SUCCESS != code) goto _end; From 908d69faa3443a257efbc4b6921a93c47cf1a3aa Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 3 Jan 2025 10:32:18 +0800 Subject: [PATCH 15/45] fxi: alias name is too long --- include/libs/nodes/querynodes.h | 4 ++-- source/libs/parser/src/parTranslater.c | 10 ++++++++-- tests/system-test/2-query/cols_function.py | 23 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index f3c84c11bc..e14a43b1c2 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -55,8 +55,8 @@ typedef struct SDataType { typedef struct SExprNode { ENodeType type; SDataType resType; - char aliasName[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; - char userAlias[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; + char aliasName[TSDB_COL_NAME_LEN]; + char userAlias[TSDB_COL_NAME_LEN]; SArray* pAssociation; bool asAlias; bool asParam; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 73f5569a7c..ec85c704d5 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7342,7 +7342,13 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { if (nodesIsExprNode(*pNode)) { ((SExprNode*)*pNode)->bindTupleFuncIdx = pCxt->bindTupleFuncIdx; int32_t len = strlen(((SExprNode*)*pNode)->aliasName); - tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pCxt->bindTupleFuncIdx); + if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { + parserWarn("The alias name is too long, the extra part will be truncated"); + tsnprintf(((SExprNode*)*pNode)->aliasName + TSDB_COL_NAME_EXLEN - TSDB_COL_NAME_EXLEN, TSDB_COL_NAME_EXLEN, ".%d", + pCxt->bindTupleFuncIdx); + } else { + tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pCxt->bindTupleFuncIdx); + } SFunctionNode* pFunc = (SFunctionNode*)*pNode; } return DEAL_RES_CONTINUE; @@ -7450,7 +7456,7 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } else { ((SExprNode*)pExpr)->asAlias = true; - tstrncpy(((SExprNode*)pExpr)->userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN); + tstrncpy(((SExprNode*)pExpr)->userAlias, pFunc->node.userAlias, TSDB_COL_NAME_LEN); } } if(*pCxt->selectFuncList == NULL) { diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index fd41c3a1a1..2f46051bb7 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -180,6 +180,11 @@ class TDTestCase: tdSql.checkCols(2) tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) + tdSql.query(f'select cols(last(ts), ts as t123456789t123456789t123456789t123456789t123456789t123456789t123456789, c0 cc) from {from_table}') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) tdSql.query(f'select cols(last(ts), c0, c1, c2, c3) from {from_table}') tdSql.checkRows(1) tdSql.checkCols(4) @@ -362,6 +367,24 @@ class TDTestCase: tdSql.checkData(0, 2, 1734574929000) tdSql.checkData(0, 3, 1) tdSql.checkData(0, 4, 5) + tdSql.query(f'select cols(max(c0), ts as t123456789t123456789t123456789t123456789t123456789t123456789t123456789, c1 as c11), cols(first(c0), \ + ts as t123456789t123456789t123456789t123456789t123456789t123456789t123456789, c1 c21), count(1) from {self.dbname}.meters') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 6) + tdSql.query(f'select cols(max(c0), ts as t123456789t123456789t123456789t123456789t123456789t123456789t123456789, c1 as c11), cols(first(c0), \ + ts as t123456789t123456789t123456789t123456789t123456789t123456789t123456789, c1 c21), count(1) from {self.dbname}.meters where c0 < 4') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929003) + tdSql.checkData(0, 1, 3) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 5) tdSql.query(f'select cols(max(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') tdSql.checkRows(2) tdSql.checkCols(5) From 8d45b04d69244bacdeece8a5464c394ceb694878 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 3 Jan 2025 16:34:20 +0800 Subject: [PATCH 16/45] test case --- tests/system-test/2-query/cols_function.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 2f46051bb7..d843b5959c 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -441,7 +441,11 @@ class TDTestCase: tdSql.checkData(1, 3, 2) tdSql.checkData(1, 4, 4) + # todo 1 , has same select function outof cols func #select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4; + # todo 2, sub query has cols func + # select * from (select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4); + # todo 3, cols on system table def funcSupperTableTest(self): tdSql.execute('create database if not exists db;') From fa3cd81a268bfb936c787a69e5e232cf3c6f2adb Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 3 Jan 2025 17:28:31 +0800 Subject: [PATCH 17/45] parerlog --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ec85c704d5..7e768d740a 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7343,7 +7343,7 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { ((SExprNode*)*pNode)->bindTupleFuncIdx = pCxt->bindTupleFuncIdx; int32_t len = strlen(((SExprNode*)*pNode)->aliasName); if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { - parserWarn("The alias name is too long, the extra part will be truncated"); + parserWarn("%s The alias name is too long, the extra part will be truncated", __func__); tsnprintf(((SExprNode*)*pNode)->aliasName + TSDB_COL_NAME_EXLEN - TSDB_COL_NAME_EXLEN, TSDB_COL_NAME_EXLEN, ".%d", pCxt->bindTupleFuncIdx); } else { From b1f1c2e860cf47cd3ed1bb2b5f34dc943c4452f8 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 23 Jan 2025 15:37:21 +0800 Subject: [PATCH 18/45] test case: test cols name --- tests/pytest/util/sql.py | 9 +++++ tests/system-test/2-query/cols_function.py | 41 ++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index cdfe3ce8a0..bfdbff3826 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -648,6 +648,15 @@ class TDSql: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, col_name_list, expect_col_name_list) tdLog.exit("%s(%d) failed: sql:%s, col_name_list:%s != expect_col_name_list:%s" % args) + + def checkResColNameList(self, expect_col_name_list): + col_name_list = [] + col_type_list = [] + for query_col in self.cursor.description: + col_name_list.append(query_col[0]) + col_type_list.append(query_col[1]) + + self.checkColNameList(col_name_list, expect_col_name_list) def __check_equal(self, elm, expect_elm): if elm == expect_elm: diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index d843b5959c..54dea9b4e5 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -46,15 +46,23 @@ class TDTestCase: def one_cols_1output_test(self): tdLog.info("one_cols_1output_test") tdSql.query(f'select cols(last(ts), ts) from {self.dbname}.meters') + tdSql.checkResColNameList(['ts']) tdSql.query(f'select cols(last(ts), ts) as t1 from {self.dbname}.meters') + tdSql.checkResColNameList(['t1']) tdSql.query(f'select cols(last(ts), ts as t1) from {self.dbname}.meters') + tdSql.checkResColNameList(['t1']) tdSql.query(f'select cols(last(ts), c0) from {self.dbname}.meters') + tdSql.checkResColNameList(['c0']) tdSql.query(f'select cols(last(ts), c1) from {self.dbname}.meters group by tbname') + tdSql.checkResColNameList(['c1']) tdSql.query(f'select cols(last(ts+1), ts) as t1 from {self.dbname}.meters') + tdSql.checkResColNameList(['t1']) tdSql.query(f'select cols(last(ts+1), ts+2 as t1) from {self.dbname}.meters') + tdSql.checkResColNameList(['t1']) tdSql.query(f'select cols(last(ts+1), c0+10) from {self.dbname}.meters') + tdSql.checkResColNameList(['c0+10']) def one_cols_multi_output_with_group_test(self, from_table = 'test.meters', isTmpTable = False): select_t1 = ["", ", t1", ", t1 as tag1"] @@ -89,6 +97,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'd1') tdSql.checkData(1, 2, 'st2') tdSql.query(f'select cols(last(c1), ts), tbname, t1 from {from_table} group by tbname order by t1') + tdSql.checkResColNameList(['ts', 'tbname', 't1']) tdSql.checkRows(2) tdSql.checkCols(3) tdSql.checkData(0, 0, 1734574929004) @@ -171,21 +180,25 @@ class TDTestCase: def one_cols_multi_output_test(self, from_table = 'test.meters'): tdLog.info(f"one_cols_1output_test {from_table}") tdSql.query(f'select cols(last(ts), ts, c0) from {from_table}') + tdSql.checkResColNameList(['ts', 'c0']) tdSql.checkRows(1) tdSql.checkCols(2) tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) tdSql.query(f'select cols(last(ts), ts as time, c0 cc) from {from_table}') + tdSql.checkResColNameList(['time', 'cc']) tdSql.checkRows(1) tdSql.checkCols(2) tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) tdSql.query(f'select cols(last(ts), ts as t123456789t123456789t123456789t123456789t123456789t123456789t123456789, c0 cc) from {from_table}') + tdSql.checkResColNameList(['t123456789t123456789t123456789t123456789t123456789t123456789t123', 'cc']) tdSql.checkRows(1) tdSql.checkCols(2) tdSql.checkData(0, 0, 1734574929004) tdSql.checkData(0, 1, 4) tdSql.query(f'select cols(last(ts), c0, c1, c2, c3) from {from_table}') + tdSql.checkResColNameList(['c0', 'c1', 'c2', 'c3']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 4) @@ -193,13 +206,16 @@ class TDTestCase: tdSql.checkData(0, 2, 'bbbbbbbbb2') tdSql.checkData(0, 3, False) tdSql.query(f'select cols(last(ts), c0, t1) from {from_table}') + tdSql.checkResColNameList(['c0', 't1']) tdSql.checkRows(1) tdSql.checkData(0, 0, 4) tdSql.checkData(0, 1, 'st1') tdSql.query(f'select cols(max(c0), ts) from {from_table}') + tdSql.checkResColNameList(['ts']) tdSql.checkCols(1) tdSql.checkData(0, 0, 1734574929004) tdSql.query(f'select cols(min(c1), ts, c0) from {from_table}') + tdSql.checkResColNameList(['ts', 'c0']) tdSql.checkCols(2) tdSql.checkData(0, 0, 1734574929000) tdSql.checkData(0, 1, 1) @@ -235,6 +251,7 @@ class TDTestCase: tdSql.checkData(0, 1, 1) tdSql.checkData(0, 2, 5) tdSql.query(f'select cols(min(c0), ts, c0), count(1) from {self.dbname}.normal_table') + tdSql.checkResColNameList(['ts', 'c0', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(3) tdSql.checkData(0, 0, 1734574929000) @@ -305,6 +322,7 @@ class TDTestCase: tdSql.checkData(0, 2, 6) tdSql.query(f'select cols(last(ts), ts as time, c0 cc), count(1) from {from_table}') + tdSql.checkResColNameList(['time', 'cc', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(3) tdSql.checkData(0, 0, 1734574929004) @@ -312,6 +330,7 @@ class TDTestCase: tdSql.checkData(0, 2, 6) tdSql.query(f'select cols(max(c1), ts as time, c0 cc), count(1) from {from_table}') + tdSql.checkResColNameList(['time', 'cc', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(3) tdSql.checkData(0, 0, 1734574929004) @@ -319,6 +338,7 @@ class TDTestCase: tdSql.checkData(0, 2, 6) tdSql.query(f'select cols(last(ts), c0, c1, c2, c3), count(1) from {from_table}') + tdSql.checkResColNameList(['c0', 'c1', 'c2', 'c3', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(5) tdSql.checkData(0, 0, 4) @@ -336,6 +356,7 @@ class TDTestCase: def multi_cols_output_test(self, from_table = 'test.meters', isTmpTable = False): tdLog.info("multi_cols_output_test") tdSql.query(f'select cols(last(c0), ts, c1), cols(first(c0), ts, c1), count(1) from {self.dbname}.meters') + tdSql.checkResColNameList(['ts', 'c1', 'ts', 'c1', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929004) @@ -344,6 +365,7 @@ class TDTestCase: tdSql.checkData(0, 3, 1) tdSql.checkData(0, 4, 6) tdSql.query(f'select cols(last(c0),ts lts, c1 lc1), cols(first(c0), ts fts, c1 as fc1), count(1) from test.meters') + tdSql.checkResColNameList(['lts', 'lc1', 'fts', 'fc1', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929004) @@ -352,6 +374,7 @@ class TDTestCase: tdSql.checkData(0, 3, 1) tdSql.checkData(0, 4, 6) tdSql.query(f'select cols(max(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from {self.dbname}.meters') + tdSql.checkResColNameList(['t1', 'c11', 't2', 'c21', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929004) @@ -360,6 +383,7 @@ class TDTestCase: tdSql.checkData(0, 3, 1) tdSql.checkData(0, 4, 6) tdSql.query(f'select cols(max(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from {self.dbname}.meters where c0 < 4') + tdSql.checkResColNameList(['t1', 'c11', 't2', 'c21', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929003) @@ -369,6 +393,8 @@ class TDTestCase: tdSql.checkData(0, 4, 5) tdSql.query(f'select cols(max(c0), ts as t123456789t123456789t123456789t123456789t123456789t123456789t123456789, c1 as c11), cols(first(c0), \ ts as t123456789t123456789t123456789t123456789t123456789t123456789t123456789, c1 c21), count(1) from {self.dbname}.meters') + tdSql.checkResColNameList(['t123456789t123456789t123456789t123456789t123456789t123456789t123', 'c11', \ + 't123456789t123456789t123456789t123456789t123456789t123456789t123', 'c21', 'count(1)']) tdSql.checkRows(1) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929004) @@ -386,6 +412,7 @@ class TDTestCase: tdSql.checkData(0, 3, 1) tdSql.checkData(0, 4, 5) tdSql.query(f'select cols(max(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkResColNameList(['t1', 'c11', 't2', 'c21', 'count(1)']) tdSql.checkRows(2) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929000) @@ -400,6 +427,7 @@ class TDTestCase: tdSql.checkData(1, 4, 4) tdSql.query(f'select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkResColNameList(['t1', 'c11', 't2', 'c21', 'count(1)']) tdSql.checkRows(2) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929000) @@ -414,6 +442,7 @@ class TDTestCase: tdSql.checkData(1, 4, 4) tdSql.query(f'select cols(last_row(c0), ts as t1, c1 as c11), cols(min(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkResColNameList(['t1', 'c11', 't2', 'c21', 'count(1)']) tdSql.checkRows(2) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929000) @@ -428,6 +457,7 @@ class TDTestCase: tdSql.checkData(1, 4, 4) tdSql.query(f'select cols(last_row(c0), ts as t1, c1 as c11), cols(mode(c0), ts as t2, c1 c21), count(1) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkResColNameList(['t1', 'c11', 't2', 'c21', 'count(1)']) tdSql.checkRows(2) tdSql.checkCols(5) tdSql.checkData(0, 0, 1734574929000) @@ -486,6 +516,7 @@ class TDTestCase: groupby = ["", "group by tbname order ts", "group by tbname order t1", "group by tbname order ts"] tdSql.query(f'select cols(last(c0), ts, c2), cols(first(c0), ts, c2) from db.d1') + tdSql.checkResColNameList(['ts', 'c2', 'ts', 'c2']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574930000) @@ -493,6 +524,7 @@ class TDTestCase: tdSql.checkData(0, 2, 1734574929000) tdSql.checkData(0, 3, 'a') tdSql.query(f'select cols(last(c0), ts, c1, c2, c3), cols(first(c0), ts, c1, c2, c3) from db.d1') + tdSql.checkResColNameList(['ts', 'c1', 'c2', 'c3', 'ts', 'c1', 'c2', 'c3']) tdSql.checkRows(1) tdSql.checkCols(8) tdSql.checkData(0, 0, 1734574930000) @@ -505,12 +537,14 @@ class TDTestCase: tdSql.checkData(0, 7, True) tdSql.query(f'select cols(last(ts), c1), cols(first(ts), c1) from db.d1') + tdSql.checkResColNameList(['c1', 'c1']) tdSql.checkRows(1) tdSql.checkCols(2) tdSql.checkData(0, 0, 2.2) tdSql.checkData(0, 1, 1.1) tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts), c0, c1) from db.d1') + tdSql.checkResColNameList(['c0', 'c1', 'c0', 'c1']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1) @@ -519,6 +553,7 @@ class TDTestCase: tdSql.checkData(0, 3, 1.1) tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts+1), c0, c1) from db.d1') + tdSql.checkResColNameList(['c0', 'c1', 'c0', 'c1']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1) @@ -527,6 +562,7 @@ class TDTestCase: tdSql.checkData(0, 3, 1.1) tdSql.query(f'select cols(first(ts), c0, c1), cols(first(ts), c0+1, c1+2) from db.d1') + tdSql.checkResColNameList(['c0', 'c1', 'c0+1', 'c1+2']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1) @@ -535,6 +571,7 @@ class TDTestCase: tdSql.checkData(0, 3, 3.1) tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2)) from db.d1') + tdSql.checkResColNameList(['ts', 'length(c2)', 'ts', 'length(c2)']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) @@ -542,6 +579,7 @@ class TDTestCase: tdSql.checkData(0, 2, 1734574930000) tdSql.checkData(0, 3, 36) tdSql.query(f'select cols(first(c0), ts, length(c2)), cols(last(c0), ts, length(c2) + 2) from db.d1') + tdSql.checkResColNameList(['ts', 'length(c2)', 'ts', 'length(c2) + 2']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) @@ -550,6 +588,7 @@ class TDTestCase: tdSql.checkData(0, 3, 38) tdSql.query(f'select cols(first(c0), ts, c2), cols(last(c0), ts, length(c2) + 2) from db.d1') + tdSql.checkResColNameList(['ts', 'c2', 'ts', 'length(c2) + 2']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) @@ -558,6 +597,7 @@ class TDTestCase: tdSql.checkData(0, 3, 38) tdSql.query(f'select cols(min(c0), ts, c2), cols(last(c0), ts, length(c2) + 2) from db.d1') + tdSql.checkResColNameList(['ts', 'c2', 'ts', 'length(c2) + 2']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) @@ -566,6 +606,7 @@ class TDTestCase: tdSql.checkData(0, 3, 38) tdSql.query(f'select cols(min(c0), ts, c2), cols(first(c0), ts, length(c2) + 2) from db.d1') + tdSql.checkResColNameList(['ts', 'c2', 'ts', 'length(c2) + 2']) tdSql.checkRows(1) tdSql.checkCols(4) tdSql.checkData(0, 0, 1734574929000) From 7e553f7bc905943441fe91a9b0b788d5a1942ed4 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 23 Jan 2025 19:27:32 +0800 Subject: [PATCH 19/45] fxi: use the same agg func outside of cols --- source/libs/nodes/src/nodesEqualFuncs.c | 7 ++-- tests/system-test/2-query/cols_function.py | 37 ++++++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 501a9fe17d..d255c2fd6b 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -139,10 +139,13 @@ static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_STRING_FIELD(functionName); COMPARE_NODE_LIST_FIELD(pParameterList); if (a->funcType == FUNCTION_TYPE_SELECT_VALUE) { - if ((a->node.bindTupleFuncIdx != b->node.bindTupleFuncIdx) && - (a->node.bindTupleFuncIdx != 0 || b->node.bindTupleFuncIdx != 0)) + if ((a->node.bindTupleFuncIdx != b->node.bindTupleFuncIdx)) return false; + } else { + if ((a->node.tupleFuncIdx != b->node.tupleFuncIdx)) { return false; + } } + return true; } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 54dea9b4e5..86cf2528e3 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -471,8 +471,41 @@ class TDTestCase: tdSql.checkData(1, 3, 2) tdSql.checkData(1, 4, 4) - # todo 1 , has same select function outof cols func - #select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4; + # fixed: has same select function outof cols func + tdSql.query(f'select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4 group by tbname order by t1') + tdSql.checkResColNameList(['t1', 'c11', 't2', 'c21', 'first(c0)']) + tdSql.checkRows(2) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) + tdSql.checkData(1, 0, 1734574929003) + tdSql.checkData(1, 1, 3) + tdSql.checkData(1, 2, 1734574929000) + tdSql.checkData(1, 3, 1) + tdSql.checkData(1, 4, 1) + + tdSql.query(f'select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0), cols(last(c0), ts, c1) from test.meters where c0 < 4 group by tbname order by t1;') + tdSql.checkResColNameList(['t1', 'c11', 't2', 'c21', 'first(c0)', 'ts', 'c1']) + tdSql.checkRows(2) + tdSql.checkCols(7) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) + tdSql.checkData(0, 5, 1734574929000) + tdSql.checkData(0, 6, 1) + tdSql.checkData(1, 0, 1734574929003) + tdSql.checkData(1, 1, 3) + tdSql.checkData(1, 2, 1734574929000) + tdSql.checkData(1, 3, 1) + tdSql.checkData(1, 4, 1) + tdSql.checkData(1, 5, 1734574929003) + tdSql.checkData(1, 6, 3) + # todo 2, sub query has cols func # select * from (select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4); # todo 3, cols on system table From fbe345a82e3bfd89ee044e90865fcc392b2fe0d9 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 5 Feb 2025 15:26:03 +0800 Subject: [PATCH 20/45] fix: cols funcs, sub query --- include/libs/nodes/querynodes.h | 2 + source/dnode/mnode/impl/src/mndDump.c | 2 +- source/libs/function/src/builtins.c | 4 +- source/libs/nodes/src/nodesUtilFuncs.c | 4 + source/libs/parser/src/parTranslater.c | 36 ++++-- source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/planner/src/planOptimizer.c | 2 +- source/libs/planner/src/planPhysiCreater.c | 141 +++++++++++---------- source/libs/planner/src/planUtil.c | 2 + tests/system-test/2-query/cols_function.py | 34 ++++- 10 files changed, 144 insertions(+), 85 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index e14a43b1c2..653b5ff6cd 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -699,6 +699,8 @@ char* getJoinSTypeString(EJoinSubType type); char* getFullJoinTypeString(EJoinType type, EJoinSubType stype); int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc); +int32_t rewriteExprAliasName(SExprNode* pNode, int64_t num); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index d1dd99b319..485a89f1bd 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -41,7 +41,7 @@ int32_t sendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { } char *i642str(int64_t val) { - static char str[24] = {0}; + static threadlocal char str[24] = {0}; (void)snprintf(str, sizeof(str), "%" PRId64, val); return str; } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 47ba99aca0..722f838061 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1702,7 +1702,7 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t return TSDB_CODE_SUCCESS; } -static int32_t translateColsFunction(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { +static int32_t invalidColsFunction(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; } @@ -5654,7 +5654,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "cols", - .translateFunc = translateColsFunction, + .translateFunc = invalidColsFunction, }, }; // clang-format on diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index ae5b302d2d..dd4d1c694b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -3237,3 +3237,7 @@ int32_t nodesListDeduplicate(SNodeList** ppList) { } return code; } + +int32_t rewriteExprAliasName(SExprNode* pNode, int64_t num) { + return tsnprintf(pNode->aliasName, TSDB_COL_NAME_LEN, "expr_%d", num); +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f4c1a62a17..3b97097a1a 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1574,24 +1574,30 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, STempTableNode* pTempTable = (STempTableNode*)pTable; SNodeList* pProjectList = getProjectList(pTempTable->pSubquery); SNode* pNode; + SExprNode* pFoundExpr = NULL; FOREACH(pNode, pProjectList) { SExprNode* pExpr = (SExprNode*)pNode; - if (0 == strcmp(pCol->colName, pExpr->aliasName)) { + if (0 == strcmp(pCol->colName, pExpr->aliasName) || 0 == strcmp(pCol->colName, pExpr->userAlias)) { if (*pFound) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); } - code = setColumnInfoByExpr(pTempTable, pExpr, pColRef); - if (TSDB_CODE_SUCCESS != code) { - break; - } + pFoundExpr = pExpr; *pFound = true; } else if (isPrimaryKeyImpl(pNode) && isInternalPrimaryKey(pCol)) { - code = setColumnInfoByExpr(pTempTable, pExpr, pColRef); - if (TSDB_CODE_SUCCESS != code) break; + if (*pFound) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); + } + pFoundExpr = pExpr; pCol->isPrimTs = true; *pFound = true; } } + if (pFoundExpr) { + code = setColumnInfoByExpr(pTempTable, pFoundExpr, pColRef); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } } return code; } @@ -5561,7 +5567,7 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { if ('\0' == pExpr->userAlias[0]) { tstrncpy(pExpr->userAlias, pExpr->aliasName, TSDB_COL_NAME_LEN); } - snprintf(pExpr->aliasName, TSDB_COL_NAME_LEN,"#expr_%d", no++); + rewriteExprAliasName(pExpr, no++); } return TSDB_CODE_SUCCESS; } @@ -7622,6 +7628,16 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList } static int32_t translateColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable)) { + SNode* pSubquery = ((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { + SSelectStmt* pSubSelect = (SSelectStmt*)pSubquery; + int32_t code = translateColsFunction(pCxt, pSubSelect); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } SNodeList* selectFuncList = NULL; int32_t code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &selectFuncList); if (code == TSDB_CODE_SUCCESS) { @@ -7647,9 +7663,9 @@ _end: static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = (SNode*)pSelect; pCxt->dual = false; - int32_t code = translateFrom(pCxt, &pSelect->pFromTable); + int32_t code = translateColsFunction(pCxt, pSelect); if (TSDB_CODE_SUCCESS == code) { - code = translateColsFunction(pCxt, pSelect); + code = translateFrom(pCxt, &pSelect->pFromTable); } if (TSDB_CODE_SUCCESS == code) { pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 802a0607ee..b24ef5f12b 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -152,7 +152,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { if ('\0' == ((SExprNode*)pNode)->aliasName[0]) { - snprintf(((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN, "#expr_%p", pNode); + rewriteExprAliasName((SExprNode*)pNode, (int64_t)pNode); } return DEAL_RES_IGNORE_CHILD; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 172f429a5b..715ab04676 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3035,7 +3035,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo } SExprNode exprNode; exprNode.resType = ((SExprNode*)pWsNode)->resType; - snprintf(exprNode.aliasName, TSDB_COL_NAME_LEN, "#expr_%d", index + 1); + rewriteExprAliasName(&exprNode, index + 1); SColumnNode* pkNode = NULL; code = smaIndexOptCreateSmaCol((SNode*)&exprNode, tableId, PRIMARYKEY_TIMESTAMP_COL_ID, &pkNode); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 31d51fad9b..5902f1c3e9 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "nodes.h" #include "planInt.h" #include "catalog.h" @@ -30,98 +31,106 @@ typedef struct SSlotIndex { SArray* pSlotIdsInfo; // duplicate name slot } SSlotIndex; +static int64_t getExprBindIndexStr(SNode* pNode, char* bindInfo) { + if (nodeType(pNode) == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)pNode; + if (pCol->dataBlockId == 0) { + return 0; + } + } + if (nodesIsExprNode(pNode)) { + SExprNode* pExpr = (SExprNode*)pNode; + if (pExpr->bindTupleFuncIdx > 0 && pExpr->bindTupleFuncIdx <= 9) { + bindInfo[0] = '0' + pExpr->bindTupleFuncIdx; + return 1; + } else if (pExpr->bindTupleFuncIdx != 0) { + return tsnprintf(bindInfo, sizeof(bindInfo), "%d", pExpr->bindTupleFuncIdx); + } + } + return 0; +} + +enum { + SLOT_KEY_TYPE_ALL = 1, + SLOT_KEY_TYPE_COLNAME = 2, +}; + +static int32_t getSlotKeyHelper(SNode* pNode, const char* pPreName, const char* name, char** ppKey, int32_t callocLen, + int32_t* pLen, uint16_t extraBufLen, int8_t slotKeyType) { + int32_t code = 0; + char bindInfo[16] = {0}; + //int exBindInfoLen = getExprBindIndexStr(pNode, bindInfo); + *ppKey = taosMemoryCalloc(1, callocLen); + if (!*ppKey) { + return terrno; + } + if (slotKeyType == SLOT_KEY_TYPE_ALL) { + TAOS_STRNCAT(*ppKey, pPreName, TSDB_TABLE_NAME_LEN); + TAOS_STRNCAT(*ppKey, ".", 2); + TAOS_STRNCAT(*ppKey, name, TSDB_COL_NAME_LEN); + // if (exBindInfoLen > 0) { + // TAOS_STRNCAT(*ppKey, bindInfo, exBindInfoLen); + // } + *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); + } else { + TAOS_STRNCAT(*ppKey, name, TSDB_COL_NAME_LEN); + *pLen = strlen(*ppKey); + } + + return code; +} + static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t* pLen, uint16_t extraBufLen) { int32_t code = 0; + int32_t callocLen = 0; if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; if (NULL != pStmtName) { if ('\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); - if (!*ppKey) { - return terrno; - } - TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); - TAOS_STRNCAT(*ppKey, ".", 2); - TAOS_STRNCAT(*ppKey, pCol->node.aliasName, TSDB_COL_NAME_LEN); - *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); - return code; + callocLen = TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen; + return getSlotKeyHelper(pNode, pStmtName, pCol->node.aliasName, ppKey, callocLen, pLen, extraBufLen, + SLOT_KEY_TYPE_ALL); } else { - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); - if (!*ppKey) { - return terrno; - } - TAOS_STRNCAT(*ppKey, pCol->node.aliasName, TSDB_COL_NAME_LEN); - *pLen = strlen(*ppKey); - return code; + callocLen = TSDB_COL_NAME_LEN + 1 + extraBufLen; + return getSlotKeyHelper(pNode, pStmtName, pCol->node.aliasName, ppKey, callocLen, pLen, extraBufLen, + SLOT_KEY_TYPE_COLNAME); } } if ('\0' == pCol->tableAlias[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); - if (!*ppKey) { - return terrno; - } - TAOS_STRNCAT(*ppKey, pCol->colName, TSDB_COL_NAME_LEN); - *pLen = strlen(*ppKey); - return code; + callocLen = TSDB_COL_NAME_LEN + 1 + extraBufLen; + return getSlotKeyHelper(pNode, pStmtName, pCol->colName, ppKey, callocLen, pLen, extraBufLen, + SLOT_KEY_TYPE_COLNAME); } - - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); - if (!*ppKey) { - return terrno; - } - TAOS_STRNCAT(*ppKey, pCol->tableAlias, TSDB_TABLE_NAME_LEN); - TAOS_STRNCAT(*ppKey, ".", 2); - TAOS_STRNCAT(*ppKey, pCol->colName, TSDB_COL_NAME_LEN); - *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); - return code; + callocLen = TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen; + return getSlotKeyHelper(pNode, pCol->tableAlias, pCol->colName, ppKey, callocLen, pLen, extraBufLen, + SLOT_KEY_TYPE_ALL); } else if (QUERY_NODE_FUNCTION == nodeType(pNode)) { SFunctionNode* pFunc = (SFunctionNode*)pNode; if (FUNCTION_TYPE_TBNAME == pFunc->funcType) { SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); if (pVal) { if (NULL != pStmtName && '\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); - if (!*ppKey) { - return terrno; - } - TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); - TAOS_STRNCAT(*ppKey, ".", 2); - TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); - *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); - return code; + callocLen = TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen; + return getSlotKeyHelper(pNode, pStmtName, ((SExprNode*)pNode)->aliasName, ppKey, callocLen, pLen, extraBufLen, + SLOT_KEY_TYPE_ALL); } int32_t literalLen = strlen(pVal->literal); - *ppKey = taosMemoryCalloc(1, literalLen + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); - if (!*ppKey) { - return terrno; - } - TAOS_STRNCAT(*ppKey, pVal->literal, literalLen); - TAOS_STRNCAT(*ppKey, ".", 2); - TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); - *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); - return code; + callocLen = literalLen + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen; + return getSlotKeyHelper(pNode, pVal->literal, ((SExprNode*)pNode)->aliasName, ppKey, callocLen, pLen, + extraBufLen, SLOT_KEY_TYPE_ALL); } } } if (NULL != pStmtName && '\0' != pStmtName[0]) { - *ppKey = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen); - if (!*ppKey) { - return terrno; - } - TAOS_STRNCAT(*ppKey, pStmtName, TSDB_TABLE_NAME_LEN); - TAOS_STRNCAT(*ppKey, ".", 2); - TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); - *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); - return code; + callocLen = TSDB_TABLE_NAME_LEN + 1 + TSDB_COL_NAME_LEN + 1 + extraBufLen; + return getSlotKeyHelper(pNode, pStmtName, ((SExprNode*)pNode)->aliasName, ppKey, callocLen, pLen, extraBufLen, + SLOT_KEY_TYPE_ALL); } - *ppKey = taosMemoryCalloc(1, TSDB_COL_NAME_LEN + 1 + extraBufLen); - if (!*ppKey) { - return terrno; - } - TAOS_STRNCAT(*ppKey, ((SExprNode*)pNode)->aliasName, TSDB_COL_NAME_LEN); - *pLen = strlen(*ppKey); + callocLen = TSDB_COL_NAME_LEN + 1 + extraBufLen; + return getSlotKeyHelper(pNode, pStmtName, ((SExprNode*)pNode)->aliasName, ppKey, callocLen, pLen, extraBufLen, + SLOT_KEY_TYPE_COLNAME); return code; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index f03e2d8ab0..49559719a7 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -79,6 +79,8 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { } } } + pCol->node.bindTupleFuncIdx = pExpr->bindTupleFuncIdx; + pCol->node.tupleFuncIdx = pExpr->tupleFuncIdx; return (TSDB_CODE_SUCCESS == nodesListStrictAppend(pCxt->pList, (SNode*)pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 86cf2528e3..9bccee1f73 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -506,9 +506,35 @@ class TDTestCase: tdSql.checkData(1, 5, 1734574929003) tdSql.checkData(1, 6, 3) - # todo 2, sub query has cols func - # select * from (select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4); - # todo 3, cols on system table + # sub query has cols func + tdSql.query(f'select c11 from (select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4)') + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.checkData(0, 0, 3) + tdSql.query(f'select c11, c21 from (select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4)') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 3) + tdSql.checkData(0, 1, 1) + tdSql.query(f'select c1, c21 from (select cols(last_row(c0), ts as t1, c1), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4)') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 3) + tdSql.checkData(0, 1, 1) + tdSql.error(f'select c1 from (select cols(last_row(c0), ts as t1, c1), cols(first(c0), ts as t2, c1), first(c0) from test.meters where c0 < 4)') + + # cols on system table + tdSql.query(f'select cols(max(vgroup_id), uid) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.query(f'select cols(max(vgroup_id), uid, `ttl`, create_time) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.query(f'select cols(max(vgroup_id), uid as uidname) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.error(f'select cols(last(vgroup_id), uid, `ttl`, create_time) from information_schema.ins_tables') + tdSql.error(f'select cols(first(vgroup_id), uid, `ttl`, create_time) from information_schema.ins_tables') def funcSupperTableTest(self): tdSql.execute('create database if not exists db;') @@ -534,7 +560,7 @@ class TDTestCase: tdSql.checkData(0, 6, 'c2') tdSql.checkData(0, 7, True) - #tdSql.execute(f'drop table if exists db.st') + tdSql.execute(f'drop table if exists db.st') def funcNestTest(self): From b461ec002a9fda53f9ffc838f8ab0b48c49516cf Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 5 Feb 2025 17:04:50 +0800 Subject: [PATCH 21/45] fix: ts in subquery --- source/libs/parser/src/parTranslater.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3b97097a1a..c164f08ca6 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1584,9 +1584,6 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, pFoundExpr = pExpr; *pFound = true; } else if (isPrimaryKeyImpl(pNode) && isInternalPrimaryKey(pCol)) { - if (*pFound) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); - } pFoundExpr = pExpr; pCol->isPrimTs = true; *pFound = true; @@ -7628,7 +7625,7 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList } static int32_t translateColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable)) { + if (pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable)) { SNode* pSubquery = ((STempTableNode*)pSelect->pFromTable)->pSubquery; if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { SSelectStmt* pSubSelect = (SSelectStmt*)pSubquery; From 6ba3edfd0b21a517f3580a3a412981fe04060ce5 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 6 Feb 2025 09:39:39 +0800 Subject: [PATCH 22/45] test case --- source/libs/parser/src/parTranslater.c | 5 +-- tests/system-test/2-query/cols_function.py | 48 +++++++++++++++++++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c164f08ca6..2842ea2d42 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7375,9 +7375,8 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { ((SExprNode*)*pNode)->bindTupleFuncIdx = pCxt->bindTupleFuncIdx; int32_t len = strlen(((SExprNode*)*pNode)->aliasName); if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { - parserWarn("%s The alias name is too long, the extra part will be truncated", __func__); - tsnprintf(((SExprNode*)*pNode)->aliasName + TSDB_COL_NAME_EXLEN - TSDB_COL_NAME_EXLEN, TSDB_COL_NAME_EXLEN, ".%d", - pCxt->bindTupleFuncIdx); + parserError("%s The alias name is too long, the extra part will be truncated", __func__); + return DEAL_RES_ERROR; } else { tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pCxt->bindTupleFuncIdx); } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 9bccee1f73..085c95dd9a 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -890,8 +890,53 @@ class TDTestCase: def window_test(self): tdSql.query(f'select tbname, _wstart,_wend, max(c0), max(c1), cols( max(c0), c1) from test.meters partition \ - by tbname count_window(2) order by tbname') + by tbname count_window(2) order by tbname') + tdSql.checkRows(4) + tdSql.checkCols(6) + tdSql.checkData(0, 0, 'd0') + tdSql.checkData(3, 0, 'd1') + tdSql.checkData(3, 5, 1) + tdSql.query(f'select _wstart,_wend, max(c0), max(c1), cols( max(c0), c1) from test.d1 count_window(2);') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 4, 1) + tdSql.query(f'select _wstart,_wend, max(c0), max(c1), cols( max(c0), c1) from test.normal_table count_window(2);') + tdSql.checkRows(3) + tdSql.checkCols(5) + tdSql.checkData(0, 4, 2) + + def join_test(self): + tdSql.query(f'select cols(last(a.ts), a.c0) from test.d0 a join test.d1 b on a.ts = b.ts;') + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.checkData(0, 0, 1) + tdSql.query(f'select cols(last(a.ts), a.c0) from test.d0 a join test.d1 b on a.ts = b.ts and a.c0 = b.c0;') + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.checkData(0, 0, 1) + tdSql.query(f'select cols(last(a.ts), a.c0) from test.d0 a join test.d1 b on a.ts = b.ts and a.c0 > b.c0;') + tdSql.checkRows(0) + tdSql.query(f'select tbname, ts, c0 from (select cols(last(a.ts), a.tbname, a.ts, a.c0) from test.d0 a join test.d1 b on a.ts = b.ts and a.c0=b.c0)') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'd0') + tdSql.checkData(0, 1, 1734574929000) + tdSql.checkData(0, 2, 1) + tdSql.query(f'select tbname, ts, c0 from (select cols(first(a.ts), a.tbname, a.ts, a.c0) from test.d0 a join test.d1 b on a.ts = b.ts and a.c0=b.c0)') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'd0') + tdSql.checkData(0, 1, 1734574929000) + tdSql.checkData(0, 2, 1) + tdSql.error(f'select tbname, ts, c0 from (select cols(first(a.ts), a.tbname, a.ts, a.c0), cols(last(a.ts), b.tbname, b.ts, b.c0) from test.d0 a join test.d1 b on a.ts = b.ts and a.c0=b.c0)') + tdSql.query(f'select tbname, ts, c0 from (select cols(first(a.ts), a.tbname, a.ts, a.c0), cols(last(a.ts), b.tbname tbname1, b.ts ts2, b.c0 c02) from test.d0 a join test.d1 b on a.ts = b.ts and a.c0=b.c0)') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'd0') + tdSql.checkData(0, 1, 1734574929000) + tdSql.checkData(0, 2, 1) + def run(self): self.funcNestTest() self.funcSupperTableTest() @@ -901,6 +946,7 @@ class TDTestCase: self.multi_cols_output_test() self.subquery_test() self.window_test() + self.join_test() def stop(self): tdSql.close() From 3c0121ef1ad5fe422071d520e73b5e321339db00 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 11 Feb 2025 07:38:12 +0000 Subject: [PATCH 23/45] unused code --- include/libs/function/function.h | 2 +- include/util/taoserror.h | 2 +- source/libs/executor/src/executil.c | 6 ++++-- source/libs/parser/inc/parAst.h | 1 - source/libs/parser/src/parAstCreater.c | 19 +----------------- source/libs/planner/src/planPhysiCreater.c | 23 ---------------------- 6 files changed, 7 insertions(+), 46 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 715f257688..8c299d2c8c 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -276,13 +276,13 @@ typedef struct tExprNode { int32_t num; struct SFunctionNode *pFunctNode; int32_t functionType; + int32_t bindTupleFuncIdx; } _function; struct { struct SNode *pRootNode; } _optrRoot; }; - int32_t bindTupleFuncIdx; int32_t tupleFuncIdx; } tExprNode; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 5c83aa90b2..c1c85ea060 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -911,7 +911,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_PAR_INVALID_COLS_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2687) #define TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC TAOS_DEF_ERROR_CODE(0, 0x2688) #define TSDB_CODE_INVALID_MULITI_COLS_FUNC TAOS_DEF_ERROR_CODE(0, 0x2689) -#define TSDB_CODE_INVALID_COLS_ALIAS TAOS_DEF_ERROR_CODE(0, 0x2690) +#define TSDB_CODE_INVALID_COLS_ALIAS TAOS_DEF_ERROR_CODE(0, 0x268A) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c57accfa66..95e76e8599 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1992,7 +1992,9 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; QUERY_CHECK_CODE(code, lino, _end); } - pExp->pExpr->bindTupleFuncIdx = ((SExprNode*)pNode)->bindTupleFuncIdx; + if(type == QUERY_NODE_FUNCTION) { + pExp->pExpr->_function.bindTupleFuncIdx = ((SExprNode*)pNode)->bindTupleFuncIdx; + } pExp->pExpr->tupleFuncIdx = ((SExprNode*)pNode)->tupleFuncIdx; _end: if (code != TSDB_CODE_SUCCESS) { @@ -2132,7 +2134,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu if (pValCtxArray == NULL) { pValCtx[num++] = &pCtx[i]; } else { - int32_t bindFuncIndex = pCtx[i].pExpr->pExpr->bindTupleFuncIdx; // start from index 1; + int32_t bindFuncIndex = pCtx[i].pExpr->pExpr->_function.bindTupleFuncIdx; // start from index 1; if (bindFuncIndex > 0) { // 0 is default index related to the select function bindFuncIndex -= 1; } diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index f2b1059a84..857c2e03bc 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -170,7 +170,6 @@ SNode* createInterpTimeAround(SAstCreateContext* pCxt, SNode* pTimepoint, SN SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); SNode* createAlterSingleTagColumnNode(SAstCreateContext* pCtx, SToken* token, SNode* pVal); -SNode* createColsFunctionNode(SAstCreateContext* pCxt, SNode* pFunc, SNodeList* pList); SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 478d277d83..caf409338d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -375,6 +375,7 @@ SNodeList* createColsFuncParamNodeList(SAstCreateContext* pCxt, SNode* pNode, SN pCxt->errCode = nodesListAppend(list, pFuncNode); CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListAppendList(list, pNodeList); + CHECK_PARSER_STATUS(pCxt); return list; _err: @@ -1177,24 +1178,6 @@ _err: return NULL; } -SNode* createColsFunctionNode(SAstCreateContext* pCxt, SNode* pColFunc, SNodeList* pExpr){ - SFunctionNode* func = NULL; - CHECK_PARSER_STATUS(pCxt); - pCxt->errCode = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&func); - CHECK_MAKE_NODE(func); - strcpy(func->functionName, "cols"); - pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pColFunc); - CHECK_PARSER_STATUS(pCxt); - pCxt->errCode = nodesListMakeStrictAppendList(&func->pParameterList, pExpr); - CHECK_PARSER_STATUS(pCxt); - return (SNode*)func; -_err: - nodesDestroyNode((SNode*)func); - nodesDestroyNode(pColFunc); - nodesDestroyList(pExpr); - return NULL; -} - SNode* createSubstrFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* pExpr2, SNode* pExpr3) { SFunctionNode* func = NULL; CHECK_PARSER_STATUS(pCxt); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 5902f1c3e9..33818feb39 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -31,25 +31,6 @@ typedef struct SSlotIndex { SArray* pSlotIdsInfo; // duplicate name slot } SSlotIndex; -static int64_t getExprBindIndexStr(SNode* pNode, char* bindInfo) { - if (nodeType(pNode) == QUERY_NODE_COLUMN) { - SColumnNode* pCol = (SColumnNode*)pNode; - if (pCol->dataBlockId == 0) { - return 0; - } - } - if (nodesIsExprNode(pNode)) { - SExprNode* pExpr = (SExprNode*)pNode; - if (pExpr->bindTupleFuncIdx > 0 && pExpr->bindTupleFuncIdx <= 9) { - bindInfo[0] = '0' + pExpr->bindTupleFuncIdx; - return 1; - } else if (pExpr->bindTupleFuncIdx != 0) { - return tsnprintf(bindInfo, sizeof(bindInfo), "%d", pExpr->bindTupleFuncIdx); - } - } - return 0; -} - enum { SLOT_KEY_TYPE_ALL = 1, SLOT_KEY_TYPE_COLNAME = 2, @@ -59,7 +40,6 @@ static int32_t getSlotKeyHelper(SNode* pNode, const char* pPreName, const char* int32_t* pLen, uint16_t extraBufLen, int8_t slotKeyType) { int32_t code = 0; char bindInfo[16] = {0}; - //int exBindInfoLen = getExprBindIndexStr(pNode, bindInfo); *ppKey = taosMemoryCalloc(1, callocLen); if (!*ppKey) { return terrno; @@ -68,9 +48,6 @@ static int32_t getSlotKeyHelper(SNode* pNode, const char* pPreName, const char* TAOS_STRNCAT(*ppKey, pPreName, TSDB_TABLE_NAME_LEN); TAOS_STRNCAT(*ppKey, ".", 2); TAOS_STRNCAT(*ppKey, name, TSDB_COL_NAME_LEN); - // if (exBindInfoLen > 0) { - // TAOS_STRNCAT(*ppKey, bindInfo, exBindInfoLen); - // } *pLen = taosHashBinary(*ppKey, strlen(*ppKey)); } else { TAOS_STRNCAT(*ppKey, name, TSDB_COL_NAME_LEN); From fa203ca2491b1afd09bda42e4248cda8a01ae07c Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 11 Feb 2025 09:33:34 +0000 Subject: [PATCH 24/45] cols in stream --- source/libs/parser/src/parTranslater.c | 3 +++ tests/pytest/util/tserror.py | 2 ++ tests/system-test/2-query/cols_function.py | 10 ++++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2842ea2d42..9b5ddcdb6e 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7548,6 +7548,9 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList SNodeList* pNewNodeList = NULL; if (needRewrite) { + if (pCxt->createStream) { + return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + } code = nodesMakeList(&pNewNodeList); if (NULL == pNewNodeList) { return code; diff --git a/tests/pytest/util/tserror.py b/tests/pytest/util/tserror.py index 0d88d78e80..6dca0012ef 100644 --- a/tests/pytest/util/tserror.py +++ b/tests/pytest/util/tserror.py @@ -15,3 +15,5 @@ TSDB_CODE_UDF_FUNC_EXEC_FAILURE = (TAOS_DEF_ERROR_CODE | 0x290A) TSDB_CODE_TSC_INTERNAL_ERROR = (TAOS_DEF_ERROR_CODE | 0x02FF) TSDB_CODE_PAR_SYNTAX_ERROR = (TAOS_DEF_ERROR_CODE | 0x2600) + +TSDB_CODE_PAR_INVALID_COLS_FUNCTION = (TAOS_DEF_ERROR_CODE | 0x2687) diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 085c95dd9a..7707fbccb5 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -4,6 +4,7 @@ from util.log import * from util.cases import * from util.sql import * from util.common import * +from util.tserror import * import numpy as np @@ -935,8 +936,12 @@ class TDTestCase: tdSql.checkCols(3) tdSql.checkData(0, 0, 'd0') tdSql.checkData(0, 1, 1734574929000) - tdSql.checkData(0, 2, 1) - + tdSql.checkData(0, 2, 1) + + def stream_cols_test(self): + tdSql.error(f'CREATE STREAM last_col_s1 INTO last_col1 AS SELECT cols(last(ts), ts, c0) FROM meters PARTITION BY tbname INTERVAL(1s) SLIDING(1s);', TSDB_CODE_PAR_INVALID_COLS_FUNCTION) + tdSql.query(f'CREATE STREAM last_col_s INTO last_col AS SELECT last(ts), c0 FROM meters PARTITION BY tbname INTERVAL(1s) SLIDING(1s);') + def run(self): self.funcNestTest() self.funcSupperTableTest() @@ -947,6 +952,7 @@ class TDTestCase: self.subquery_test() self.window_test() self.join_test() + self.stream_cols_test() def stop(self): tdSql.close() From 44f6859d65abb0d8426bc402f943c3cc689dcce9 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Wed, 12 Feb 2025 07:09:51 +0000 Subject: [PATCH 25/45] unused code --- source/libs/parser/src/parTranslater.c | 1 - source/libs/planner/src/planOptimizer.c | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9b5ddcdb6e..3fedf49ac9 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2500,7 +2500,6 @@ static int32_t rewriteCountTbname(STranslateContext* pCxt, SFunctionNode* pCount } static bool hasInvalidFuncNesting(SFunctionNode* pFunc) { - if(pFunc->funcType == FUNCTION_TYPE_COLS) return false; bool hasInvalidFunc = false; nodesWalkExprs(pFunc->pParameterList, haveVectorFunction, &hasInvalidFunc); return hasInvalidFunc; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 715ab04676..1ffd8ae59b 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3478,10 +3478,10 @@ static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild, nodesWalkExpr(pJoinLogicNode->pFullOnCond, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); if (!cxt.canUse) return false; } - if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pChild) && - ((SAggLogicNode*)pChild)->node.pTargets->length != pNewChildTargets->length) { - return false; - } + //if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pChild) && + // ((SAggLogicNode*)pChild)->node.pTargets->length != pNewChildTargets->length) { + // return false; + //} return true; } From e16b96db723ff5b1c4564ce271c56b8cd60dd168 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Thu, 13 Feb 2025 03:21:27 +0000 Subject: [PATCH 26/45] enh: translate cols in clauses --- docs/en/14-reference/09-error-code.md | 2 +- docs/zh/14-reference/09-error-code.md | 2 +- source/libs/parser/src/parTranslater.c | 124 +++++++++++++++++-------- 3 files changed, 85 insertions(+), 43 deletions(-) diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index c35df8ebc0..857f84da34 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -461,7 +461,7 @@ This document details the server error codes that may be encountered when using | 0x80002687 | Invalid using cols function | Illegal using cols function | Check and correct the SQL statement | | 0x80002688 | Cols function's first param must be a select function | The first parameter of the cols function should be a selection function | Check and correct the SQL statement | | 0x80002689 | Invalid using cols function with multiple output columns | Illegal using the cols function for multiple column output | Check and correct the SQL statement | -| 0x80002690 | Invalid using alias for cols function | Illegal cols function alias | Check and correct the SQL statement | +| 0x8000268A | Invalid using alias for cols function | Illegal cols function alias | Check and correct the SQL statement | | 0x800026FF | Parser internal error | Internal error in parser | Preserve the scene and logs, report issue on GitHub | | 0x80002700 | Planner internal error | Internal error in planner | Preserve the scene and logs, report issue on GitHub | | 0x80002701 | Expect ts equal | JOIN condition validation failed | Preserve the scene and logs, report issue on GitHub | diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index a1a3fa050d..74fe39421d 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -478,7 +478,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80002687 | Invalid using cols function | cols函数使用错误 | 检查并修正SQL语句 | | 0x80002688 | Cols function's first param must be a select function | cols函数第一个参数应该为选择函数 | 检查并修正SQL语句 | | 0x80002689 | Invalid using cols function with multiple output columns | 多列输出的 cols 函数使用错误 | 检查并修正SQL语句 | -| 0x80002690 | Invalid using alias for cols function | cols 函数输出列重命名错误 | 检查并修正SQL语句 | +| 0x8000268A | Invalid using alias for cols function | cols 函数输出列重命名错误 | 检查并修正SQL语句 | | 0x800026FF | Parser internal error | 解析器内部错误 | 保留现场和日志，github上报issue | | 0x80002700 | Planner internal error | 计划期内部错误 | 保留现场和日志，github上报issue | | 0x80002701 | Expect ts equal | JOIN条件校验失败 | 保留现场和日志，github上报issue | diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3fedf49ac9..7ab0261458 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5460,9 +5460,83 @@ static int32_t translateClausePosition(STranslateContext* pCxt, SNodeList* pProj return TSDB_CODE_SUCCESS; } +static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList, SNodeList** selectFuncList); +static int32_t translateSelectColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { + SNodeList* selectFuncList = NULL; + int32_t code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &selectFuncList); + if (TSDB_CODE_SUCCESS != code) { + goto _end; + } + if (selectFuncList != NULL) { + code = nodesListAppendList(pSelect->pProjectionList, selectFuncList); + if (TSDB_CODE_SUCCESS != code) { + goto _end; + } + selectFuncList = NULL; + } +_end: + if (selectFuncList) { + nodesDestroyList(selectFuncList); + } + return code; +} + +static int32_t getBindedExprList(STranslateContext* pCxt, SNodeList* projectionList, SNodeList** selectFuncList) { + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < LIST_LENGTH(projectionList); ++i) { + SNode* pNode = nodesListGetNode(projectionList, i); + if (((SExprNode*)pNode)->bindTupleFuncIdx > 0) { + if (NULL == *selectFuncList) { + code = nodesMakeList(selectFuncList); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + code = nodesListStrictAppend(*selectFuncList, pNode); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + return code; +} + +static int32_t translateOrderbyColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { + SNodeList* selectFuncList = NULL; + int32_t code = getBindedExprList(pCxt, pSelect->pProjectionList, &selectFuncList); + if (TSDB_CODE_SUCCESS != code) { + goto _end; + } + int len = LIST_LENGTH(selectFuncList); + code = rewriteColsFunction(pCxt, &pSelect->pOrderByList, &selectFuncList); + if (TSDB_CODE_SUCCESS != code) { + goto _end; + } + if(LIST_LENGTH(selectFuncList) - len > 0) { + for (int i = len; i < LIST_LENGTH(selectFuncList); ++i) { + SNode* pNode = nodesListGetNode(selectFuncList, i); + int32_t code = nodesListStrictAppend(pSelect->pProjectionList, pNode); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + nodesClearList(selectFuncList); + return code; +_end: + if (selectFuncList) { + nodesDestroyList(selectFuncList); + } + return code; +} + static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { bool other; - int32_t code = translateClausePosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other); + int32_t code = translateOrderbyColsFunction(pCxt, pSelect); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = translateClausePosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other); if (TSDB_CODE_SUCCESS == code) { if (0 == LIST_LENGTH(pSelect->pOrderByList)) { NODES_DESTORY_LIST(pSelect->pOrderByList); @@ -5681,7 +5755,11 @@ static int32_t translatePartitionByList(STranslateContext* pCxt, SSelectStmt* pS static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_SELECT; - int32_t code = translateExprList(pCxt, pSelect->pProjectionList); + int32_t code = translateSelectColsFunction(pCxt, pSelect); + code = translateOrderbyColsFunction(pCxt, pSelect); + if (TSDB_CODE_SUCCESS == code) { + code = translateExprList(pCxt, pSelect->pProjectionList); + } if (TSDB_CODE_SUCCESS == code) { code = translateStar(pCxt, pSelect); } @@ -7432,7 +7510,7 @@ static int32_t checkMultColsFuncParam(SNodeList* pParameterList) { SNode* pNode = NULL; FOREACH(pNode, pParameterList) { if (index == 0) { // the first parameter is select function - if (QUERY_NODE_FUNCTION != nodeType(pNode)) { + if (QUERY_NODE_FUNCTION != nodeType(pNode) || isColsFuncByName((SFunctionNode*)pNode)) { return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; } SFunctionNode* pFunc = (SFunctionNode*)pNode; @@ -7474,7 +7552,7 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { } SNode* pSelectFunc = nodesListGetNode(pFunc->pParameterList, 0); SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 1); - if (nodeType(pSelectFunc) != QUERY_NODE_FUNCTION) { + if (nodeType(pSelectFunc) != QUERY_NODE_FUNCTION || isColsFuncByName((SFunctionNode*)pSelectFunc)) { pCxt->status = TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC; parserError("%s Invalid cols function, the first parameter must be a select function", __func__); return DEAL_RES_ERROR; @@ -7625,46 +7703,10 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList return code; } -static int32_t translateColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable)) { - SNode* pSubquery = ((STempTableNode*)pSelect->pFromTable)->pSubquery; - if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { - SSelectStmt* pSubSelect = (SSelectStmt*)pSubquery; - int32_t code = translateColsFunction(pCxt, pSubSelect); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - } - } - SNodeList* selectFuncList = NULL; - int32_t code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &selectFuncList); - if (code == TSDB_CODE_SUCCESS) { - code = rewriteColsFunction(pCxt, &pSelect->pOrderByList, &selectFuncList); - } - if (TSDB_CODE_SUCCESS != code) { - goto _end; - } - - if (selectFuncList != NULL) { - nodesListAppendList(pSelect->pProjectionList, selectFuncList); - selectFuncList = NULL; - } - -_end: - if (selectFuncList) { - nodesDestroyList(selectFuncList); - } - - return code; -} - static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = (SNode*)pSelect; pCxt->dual = false; - int32_t code = translateColsFunction(pCxt, pSelect); - if (TSDB_CODE_SUCCESS == code) { - code = translateFrom(pCxt, &pSelect->pFromTable); - } + int32_t code = translateFrom(pCxt, &pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision; code = translateWhere(pCxt, pSelect); From a391cc49df3f1951842bdb15fc94c4739e13fa6b Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Thu, 13 Feb 2025 11:08:58 +0000 Subject: [PATCH 27/45] cols in order clause --- source/libs/nodes/src/nodesEqualFuncs.c | 1 + source/libs/parser/src/parTranslater.c | 61 +++------------------- tests/system-test/2-query/cols_function.py | 17 ++++++ 3 files changed, 24 insertions(+), 55 deletions(-) diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index d255c2fd6b..e1c17f5c35 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -141,6 +141,7 @@ static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { if (a->funcType == FUNCTION_TYPE_SELECT_VALUE) { if ((a->node.bindTupleFuncIdx != b->node.bindTupleFuncIdx)) return false; } else { + // select cols(cols(first(c0), ts), first(c0) from meters; if ((a->node.tupleFuncIdx != b->node.tupleFuncIdx)) { return false; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7ab0261458..d890c8ed04 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5461,12 +5461,17 @@ static int32_t translateClausePosition(STranslateContext* pCxt, SNodeList* pProj } static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList, SNodeList** selectFuncList); +static int32_t preGetBindCols(STranslateContext* pCxt, SSelectStmt* pSelect, SNodeList** selectFuncList) { + return rewriteColsFunction(pCxt, &pSelect->pOrderByList, selectFuncList); +} + static int32_t translateSelectColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { SNodeList* selectFuncList = NULL; int32_t code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &selectFuncList); if (TSDB_CODE_SUCCESS != code) { goto _end; } + code = preGetBindCols(pCxt, pSelect, &selectFuncList); if (selectFuncList != NULL) { code = nodesListAppendList(pSelect->pProjectionList, selectFuncList); if (TSDB_CODE_SUCCESS != code) { @@ -5481,62 +5486,9 @@ _end: return code; } -static int32_t getBindedExprList(STranslateContext* pCxt, SNodeList* projectionList, SNodeList** selectFuncList) { - int32_t code = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < LIST_LENGTH(projectionList); ++i) { - SNode* pNode = nodesListGetNode(projectionList, i); - if (((SExprNode*)pNode)->bindTupleFuncIdx > 0) { - if (NULL == *selectFuncList) { - code = nodesMakeList(selectFuncList); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - } - code = nodesListStrictAppend(*selectFuncList, pNode); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - } - } - return code; -} - -static int32_t translateOrderbyColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { - SNodeList* selectFuncList = NULL; - int32_t code = getBindedExprList(pCxt, pSelect->pProjectionList, &selectFuncList); - if (TSDB_CODE_SUCCESS != code) { - goto _end; - } - int len = LIST_LENGTH(selectFuncList); - code = rewriteColsFunction(pCxt, &pSelect->pOrderByList, &selectFuncList); - if (TSDB_CODE_SUCCESS != code) { - goto _end; - } - if(LIST_LENGTH(selectFuncList) - len > 0) { - for (int i = len; i < LIST_LENGTH(selectFuncList); ++i) { - SNode* pNode = nodesListGetNode(selectFuncList, i); - int32_t code = nodesListStrictAppend(pSelect->pProjectionList, pNode); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - } - } - nodesClearList(selectFuncList); - return code; -_end: - if (selectFuncList) { - nodesDestroyList(selectFuncList); - } - return code; -} - static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { bool other; - int32_t code = translateOrderbyColsFunction(pCxt, pSelect); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - code = translateClausePosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other); + int32_t code = translateClausePosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other); if (TSDB_CODE_SUCCESS == code) { if (0 == LIST_LENGTH(pSelect->pOrderByList)) { NODES_DESTORY_LIST(pSelect->pOrderByList); @@ -5756,7 +5708,6 @@ static int32_t translatePartitionByList(STranslateContext* pCxt, SSelectStmt* pS static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_SELECT; int32_t code = translateSelectColsFunction(pCxt, pSelect); - code = translateOrderbyColsFunction(pCxt, pSelect); if (TSDB_CODE_SUCCESS == code) { code = translateExprList(pCxt, pSelect->pProjectionList); } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 7707fbccb5..7f128e28d0 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -941,6 +941,22 @@ class TDTestCase: def stream_cols_test(self): tdSql.error(f'CREATE STREAM last_col_s1 INTO last_col1 AS SELECT cols(last(ts), ts, c0) FROM meters PARTITION BY tbname INTERVAL(1s) SLIDING(1s);', TSDB_CODE_PAR_INVALID_COLS_FUNCTION) tdSql.query(f'CREATE STREAM last_col_s INTO last_col AS SELECT last(ts), c0 FROM meters PARTITION BY tbname INTERVAL(1s) SLIDING(1s);') + + def include_null_test(self): + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929010, 0, NULL, NULL, NULL)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929011, NULL, 1, NULL, NULL)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929012, NULL, NULL, 2, NULL)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929013, NULL, NULL, NULL, false)') + tdSql.execute(f'insert into {self.dbname}.d0 values(1734574929014, NULL, NULL, NULL, NULL)') + + tdSql.query(f'select cols(last(c0), ts), cols(last(c1), ts), cols(last(c2), ts), cols(last(c3), ts) from {self.dbname}.d0') + tdSql.checkRows(1) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 1734574929010) + tdSql.checkData(0, 1, 1734574929011) + tdSql.checkData(0, 2, 1734574929012) + tdSql.checkData(0, 3, 1734574929013) + def run(self): self.funcNestTest() @@ -953,6 +969,7 @@ class TDTestCase: self.window_test() self.join_test() self.stream_cols_test() + self.include_null_test() def stop(self): tdSql.close() From 26fb867d4c0f38b734d91eb8d784df92ae7e6a17 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Fri, 14 Feb 2025 18:38:25 +0800 Subject: [PATCH 28/45] cols test --- include/libs/nodes/querynodes.h | 1 + source/libs/nodes/src/nodesTraverseFuncs.c | 2 + source/libs/nodes/src/nodesUtilFuncs.c | 4 +- source/libs/parser/src/parTranslater.c | 66 ++++++++-------------- source/libs/planner/src/planLogicCreater.c | 7 +-- source/libs/planner/src/planOptimizer.c | 4 -- tests/system-test/2-query/cols_function.py | 2 +- 7 files changed, 33 insertions(+), 53 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 653b5ff6cd..9a118bdeab 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -431,6 +431,7 @@ typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; SNodeList* pProjectionList; + SNodeList* pProjectionBindList; SNode* pFromTable; SNode* pWhere; SNodeList* pPartitionByList; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index f3f7395a37..7b6dfe8aa3 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -475,6 +475,7 @@ void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalke nodesWalkExprs(pSelect->pOrderByList, walker, pContext); case SQL_CLAUSE_ORDER_BY: nodesWalkExprs(pSelect->pProjectionList, walker, pContext); + nodesWalkExprs(pSelect->pProjectionBindList, walker, pContext); default: break; } @@ -515,6 +516,7 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext); case SQL_CLAUSE_ORDER_BY: nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext); + nodesRewriteExprs(pSelect->pProjectionBindList, rewriter, pContext); default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index dd4d1c694b..e0dfe69eb7 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -15,6 +15,7 @@ #include "cmdnodes.h" #include "functionMgt.h" +#include "nodes.h" #include "nodesUtil.h" #include "plannodes.h" #include "querynodes.h" @@ -1287,6 +1288,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_SELECT_STMT: { SSelectStmt* pStmt = (SSelectStmt*)pNode; nodesDestroyList(pStmt->pProjectionList); + nodesDestroyList(pStmt->pProjectionBindList); nodesDestroyNode(pStmt->pFromTable); nodesDestroyNode(pStmt->pWhere); nodesDestroyList(pStmt->pPartitionByList); @@ -3239,5 +3241,5 @@ int32_t nodesListDeduplicate(SNodeList** ppList) { } int32_t rewriteExprAliasName(SExprNode* pNode, int64_t num) { - return tsnprintf(pNode->aliasName, TSDB_COL_NAME_LEN, "expr_%d", num); + return tsnprintf(pNode->aliasName, TSDB_COL_NAME_LEN, "expr_%ld", num); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d890c8ed04..342b2dba79 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5461,34 +5461,31 @@ static int32_t translateClausePosition(STranslateContext* pCxt, SNodeList* pProj } static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList, SNodeList** selectFuncList); -static int32_t preGetBindCols(STranslateContext* pCxt, SSelectStmt* pSelect, SNodeList** selectFuncList) { - return rewriteColsFunction(pCxt, &pSelect->pOrderByList, selectFuncList); -} -static int32_t translateSelectColsFunction(STranslateContext* pCxt, SSelectStmt* pSelect) { - SNodeList* selectFuncList = NULL; - int32_t code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &selectFuncList); - if (TSDB_CODE_SUCCESS != code) { - goto _end; - } - code = preGetBindCols(pCxt, pSelect, &selectFuncList); - if (selectFuncList != NULL) { - code = nodesListAppendList(pSelect->pProjectionList, selectFuncList); - if (TSDB_CODE_SUCCESS != code) { - goto _end; +static int32_t prepareColumnExpansion(STranslateContext* pCxt, ESqlClause clause, SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; + if (clause == SQL_CLAUSE_SELECT) { + code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &pSelect->pProjectionBindList); + code = translateExprList(pCxt, pSelect->pProjectionBindList); + } else if (clause == SQL_CLAUSE_ORDER_BY) { + code = rewriteColsFunction(pCxt, &pSelect->pOrderByList, &pSelect->pProjectionBindList); + if (TSDB_CODE_SUCCESS == code) { + code = translateExprList(pCxt, pSelect->pProjectionBindList); } - selectFuncList = NULL; - } -_end: - if (selectFuncList) { - nodesDestroyList(selectFuncList); + } else { + code = + generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Invalid clause for column expansion"); } return code; } static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { bool other; - int32_t code = translateClausePosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other); + int32_t code = prepareColumnExpansion(pCxt, SQL_CLAUSE_ORDER_BY, pSelect); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = translateClausePosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other); if (TSDB_CODE_SUCCESS == code) { if (0 == LIST_LENGTH(pSelect->pOrderByList)) { NODES_DESTORY_LIST(pSelect->pOrderByList); @@ -5618,14 +5615,14 @@ static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjection } static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSelect) { + SNode* pNode; + int32_t projIdx = 1; + FOREACH(pNode, pSelect->pProjectionList) { ((SExprNode*)pNode)->projIdx = projIdx++; } + if (!pSelect->isSubquery) { return rewriteProjectAlias(pSelect->pProjectionList); - } else { - SNode* pNode; - int32_t projIdx = 1; - FOREACH(pNode, pSelect->pProjectionList) { ((SExprNode*)pNode)->projIdx = projIdx++; } - return TSDB_CODE_SUCCESS; } + return TSDB_CODE_SUCCESS; } typedef struct SReplaceGroupByAliasCxt { @@ -5707,7 +5704,7 @@ static int32_t translatePartitionByList(STranslateContext* pCxt, SSelectStmt* pS static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_SELECT; - int32_t code = translateSelectColsFunction(pCxt, pSelect); + int32_t code = prepareColumnExpansion(pCxt, SQL_CLAUSE_SELECT, pSelect); if (TSDB_CODE_SUCCESS == code) { code = translateExprList(pCxt, pSelect->pProjectionList); } @@ -7413,23 +7410,6 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool invalidColsAlias(SFunctionNode* pFunc) { - if (pFunc->node.asAlias) { - if (pFunc->pParameterList->length > 2) { - return true; - } else { - SNode* pNode; - FOREACH(pNode, pFunc->pParameterList) { - SExprNode* pExpr = (SExprNode*)pNode; - if (pExpr->asAlias) { - return true; - } - } - } - } - return false; -} - static int32_t getSelectFuncIndex(SNodeList* FuncNodeList, SNode* pSelectFunc) { SNode* pNode = NULL; int32_t selectFuncIndex = 0; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b24ef5f12b..cd0355008a 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -713,7 +713,6 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName); } pCol->node.bindTupleFuncIdx = pExpr->bindTupleFuncIdx; - pCol->node.tupleFuncIdx = pExpr->tupleFuncIdx; return pCol; } @@ -819,7 +818,7 @@ static int32_t addWinJoinPrimKeyToAggFuncs(SSelectStmt* pSelect, SNodeList** pLi } static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { - if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { + if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList && !pSelect->pProjectionBindList) { return TSDB_CODE_SUCCESS; } @@ -1599,10 +1598,10 @@ static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pS SNode* pNode; int32_t projIdx = 1; FOREACH(pNode, pExprs) { - SColumnNode* pCol = createColumnByExpr(pStmtName, (SExprNode*)pNode); - if (pCol->node.tupleFuncIdx != 0) { + if (((SExprNode*)pNode)->tupleFuncIdx != 0) { continue; } + SColumnNode* pCol = createColumnByExpr(pStmtName, (SExprNode*)pNode); if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)pCol))) { nodesDestroyList(pList); return code; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 1ffd8ae59b..8dec06137c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3478,10 +3478,6 @@ static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild, nodesWalkExpr(pJoinLogicNode->pFullOnCond, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); if (!cxt.canUse) return false; } - //if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pChild) && - // ((SAggLogicNode*)pChild)->node.pTargets->length != pNewChildTargets->length) { - // return false; - //} return true; } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 7f128e28d0..a418e1f8a5 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -966,7 +966,7 @@ class TDTestCase: self.one_cols_1output_test() self.multi_cols_output_test() self.subquery_test() - self.window_test() + #self.window_test() self.join_test() self.stream_cols_test() self.include_null_test() From c4cc51b583162bbca8ce4e9aa80c2f5f3acb9a92 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Sun, 16 Feb 2025 01:15:12 +0800 Subject: [PATCH 29/45] cols: use aggfunc --- include/libs/function/function.h | 4 +-- include/libs/nodes/querynodes.h | 5 +-- source/libs/executor/src/aggregateoperator.c | 2 +- source/libs/executor/src/executil.c | 10 +++--- source/libs/function/src/functionMgt.c | 4 +-- source/libs/nodes/src/nodesCloneFuncs.c | 4 +-- source/libs/nodes/src/nodesEqualFuncs.c | 4 +-- source/libs/nodes/src/nodesMsgFuncs.c | 16 ++++----- source/libs/nodes/src/nodesUtilFuncs.c | 4 +++ source/libs/parser/src/parTranslater.c | 38 ++++++++++---------- source/libs/planner/src/planLogicCreater.c | 11 +++--- source/libs/planner/src/planOptimizer.c | 3 +- source/libs/planner/src/planUtil.c | 3 +- tests/system-test/2-query/cols_function.py | 9 ++++- 14 files changed, 61 insertions(+), 56 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 8c299d2c8c..b2bde42463 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -276,14 +276,14 @@ typedef struct tExprNode { int32_t num; struct SFunctionNode *pFunctNode; int32_t functionType; - int32_t bindTupleFuncIdx; + int32_t bindExprID; } _function; struct { struct SNode *pRootNode; } _optrRoot; }; - int32_t tupleFuncIdx; + int32_t relatedTo; } tExprNode; struct SScalarParam { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 9a118bdeab..5fae336d22 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -62,8 +62,8 @@ typedef struct SExprNode { bool asParam; bool asPosition; int32_t projIdx; - int32_t bindTupleFuncIdx; - int32_t tupleFuncIdx; + int32_t relatedTo; + int32_t bindExprID; } SExprNode; typedef enum EColumnType { @@ -701,6 +701,7 @@ char* getFullJoinTypeString(EJoinType type, EJoinSubType stype); int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc); int32_t rewriteExprAliasName(SExprNode* pNode, int64_t num); +bool isRelatedToOtherExpr(SExprNode* pExpr); #ifdef __cplusplus } diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index bec8bf7f6f..bd4eb2e61d 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -397,7 +397,7 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc // if the last expression is a tuple function, we don't need to create a empty data block int32_t lastExprIndex = pOperator->exprSupp.numOfExprs - 1; - if(pOperator->exprSupp.pExprInfo[lastExprIndex].pExpr->tupleFuncIdx > 0) { + if(pOperator->exprSupp.pExprInfo[lastExprIndex].pExpr->_function.bindExprID > 0) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 95e76e8599..c2baf75881 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1992,10 +1992,10 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; QUERY_CHECK_CODE(code, lino, _end); } - if(type == QUERY_NODE_FUNCTION) { - pExp->pExpr->_function.bindTupleFuncIdx = ((SExprNode*)pNode)->bindTupleFuncIdx; + if (type == QUERY_NODE_FUNCTION) { + pExp->pExpr->_function.bindExprID = ((SExprNode*)pNode)->bindExprID; } - pExp->pExpr->tupleFuncIdx = ((SExprNode*)pNode)->tupleFuncIdx; + pExp->pExpr->relatedTo = ((SExprNode*)pNode)->relatedTo; _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); @@ -2091,7 +2091,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu SArray* pValCtxArray = NULL; for (int32_t i = numOfOutput - 1; i > 0; --i) { // select Func is at the end of the list - int32_t funcIdx = pCtx[i].pExpr->pExpr->tupleFuncIdx; + int32_t funcIdx = pCtx[i].pExpr->pExpr->_function.bindExprID; if (funcIdx > 0) { if (pValCtxArray == NULL) { // the end of the list is the select function of biggest index @@ -2134,7 +2134,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu if (pValCtxArray == NULL) { pValCtx[num++] = &pCtx[i]; } else { - int32_t bindFuncIndex = pCtx[i].pExpr->pExpr->_function.bindTupleFuncIdx; // start from index 1; + int32_t bindFuncIndex = pCtx[i].pExpr->pExpr->relatedTo; // start from index 1; if (bindFuncIndex > 0) { // 0 is default index related to the select function bindFuncIndex -= 1; } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index a88bee48fa..03935bf1a0 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -443,8 +443,8 @@ int32_t createFunctionWithSrcFunc(const char* pName, const SFunctionNode* pSrcFu return code; } resetOutputChangedFunc(*ppFunc, pSrcFunc); - (*ppFunc)->node.bindTupleFuncIdx = pSrcFunc->node.bindTupleFuncIdx; - (*ppFunc)->node.tupleFuncIdx = pSrcFunc->node.tupleFuncIdx; + (*ppFunc)->node.relatedTo = pSrcFunc->node.relatedTo; + (*ppFunc)->node.bindExprID = pSrcFunc->node.bindExprID; return code; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 11c9b322d2..c27f4a4859 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -106,8 +106,8 @@ static int32_t exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { COPY_SCALAR_FIELD(asParam); COPY_SCALAR_FIELD(asPosition); COPY_SCALAR_FIELD(projIdx); - COPY_SCALAR_FIELD(bindTupleFuncIdx); - COPY_SCALAR_FIELD(tupleFuncIdx); + COPY_SCALAR_FIELD(relatedTo); + COPY_SCALAR_FIELD(bindExprID); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index e1c17f5c35..fa15c9f2fa 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -139,10 +139,10 @@ static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_STRING_FIELD(functionName); COMPARE_NODE_LIST_FIELD(pParameterList); if (a->funcType == FUNCTION_TYPE_SELECT_VALUE) { - if ((a->node.bindTupleFuncIdx != b->node.bindTupleFuncIdx)) return false; + if ((a->node.relatedTo != b->node.relatedTo)) return false; } else { // select cols(cols(first(c0), ts), first(c0) from meters; - if ((a->node.tupleFuncIdx != b->node.tupleFuncIdx)) { + if ((a->node.bindExprID != b->node.bindExprID)) { return false; } } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 5bd773796f..d6348f4184 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -670,10 +670,10 @@ static int32_t exprNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { const SExprNode* pNode = (const SExprNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, EXPR_CODE_RES_TYPE, dataTypeToMsg, &pNode->resType); if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI32(pEncoder, EXPR_CODE_BIND_TUPLE_FUNC_IDX, pNode->bindTupleFuncIdx); + code = tlvEncodeI32(pEncoder, EXPR_CODE_BIND_TUPLE_FUNC_IDX, pNode->relatedTo); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI32(pEncoder, EXPR_CODE_TUPLE_FUNC_IDX, pNode->tupleFuncIdx); + code = tlvEncodeI32(pEncoder, EXPR_CODE_TUPLE_FUNC_IDX, pNode->bindExprID); } return code; } @@ -689,10 +689,10 @@ static int32_t msgToExprNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeObjFromTlv(pTlv, msgToDataType, &pNode->resType); break; case EXPR_CODE_BIND_TUPLE_FUNC_IDX: - code = tlvDecodeI32(pTlv, &pNode->bindTupleFuncIdx); + code = tlvDecodeI32(pTlv, &pNode->relatedTo); break; case EXPR_CODE_TUPLE_FUNC_IDX: - code = tlvDecodeI32(pTlv, &pNode->tupleFuncIdx); + code = tlvDecodeI32(pTlv, &pNode->bindExprID); break; default: break; @@ -709,10 +709,10 @@ static int32_t columnNodeInlineToMsg(const void* pObj, STlvEncoder* pEncoder) { int32_t code = dataTypeInlineToMsg(&pNode->node.resType, pEncoder); if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeValueI32(pEncoder, pNode->node.bindTupleFuncIdx); + code = tlvEncodeValueI32(pEncoder, pNode->node.relatedTo); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeValueI32(pEncoder, pNode->node.tupleFuncIdx); + code = tlvEncodeValueI32(pEncoder, pNode->node.bindExprID); } if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueU64(pEncoder, pNode->tableId); @@ -765,10 +765,10 @@ static int32_t msgToColumnNodeInline(STlvDecoder* pDecoder, void* pObj) { int32_t code = msgToDataTypeInline(pDecoder, &pNode->node.resType); if (TSDB_CODE_SUCCESS == code) { - code = tlvDecodeValueI32(pDecoder, &pNode->node.bindTupleFuncIdx); + code = tlvDecodeValueI32(pDecoder, &pNode->node.relatedTo); } if (TSDB_CODE_SUCCESS == code) { - code = tlvDecodeValueI32(pDecoder, &pNode->node.tupleFuncIdx); + code = tlvDecodeValueI32(pDecoder, &pNode->node.bindExprID); } if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueU64(pDecoder, &pNode->tableId); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index e0dfe69eb7..b96a1d3a12 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -3243,3 +3243,7 @@ int32_t nodesListDeduplicate(SNodeList** ppList) { int32_t rewriteExprAliasName(SExprNode* pNode, int64_t num) { return tsnprintf(pNode->aliasName, TSDB_COL_NAME_LEN, "expr_%ld", num); } + +bool isRelatedToOtherExpr(SExprNode* pExpr) { + return pExpr->relatedTo != 0; +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 342b2dba79..b0be691240 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1105,11 +1105,11 @@ static bool isForecastPseudoColumnFunc(const SNode* pNode) { } static bool isColsFunctionResult(const SNode* pNode) { - return ((nodesIsExprNode(pNode)) && ((SExprNode*)pNode)->bindTupleFuncIdx > 0); + return ((nodesIsExprNode(pNode)) && ((SExprNode*)pNode)->relatedTo > 0); } static bool isInvalidColsBindFunction(const SFunctionNode* pFunc) { - return (!fmIsSelectFunc(pFunc->funcId) && pFunc->node.tupleFuncIdx != 0); + return (!fmIsSelectFunc(pFunc->funcId) && pFunc->node.bindExprID != 0); } #ifdef BUILD_NO_CALL @@ -3582,8 +3582,8 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode tstrncpy(pFunc->functionName, "_select_value", TSDB_FUNC_NAME_LEN); tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); - pFunc->node.bindTupleFuncIdx = ((SExprNode*)*pNode)->bindTupleFuncIdx; - pFunc->node.tupleFuncIdx = ((SExprNode*)*pNode)->tupleFuncIdx; + pFunc->node.relatedTo = ((SExprNode*)*pNode)->relatedTo; + pFunc->node.bindExprID = ((SExprNode*)*pNode)->bindExprID; pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = getFuncInfo(pCxt, pFunc); @@ -3894,7 +3894,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if ((pSelect->selectFuncNum > 1 || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) && - ((SExprNode*)*pNode)->bindTupleFuncIdx == 0) { + !isRelatedToOtherExpr((SExprNode*)*pNode)) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } if (isWindowJoinStmt(pSelect) && @@ -3903,7 +3903,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } - if ((pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) && ((SExprNode*)*pNode)->bindTupleFuncIdx == 0) { + if ((pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) && !isRelatedToOtherExpr((SExprNode*)*pNode)) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } @@ -3983,7 +3983,7 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { return rewriteExprToSelectTagFunc(pCxt->pTranslateCxt, pNode); } if ((isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) && - ((!nodesIsExprNode(*pNode) || ((SExprNode*)*pNode)->bindTupleFuncIdx == 0))) { + ((!nodesIsExprNode(*pNode) || !isRelatedToOtherExpr((SExprNode*)*pNode)))) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -5466,16 +5466,18 @@ static int32_t prepareColumnExpansion(STranslateContext* pCxt, ESqlClause clause int32_t code = TSDB_CODE_SUCCESS; if (clause == SQL_CLAUSE_SELECT) { code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &pSelect->pProjectionBindList); - code = translateExprList(pCxt, pSelect->pProjectionBindList); } else if (clause == SQL_CLAUSE_ORDER_BY) { code = rewriteColsFunction(pCxt, &pSelect->pOrderByList, &pSelect->pProjectionBindList); - if (TSDB_CODE_SUCCESS == code) { - code = translateExprList(pCxt, pSelect->pProjectionBindList); - } } else { code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Invalid clause for column expansion"); } + if (TSDB_CODE_SUCCESS == code) { + code = translateExprList(pCxt, pSelect->pProjectionBindList); + } + if (pSelect->pProjectionBindList != NULL) { + pSelect->hasAggFuncs = true; + } return code; } @@ -7391,19 +7393,19 @@ static bool isMultiColsFuncNode(SNode* pNode) { } typedef struct SBindTupleFuncCxt { - int32_t bindTupleFuncIdx; + int32_t bindExprID; } SBindTupleFuncCxt; static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { SBindTupleFuncCxt* pCxt = pContext; if (nodesIsExprNode(*pNode)) { - ((SExprNode*)*pNode)->bindTupleFuncIdx = pCxt->bindTupleFuncIdx; + ((SExprNode*)*pNode)->relatedTo = pCxt->bindExprID; int32_t len = strlen(((SExprNode*)*pNode)->aliasName); if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { parserError("%s The alias name is too long, the extra part will be truncated", __func__); return DEAL_RES_ERROR; } else { - tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pCxt->bindTupleFuncIdx); + tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pCxt->bindExprID); } SFunctionNode* pFunc = (SFunctionNode*)*pNode; } @@ -7512,7 +7514,7 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { selectFuncIndex = selectFuncCount; SNode* pNewNode = NULL; code = nodesCloneNode(pSelectFunc, &pNewNode); - ((SExprNode*)pNewNode)->tupleFuncIdx = selectFuncIndex; + ((SExprNode*)pNewNode)->bindExprID = selectFuncIndex; nodesListMakeStrictAppend(pCxt->selectFuncList, pNewNode); } @@ -7595,7 +7597,7 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList ++selectFuncCount; selectFuncIndex = selectFuncCount; code = nodesCloneNode(pSelectFunc, &pNewNode); - ((SExprNode*)pNewNode)->tupleFuncIdx = selectFuncIndex; + ((SExprNode*)pNewNode)->bindExprID = selectFuncIndex; nodesListMakeStrictAppend(selectFuncList, pNewNode); } // start from index 1, because the first parameter is select function which needn't to output. @@ -13905,10 +13907,6 @@ static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* int32_t index = 0; FOREACH(pNode, pProjections) { SExprNode* pExpr = (SExprNode*)pNode; - if(pExpr->tupleFuncIdx != 0) { - *numOfCols -= 1; - continue; - } if (TSDB_DATA_TYPE_NULL == pExpr->resType.type) { (*pSchema)[index].type = TSDB_DATA_TYPE_VARCHAR; (*pSchema)[index].bytes = VARSTR_HEADER_SIZE; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index cd0355008a..2fd8d863fb 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -123,8 +123,8 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { tstrncpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias, TSDB_COL_NAME_LEN); tstrncpy(pCol->colName, ((SExprNode*)pExpr)->aliasName, TSDB_COL_NAME_LEN); pCol->node.projIdx = ((SExprNode*)(*pNode))->projIdx; - pCol->node.bindTupleFuncIdx = ((SExprNode*)(*pNode))->bindTupleFuncIdx; - pCol->node.tupleFuncIdx = ((SExprNode*)(*pNode))->tupleFuncIdx; + pCol->node.relatedTo = ((SExprNode*)(*pNode))->relatedTo; + //pCol->node.bindExprID = ((SExprNode*)(*pNode))->bindExprID; if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } @@ -712,7 +712,7 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) if (NULL != pStmtName) { snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName); } - pCol->node.bindTupleFuncIdx = pExpr->bindTupleFuncIdx; + pCol->node.relatedTo = pExpr->relatedTo; return pCol; } @@ -818,7 +818,7 @@ static int32_t addWinJoinPrimKeyToAggFuncs(SSelectStmt* pSelect, SNodeList** pLi } static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { - if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList && !pSelect->pProjectionBindList) { + if (!pSelect->hasAggFuncs && NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; } @@ -1598,9 +1598,6 @@ static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pS SNode* pNode; int32_t projIdx = 1; FOREACH(pNode, pExprs) { - if (((SExprNode*)pNode)->tupleFuncIdx != 0) { - continue; - } SColumnNode* pCol = createColumnByExpr(pStmtName, (SExprNode*)pNode); if (TSDB_CODE_SUCCESS != (code = nodesListStrictAppend(pList, (SNode*)pCol))) { nodesDestroyList(pList); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8dec06137c..f3a80af168 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3606,8 +3606,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* } else { FOREACH(pProjection, pProjectNode->pProjections) { FOREACH(pChildTarget, pChild->pTargets) { - if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName) - && ((SColumnNode*)pProjection)->node.tupleFuncIdx == 0) { + if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { SNode* pNew = NULL; code = nodesCloneNode(pChildTarget, &pNew); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 49559719a7..4cac6ae1a8 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -79,8 +79,7 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { } } } - pCol->node.bindTupleFuncIdx = pExpr->bindTupleFuncIdx; - pCol->node.tupleFuncIdx = pExpr->tupleFuncIdx; + pCol->node.relatedTo = pExpr->relatedTo; return (TSDB_CODE_SUCCESS == nodesListStrictAppend(pCxt->pList, (SNode*)pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index a418e1f8a5..29ca667e3a 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -522,6 +522,12 @@ class TDTestCase: tdSql.checkCols(2) tdSql.checkData(0, 0, 3) tdSql.checkData(0, 1, 1) + tdSql.query(f'select * from (select cols(last_row(c0), ts as t1, c1), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4)') + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 1, 3) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) tdSql.error(f'select c1 from (select cols(last_row(c0), ts as t1, c1), cols(first(c0), ts as t2, c1), first(c0) from test.meters where c0 < 4)') # cols on system table @@ -966,11 +972,12 @@ class TDTestCase: self.one_cols_1output_test() self.multi_cols_output_test() self.subquery_test() - #self.window_test() + self.window_test() self.join_test() self.stream_cols_test() self.include_null_test() + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 9d86f486228b1ed1942650ca8c27357b0c26187b Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Mon, 17 Feb 2025 11:00:05 +0800 Subject: [PATCH 30/45] fix: select_value node name --- source/libs/parser/src/parTranslater.c | 17 +++++++++-------- source/libs/planner/src/planLogicCreater.c | 1 - 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b0be691240..a318eaa2db 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1105,7 +1105,7 @@ static bool isForecastPseudoColumnFunc(const SNode* pNode) { } static bool isColsFunctionResult(const SNode* pNode) { - return ((nodesIsExprNode(pNode)) && ((SExprNode*)pNode)->relatedTo > 0); + return ((nodesIsExprNode(pNode)) && (isRelatedToOtherExpr((SExprNode*)pNode))); } static bool isInvalidColsBindFunction(const SFunctionNode* pFunc) { @@ -3582,6 +3582,14 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode tstrncpy(pFunc->functionName, "_select_value", TSDB_FUNC_NAME_LEN); tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); + if (isRelatedToOtherExpr((SExprNode*)*pNode)) { + int len = strlen(((SExprNode*)*pNode)->aliasName); + if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { + parserError("%s The alias name is too long, the extra part will be truncated", __func__); + return DEAL_RES_ERROR; + } + tsnprintf(pFunc->node.aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", ((SExprNode*)*pNode)->relatedTo); + } pFunc->node.relatedTo = ((SExprNode*)*pNode)->relatedTo; pFunc->node.bindExprID = ((SExprNode*)*pNode)->bindExprID; pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); @@ -7400,13 +7408,6 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { SBindTupleFuncCxt* pCxt = pContext; if (nodesIsExprNode(*pNode)) { ((SExprNode*)*pNode)->relatedTo = pCxt->bindExprID; - int32_t len = strlen(((SExprNode*)*pNode)->aliasName); - if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { - parserError("%s The alias name is too long, the extra part will be truncated", __func__); - return DEAL_RES_ERROR; - } else { - tsnprintf(((SExprNode*)*pNode)->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pCxt->bindExprID); - } SFunctionNode* pFunc = (SFunctionNode*)*pNode; } return DEAL_RES_CONTINUE; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2fd8d863fb..834c894585 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -124,7 +124,6 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { tstrncpy(pCol->colName, ((SExprNode*)pExpr)->aliasName, TSDB_COL_NAME_LEN); pCol->node.projIdx = ((SExprNode*)(*pNode))->projIdx; pCol->node.relatedTo = ((SExprNode*)(*pNode))->relatedTo; - //pCol->node.bindExprID = ((SExprNode*)(*pNode))->bindExprID; if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } From 8a7b6330363d787c9d437f9a64e029d4d3202822 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 18 Feb 2025 00:20:40 +0800 Subject: [PATCH 31/45] fix: alias name length --- include/libs/nodes/querynodes.h | 4 ++-- source/libs/executor/src/aggregateoperator.c | 11 +++++++---- source/libs/parser/src/parTranslater.c | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5fae336d22..6224345a39 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -55,8 +55,8 @@ typedef struct SDataType { typedef struct SExprNode { ENodeType type; SDataType resType; - char aliasName[TSDB_COL_NAME_LEN]; - char userAlias[TSDB_COL_NAME_LEN]; + char aliasName[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; + char userAlias[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; SArray* pAssociation; bool asAlias; bool asParam; diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index bd4eb2e61d..2b9db7542f 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -15,6 +15,7 @@ #include "filter.h" #include "function.h" +#include "nodes.h" #include "os.h" #include "querynodes.h" #include "tfill.h" @@ -395,10 +396,12 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc return TSDB_CODE_SUCCESS; } - // if the last expression is a tuple function, we don't need to create a empty data block - int32_t lastExprIndex = pOperator->exprSupp.numOfExprs - 1; - if(pOperator->exprSupp.pExprInfo[lastExprIndex].pExpr->_function.bindExprID > 0) { - return TSDB_CODE_SUCCESS; + // if the expression is a bind expression, we don't need to create a empty data block + for (int i = pOperator->exprSupp.numOfExprs - 1; i >= 0; i--) { + SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[i]; + if (nodeType(pExprInfo->pExpr) == QUERY_NODE_FUNCTION && pExprInfo->pExpr->_function.bindExprID > 0) { + return TSDB_CODE_SUCCESS; + } } code = createDataBlock(&pBlock); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a318eaa2db..27688d7366 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3585,8 +3585,7 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode if (isRelatedToOtherExpr((SExprNode*)*pNode)) { int len = strlen(((SExprNode*)*pNode)->aliasName); if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { - parserError("%s The alias name is too long, the extra part will be truncated", __func__); - return DEAL_RES_ERROR; + parserWarn("%s The alias name is too long, the extra part will be truncated", __func__); } tsnprintf(pFunc->node.aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", ((SExprNode*)*pNode)->relatedTo); } From 00473a9c64388c15040b57d5878901d15f80da48 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 18 Feb 2025 13:06:13 +0800 Subject: [PATCH 32/45] fix: aliasName too long --- include/libs/nodes/querynodes.h | 4 +- source/libs/parser/src/parTranslater.c | 27 +++-- tests/script/tsim/stream/basic1.sim | 5 +- tests/system-test/2-query/cols_function.py | 123 +++++++++++++++------ 4 files changed, 114 insertions(+), 45 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 6224345a39..5fae336d22 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -55,8 +55,8 @@ typedef struct SDataType { typedef struct SExprNode { ENodeType type; SDataType resType; - char aliasName[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; - char userAlias[TSDB_COL_NAME_LEN + TSDB_COL_NAME_EXLEN]; + char aliasName[TSDB_COL_NAME_LEN]; + char userAlias[TSDB_COL_NAME_LEN]; SArray* pAssociation; bool asAlias; bool asParam; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 27688d7366..716f16f6c6 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3572,6 +3572,23 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { return TSDB_CODE_PAR_INVALID_OPTR_USAGE; } +static void rewriteAliasNameWithRelate(SFunctionNode* pFunc, SExprNode* pExpr) { + if (isRelatedToOtherExpr(pExpr)) { + int len = strlen(pExpr->aliasName); + if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { + char buffer[TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1] = {0}; + tsnprintf(buffer, sizeof(buffer), "%s.%d", pExpr->aliasName, pExpr->relatedTo); + uint64_t hashVal = MurmurHash3_64(buffer, TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1); + tsnprintf(pFunc->node.aliasName, TSDB_COL_NAME_EXLEN, "%" PRIu64, hashVal); + } else { + tstrncpy(pFunc->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN); + tsnprintf(pFunc->node.aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pExpr->relatedTo); + } + } else { + tstrncpy(pFunc->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN); + } +} + static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) { SFunctionNode* pFunc = NULL; int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); @@ -3580,15 +3597,9 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode return DEAL_RES_ERROR; } tstrncpy(pFunc->functionName, "_select_value", TSDB_FUNC_NAME_LEN); - tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); - if (isRelatedToOtherExpr((SExprNode*)*pNode)) { - int len = strlen(((SExprNode*)*pNode)->aliasName); - if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { - parserWarn("%s The alias name is too long, the extra part will be truncated", __func__); - } - tsnprintf(pFunc->node.aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", ((SExprNode*)*pNode)->relatedTo); - } + rewriteAliasNameWithRelate(pFunc, (SExprNode*)*pNode); + pFunc->node.relatedTo = ((SExprNode*)*pNode)->relatedTo; pFunc->node.bindExprID = ((SExprNode*)*pNode)->bindExprID; pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index 0d89f528d1..9d31b1e073 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -809,7 +809,6 @@ sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc i sql create table ts1 using st tags(1,1,1); sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt5 as select count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt6 as select count(*), _wstart, _wend, max(a), _wstart as ts from ts1 interval(10s) ; - run tsim/stream/checkTaskStatus.sim sql_error create stream streams7 trigger at_once into streamt7 as select _wstart, count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; @@ -833,14 +832,14 @@ if $loop_count == 10 then endi if $rows != 1 then - print =====rows=$rows + print ===== streamt5: rows=$rows goto loop170 endi sql select * from streamt6; if $rows != 1 then - print =====rows=$rows + print ===== streamt6: rows=$rows goto loop170 endi diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 29ca667e3a..1dd0fec40a 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -530,6 +530,29 @@ class TDTestCase: tdSql.checkData(0, 4, 1) tdSql.error(f'select c1 from (select cols(last_row(c0), ts as t1, c1), cols(first(c0), ts as t2, c1), first(c0) from test.meters where c0 < 4)') + tdSql.query(f'select c11, _rowts from (select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4)') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 3) + tdSql.query(f'select c11, c21, _rowts from (select cols(last_row(c0), ts as t1, c1 as c11), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4)') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 3) + tdSql.checkData(0, 1, 1) + tdSql.query(f'select c1, c21, _rowts from (select cols(last_row(c0), ts as t1, c1), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4)') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 3) + tdSql.checkData(0, 1, 1) + tdSql.query(f'select *, _rowts from (select cols(last_row(c0), ts as t1, c1), cols(first(c0), ts as t2, c1 c21), first(c0) from test.meters where c0 < 4)') + tdSql.checkRows(1) + tdSql.checkCols(6) + tdSql.checkData(0, 1, 3) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) + tdSql.error(f'select c1, _rowts from (select cols(last_row(c0), ts as t1, c1), cols(first(c0), ts as t2, c1), first(c0) from test.meters where c0 < 4)') + + # cols on system table tdSql.query(f'select cols(max(vgroup_id), uid) from information_schema.ins_tables') tdSql.checkRows(1) @@ -680,25 +703,25 @@ class TDTestCase: tdSql.checkData(0, 2, 1734574929000) tdSql.checkData(0, 3, 6) - def orderby_test(self, from_table = 'test.meters', isTmpTable = False): + def orderby_test(self, from_table = 'test.meters', col_name='c0', isTmpTable = False): select_t1 = ["", ", t1", ", t1 as tag1"] for t1 in select_t1: if t1 != "" and isTmpTable: # Not a GROUP BY expression - tdSql.error(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') - tdSql.error(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') - tdSql.error(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') - tdSql.error(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') - tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') - tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') - tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') - tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') - tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') - tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') - tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') - tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.error(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2)') + tdSql.error(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2) desc') + tdSql.error(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name})') + tdSql.error(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name}), cols(last({col_name}), c1)') + tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2)') + tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2) desc') + tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name})') + tdSql.error(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name}), cols(last({col_name}), c1)') + tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2)') + tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2) desc') + tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name})') + tdSql.error(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name}), cols(last({col_name}), c1)') continue - tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.query(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2)') tdSql.checkRows(2) tags_count = 0 if t1 == "" else 1 tdLog.debug(f'tags_count: {tags_count}') @@ -710,7 +733,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'c2') self.condition_check(t1 != "", 1, 2, 'st2') - tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.query(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2) desc') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -720,7 +743,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'bbbbbbbbb2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.query(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name})') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -730,7 +753,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'bbbbbbbbb2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by 1') + tdSql.query(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by 1') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -740,7 +763,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'bbbbbbbbb2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by 2') + tdSql.query(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by 2') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 5) @@ -750,7 +773,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'c2') self.condition_check(t1 != "", 1, 2, 'st2') - tdSql.query(f'select count(1), cols(last(c0),c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.query(f'select count(1), cols(last({col_name}),c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name}), cols(last({col_name}), c1)') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -760,7 +783,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'bbbbbbbbb2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2)') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 5) @@ -770,7 +793,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'c2') self.condition_check(t1 != "", 1, 2, 'st2') - tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2) desc') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -780,7 +803,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'bbbbbbbbb2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name})') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -790,7 +813,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'bbbbbbbbb2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.query(f'select count(1), last(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name}), cols(last({col_name}), c1)') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -800,7 +823,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'bbbbbbbbb2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2)') + tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2)') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 5) @@ -810,7 +833,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'c2') self.condition_check(t1 != "", 1, 2, 'st2') - tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c2) desc') + tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), c2) desc') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -820,7 +843,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'c2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0)') + tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name})') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -830,7 +853,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'c2') self.condition_check(t1 != "", 1, 2, 'st1') - tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last(c0), c0), cols(last(c0), c1)') + tdSql.query(f'select count(1), max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name}), cols(last({col_name}), c1)') tdSql.checkRows(2) tdSql.checkCols(2 + tags_count) tdSql.checkData(0, 0, 1) @@ -838,7 +861,15 @@ class TDTestCase: self.condition_check(t1 != "", 0, 2, 'st2') tdSql.checkData(1, 0, 5) tdSql.checkData(1, 1, 'c2') - self.condition_check(t1 != "", 1, 2, 'st1') + self.condition_check(t1 != "", 1, 2, 'st1') + + tdSql.query(f'select _rowts, max(c2) {t1} from {from_table} group by tbname order by cols(last({col_name}), {col_name}), cols(last({col_name}), c1)') + tdSql.checkRows(2) + tdSql.checkCols(2 + tags_count) + tdSql.checkData(0, 1, 'c2') + self.condition_check(t1 != "", 0, 2, 'st2') + tdSql.checkData(1, 1, 'c2') + self.condition_check(t1 != "", 1, 2, 'st1') def parse_test(self): tdLog.info("parse test") @@ -883,9 +914,9 @@ class TDTestCase: tdSql.query(f'select count(1), cols(last(c0),c0) from (select *, tbname from test.meters) group by tbname') tdLog.info("subquery_test: orderby_test from meters") - self.orderby_test("test.meters", False) + self.orderby_test("test.meters", "c0", False) tdLog.info("subquery_test: orderby_test from (select *, tbname from meters)") - self.orderby_test("(select *, tbname from test.meters)", True) + self.orderby_test("(select *, tbname from test.meters)", "c0", True) tdLog.info("subquery_test: one_cols_multi_output_with_group_test from meters") self.one_cols_multi_output_with_group_test("test.meters", False) tdLog.info("subquery_test: one_cols_multi_output_with_group_test from (select *, tbname from meters)") @@ -893,8 +924,7 @@ class TDTestCase: self.one_cols_multi_output_test("test.meters") self.one_cols_multi_output_test("(select *, tbname from test.meters)") - - + def window_test(self): tdSql.query(f'select tbname, _wstart,_wend, max(c0), max(c1), cols( max(c0), c1) from test.meters partition \ by tbname count_window(2) order by tbname') @@ -962,7 +992,35 @@ class TDTestCase: tdSql.checkData(0, 1, 1734574929011) tdSql.checkData(0, 2, 1734574929012) tdSql.checkData(0, 3, 1734574929013) + + def long_column_name_test(self): + tdSql.execute(f'use {self.dbname}') + tdSql.execute(f'create table {self.dbname}.long_col_test (ts timestamp, longcolumntestlongcolumntestlongcolumntestlongcolumntest88888888 int, c1 float, c2 nchar(30), c3 bool) tags (t1 nchar(30))') + tdSql.execute(f'create table {self.dbname}.dl0 using {self.dbname}.long_col_test tags("st1")') + tdSql.execute(f'create table {self.dbname}.dl1 using {self.dbname}.long_col_test tags("st2")') + tdSql.execute(f'insert into {self.dbname}.dl0 values(1734574929000, 1, 1, "c2", true)') + tdSql.execute(f'insert into {self.dbname}.dl0 values(1734574929001, 2, 2, "bbbbbbbbb1", false)') + tdSql.execute(f'insert into {self.dbname}.dl0 values(1734574929002, 2, 2, "bbbbbbbbb1", false)') + tdSql.execute(f'insert into {self.dbname}.dl0 values(1734574929003, 3, 3, "a2", true)') + tdSql.execute(f'insert into {self.dbname}.dl0 values(1734574929004, 4, 4, "bbbbbbbbb2", false)') + tdSql.execute(f'insert into {self.dbname}.dl1 values(1734574929000, 1, 1, "c2", true)') + + tdSql.query(f'select cols(last(longcolumntestlongcolumntestlongcolumntestlongcolumntest88888888),ts lts, c1 lc1), cols(first(longcolumntestlongcolumntestlongcolumntestlongcolumntest88888888), ts fts, c1 as fc1), count(1) from test.long_col_test') + tdSql.checkResColNameList(['lts', 'lc1', 'fts', 'fc1', 'count(1)']) + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, 1734574929004) + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 6) + + tdLog.info("long_column_name_test subquery_test: orderby_test from meters") + self.orderby_test("test.long_col_test", "longcolumntestlongcolumntestlongcolumntestlongcolumntest88888888", False) + tdLog.info("long_column_name_test subquery_test: orderby_test from (select *, tbname from meters)") + self.orderby_test("(select *, tbname from test.long_col_test)", "longcolumntestlongcolumntestlongcolumntestlongcolumntest88888888", True) + tdLog.info("long_column_name_test subquery_test: one_cols_multi_output_with_group_test from meters") def run(self): self.funcNestTest() @@ -976,6 +1034,7 @@ class TDTestCase: self.join_test() self.stream_cols_test() self.include_null_test() + self.long_column_name_test() def stop(self): From bd3a36bd129b1caa60b8766dde1fab3eed67d063 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 18 Feb 2025 18:01:32 +0800 Subject: [PATCH 33/45] rewrite aliasname --- source/libs/parser/src/parTranslater.c | 42 ++++++++++++-------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 716f16f6c6..f362dba557 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1577,7 +1577,7 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, SExprNode* pFoundExpr = NULL; FOREACH(pNode, pProjectList) { SExprNode* pExpr = (SExprNode*)pNode; - if (0 == strcmp(pCol->colName, pExpr->aliasName) || 0 == strcmp(pCol->colName, pExpr->userAlias)) { + if (0 == strcmp(pCol->colName, pExpr->aliasName)) { if (*pFound) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); } @@ -3572,23 +3572,6 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { return TSDB_CODE_PAR_INVALID_OPTR_USAGE; } -static void rewriteAliasNameWithRelate(SFunctionNode* pFunc, SExprNode* pExpr) { - if (isRelatedToOtherExpr(pExpr)) { - int len = strlen(pExpr->aliasName); - if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { - char buffer[TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1] = {0}; - tsnprintf(buffer, sizeof(buffer), "%s.%d", pExpr->aliasName, pExpr->relatedTo); - uint64_t hashVal = MurmurHash3_64(buffer, TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1); - tsnprintf(pFunc->node.aliasName, TSDB_COL_NAME_EXLEN, "%" PRIu64, hashVal); - } else { - tstrncpy(pFunc->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN); - tsnprintf(pFunc->node.aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pExpr->relatedTo); - } - } else { - tstrncpy(pFunc->node.aliasName, pExpr->aliasName, TSDB_COL_NAME_LEN); - } -} - static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) { SFunctionNode* pFunc = NULL; int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); @@ -3597,8 +3580,8 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode return DEAL_RES_ERROR; } tstrncpy(pFunc->functionName, "_select_value", TSDB_FUNC_NAME_LEN); + tstrncpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName, TSDB_COL_NAME_LEN); tstrncpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias, TSDB_COL_NAME_LEN); - rewriteAliasNameWithRelate(pFunc, (SExprNode*)*pNode); pFunc->node.relatedTo = ((SExprNode*)*pNode)->relatedTo; pFunc->node.bindExprID = ((SExprNode*)*pNode)->bindExprID; @@ -7411,14 +7394,27 @@ static bool isMultiColsFuncNode(SNode* pNode) { } typedef struct SBindTupleFuncCxt { + SNode* root; int32_t bindExprID; } SBindTupleFuncCxt; static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { SBindTupleFuncCxt* pCxt = pContext; if (nodesIsExprNode(*pNode)) { - ((SExprNode*)*pNode)->relatedTo = pCxt->bindExprID; - SFunctionNode* pFunc = (SFunctionNode*)*pNode; + SExprNode* pExpr = (SExprNode*)*pNode; + pExpr->relatedTo = pCxt->bindExprID; + + if (isRelatedToOtherExpr(pExpr) && *pNode != pCxt->root) { + int len = strlen(pExpr->aliasName); + if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { + char buffer[TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1] = {0}; + tsnprintf(buffer, sizeof(buffer), "%s.%d", pExpr->aliasName, pExpr->relatedTo); + uint64_t hashVal = MurmurHash3_64(buffer, TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1); + tsnprintf(pExpr->aliasName, TSDB_COL_NAME_EXLEN, "%" PRIu64, hashVal); + } else { + tsnprintf(pExpr->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pExpr->relatedTo); + } + } } return DEAL_RES_CONTINUE; } @@ -7532,7 +7528,7 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { SNode* pNewNode = NULL; code = nodesCloneNode(pExpr, &pNewNode); if (nodesIsExprNode(pNewNode)) { - SBindTupleFuncCxt pCxt = {selectFuncIndex}; + SBindTupleFuncCxt pCxt = {pNewNode, selectFuncIndex}; nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); } else { pCxt->status = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; @@ -7617,7 +7613,7 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList code = nodesCloneNode(pExpr, &pNewNode); if (nodesIsExprNode(pNewNode)) { - SBindTupleFuncCxt pCxt = {selectFuncIndex}; + SBindTupleFuncCxt pCxt = {pNewNode, selectFuncIndex}; nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); } else { code = TSDB_CODE_PAR_INVALID_COLS_FUNCTION; From 5c646849653ba3ecd63510010b2b5b9ca1590029 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 18 Feb 2025 20:14:05 +0800 Subject: [PATCH 34/45] fix: select tbname when use multi select function --- source/libs/parser/src/parTranslater.c | 2 +- tests/system-test/2-query/cols_function.py | 75 ++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f362dba557..0df1fd17ee 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3894,7 +3894,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { - if ((pSelect->selectFuncNum > 1 || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) && + if (((pSelect->selectFuncNum > 1 && pCxt->stableQuery) || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) && !isRelatedToOtherExpr((SExprNode*)*pNode)) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 1dd0fec40a..ad008d71cf 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -1022,7 +1022,82 @@ class TDTestCase: self.orderby_test("(select *, tbname from test.long_col_test)", "longcolumntestlongcolumntestlongcolumntestlongcolumntest88888888", True) tdLog.info("long_column_name_test subquery_test: one_cols_multi_output_with_group_test from meters") + def test1(self): + dbname = "db1" + tdSql.execute(f"drop database if exists {dbname} ") + tdSql.execute(f"create database {dbname} vgroups 6") + tdSql.execute(f"use {dbname}") + + tdSql.execute(f" create stable {dbname}.sta (ts timestamp, f1 int, f2 binary(10), f3 bool) tags(t1 int, t2 bool, t3 binary(10));") + tdSql.execute(f" create table {dbname}.tba1 using sta tags(0, false, '0');") + tdSql.execute(f" create table {dbname}.tba2 using sta tags(1, true, '1');") + tdSql.execute(f" create table {dbname}.tba3 using sta tags(null, null, '');") + tdSql.execute(f" create table {dbname}.tba4 using sta tags(1, false, null);") + tdSql.execute(f" create table {dbname}.tba5 using sta tags(3, true, 'aa');") + + tdSql.execute(f"insert into {dbname}.tba1 values ('2022-09-26 15:15:01', 0, 'a', false);") + tdSql.execute(f"insert into {dbname}.tba1 values ('2022-09-26 15:15:02', 1, '0', true);") + tdSql.execute(f"insert into {dbname}.tba1 values ('2022-09-26 15:15:03', 5, '5', false);") + tdSql.execute(f"insert into {dbname}.tba1 values ('2022-09-26 15:15:04', null, null, null);") + + tdSql.execute(f"insert into {dbname}.tba2 values ('2022-09-27 15:15:01', 0, 'a', false);") + tdSql.execute(f"insert into {dbname}.tba2 values ('2022-09-27 15:15:02', 1, '0', true);") + tdSql.execute(f"insert into {dbname}.tba2 values ('2022-09-27 15:15:03', 5, '5', false);") + tdSql.execute(f"insert into {dbname}.tba2 values ('2022-09-27 15:15:04', null, null, null);") + + tdSql.execute(f"insert into {dbname}.tba3 values ('2022-09-28 15:15:01', 0, 'a', false);") + tdSql.execute(f"insert into {dbname}.tba3 values ('2022-09-28 15:15:02', 1, '0', true);") + tdSql.execute(f"insert into {dbname}.tba3 values ('2022-09-28 15:15:03', 5, '5', false);") + tdSql.execute(f"insert into {dbname}.tba3 values ('2022-09-28 15:15:04', null, null, null);") + + tdSql.execute(f"insert into {dbname}.tba4 values ('2022-09-29 15:15:01', 0, 'a', false);") + tdSql.execute(f"insert into {dbname}.tba4 values ('2022-09-29 15:15:02', 1, '0', true);") + tdSql.execute(f"insert into {dbname}.tba4 values ('2022-09-29 15:15:03', 5, '5', false);") + tdSql.execute(f"insert into {dbname}.tba4 values ('2022-09-29 15:15:04', null, null, null);") + + tdSql.execute(f"insert into {dbname}.tba5 values ('2022-09-30 15:15:01', 0, 'a', false);") + tdSql.execute(f"insert into {dbname}.tba5 values ('2022-09-30 15:15:02', 1, '0', true);") + tdSql.execute(f"insert into {dbname}.tba5 values ('2022-09-30 15:15:03', 5, '5', false);") + tdSql.execute(f"insert into {dbname}.tba5 values ('2022-09-30 15:15:04', null, null, null);") + + tdSql.query(f'select tbname, cols(last(ts), ts as ts1), cols(last(f1), ts as ts2) from tba5 interval(1s);') + tdSql.checkRows(4) + tdSql.checkCols(3) + tdSql.checkData(0, 1, "2022-09-30 15:15:01") + tdSql.checkData(0, 2, "2022-09-30 15:15:01") + tdSql.checkData(1, 1, "2022-09-30 15:15:02") + tdSql.checkData(1, 2, "2022-09-30 15:15:02") + tdSql.checkData(2, 1, "2022-09-30 15:15:03") + tdSql.checkData(2, 2, "2022-09-30 15:15:03") + tdSql.checkData(3, 1, "2022-09-30 15:15:04") + tdSql.checkData(3, 2, "2022-09-30 15:15:04") + + tdSql.query(f'select tbname, last(ts), last(ts) from tba5 interval(1s)') + tdSql.checkData(0, 1, "2022-09-30 15:15:01") + tdSql.checkData(0, 2, "2022-09-30 15:15:01") + tdSql.checkData(1, 1, "2022-09-30 15:15:02") + tdSql.checkData(1, 2, "2022-09-30 15:15:02") + tdSql.checkData(2, 1, "2022-09-30 15:15:03") + tdSql.checkData(2, 2, "2022-09-30 15:15:03") + tdSql.checkData(3, 1, "2022-09-30 15:15:04") + tdSql.checkData(3, 2, "2022-09-30 15:15:04") + + # tdSql.query(f'select tbname, cols(last(ts), ts as ts1), cols(last(f1), ts as ts2) from {dbname}.sta group by tbname;') + # tdSql.checkRows(5) + # tdSql.checkCols(3) + # tdSql.checkData(0, 1, "2022-09-30 15:15:04") + # tdSql.checkData(0, 2, "2022-09-30 15:15:03") + # tdSql.checkData(1, 1, "2022-09-30 15:15:04") + # tdSql.checkData(1, 2, "2022-09-30 15:15:03") + # tdSql.checkData(2, 1, "2022-09-30 15:15:04") + # tdSql.checkData(2, 2, "2022-09-30 15:15:03") + # tdSql.checkData(3, 1, "2022-09-30 15:15:04") + # tdSql.checkData(3, 2, "2022-09-30 15:15:03") + # tdSql.checkData(4, 1, "2022-09-30 15:15:04") + # tdSql.checkData(4, 2, "2022-09-30 15:15:03") + def run(self): + self.test1() self.funcNestTest() self.funcSupperTableTest() self.create_test_data() From a29ddccb5a84ecabdb5d018a21ff6735ef28d824 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Wed, 19 Feb 2025 01:32:02 +0800 Subject: [PATCH 35/45] fix: rewirte to groupkey func --- source/libs/parser/src/parTranslater.c | 22 +++++++---- tests/system-test/2-query/cols_function.py | 46 +++++++++++++++------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0df1fd17ee..dec7d18771 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3860,10 +3860,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { SNode* pGroupNode = NULL; FOREACH(pGroupNode, getGroupByList(pCxt)) { SNode* pActualNode = getGroupByNode(pGroupNode); - if (nodesEqualNode(pActualNode, *pNode)) { - return DEAL_RES_IGNORE_CHILD; - } - if (IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) { + if (nodesEqualNode(pActualNode, *pNode) || IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) && @@ -3893,11 +3890,22 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } } - if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { - if (((pSelect->selectFuncNum > 1 && pCxt->stableQuery) || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) && - !isRelatedToOtherExpr((SExprNode*)*pNode)) { + if (isScanPseudoColumnFunc(*pNode)) { + if (((pSelect->selectFuncNum > 1 && pCxt->stableQuery) || + (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) && + !isRelatedToOtherExpr((SExprNode*)*pNode)) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } + } + + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (((pSelect->selectFuncNum > 1) || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) && + !isRelatedToOtherExpr((SExprNode*)*pNode)) { + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); + } + } + + if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (isWindowJoinStmt(pSelect) && (isWindowJoinProbeTableCol(pSelect, *pNode) || isWindowJoinGroupCol(pSelect, *pNode) || (isWindowJoinSubTbname(pSelect, *pNode)) || isWindowJoinSubTbTag(pSelect, *pNode))) { diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index ad008d71cf..2d204e514b 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -1082,22 +1082,39 @@ class TDTestCase: tdSql.checkData(3, 1, "2022-09-30 15:15:04") tdSql.checkData(3, 2, "2022-09-30 15:15:04") - # tdSql.query(f'select tbname, cols(last(ts), ts as ts1), cols(last(f1), ts as ts2) from {dbname}.sta group by tbname;') - # tdSql.checkRows(5) - # tdSql.checkCols(3) - # tdSql.checkData(0, 1, "2022-09-30 15:15:04") - # tdSql.checkData(0, 2, "2022-09-30 15:15:03") - # tdSql.checkData(1, 1, "2022-09-30 15:15:04") - # tdSql.checkData(1, 2, "2022-09-30 15:15:03") - # tdSql.checkData(2, 1, "2022-09-30 15:15:04") - # tdSql.checkData(2, 2, "2022-09-30 15:15:03") - # tdSql.checkData(3, 1, "2022-09-30 15:15:04") - # tdSql.checkData(3, 2, "2022-09-30 15:15:03") - # tdSql.checkData(4, 1, "2022-09-30 15:15:04") - # tdSql.checkData(4, 2, "2022-09-30 15:15:03") + tdSql.query(f'select tbname, cols(last(ts), ts as ts1), cols(last(f1), ts as ts2) from {dbname}.sta group by tbname;') + tdSql.checkRows(5) + tdSql.checkCols(3) + tdSql.query(f'select tbname, cols(last(ts), ts as ts1), cols(last(f1), ts as ts2) from {dbname}.sta group by tbname order by ts1;') + tdSql.checkRows(5) + tdSql.checkCols(3) + tdSql.checkData(0, 1, "2022-09-26 15:15:04") + tdSql.checkData(0, 2, "2022-09-26 15:15:03") + tdSql.checkData(1, 1, "2022-09-27 15:15:04") + tdSql.checkData(1, 2, "2022-09-27 15:15:03") + tdSql.checkData(2, 1, "2022-09-28 15:15:04") + tdSql.checkData(2, 2, "2022-09-28 15:15:03") + tdSql.checkData(3, 1, "2022-09-29 15:15:04") + tdSql.checkData(3, 2, "2022-09-29 15:15:03") + tdSql.checkData(4, 1, "2022-09-30 15:15:04") + tdSql.checkData(4, 2, "2022-09-30 15:15:03") + + tdSql.query(f'select t1, cols(last(ts), ts as ts1), cols(last(f1), ts as ts2) from {dbname}.sta group by t1;') + tdSql.checkRows(4) + tdSql.checkCols(3) + tdSql.query(f'select t1, cols(last(ts), ts as ts1), cols(last(f1), ts as ts2) from {dbname}.sta group by t1 order by ts1;') + tdSql.checkRows(4) + tdSql.checkCols(3) + tdSql.checkData(0, 1, "2022-09-26 15:15:04") + tdSql.checkData(0, 2, "2022-09-26 15:15:03") + tdSql.checkData(1, 1, "2022-09-28 15:15:04") + tdSql.checkData(1, 2, "2022-09-28 15:15:03") + tdSql.checkData(2, 1, "2022-09-29 15:15:04") + tdSql.checkData(2, 2, "2022-09-29 15:15:03") + tdSql.checkData(3, 1, "2022-09-30 15:15:04") + tdSql.checkData(3, 2, "2022-09-30 15:15:03") def run(self): - self.test1() self.funcNestTest() self.funcSupperTableTest() self.create_test_data() @@ -1110,6 +1127,7 @@ class TDTestCase: self.stream_cols_test() self.include_null_test() self.long_column_name_test() + self.test1() def stop(self): From 01518aae39196e229647e86d82a7725a3487d120 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Wed, 19 Feb 2025 11:09:32 +0800 Subject: [PATCH 36/45] fix: result error --- source/libs/executor/src/executil.c | 2 -- source/libs/parser/src/parTranslater.c | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c2baf75881..c3d21152a8 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2113,8 +2113,6 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu } pSubsidiary->num = 0; taosArraySet(pValCtxArray, funcIdx - 1, &pSubsidiary); - } else { - break; } } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dec7d18771..6da6d84682 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3860,7 +3860,10 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { SNode* pGroupNode = NULL; FOREACH(pGroupNode, getGroupByList(pCxt)) { SNode* pActualNode = getGroupByNode(pGroupNode); - if (nodesEqualNode(pActualNode, *pNode) || IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) { + if (nodesEqualNode(pActualNode, *pNode)) { + return DEAL_RES_IGNORE_CHILD; + } + if (IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) && From 4a3343a667776590146c20d709fb23b21a6785e6 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Wed, 19 Feb 2025 13:15:25 +0800 Subject: [PATCH 37/45] enh: cols in having clause --- include/util/tdef.h | 2 +- source/libs/parser/src/parTranslater.c | 38 ++++++++++++++++--- tests/system-test/2-query/cols_function.py | 43 ++++++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 1569e9eee3..394af567c9 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -272,7 +272,7 @@ typedef enum ELogicConditionType { #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) #define TSDB_COL_NAME_LEN 65 -#define TSDB_COL_NAME_EXLEN 4 +#define TSDB_COL_NAME_EXLEN 8 #define TSDB_COL_FNAME_LEN (TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6da6d84682..83ea444912 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4111,7 +4111,7 @@ static int32_t checkWinJoinAggColCoexist(STranslateContext* pCxt, SSelectStmt* p } static int32_t checkHavingGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; if (NULL == getGroupByList(pCxt) && NULL == pSelect->pPartitionByList && NULL == pSelect->pWindow && !isWindowJoinStmt(pSelect)) { return code; @@ -5473,22 +5473,26 @@ static int32_t translateClausePosition(STranslateContext* pCxt, SNodeList* pProj } static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList, SNodeList** selectFuncList); +static int32_t rewriteHavingColsNode(STranslateContext* pCxt, SNode** pNode, SNodeList** selectFuncList); static int32_t prepareColumnExpansion(STranslateContext* pCxt, ESqlClause clause, SSelectStmt* pSelect) { int32_t code = TSDB_CODE_SUCCESS; + int32_t len = LIST_LENGTH(pSelect->pProjectionBindList); if (clause == SQL_CLAUSE_SELECT) { code = rewriteColsFunction(pCxt, &pSelect->pProjectionList, &pSelect->pProjectionBindList); + } else if (clause == SQL_CLAUSE_HAVING) { + code = rewriteHavingColsNode(pCxt, &pSelect->pHaving, &pSelect->pProjectionBindList); } else if (clause == SQL_CLAUSE_ORDER_BY) { code = rewriteColsFunction(pCxt, &pSelect->pOrderByList, &pSelect->pProjectionBindList); } else { code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Invalid clause for column expansion"); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pSelect->pProjectionBindList) > len) { code = translateExprList(pCxt, pSelect->pProjectionBindList); } if (pSelect->pProjectionBindList != NULL) { - pSelect->hasAggFuncs = true; + pSelect->hasAggFuncs = true; } return code; } @@ -5745,10 +5749,18 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect } static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; if (NULL == pSelect->pGroupByList && NULL == pSelect->pPartitionByList && NULL == pSelect->pWindow && !isWindowJoinStmt(pSelect) && NULL != pSelect->pHaving) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); } + pCxt->currClause = SQL_CLAUSE_HAVING; + if (NULL != pSelect->pHaving) { + code = prepareColumnExpansion(pCxt, SQL_CLAUSE_HAVING, pSelect); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } if (isWindowJoinStmt(pSelect)) { if (NULL != pSelect->pHaving) { bool hasFunc = false; @@ -5758,9 +5770,7 @@ static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { } } } - pCxt->currClause = SQL_CLAUSE_HAVING; - int32_t code = translateExpr(pCxt, &pSelect->pHaving); - return code; + return translateExpr(pCxt, &pSelect->pHaving); } static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -7554,6 +7564,22 @@ _end: return code; } +static int32_t rewriteHavingColsNode(STranslateContext* pCxt, SNode** pNode, SNodeList** selectFuncList) { + int32_t code = TSDB_CODE_SUCCESS; + if(!pNode || *pNode == NULL) return code; + if (isMultiColsFuncNode(*pNode)) { + parserWarn("%s Invalid using multi cols func in having.", __func__); + return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; + } else { + SCheckColsFuncCxt pSelectFuncCxt = {false, selectFuncList, TSDB_CODE_SUCCESS}; + nodesRewriteExpr(pNode, rewriteSingleColsFunc, &pSelectFuncCxt); + if (pSelectFuncCxt.status != TSDB_CODE_SUCCESS) { + return pSelectFuncCxt.status; + } + } + return code; +} + static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList, SNodeList** selectFuncList) { int32_t code = TSDB_CODE_SUCCESS; bool needRewrite = false; diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 2d204e514b..8b318cefb1 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -1113,6 +1113,47 @@ class TDTestCase: tdSql.checkData(2, 2, "2022-09-29 15:15:03") tdSql.checkData(3, 1, "2022-09-30 15:15:04") tdSql.checkData(3, 2, "2022-09-30 15:15:03") + + def having_test(self, table_name, is_subquery): + tdLog.info("having_test") + t1 = f"from {table_name} " + tdSql.query(f'select tbname, cols(last(ts), ts) {t1} group by tbname having cols(last(ts), ts) > 1734574929000') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 'd0') + tdSql.checkData(0, 1, 1734574929014) + tdSql.query(f'select tbname, cols(last(ts), ts) {t1} group by tbname having cols(last(ts), ts) = 1734574929000') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 'd1') + tdSql.checkData(0, 1, 1734574929000) + tdSql.query(f'select tbname, cols(last(ts), ts) {t1} group by tbname having cols(last(ts), ts) < 1734574929000') + tdSql.checkRows(0) + tdSql.query(f'select tbname, cols(last(ts), ts) {t1} group by tbname having cols(last(ts), ts) != 1734574929000') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 'd0') + tdSql.checkData(0, 1, 1734574929014) + tdSql.query(f'select tbname, cols(last(ts), ts) {t1} group by tbname having cols(last(ts), ts) >= 1734574929000') + tdSql.checkRows(2) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 'd1') + tdSql.checkData(0, 1, 1734574929000) + tdSql.checkData(1, 0, 'd0') + tdSql.checkData(1, 1, 1734574929014) + tdSql.query(f'select tbname, cols(last(ts), ts) {t1} group by tbname having cols(last(ts), ts) <= 1734574929000') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 'd1') + tdSql.checkData(0, 1, 1734574929000) + tdSql.query(f'select tbname, cols(last(ts), ts) {t1} group by tbname having cols(last(c0), ts) between 1734574929000 and 1734574929014') + tdSql.checkRows(2) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 'd1') + tdSql.checkData(0, 1, 1734574929000) + tdSql.checkData(1, 0, 'd0') + tdSql.checkData(1, 1, 1734574929014) + def run(self): self.funcNestTest() @@ -1128,6 +1169,8 @@ class TDTestCase: self.include_null_test() self.long_column_name_test() self.test1() + self.having_test("test.meters", False) + self.having_test("(select tbname, * from test.meters)", True) def stop(self): From 71bdb6fcae47cd60ea702deb30d10748097cec6e Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Thu, 20 Feb 2025 11:47:03 +0800 Subject: [PATCH 38/45] nullTuple pos --- source/libs/executor/src/aggregateoperator.c | 8 ---- source/libs/function/src/builtinsimpl.c | 40 ++++++++++---------- source/libs/planner/src/planPhysiCreater.c | 1 - 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 2b9db7542f..1b8e8298b4 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -396,14 +396,6 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc return TSDB_CODE_SUCCESS; } - // if the expression is a bind expression, we don't need to create a empty data block - for (int i = pOperator->exprSupp.numOfExprs - 1; i >= 0; i--) { - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[i]; - if (nodeType(pExprInfo->pExpr) == QUERY_NODE_FUNCTION && pExprInfo->pExpr->_function.bindExprID > 0) { - return TSDB_CODE_SUCCESS; - } - } - code = createDataBlock(&pBlock); if (code) { return code; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 8618c23a79..c877ca3ae3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -912,14 +912,13 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pEntryInfo->numOfRes > 0) { code = setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); } else { - code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); + code = setNullSelectivityValue(pCtx, pBlock, currentRow); } } return code; } -#ifdef BUILD_NO_CALL int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex) { if (pCtx->subsidiaries.num <= 0) { return TSDB_CODE_SUCCESS; @@ -930,12 +929,14 @@ int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32 int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + if (NULL == pDstCol) { + return TSDB_CODE_OUT_OF_RANGE; + } colDataSetNULL(pDstCol, rowIndex); } return TSDB_CODE_SUCCESS; } -#endif int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { if (pCtx->subsidiaries.num <= 0) { @@ -2432,8 +2433,6 @@ int32_t firstLastFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResIn SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - pRes->nullTupleSaved = false; - pRes->nullTuplePos.pageId = -1; return TSDB_CODE_SUCCESS; } @@ -2463,10 +2462,10 @@ static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowI } if (!pInfo->hasResult) { - code = saveTupleData(pCtx, rowIndex, pSrcBlock, noElements ? &pInfo->nullTuplePos : &pInfo->pos); - } else { + code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + } else if (!noElements) { code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); - } + } else { } // dothing return code; } @@ -2537,7 +2536,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, true); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2632,7 +2631,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (numOfElems == 0) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, true); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2653,11 +2652,11 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; - pInfo->bytes = bytes; if (IS_NULL_TYPE(type)) { return TSDB_CODE_SUCCESS; } + pInfo->bytes = bytes; SColumnInfoData* pkCol = pInput->pPrimaryKey; pInfo->pkType = -1; @@ -2672,7 +2671,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, true); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2815,7 +2814,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { // save selectivity value for column consisted of all null values if (numOfElems == 0) { - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, true); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2864,7 +2863,7 @@ static bool firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOut static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, int32_t rowIndex) { if (firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput, pOutput->nullTupleSaved); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput, false); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -2913,7 +2912,7 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer } if (numOfElems == 0) { - int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, !pInfo->nullTupleSaved); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo, true); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2943,13 +2942,12 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pResInfo->isNullRes) { colDataSetNULL(pCol, pBlock->info.rows); - return setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, pBlock->info.rows); + return setNullSelectivityValue(pCtx, pBlock, pBlock->info.rows); } code = colDataSetVal(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); if (TSDB_CODE_SUCCESS != code) { return code; } - // handle selectivity code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); @@ -2982,7 +2980,7 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pEntryInfo->numOfRes == 0) { colDataSetNULL(pCol, pBlock->info.rows); - code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, pBlock->info.rows); + code = setNullSelectivityValue(pCtx, pBlock, pBlock->info.rows); } else { code = colDataSetVal(pCol, pBlock->info.rows, res, false); if (TSDB_CODE_SUCCESS != code) { @@ -3984,7 +3982,7 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pEntryInfo->numOfRes <= 0) { colDataSetNULL(pCol, currentRow); - code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); + code = setNullSelectivityValue(pCtx, pBlock, currentRow); return code; } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { @@ -5610,7 +5608,7 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pInfo->numSampled == 0) { colDataSetNULL(pCol, currentRow); - code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); + code = setNullSelectivityValue(pCtx, pBlock, currentRow); return code; } for (int32_t i = 0; i < pInfo->numSampled; ++i) { @@ -6062,7 +6060,7 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { code = setSelectivityValue(pCtx, pBlock, &resTuplePos, currentRow); } else { colDataSetNULL(pCol, currentRow); - code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); + code = setNullSelectivityValue(pCtx, pBlock, currentRow); } modeFunctionCleanup(pInfo); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 33818feb39..1db541624c 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -39,7 +39,6 @@ enum { static int32_t getSlotKeyHelper(SNode* pNode, const char* pPreName, const char* name, char** ppKey, int32_t callocLen, int32_t* pLen, uint16_t extraBufLen, int8_t slotKeyType) { int32_t code = 0; - char bindInfo[16] = {0}; *ppKey = taosMemoryCalloc(1, callocLen); if (!*ppKey) { return terrno; From 75984e6d1e0a1908f602e79299768e596d744aac Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Thu, 20 Feb 2025 16:29:23 +0800 Subject: [PATCH 39/45] fix: group const value --- source/libs/executor/src/executil.c | 3 +- source/libs/function/src/builtinsimpl.c | 16 ++--- source/libs/parser/src/parTranslater.c | 6 +- tests/system-test/2-query/cols_function.py | 82 +++++++++++++++++++--- 4 files changed, 85 insertions(+), 22 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c3d21152a8..7a1cf4beb6 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2127,8 +2127,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; - if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0) || - (strcmp(pName, "_group_const_value") == 0)) { + if ((strcmp(pName, "_select_value") == 0)) { if (pValCtxArray == NULL) { pValCtx[num++] = &pCtx[i]; } else { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c877ca3ae3..eb19c9400c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -962,12 +962,6 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; - // group_key function has its own process function - // do not process there - if (fmIsGroupKeyFunc(pc->functionId)) { - continue; - } - SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); if (NULL == pDstCol) { return TSDB_CODE_OUT_OF_RANGE; @@ -2432,7 +2426,8 @@ int32_t firstLastFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResIn } SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - + pRes->nullTupleSaved = false; + pRes->nullTuplePos.pageId = -1; return TSDB_CODE_SUCCESS; } @@ -2948,6 +2943,7 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (TSDB_CODE_SUCCESS != code) { return code; } + // handle selectivity code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); @@ -3982,7 +3978,7 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pEntryInfo->numOfRes <= 0) { colDataSetNULL(pCol, currentRow); - code = setNullSelectivityValue(pCtx, pBlock, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); return code; } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { @@ -5608,7 +5604,7 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pInfo->numSampled == 0) { colDataSetNULL(pCol, currentRow); - code = setNullSelectivityValue(pCtx, pBlock, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); return code; } for (int32_t i = 0; i < pInfo->numSampled; ++i) { @@ -6060,7 +6056,7 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { code = setSelectivityValue(pCtx, pBlock, &resTuplePos, currentRow); } else { colDataSetNULL(pCol, currentRow); - code = setNullSelectivityValue(pCtx, pBlock, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); } modeFunctionCleanup(pInfo); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 83ea444912..342d5bdfa1 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3857,6 +3857,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (isVectorFunc(*pNode) && !isDistinctOrderBy(pCxt)) { return DEAL_RES_IGNORE_CHILD; } + bool isSingleTable = fromSingleTable(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable); SNode* pGroupNode = NULL; FOREACH(pGroupNode, getGroupByList(pCxt)) { SNode* pActualNode = getGroupByNode(pGroupNode); @@ -3866,10 +3867,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } - if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) && + if ((isTbnameFuction(pActualNode) || isSingleTable) && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { return rewriteExprToSelectTagFunc(pCxt, pNode); } + if(isSingleTable && isTbnameFuction(*pNode)) { + return rewriteExprToSelectTagFunc(pCxt, pNode); + } } SNode* pPartKey = NULL; bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 8b318cefb1..d311dedf30 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -1022,7 +1022,7 @@ class TDTestCase: self.orderby_test("(select *, tbname from test.long_col_test)", "longcolumntestlongcolumntestlongcolumntestlongcolumntest88888888", True) tdLog.info("long_column_name_test subquery_test: one_cols_multi_output_with_group_test from meters") - def test1(self): + def test_in_interval(self): dbname = "db1" tdSql.execute(f"drop database if exists {dbname} ") tdSql.execute(f"create database {dbname} vgroups 6") @@ -1070,17 +1070,23 @@ class TDTestCase: tdSql.checkData(2, 1, "2022-09-30 15:15:03") tdSql.checkData(2, 2, "2022-09-30 15:15:03") tdSql.checkData(3, 1, "2022-09-30 15:15:04") - tdSql.checkData(3, 2, "2022-09-30 15:15:04") + tdSql.checkData(3, 2, None) - tdSql.query(f'select tbname, last(ts), last(ts) from tba5 interval(1s)') - tdSql.checkData(0, 1, "2022-09-30 15:15:01") + tdSql.query(f'select tbname, last(f1), ts from tba5 interval(1s)') + tdSql.checkRows(3) + tdSql.checkCols(3) + tdSql.checkData(0, 1, 0) tdSql.checkData(0, 2, "2022-09-30 15:15:01") - tdSql.checkData(1, 1, "2022-09-30 15:15:02") + tdSql.checkData(1, 1, 1) tdSql.checkData(1, 2, "2022-09-30 15:15:02") - tdSql.checkData(2, 1, "2022-09-30 15:15:03") + tdSql.checkData(2, 1, 5) tdSql.checkData(2, 2, "2022-09-30 15:15:03") + + tdSql.query(f'select tbname, last(ts) from tba5 interval(1s)') + tdSql.checkData(0, 1, "2022-09-30 15:15:01") + tdSql.checkData(1, 1, "2022-09-30 15:15:02") + tdSql.checkData(2, 1, "2022-09-30 15:15:03") tdSql.checkData(3, 1, "2022-09-30 15:15:04") - tdSql.checkData(3, 2, "2022-09-30 15:15:04") tdSql.query(f'select tbname, cols(last(ts), ts as ts1), cols(last(f1), ts as ts2) from {dbname}.sta group by tbname;') tdSql.checkRows(5) @@ -1154,7 +1160,63 @@ class TDTestCase: tdSql.checkData(1, 0, 'd0') tdSql.checkData(1, 1, 1734574929014) - + def test_null2(self): + dbname = "test_null2" + tdSql.execute(f"drop database if exists {dbname}") + tdSql.execute(f"create database test_null2 vgroups 5") + tdSql.execute(f"use test_null2") + tdSql.execute(f"create stable {dbname}.stb_null1 (ts timestamp, c0 int, c1 int, c2 nchar(30), c3 bool) tags (t1 nchar(30))") + tdSql.execute(f"create table {dbname}.sub_null_1 using {dbname}.stb_null1 tags('st1')") + tdSql.execute(f"create table {dbname}.sub_null_2 using {dbname}.stb_null1 tags('st2')") + + tdSql.execute(f"insert into {dbname}.sub_null_1 values(1734574929000, 1, null, null, null)") + tdSql.execute(f"insert into {dbname}.sub_null_1 values(1734574929001, 2, null, null, null)") + + tdSql.execute(f"insert into {dbname}.sub_null_2 values(1734574929000, 21, null, null, null)") + tdSql.execute(f"insert into {dbname}.sub_null_2 values(1734574929001, 22, null, null, null)") + + tdSql.query(f'select cols(last(ts), ts), cols(last(c2), ts) from {dbname}.stb_null1') + tdSql.checkRows(1) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 1734574929001) + tdSql.checkData(0, 1, None) + + tdSql.query(f'select tbname, cols(last(ts), c0), cols(last(c2), c0) from {dbname}.stb_null1 group by tbname order by tbname') + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'sub_null_1') + tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 'sub_null_2') + tdSql.checkData(1, 1, 22) + tdSql.checkData(1, 2, None) + + tdSql.query(f'select tbname, t1, cols(last(ts), c0), cols(last(c2), c0) from {dbname}.sub_null_1') + tdSql.checkRows(1) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 'sub_null_1') + tdSql.checkData(0, 1, "st1") + tdSql.checkData(0, 2, 2) + tdSql.checkData(0, 3, None) + + tdSql.query(f'select tbname, t1, cols(last(ts), c0), cols(last(c2), ts) from {dbname}.sub_null_2') + tdSql.checkRows(1) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 'sub_null_2') + tdSql.checkData(0, 1, "st2") + tdSql.checkData(0, 2, 22) + tdSql.checkData(0, 3, None) + + tdSql.query(f'select tbname, cols(last(ts), c0), cols(last(c2), c0) from {dbname}.sub_null_1') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'sub_null_1') + tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 2, None) + + tdSql.error(f'select tbname, cols(last(ts), c0), cols(last(c2), c0) from {dbname}.stb_null1') + tdSql.error(f'select t1, cols(last(ts), c0), cols(last(c2), c0) from {dbname}.stb_null1') + def run(self): self.funcNestTest() self.funcSupperTableTest() @@ -1166,11 +1228,13 @@ class TDTestCase: self.window_test() self.join_test() self.stream_cols_test() + self.test_in_interval() self.include_null_test() self.long_column_name_test() - self.test1() + self.having_test("test.meters", False) self.having_test("(select tbname, * from test.meters)", True) + self.test_null2() def stop(self): From d6f05d48c89d2aa145b58fc8cc77860ce58d97b5 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Thu, 20 Feb 2025 17:17:41 +0800 Subject: [PATCH 40/45] return code --- include/libs/nodes/querynodes.h | 2 +- source/libs/nodes/src/nodesUtilFuncs.c | 5 +++-- source/libs/parser/src/parTranslater.c | 18 +++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5fae336d22..dab666a27b 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -700,7 +700,7 @@ char* getJoinSTypeString(EJoinSubType type); char* getFullJoinTypeString(EJoinType type, EJoinSubType stype); int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc); -int32_t rewriteExprAliasName(SExprNode* pNode, int64_t num); +void rewriteExprAliasName(SExprNode* pNode, int64_t num); bool isRelatedToOtherExpr(SExprNode* pExpr); #ifdef __cplusplus diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index b96a1d3a12..48e9f12c27 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -3240,8 +3240,9 @@ int32_t nodesListDeduplicate(SNodeList** ppList) { return code; } -int32_t rewriteExprAliasName(SExprNode* pNode, int64_t num) { - return tsnprintf(pNode->aliasName, TSDB_COL_NAME_LEN, "expr_%ld", num); +void rewriteExprAliasName(SExprNode* pNode, int64_t num) { + tsnprintf(pNode->aliasName, TSDB_COL_NAME_LEN, "expr_%x", num); + return; } bool isRelatedToOtherExpr(SExprNode* pExpr) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 342d5bdfa1..51c13a5eee 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7433,11 +7433,11 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { int len = strlen(pExpr->aliasName); if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { char buffer[TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1] = {0}; - tsnprintf(buffer, sizeof(buffer), "%s.%d", pExpr->aliasName, pExpr->relatedTo); + (void)tsnprintf(buffer, sizeof(buffer), "%s.%d", pExpr->aliasName, pExpr->relatedTo); uint64_t hashVal = MurmurHash3_64(buffer, TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1); - tsnprintf(pExpr->aliasName, TSDB_COL_NAME_EXLEN, "%" PRIu64, hashVal); + (void)tsnprintf(pExpr->aliasName, TSDB_COL_NAME_EXLEN, "%" PRIu64, hashVal); } else { - tsnprintf(pExpr->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pExpr->relatedTo); + (void)tsnprintf(pExpr->aliasName + len, TSDB_COL_NAME_EXLEN, ".%d", pExpr->relatedTo); } } } @@ -7533,9 +7533,9 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { } } if(*pCxt->selectFuncList == NULL) { - nodesMakeList(pCxt->selectFuncList); + code = nodesMakeList(pCxt->selectFuncList); if (NULL == *pCxt->selectFuncList) { - pCxt->status = terrno; + pCxt->status = code; return DEAL_RES_ERROR; } } @@ -7546,12 +7546,15 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { selectFuncIndex = selectFuncCount; SNode* pNewNode = NULL; code = nodesCloneNode(pSelectFunc, &pNewNode); + if(code) goto _end; ((SExprNode*)pNewNode)->bindExprID = selectFuncIndex; - nodesListMakeStrictAppend(pCxt->selectFuncList, pNewNode); + code = nodesListMakeStrictAppend(pCxt->selectFuncList, pNewNode); + if(code) goto _end; } SNode* pNewNode = NULL; code = nodesCloneNode(pExpr, &pNewNode); + if(code) goto _end; if (nodesIsExprNode(pNewNode)) { SBindTupleFuncCxt pCxt = {pNewNode, selectFuncIndex}; nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); @@ -7565,7 +7568,8 @@ static EDealRes rewriteSingleColsFunc(SNode** pNode, void* pContext) { } return DEAL_RES_CONTINUE; _end: - return code; + pCxt->status = code; + return DEAL_RES_ERROR; } static int32_t rewriteHavingColsNode(STranslateContext* pCxt, SNode** pNode, SNodeList** selectFuncList) { From d9bbe0d888205e2635ee2f73227690f6b6ebe687 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Thu, 20 Feb 2025 19:15:22 +0800 Subject: [PATCH 41/45] return code --- source/libs/executor/src/executil.c | 4 +--- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/src/parTranslater.c | 13 ++++++++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 7a1cf4beb6..cf2dd0a283 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1955,6 +1955,7 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { QUERY_CHECK_CODE(code, lino, _end); } } + pExp->pExpr->_function.bindExprID = ((SExprNode*)pNode)->bindExprID; } else if (type == QUERY_NODE_OPERATOR) { pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; SOperatorNode* pOpNode = (SOperatorNode*)pNode; @@ -1992,9 +1993,6 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; QUERY_CHECK_CODE(code, lino, _end); } - if (type == QUERY_NODE_FUNCTION) { - pExp->pExpr->_function.bindExprID = ((SExprNode*)pNode)->bindExprID; - } pExp->pExpr->relatedTo = ((SExprNode*)pNode)->relatedTo; _end: if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 48e9f12c27..d581283a04 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -3241,7 +3241,7 @@ int32_t nodesListDeduplicate(SNodeList** ppList) { } void rewriteExprAliasName(SExprNode* pNode, int64_t num) { - tsnprintf(pNode->aliasName, TSDB_COL_NAME_LEN, "expr_%x", num); + (void)tsnprintf(pNode->aliasName, TSDB_COL_NAME_LEN, "expr_%x", num); return; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 51c13a5eee..26857403e2 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7428,8 +7428,11 @@ static EDealRes pushDownBindSelectFunc(SNode** pNode, void* pContext) { if (nodesIsExprNode(*pNode)) { SExprNode* pExpr = (SExprNode*)*pNode; pExpr->relatedTo = pCxt->bindExprID; + if (nodeType(*pNode) != QUERY_NODE_COLUMN) { + return DEAL_RES_CONTINUE; + } - if (isRelatedToOtherExpr(pExpr) && *pNode != pCxt->root) { + if (*pNode != pCxt->root) { int len = strlen(pExpr->aliasName); if (len + TSDB_COL_NAME_EXLEN >= TSDB_COL_NAME_LEN) { char buffer[TSDB_COL_NAME_EXLEN + TSDB_COL_NAME_LEN + 1] = {0}; @@ -7609,6 +7612,7 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList } SNodeList* pNewNodeList = NULL; + SNode* pNewNode = NULL; if (needRewrite) { if (pCxt->createStream) { return TSDB_CODE_PAR_INVALID_COLS_FUNCTION; @@ -7625,7 +7629,6 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList } } - SNode* pNewNode = NULL; int32_t nums = 0; int32_t selectFuncCount = (*selectFuncList)->length; SNode* pTmpNode = NULL; @@ -7649,14 +7652,17 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList ++selectFuncCount; selectFuncIndex = selectFuncCount; code = nodesCloneNode(pSelectFunc, &pNewNode); + if(TSDB_CODE_SUCCESS != code) goto _end; ((SExprNode*)pNewNode)->bindExprID = selectFuncIndex; - nodesListMakeStrictAppend(selectFuncList, pNewNode); + code = nodesListMakeStrictAppend(selectFuncList, pNewNode); + if(TSDB_CODE_SUCCESS != code) goto _end; } // start from index 1, because the first parameter is select function which needn't to output. for (int i = 1; i < pFunc->pParameterList->length; ++i) { SNode* pExpr = nodesListGetNode(pFunc->pParameterList, i); code = nodesCloneNode(pExpr, &pNewNode); + if(TSDB_CODE_SUCCESS != code) goto _end; if (nodesIsExprNode(pNewNode)) { SBindTupleFuncCxt pCxt = {pNewNode, selectFuncIndex}; nodesRewriteExpr(&pNewNode, pushDownBindSelectFunc, &pCxt); @@ -7683,6 +7689,7 @@ static int32_t rewriteColsFunction(STranslateContext* pCxt, SNodeList** nodeList } _end: if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNewNode); nodesDestroyList(pNewNodeList); } return code; From fa61c13a75e31ab9f44e4582becd498cf40334bf Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Fri, 21 Feb 2025 00:40:11 +0800 Subject: [PATCH 42/45] fix: _group_key not select --- source/libs/function/src/builtins.c | 4 ++-- tests/script/tsim/parser/having_child.sim | 5 ++++- tests/system-test/2-query/last_cache_scan.py | 9 ++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 722f838061..5a6b042cd0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -4236,7 +4236,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_group_key", .type = FUNCTION_TYPE_GROUP_KEY, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC, .translateFunc = translateGroupKey, .getEnvFunc = getGroupKeyFuncEnv, .initFunc = functionSetup, @@ -4956,7 +4956,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_group_const_value", .type = FUNCTION_TYPE_GROUP_CONST_VALUE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .parameters = {.minParamNum = 0, .maxParamNum = 0, .paramInfoPattern = 0, diff --git a/tests/script/tsim/parser/having_child.sim b/tests/script/tsim/parser/having_child.sim index db9a25365e..093877c3dd 100644 --- a/tests/script/tsim/parser/having_child.sim +++ b/tests/script/tsim/parser/having_child.sim @@ -1294,7 +1294,10 @@ sql_error select avg(f1), spread(f1), spread(f2), spread(tb1.f1) from tb1 group sql_error select avg(f1), spread(f1), spread(f2), spread(tb1.f1) from tb1 group by f1 having spread(f1) > id1 and sum(f1); -sql_error select avg(f1), spread(f1), spread(f2), spread(tb1.f1) from tb1 group by f1 having spread(f1) > id1 and sum(f1) > 1; +sql select avg(f1), spread(f1), spread(f2), spread(tb1.f1) from tb1 group by f1 having spread(f1) > id1 and sum(f1) > 1; +if $rows != 0 then + return -1 +endi sql select avg(f1), spread(f1), spread(f2), spread(tb1.f1) from tb1 group by f1 having spread(f1) > 2 and sum(f1) > 1 order by f1; if $rows != 0 then diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 049fe60f16..097e3903eb 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -295,7 +295,7 @@ class TDTestCase: tdSql.query("select last(ts) from meters partition by tbname") tdSql.query("select last(ts) from meters partition by t1") sql_template = 'select %s from meters partition by tbname' - select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, tbname", "last(c10), c10, ts"] + select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, tbname", "last(c10), c10, ts", "ts, ts, last(c10), c10, t1" ] has_last_row_scan_res = [1,1,1] sqls = self.format_sqls(sql_template, select_items) self.explain_and_check_res(sqls, has_last_row_scan_res) @@ -312,6 +312,13 @@ class TDTestCase: tdSql.checkData(0,1, '2018-11-25 19:30:00.000') tdSql.checkData(0,2, '2018-11-25 19:30:01.000') tdSql.checkData(0,3, '2018-11-25 19:30:01.000') + + tdSql.query(sqls[3], queryTimes=1) + tdSql.checkRows(10) + tdSql.checkData(0,0, '2018-11-25 19:30:00.000') + tdSql.checkData(0,1, '2018-11-25 19:30:00.000') + tdSql.checkData(0,2, '2018-11-25 19:30:01.000') + tdSql.checkData(0,3, '2018-11-25 19:30:01.000') sql_template = 'select %s from meters partition by t1' select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, t1", "last(c10), c10, ts"] From 450d262415bcc5937648ce267ce560861c2af55b Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Fri, 21 Feb 2025 16:32:39 +0800 Subject: [PATCH 43/45] test cases: window --- source/libs/function/src/builtinsimpl.c | 4 +- tests/system-test/2-query/cols_function.py | 81 ++++++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index eb19c9400c..91be791dff 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -930,7 +930,7 @@ int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32 SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); if (NULL == pDstCol) { - return TSDB_CODE_OUT_OF_RANGE; + return terrno; } colDataSetNULL(pDstCol, rowIndex); } @@ -964,7 +964,7 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); if (NULL == pDstCol) { - return TSDB_CODE_OUT_OF_RANGE; + return terrno; } if (nullList[j]) { colDataSetNULL(pDstCol, rowIndex); diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index d311dedf30..53c7f33174 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -942,6 +942,86 @@ class TDTestCase: tdSql.checkCols(5) tdSql.checkData(0, 4, 2) + tdSql.query(f'select c0, cols(first(ts), c0, c1) from test.meters state_window(c0) order by c0') + tdSql.checkRows(4) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, 2) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, 3) + + tdSql.query(f'select c0, cols(first(ts), c0, c1), tbname from test.meters state_window(c0)') + tdSql.checkRows(4) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, 'd0') + tdSql.checkData(1, 0, 2) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, 3) + + tdSql.query(f'select c0, cols(first(ts), c0, c1), t1 from test.meters state_window(c0)') + tdSql.checkRows(4) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, 'st1') + tdSql.checkData(1, 0, 2) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, 3) + + tdSql.error(f'select tbname, c0, cols(first(ts), c0, c1), cols(last(ts), c0) from test.meters state_window(c0)') + + def window_test2(self): + tdSql.query(f'select cols(first(ts), c0, c1), count(1) from test.meters session(ts, 3)') + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 6) + tdSql.checkData(1, 0, 0) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, 5) + + tdSql.query(f'select cols(max(c0), c1), count(1) from test.meters session(ts, 3)') + tdSql.checkRows(2) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 4) + tdSql.checkData(0, 1, 6) + tdSql.checkData(1, 0, None) + tdSql.checkData(1, 1, 5) + + tdSql.query(f'select cols(first(ts), c0, c1), first(ts), count(1) from test.meters session(ts, 3)') + tdSql.checkRows(2) + tdSql.checkCols(4) + + tdSql.query(f'select cols(first(ts), c0, c1), first(ts), count(1) from test.meters event_window start with c3=true end with c3!=true') + tdSql.checkRows(2) + tdSql.checkCols(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1734574929000) + tdSql.checkData(0, 3, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(1, 1, 3) + tdSql.checkData(1, 2, 1734574929003) + tdSql.checkData(1, 3, 2) + + tdSql.query(f'select first(ts), count(1) from test.meters event_window start with c3=true end with c3!=true') + tdSql.checkRows(2) + tdSql.checkCols(2) + tdSql.checkData(0, 0, 1734574929000) + tdSql.checkData(0, 1, 3) + tdSql.checkData(1, 0, 1734574929003) + tdSql.checkData(1, 1, 2) + def join_test(self): tdSql.query(f'select cols(last(a.ts), a.c0) from test.d0 a join test.d1 b on a.ts = b.ts;') @@ -1235,6 +1315,7 @@ class TDTestCase: self.having_test("test.meters", False) self.having_test("(select tbname, * from test.meters)", True) self.test_null2() + self.window_test2() def stop(self): From 29b7b27e53ec51cfa266e0aa51133bdb26b13a7c Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Fri, 21 Feb 2025 11:05:08 +0800 Subject: [PATCH 44/45] cols desc --- .../14-reference/03-taos-sql/10-function.md | 21 +++++++++++++++++++ .../14-reference/03-taos-sql/10-function.md | 20 ++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index 8397c59177..80596621bf 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -2125,6 +2125,27 @@ UNIQUE(expr) **Applicable to**: Tables and supertables. +### COLS​ + +```sql​ +COLS​(select_function(expr), output_expr1, [, output_expr2] ... )​ +``` + +**Function Description**: On the data row where the execution result of function func (expr) is located, execute the expression output_expr1, [, output_expr2], return its result, and the result of func (expr) is not output.​ + +**Return Data Type**: Returns multiple columns of data, and the data type of each column is the type of the result returned by the corresponding expression.​ + +**Applicable Data Types**: All type fields.​ + +**Applicable to**: Tables and Super Tables.​ + +**Usage Instructions**: +- Func function type: must be a single-line selection function (output result is a single-line selection function, for example, last is a single-line selection function, but top is a multi-line selection function).​ +- The result of the parameter func is not returned. If you want to output the result of func, you need to add the corresponding output_expr parameter to the letter. +- When there is only one column in the output, you can set an alias for the function. For example, you can do it like this: "select cols(first (ts), c1) as c11 from ...". +- Output one or more columns, and you can set an alias for each output column of the function. For example, you can do it like this: "select (first (ts), c1 as c11, c2 as c22) from ...". + + ## Time-Series Specific Functions Time-Series specific functions are tailor-made by TDengine to meet the query scenarios of time-series data. In general databases, implementing similar functionalities usually requires complex query syntax and is inefficient. TDengine has built these functionalities into functions, greatly reducing the user's cost of use. diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index c0e80e80df..6c68d8f62c 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -2051,6 +2051,26 @@ UNIQUE(expr) **适用于**: 表和超级表。 +### COLS + +```sql +COLS(select_function(expr), output_expr1, [, output_expr2] ... ) +``` + +**功能说明**：在选择函数 func(expr) 执行结果所在数据行上，执行表达式 output_expr1, [,output_expr2]，返回其结果，func(expr)结果不输出. + +**返回数据类型**：返回多列数据，每列数据类型为对应表达式返回结果的类型。 + +**适用数据类型**：全部类型字段。 + +**适用于**：表和超级表。 + +**使用说明:** +- func 函数类型：必须是单行选择函数（输出结果为一行的选择函数，例如 last 是单行选择函数， 但 top 是多行选择函数）。 +- 注意, 参数 func 的结果并没有返回，如需输出 func 结果，需要在 cols 中加上相应的 output_expr 参数。 +- 输出只有一列时，可以对 cols 函数设置别名。例如: "select cols(first(ts), c1) as c11 from ..." +- 输出一列或者多列，可以对 cols 函数的每个输出列设置命名。例如: "select cols(first(ts), c1 as c11, c2 as c22)" + ## 时序数据特有函数 From 63dc2e1299d8f730fa06363c752145852199e648 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Mon, 24 Feb 2025 00:06:22 +0800 Subject: [PATCH 45/45] cols desc --- docs/en/14-reference/03-taos-sql/10-function.md | 7 ++++--- docs/en/14-reference/09-error-code.md | 2 +- docs/zh/14-reference/03-taos-sql/10-function.md | 5 +++-- docs/zh/14-reference/09-error-code.md | 2 +- source/libs/parser/src/parTranslater.c | 3 ++- source/util/src/terror.c | 2 +- tests/system-test/2-query/cols_function.py | 8 ++++++-- 7 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index 80596621bf..2292e40f75 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -2128,10 +2128,10 @@ UNIQUE(expr) ### COLS​ ```sql​ -COLS​(select_function(expr), output_expr1, [, output_expr2] ... )​ +COLS​(func(expr), output_expr1, [, output_expr2] ... )​ ``` -**Function Description**: On the data row where the execution result of function func (expr) is located, execute the expression output_expr1, [, output_expr2], return its result, and the result of func (expr) is not output.​ +**Function Description**: On the data row where the execution result of function func(expr) is located, execute the expression output_expr1, [, output_expr2], return its result, and the result of func (expr) is not output.​ **Return Data Type**: Returns multiple columns of data, and the data type of each column is the type of the result returned by the corresponding expression.​ @@ -2141,7 +2141,8 @@ COLS​(select_function(expr), output_expr1, [, output_expr2] ... )​ **Usage Instructions**: - Func function type: must be a single-line selection function (output result is a single-line selection function, for example, last is a single-line selection function, but top is a multi-line selection function).​ -- The result of the parameter func is not returned. If you want to output the result of func, you need to add the corresponding output_expr parameter to the letter. +- Mainly used to obtain the associated columns of multiple selection function results in a single SQL query. For example: select cols(max(c0), ts), cols(max(c1), ts) from ... can be used to get the different ts values of the maximum values of columns c0 and c1. +- The result of the parameter func is not returned. If you need to output the result of func, you can add additional output columns, such as: select first(ts), cols(first(ts), c1) from .. - When there is only one column in the output, you can set an alias for the function. For example, you can do it like this: "select cols(first (ts), c1) as c11 from ...". - Output one or more columns, and you can set an alias for each output column of the function. For example, you can do it like this: "select (first (ts), c1 as c11, c2 as c22) from ...". diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 857f84da34..38ce3a4d39 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -459,7 +459,7 @@ This document details the server error codes that may be encountered when using | 0x80002666 | Subquery does not output primary timestamp column | Check and correct the SQL statement | | | 0x80002667 | Invalid usage of expr: %s | Illegal expression | Check and correct the SQL statement | | 0x80002687 | Invalid using cols function | Illegal using cols function | Check and correct the SQL statement | -| 0x80002688 | Cols function's first param must be a select function | The first parameter of the cols function should be a selection function | Check and correct the SQL statement | +| 0x80002688 | Cols function's first param must be a select function that output a single row | The first parameter of the cols function should be a selection function | Check and correct the SQL statement | | 0x80002689 | Invalid using cols function with multiple output columns | Illegal using the cols function for multiple column output | Check and correct the SQL statement | | 0x8000268A | Invalid using alias for cols function | Illegal cols function alias | Check and correct the SQL statement | | 0x800026FF | Parser internal error | Internal error in parser | Preserve the scene and logs, report issue on GitHub | diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 6c68d8f62c..2e0fd41dc4 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -2054,7 +2054,7 @@ UNIQUE(expr) ### COLS ```sql -COLS(select_function(expr), output_expr1, [, output_expr2] ... ) +COLS(func(expr), output_expr1, [, output_expr2] ... ) ``` **功能说明**：在选择函数 func(expr) 执行结果所在数据行上，执行表达式 output_expr1, [,output_expr2]，返回其结果，func(expr)结果不输出. @@ -2067,7 +2067,8 @@ COLS(select_function(expr), output_expr1, [, output_expr2] ... ) **使用说明:** - func 函数类型：必须是单行选择函数（输出结果为一行的选择函数，例如 last 是单行选择函数， 但 top 是多行选择函数）。 -- 注意, 参数 func 的结果并没有返回，如需输出 func 结果，需要在 cols 中加上相应的 output_expr 参数。 +- 主要用于一个 sql 中获取多个选择函数结果关联列的场景，例如: select cols(max(c0), ts), cols(max(c1), ts) from ...可用于获取 c0, c1 列最大值的不同 ts 值。 +- 注意, 参数 func 的结果并没有返回，如需输出func结果，可额外增加输出列，如: select fist(ts), cols(first(ts), c1) from ... - 输出只有一列时，可以对 cols 函数设置别名。例如: "select cols(first(ts), c1) as c11 from ..." - 输出一列或者多列，可以对 cols 函数的每个输出列设置命名。例如: "select cols(first(ts), c1 as c11, c2 as c22)" diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index 74fe39421d..7481903397 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -476,7 +476,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80002666 | 子查询不含主键时间戳列输出 | 检查并修正SQL语句 | | 0x80002667 | Invalid usage of expr: %s | 非法表达式 | 检查并修正SQL语句 | | 0x80002687 | Invalid using cols function | cols函数使用错误 | 检查并修正SQL语句 | -| 0x80002688 | Cols function's first param must be a select function | cols函数第一个参数应该为选择函数 | 检查并修正SQL语句 | +| 0x80002688 | Cols function's first param must be a select function that output a single row | cols函数第一个参数应该为选择函数 | 检查并修正SQL语句 | | 0x80002689 | Invalid using cols function with multiple output columns | 多列输出的 cols 函数使用错误 | 检查并修正SQL语句 | | 0x8000268A | Invalid using alias for cols function | cols 函数输出列重命名错误 | 检查并修正SQL语句 | | 0x800026FF | Parser internal error | 解析器内部错误 | 保留现场和日志，github上报issue | diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 26857403e2..c776fc6eee 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1109,7 +1109,8 @@ static bool isColsFunctionResult(const SNode* pNode) { } static bool isInvalidColsBindFunction(const SFunctionNode* pFunc) { - return (!fmIsSelectFunc(pFunc->funcId) && pFunc->node.bindExprID != 0); + return (pFunc->node.bindExprID != 0 && (!fmIsSelectFunc(pFunc->funcId) || fmIsMultiRowsFunc(pFunc->funcId) || + fmIsIndefiniteRowsFunc(pFunc->funcId))); } #ifdef BUILD_NO_CALL diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 57150cdc9d..82c48a1fc2 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -753,7 +753,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE, "Invalid forecast cl TAOS_DEFINE_ERROR(TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR, "Syntax error in regular expression") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VGID_LIST, "Invalid vgid list") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLS_FUNCTION, "Invalid cols function") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC, "cols function's first param must be a select function") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC, "cols function's first param must be a select function that output a single row") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MULITI_COLS_FUNC, "Improper use of cols function with multiple output columns") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_COLS_ALIAS, "Invalid using alias for cols function") diff --git a/tests/system-test/2-query/cols_function.py b/tests/system-test/2-query/cols_function.py index 53c7f33174..bacab811ea 100644 --- a/tests/system-test/2-query/cols_function.py +++ b/tests/system-test/2-query/cols_function.py @@ -288,7 +288,6 @@ class TDTestCase: tdSql.checkData(0, 2, 4) tdSql.checkData(0, 3, 1) - # there's some error on last_row func when using sub query. tdSql.query(f'select count(1), cols(last_row(ts), ts, c0), min(c0) from test.meters') tdSql.checkRows(1) tdSql.checkCols(4) @@ -297,7 +296,6 @@ class TDTestCase: tdSql.checkData(0, 2, 4) tdSql.checkData(0, 3, 1) - # there's some error on last_row func when using sub query. tdSql.query(f'select count(1), cols(last_row(ts), ts, c0), last_row(c1), last_row(c3) from test.meters') tdSql.checkRows(1) tdSql.checkCols(5) @@ -896,6 +894,12 @@ class TDTestCase: tdSql.error(f'select cols(first(ts+1), c0+2 cc0, c1 cc1) cc from {self.dbname}.meters') tdSql.error(f'select cols(last(ts)+1, c0+2 as cc0) as cc from {self.dbname}.meters') tdSql.error(f'select cols(ABS(c0), c1) from {self.dbname}.meters group by tbname') + tdSql.error(f'select cols(top(c0, 5), c1) from {self.dbname}.meters') + tdSql.error(f'select cols(tail(c0, 5),c1) from {self.dbname}.meters') + tdSql.error(f'select cols(BOTTOM(c0, 5),c1) from {self.dbname}.meters') + tdSql.error(f'select cols(tail(ts, 5),c1) from {self.dbname}.meters') + tdSql.error(f'select cols(UNIQUE(ts),c1) from {self.dbname}.meters') + tdSql.error(f'select cols(sample(ts, 5),c1) from {self.dbname}.meters') tdSql.error(f'select cols(last(ts)+1, ts) from {self.dbname}.meters') tdSql.error(f'select cols(last(ts)+10, c1+10) from {self.dbname}.meters group by tbname')