From b734506363963515f03b6f2f495432a0aca6d1bf Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 6 Jul 2023 14:54:01 +0800 Subject: [PATCH 1/6] enh: add procedures on server for udf/udaf in nested queries where outer query is constant table --- source/libs/executor/src/projectoperator.c | 59 +++++++++++++++++++--- source/libs/scalar/src/scalar.c | 3 +- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 412a4bfbc0..011df0ff31 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -630,14 +630,59 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { for (int32_t k = 0; k < pSup->numOfExprs; ++k) { int32_t outputSlotId = pExpr[k].base.resSchema.slotId; - ASSERT(pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE); - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, outputSlotId); + if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) { + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, outputSlotId); - int32_t type = pExpr[k].base.pParam[0].param.nType; - if (TSDB_DATA_TYPE_NULL == type) { - colDataSetNNULL(pColInfoData, 0, 1); - } else { - colDataSetVal(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); + int32_t type = pExpr[k].base.pParam[0].param.nType; + if (TSDB_DATA_TYPE_NULL == type) { + colDataSetNNULL(pColInfoData, 0, 1); + } else { + colDataSetVal(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); + } + } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) { + SqlFunctionCtx* pfCtx = &pSup->pCtx[k]; + + if (fmIsAggFunc(pfCtx->functionId)) { + // selective value output should be set during corresponding function execution + if (fmIsSelectValueFunc(pfCtx->functionId)) { + continue; + } + + SColumnInfoData* pOutput = taosArrayGet(pRes->pDataBlock, outputSlotId); + int32_t slotId = pfCtx->param[0].pCol->slotId; + + // todo handle the json tag + //SColumnInfoData* pInput = taosArrayGet(pSrcBlock->pDataBlock, slotId); + //for (int32_t f = 0; f < pSrcBlock->info.rows; ++f) { + // bool isNull = colDataIsNull_s(pInput, f); + // if (isNull) { + // colDataSetNULL(pOutput, pRes->info.rows + f); + // } else { + // char* data = colDataGetData(pInput, f); + // colDataSetVal(pOutput, pRes->info.rows + f, data, isNull); + // } + //} + } else { + SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); + taosArrayPush(pBlockList, &pRes); + + SColumnInfoData* pResColData = taosArrayGet(pRes->pDataBlock, outputSlotId); + SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; + + SScalarParam dest = {.columnData = &idata}; + int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pBlockList); + return NULL; + } + + int32_t startOffset = pRes->info.rows; + ASSERT(pRes->info.capacity > 0); + colDataMergeCol(pResColData, startOffset, (int32_t*)&pRes->info.capacity, &idata, dest.numOfRows); + colDataDestroy(&idata); + + taosArrayDestroy(pBlockList); + } } } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index d9295656e8..4eb0f0e1bc 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1694,7 +1694,8 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } - if (1 == res->numOfRows) { + SSDataBlock *pb = taosArrayGetP(pBlockList, 0); + if (1 == res->numOfRows && pb->info.rows > 0) { SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList)); } else { colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows, true); From d4e07c551e8396381649144b1a36133836408ac3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 6 Jul 2023 15:05:49 +0800 Subject: [PATCH 2/6] return error code of udf execution failure --- source/libs/executor/src/projectoperator.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 011df0ff31..d9f065b04f 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -38,7 +38,7 @@ typedef struct SIndefOperatorInfo { SSDataBlock* pNextGroupRes; } SIndefOperatorInfo; -static SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator); +static int32_t doGenerateSourceData(SOperatorInfo* pOperator); static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols); @@ -215,7 +215,7 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S if (newGroup) { resetLimitInfoForNextGroup(pLimitInfo); } - + return PROJECT_RETRIEVE_CONTINUE; } @@ -267,7 +267,12 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo; if (downstream == NULL) { - return doGenerateSourceData(pOperator); + code = doGenerateSourceData(pOperator); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + + return (pRes->info.rows > 0) ? pRes : NULL; } while (1) { @@ -616,7 +621,7 @@ SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { return pList; } -SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { +int32_t doGenerateSourceData(SOperatorInfo* pOperator) { SProjectOperatorInfo* pProjectInfo = pOperator->info; SExprSupp* pSup = &pOperator->exprSupp; @@ -673,7 +678,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pBlockList); - return NULL; + return code; } int32_t startOffset = pRes->info.rows; @@ -683,6 +688,8 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { taosArrayDestroy(pBlockList); } + } else { + return TSDB_CODE_OPS_NOT_SUPPORT; } } @@ -698,7 +705,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } - return (pRes->info.rows > 0) ? pRes : NULL; + return TSDB_CODE_SUCCESS; } static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, SArray* pPseudoList) { From 365993d7752a46bfabc0c7cc579214cb22c6c151 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 6 Jul 2023 16:11:41 +0800 Subject: [PATCH 3/6] remove udfd agg function handling --- source/libs/executor/src/projectoperator.c | 26 +++++----------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index d9f065b04f..8ab0efbacf 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -647,27 +647,9 @@ int32_t doGenerateSourceData(SOperatorInfo* pOperator) { } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) { SqlFunctionCtx* pfCtx = &pSup->pCtx[k]; - if (fmIsAggFunc(pfCtx->functionId)) { - // selective value output should be set during corresponding function execution - if (fmIsSelectValueFunc(pfCtx->functionId)) { - continue; - } - - SColumnInfoData* pOutput = taosArrayGet(pRes->pDataBlock, outputSlotId); - int32_t slotId = pfCtx->param[0].pCol->slotId; - - // todo handle the json tag - //SColumnInfoData* pInput = taosArrayGet(pSrcBlock->pDataBlock, slotId); - //for (int32_t f = 0; f < pSrcBlock->info.rows; ++f) { - // bool isNull = colDataIsNull_s(pInput, f); - // if (isNull) { - // colDataSetNULL(pOutput, pRes->info.rows + f); - // } else { - // char* data = colDataGetData(pInput, f); - // colDataSetVal(pOutput, pRes->info.rows + f, data, isNull); - // } - //} - } else { + // UDF scalar functions will be calculated here, for example, select foo(n) from (select 1 n). + // UDF aggregate functions will be handled in agg operator. + if (fmIsScalarFunc(pfCtx->functionId)) { SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); taosArrayPush(pBlockList, &pRes); @@ -687,6 +669,8 @@ int32_t doGenerateSourceData(SOperatorInfo* pOperator) { colDataDestroy(&idata); taosArrayDestroy(pBlockList); + } else { + return TSDB_CODE_OPS_NOT_SUPPORT; } } else { return TSDB_CODE_OPS_NOT_SUPPORT; From 7345d7953399ab305514a96fd612577051803df4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 6 Jul 2023 16:44:57 +0800 Subject: [PATCH 4/6] add test cases --- tests/system-test/0-others/udfTest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 78020cb958..88d0d420f7 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -234,6 +234,11 @@ class TDTestCase: tdSql.checkData(20,6,88) tdSql.checkData(20,7,1) + tdSql.query("select udf1(1) from (select 1)") + tdSql.checkData(0,0,1) + + tdSql.query("select udf1(n) from (select 1 n)") + tdSql.checkData(0,0,1) # aggregate functions tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb") From 340e4de30d21df52f05d30e706efb9ae1d1a700a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 10 Jul 2023 09:12:21 +0800 Subject: [PATCH 5/6] fix an issue --- source/libs/executor/src/projectoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 8ab0efbacf..1cc377b3ee 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -665,7 +665,7 @@ int32_t doGenerateSourceData(SOperatorInfo* pOperator) { int32_t startOffset = pRes->info.rows; ASSERT(pRes->info.capacity > 0); - colDataMergeCol(pResColData, startOffset, (int32_t*)&pRes->info.capacity, &idata, dest.numOfRows); + colDataAssign(pResColData, &idata, dest.numOfRows, &pRes->info); colDataDestroy(&idata); taosArrayDestroy(pBlockList); From eb2cc3cb2b4c95efb027a2e113af460196de4459 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 10 Jul 2023 10:28:48 +0800 Subject: [PATCH 6/6] docs: add info for ttlChangeOnWrite --- docs/en/14-reference/12-config/index.md | 13 +++++++++++-- docs/zh/14-reference/12-config/index.md | 14 ++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index cbff7301d2..7522744469 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -102,7 +102,7 @@ Ensure that your firewall rules do not block TCP port 6042 on any host in the c | Value Range | 10-50000000 | | Default Value | 5000 | -### numOfRpcSessions +### numOfRpcSessions | Attribute | Description | | ------------- | ------------------------------------------ | @@ -202,7 +202,7 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo | Default Value | 0 | | Notes | 0: Disable SMA indexing and perform all queries on non-indexed data; 1: Enable SMA indexing and perform queries from suitable statements on precomputation results. | -### countAlwaysReturnValue +### countAlwaysReturnValue | Attribute | Description | | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -713,6 +713,14 @@ The charset that takes effect is UTF-8. | Value Range | 0: disable UDF; 1: enabled UDF | | Default Value | 1 | +### ttlChangeOnWrite + +| Attribute | Description | +| ------------- | ----------------------------------------------------------------------------- | +| Applicable | Server Only | +| Meaning | Whether the ttl expiration time changes with the table modification operation | +| Value Range | 0: not change; 1: change by modification | +| Default Value | 0 | ## 3.0 Parameters @@ -770,3 +778,4 @@ The charset that takes effect is UTF-8. | 52 | charset | Yes | Yes | | | 53 | udf | Yes | Yes | | | 54 | enableCoreFile | Yes | Yes | | +| 55 | ttlChangeOnWrite | No | Yes | | diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index a637b52bf8..d57ee02868 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -101,7 +101,7 @@ taos -C | 取值范围 | 10-50000000 | | 缺省值 | 5000 | -### numOfRpcSessions +### numOfRpcSessions | 属性 | 说明 | | --------| ---------------------- | @@ -120,7 +120,7 @@ taos -C | 缺省值 | 500000 | -### numOfRpcSessions +### numOfRpcSessions | 属性 | 说明 | | -------- | ---------------------------- | @@ -717,6 +717,15 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0: 不启动；1：启动 | | 缺省值 | 1 | +### ttlChangeOnWrite + +| 属性 | 说明 | +| -------- | ------------------ | +| 适用范围 | 仅服务端适用 | +| 含义 | ttl 到期时间是否伴随表的修改操作改变 | +| 取值范围 | 0: 不改变；1：改变 | +| 缺省值 | 0 | + ## 压缩参数 ### compressMsgSize @@ -784,6 +793,7 @@ charset 的有效值是 UTF-8。 | 52 | charset | 是 | 是 | | | 53 | udf | 是 | 是 | | | 54 | enableCoreFile | 是 | 是 | | +| 55 | ttlChangeOnWrite | 否 | 是 | | ## 2.x->3.0 的废弃参数