From f45f3b460f292fdeeb399a82aa4771f6a787bce0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 14:30:58 +0800 Subject: [PATCH 1/8] enh: optimize count(1) performance --- source/libs/parser/src/parTranslater.c | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 25e54fc5c4..ded40f7523 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1331,6 +1331,32 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) return code; } +static bool isCountNotNullValue(SFunctionNode* pFunc) { + if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { + return false; + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + return (QUERY_NODE_VALUE == nodeType(pPara) && !((SValueNode*)pPara)->isNull); +} + +// count(1) is rewritten as count(ts) for scannning optimization +static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* pCount) { + SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, NULL, &pTable); + if (TSDB_CODE_SUCCESS == code) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); + NODES_DESTORY_LIST(pCount->pParameterList); + code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol); + } + } + return code; +} + static bool isCountTbname(SFunctionNode* pFunc) { if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { return false; @@ -1396,6 +1422,9 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (isCountStar(pFunc)) { return rewriteCountStar(pCxt, pFunc); } + if (isCountNotNullValue(pFunc)) { + return rewriteCountNotNullValue(pCxt, pFunc); + } if (isCountTbname(pFunc)) { return rewriteCountTbname(pCxt, pFunc); } From 2bb7d6c799ecc4fe40e746abe546d4988e8afdf3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 14:38:03 +0800 Subject: [PATCH 2/8] fix: invalid write issue --- source/client/src/clientImpl.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 70185a1395..bef7da2c33 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1132,11 +1132,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM pRequest->body.queryFp(pRequest->body.param, pRequest, -1); break; } - - // TODO weired responding code? - if (TSDB_CODE_SUCCESS != code) { - pRequest->code = terrno; - } } int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { From c2f346407dde40cde43131da2f184b9f621c8ab6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 15:17:09 +0800 Subject: [PATCH 3/8] fix: order by error info --- include/util/taoserror.h | 1 + source/libs/parser/src/parTranslater.c | 6 +++--- source/libs/parser/src/parUtil.c | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f626f49661..6489304bda 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -688,6 +688,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_NOT_SUPPORT_JOIN TAOS_DEF_ERROR_CODE(0, 0x2664) #define TSDB_CODE_PAR_INVALID_TAGS_PC TAOS_DEF_ERROR_CODE(0, 0x2665) #define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY TAOS_DEF_ERROR_CODE(0, 0x2666) +#define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 25e54fc5c4..1bc5976371 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2041,7 +2041,7 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; } - return TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN; + return TSDB_CODE_PAR_INVALID_OPTR_USAGE; } static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) { @@ -2114,13 +2114,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->aliasName); } else { return rewriteColToSelectValFunc(pCxt, pNode); } } if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->aliasName); } return DEAL_RES_CONTINUE; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index a4cf2f603d..793d05721e 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -174,6 +174,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid usage of RANGE clause, EVERY clause or FILL clause"; case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN: return "No valid function in window query"; + case TSDB_CODE_PAR_INVALID_OPTR_USAGE: + return "Invalid usage of expr: %s"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: From 859f2594ed3371f757430e096ba857d9ba2f0444 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 15:21:43 +0800 Subject: [PATCH 4/8] fix: add sma load time --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c939c11536..5c7c63f59f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4550,6 +4550,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, return TSDB_CODE_SUCCESS; } + int64_t st = taosGetTimestampUs(); + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (tDataBlkHasSma(pBlock)) { code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); @@ -4611,6 +4613,9 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, *pBlockSMA = pResBlock->pBlockAgg; pReader->cost.smaDataLoad += 1; + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + pReader->cost.smaLoadTime += elapsedTime; + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; } From 746174ea45dbfeb9df2f4db3f5a62ba95a7c01fd Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 16:16:15 +0800 Subject: [PATCH 5/8] fix: remove count optimize --- source/libs/parser/src/parTranslater.c | 29 -------------------------- 1 file changed, 29 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ded40f7523..25e54fc5c4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1331,32 +1331,6 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) return code; } -static bool isCountNotNullValue(SFunctionNode* pFunc) { - if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { - return false; - } - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - return (QUERY_NODE_VALUE == nodeType(pPara) && !((SValueNode*)pPara)->isNull); -} - -// count(1) is rewritten as count(ts) for scannning optimization -static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* pCount) { - SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0); - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, NULL, &pTable); - if (TSDB_CODE_SUCCESS == code) { - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); - NODES_DESTORY_LIST(pCount->pParameterList); - code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol); - } - } - return code; -} - static bool isCountTbname(SFunctionNode* pFunc) { if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { return false; @@ -1422,9 +1396,6 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (isCountStar(pFunc)) { return rewriteCountStar(pCxt, pFunc); } - if (isCountNotNullValue(pFunc)) { - return rewriteCountNotNullValue(pCxt, pFunc); - } if (isCountTbname(pFunc)) { return rewriteCountTbname(pCxt, pFunc); } From af5c214e16cd76fef3d5f36b8b4657dc803e1c47 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 16:36:13 +0800 Subject: [PATCH 6/8] fix: count(1) from nested query issue --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ded40f7523..1d9abe15c1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1344,7 +1344,7 @@ static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0); STableNode* pTable = NULL; int32_t code = findTable(pCxt, NULL, &pTable); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { code = TSDB_CODE_OUT_OF_MEMORY; From 28905af158279ce0a51131d031584a47b6c44909 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 17:01:46 +0800 Subject: [PATCH 7/8] fix: column alisa issue and ut case --- source/libs/parser/src/parTranslater.c | 4 ++-- source/libs/parser/test/parSelectTest.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1bc5976371..3faeae0118 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2114,13 +2114,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->aliasName); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } else { return rewriteColToSelectValFunc(pCxt, pNode); } } if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->aliasName); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } return DEAL_RES_CONTINUE; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index fcd8dd1f26..ec6c69ea8d 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -286,7 +286,7 @@ TEST_F(ParserSelectTest, interval) { TEST_F(ParserSelectTest, intervalSemanticCheck) { useDb("root", "test"); - run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN); + run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_INVALID_OPTR_USAGE); run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 " "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)", From 4685fb48704c2789b4ed00c95fb8ea9d77b9bb1e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Mar 2023 17:52:15 +0800 Subject: [PATCH 8/8] fix: no error passed issue --- source/client/src/clientImpl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index bef7da2c33..1b4292b943 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1085,6 +1085,10 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + if (TSDB_CODE_SUCCESS != code) { + pRequest->code = terrno; + } + pRequest->body.queryFp(pRequest->body.param, pRequest, code); }