From be2c16723dad5bb9deb6a55ca940144bc81eae45 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 1 Jul 2024 16:38:47 +0800 Subject: [PATCH 1/3] adj second param for last_row --- source/libs/planner/src/planOptimizer.c | 13 +++++- tests/script/tsim/query/cache_last.sim | 61 +++++++++++++++++-------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index b00767b6b6..47dbfe95d4 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -4056,6 +4056,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic pScan->igLastNull = pAgg->hasLast ? true : false; SArray* isDuplicateCol = taosArrayInit(pScan->pScanCols->length, sizeof(bool)); SNodeList* pLastRowCols = NULL; + bool adjLastRowTsColName = false; + char tsColName[TSDB_COL_NAME_LEN] = {0}; FOREACH(pNode, pAgg->pAggFuncs) { SFunctionNode* pFunc = (SFunctionNode*)pNode; @@ -4094,6 +4096,11 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic sprintf(((SColumnNode*)newColNode)->colName, "#dup_col.%p", newColNode); sprintf(((SColumnNode*)pParamNode)->colName, "#dup_col.%p", newColNode); + if (FUNCTION_TYPE_LAST_ROW == funcType && ((SColumnNode*)pParamNode)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + adjLastRowTsColName = true; + strncpy(tsColName, ((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN); + } + nodesListAppend(pScan->pScanCols, newColNode); isDup = true; taosArrayInsert(isDuplicateCol, pScan->pScanCols->length, &isDup); @@ -4112,7 +4119,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic lastRowScanBuildFuncTypes(pScan, (SColumnNode*)pColNode, pFunc->funcType); } continue; - }else if (nodeListNodeEqual(pFunc->pParameterList, pColNode)) { + } else if (nodeListNodeEqual(pFunc->pParameterList, pColNode)) { if (funcType != FUNCTION_TYPE_LAST && ((SColumnNode*)pColNode)->colId == PRIMARYKEY_TIMESTAMP_COL_ID && !nodeListNodeEqual(pLastRowCols, pColNode)) { nodesListMakeAppend(&pLastRowCols, nodesCloneNode(pColNode)); @@ -4140,6 +4147,10 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (pFunc->hasPk) { nodesListMakeAppend(&cxt.pOtherCols, nodesListGetNode(pFunc->pParameterList, LIST_LENGTH(pFunc->pParameterList) - 1)); } + if (FUNCTION_TYPE_LAST_ROW == funcType && adjLastRowTsColName) { + SNode* pParamNodeTs = nodesListGetNode(pFunc->pParameterList, 1); + strncpy(((SColumnNode*)pParamNodeTs)->colName, tsColName, TSDB_COL_NAME_LEN); + } } else { pNode = nodesListGetNode(pFunc->pParameterList, 0); nodesListMakeAppend(&cxt.pOtherCols, pNode); diff --git a/tests/script/tsim/query/cache_last.sim b/tests/script/tsim/query/cache_last.sim index b2d5ad8aa0..a06e6e1e6f 100644 --- a/tests/script/tsim/query/cache_last.sim +++ b/tests/script/tsim/query/cache_last.sim @@ -141,7 +141,7 @@ if $data01 != t2 then return -1 endi -sql select last(ts) ts1,tbname, ts from stb group by tbname; +sql select last(ts) ts1,tbname, ts from stb group by tbname order by 1 desc; if $data00 != $data02 then print $data00 @@ -156,16 +156,16 @@ endi print step 3------------------------------- sql drop database if exists test1; -sql create database test1 cachemodel 'both'; +sql create database test1 cachemodel 'both' vgroups 4; sql use test1; sql create table stb (ts timestamp,a int primary key,b int,c int) tags(ta int,tb int,tc int); -sql create table t1 using stb tags(1,1,1); -sql create table t2 using stb tags(2,2,2); -sql insert into t1 values('2024-06-05 11:00:00',1,2,3); -sql insert into t1 values('2024-06-05 12:00:00',2,2,3); -sql insert into t2 values('2024-06-05 13:00:00',3,2,3); -sql insert into t2 values('2024-06-05 14:00:00',4,2,3); +sql create table aaat1 using stb tags(1,1,1); +sql create table bbbt2 using stb tags(2,2,2); +sql insert into aaat1 values('2024-06-05 11:00:00',1,2,3); +sql insert into aaat1 values('2024-06-05 12:00:00',2,2,3); +sql insert into bbbt2 values('2024-06-05 13:00:00',3,2,3); +sql insert into bbbt2 values('2024-06-05 14:00:00',4,2,3); sql select last(ts) ts1,ts from stb; @@ -179,7 +179,7 @@ if $data00 != @24-06-05 14:00:00.000@ then return -1 endi -sql select last(ts) ts1,ts from stb group by tbname; +sql select last(ts) ts1,ts from stb group by tbname order by 1 desc; if $data00 != $data01 then print $data00 @@ -203,12 +203,12 @@ if $data00 != @24-06-05 14:00:00.000@ then return -1 endi -if $data01 != t2 then +if $data01 != bbbt2 then print $data01 return -1 endi -sql select last(ts) ts1,tbname, ts from stb group by tbname; +sql select last(ts) ts1,tbname, ts from stb group by tbname order by 1 desc; if $data00 != $data02 then print $data00 @@ -220,7 +220,7 @@ if $data00 != @24-06-05 14:00:00.000@ then return -1 endi -if $data01 != t2 then +if $data01 != bbbt2 then print $data01 return -1 endi @@ -239,7 +239,7 @@ if $data01 != @24-06-05 14:00:00.000@ then return -1 endi -sql select last(a) a,ts from stb group by tbname; +sql select last(a) a,ts from stb group by tbname order by 1 desc; if $data00 != 4 then print $data00 @@ -258,7 +258,7 @@ if $data00 != 4 then return -1 endi -if $data01 != t2 then +if $data01 != bbbt2 then print $data01 return -1 endi @@ -268,14 +268,14 @@ if $data02 != @24-06-05 14:00:00.000@ then return -1 endi -sql select last(a) a,tbname, ts from stb group by tbname; +sql select last(a) a,tbname, ts from stb group by tbname order by 1 desc; if $data00 != 4 then print $data00 return -1 endi -if $data01 != t2 then +if $data01 != bbbt2 then print $data01 return -1 endi @@ -299,7 +299,7 @@ if $data01 != 4 then return -1 endi -sql select last(ts) ts1,a from stb group by tbname; +sql select last(ts) ts1,a from stb group by tbname order by 1 desc; if $data00 != @24-06-05 14:00:00.000@ then print $data00 @@ -318,7 +318,7 @@ if $data00 != @24-06-05 14:00:00.000@ then return -1 endi -if $data01 != t2 then +if $data01 != bbbt2 then print $data01 return -1 endi @@ -328,14 +328,14 @@ if $data02 != 4 then return -1 endi -sql select last(ts) ts1,tbname, a from stb group by tbname; +sql select last(ts) ts1,tbname, a from stb group by tbname order by 1 desc; if $data00 != @24-06-05 14:00:00.000@ then print $data00 return -1 endi -if $data01 != t2 then +if $data01 != bbbt2 then print $data01 return -1 endi @@ -345,4 +345,25 @@ if $data02 != 4 then return -1 endi +$loop_count = 0 + +loop0: + +sql select last(ts), last_row(ts) from stb; + +if $data00 != $data01 then + print ====data00=$data00 + print ====data01=$data01 + return -1 +endi + +$loop_count = $loop_count + 1 +if $loop_count < 10 then + sleep 200 + print ====checktimes=$loop_count + goto loop0 +endi + +print ------------------------end + system sh/exec.sh -n dnode1 -s stop -x SIGINT From b723e0553597d0ea374c7e9cce64a3af3dcff999 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 1 Jul 2024 16:57:46 +0800 Subject: [PATCH 2/3] adj second param for last_row --- source/libs/planner/src/planOptimizer.c | 191 ++++++++++++------------ 1 file changed, 98 insertions(+), 93 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 47dbfe95d4..92b6c2bc73 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -4051,120 +4051,125 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic SNode* pNode = NULL; SColumnNode* pPKTsCol = NULL; SColumnNode* pNonPKCol = NULL; - SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0); + SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0); pScan->scanType = SCAN_TYPE_LAST_ROW; pScan->igLastNull = pAgg->hasLast ? true : false; - SArray* isDuplicateCol = taosArrayInit(pScan->pScanCols->length, sizeof(bool)); + SArray* isDuplicateCol = taosArrayInit(pScan->pScanCols->length, sizeof(bool)); SNodeList* pLastRowCols = NULL; bool adjLastRowTsColName = false; - char tsColName[TSDB_COL_NAME_LEN] = {0}; + char tsColName[TSDB_COL_NAME_LEN] = {0}; FOREACH(pNode, pAgg->pAggFuncs) { SFunctionNode* pFunc = (SFunctionNode*)pNode; int32_t funcType = pFunc->funcType; - SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, 0); - if (FUNCTION_TYPE_LAST_ROW == funcType || FUNCTION_TYPE_LAST == funcType) { - int32_t len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), - FUNCTION_TYPE_LAST_ROW == funcType ? "_cache_last_row" : "_cache_last"); - pFunc->functionName[len] = '\0'; - int32_t code = fmGetFuncInfo(pFunc, NULL, 0); - if (TSDB_CODE_SUCCESS != code) { - nodesClearList(cxt.pLastCols); - return code; - } - cxt.funcType = pFunc->funcType; - cxt.pkBytes = (pFunc->hasPk) ? pFunc->pkBytes : 0; - // add duplicate cols which be removed for both last_row, last - if (pAgg->hasLast && pAgg->hasLastRow) { - if (QUERY_NODE_COLUMN == nodeType(pParamNode)) { - SNode* pColNode = NULL; - int i = 0; - FOREACH(pColNode, pScan->pScanCols) { - bool isDup = false; - bool* isDuplicate = taosArrayGet(isDuplicateCol, i); - if (NULL == isDuplicate) { - taosArrayInsert(isDuplicateCol, i, &isDup); - isDuplicate = taosArrayGet(isDuplicateCol, i); + SNode* pParamNode = NULL; + if (FUNCTION_TYPE_LAST == funcType) { + nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1)); + nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt); + } + FOREACH(pParamNode, pFunc->pParameterList) { + if (FUNCTION_TYPE_LAST_ROW == funcType || FUNCTION_TYPE_LAST == funcType) { + int32_t len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), + FUNCTION_TYPE_LAST_ROW == funcType ? "_cache_last_row" : "_cache_last"); + pFunc->functionName[len] = '\0'; + int32_t code = fmGetFuncInfo(pFunc, NULL, 0); + if (TSDB_CODE_SUCCESS != code) { + nodesClearList(cxt.pLastCols); + return code; + } + cxt.funcType = pFunc->funcType; + cxt.pkBytes = (pFunc->hasPk) ? pFunc->pkBytes : 0; + // add duplicate cols which be removed for both last_row, last + if (pAgg->hasLast && pAgg->hasLastRow) { + if (QUERY_NODE_COLUMN == nodeType(pParamNode)) { + SNode* pColNode = NULL; + int i = 0; + FOREACH(pColNode, pScan->pScanCols) { + bool isDup = false; + bool* isDuplicate = taosArrayGet(isDuplicateCol, i); + if (NULL == isDuplicate) { + taosArrayInsert(isDuplicateCol, i, &isDup); + isDuplicate = taosArrayGet(isDuplicateCol, i); + } + i++; + if (nodesEqualNode(pParamNode, pColNode)) { + if (*isDuplicate) { + if (0 == strncmp(((SColumnNode*)pColNode)->colName, "#dup_col.", 9)) { + continue; + } + SNode* newColNode = nodesCloneNode(pColNode); + sprintf(((SColumnNode*)newColNode)->colName, "#dup_col.%p", newColNode); + sprintf(((SColumnNode*)pParamNode)->colName, "#dup_col.%p", newColNode); + if (FUNCTION_TYPE_LAST_ROW == funcType && + ((SColumnNode*)pParamNode)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (!adjLastRowTsColName) { + adjLastRowTsColName = true; + strncpy(tsColName, ((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN); + } else { + strncpy(((SColumnNode*)pParamNode)->colName, tsColName, TSDB_COL_NAME_LEN); + nodesDestroyNode(newColNode); + continue; + } + } + + nodesListAppend(pScan->pScanCols, newColNode); + isDup = true; + taosArrayInsert(isDuplicateCol, pScan->pScanCols->length, &isDup); + nodesListAppend(pScan->node.pTargets, nodesCloneNode(newColNode)); + if (funcType != FUNCTION_TYPE_LAST) { + nodesListMakeAppend(&pLastRowCols, nodesCloneNode(newColNode)); + } + + lastRowScanBuildFuncTypes(pScan, (SColumnNode*)newColNode, pFunc->funcType); + } else { + isDup = true; + *isDuplicate = isDup; + if (funcType != FUNCTION_TYPE_LAST && !nodeListNodeEqual(cxt.pLastCols, pColNode)) { + nodesListMakeAppend(&pLastRowCols, nodesCloneNode(pColNode)); + } + lastRowScanBuildFuncTypes(pScan, (SColumnNode*)pColNode, pFunc->funcType); + } + continue; + }else if (nodeListNodeEqual(pFunc->pParameterList, pColNode)) { + if (funcType != FUNCTION_TYPE_LAST && ((SColumnNode*)pColNode)->colId == PRIMARYKEY_TIMESTAMP_COL_ID && + !nodeListNodeEqual(pLastRowCols, pColNode)) { + nodesListMakeAppend(&pLastRowCols, nodesCloneNode(pColNode)); + + lastRowScanBuildFuncTypes(pScan, (SColumnNode*)pColNode, pFunc->funcType); + isDup = true; + *isDuplicate = isDup; + } + } } - i++; - if (nodesEqualNode(pParamNode, pColNode)) { - if (*isDuplicate) { - if (0 == strncmp(((SColumnNode*)pColNode)->colName, "#dup_col.", 9)) { - continue; - } - SNode* newColNode = nodesCloneNode(pColNode); - sprintf(((SColumnNode*)newColNode)->colName, "#dup_col.%p", newColNode); - sprintf(((SColumnNode*)pParamNode)->colName, "#dup_col.%p", newColNode); - - if (FUNCTION_TYPE_LAST_ROW == funcType && ((SColumnNode*)pParamNode)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - adjLastRowTsColName = true; - strncpy(tsColName, ((SColumnNode*)pParamNode)->colName, TSDB_COL_NAME_LEN); - } - - nodesListAppend(pScan->pScanCols, newColNode); - isDup = true; - taosArrayInsert(isDuplicateCol, pScan->pScanCols->length, &isDup); - nodesListAppend(pScan->node.pTargets, nodesCloneNode(newColNode)); + FOREACH(pColNode, pScan->pScanPseudoCols) { + if (nodesEqualNode(pParamNode, pColNode)) { if (funcType != FUNCTION_TYPE_LAST) { - nodesListMakeAppend(&pLastRowCols, nodesCloneNode(newColNode)); - } - - lastRowScanBuildFuncTypes(pScan, (SColumnNode*)newColNode, pFunc->funcType); - } else { - isDup = true; - *isDuplicate = isDup; - if (funcType != FUNCTION_TYPE_LAST && !nodeListNodeEqual(cxt.pLastCols, pColNode)) { nodesListMakeAppend(&pLastRowCols, nodesCloneNode(pColNode)); } - lastRowScanBuildFuncTypes(pScan, (SColumnNode*)pColNode, pFunc->funcType); - } - continue; - } else if (nodeListNodeEqual(pFunc->pParameterList, pColNode)) { - if (funcType != FUNCTION_TYPE_LAST && ((SColumnNode*)pColNode)->colId == PRIMARYKEY_TIMESTAMP_COL_ID && - !nodeListNodeEqual(pLastRowCols, pColNode)) { - nodesListMakeAppend(&pLastRowCols, nodesCloneNode(pColNode)); - - lastRowScanBuildFuncTypes(pScan, (SColumnNode*)pColNode, pFunc->funcType); - isDup = true; - *isDuplicate = isDup; - } - } - } - FOREACH(pColNode, pScan->pScanPseudoCols) { - if (nodesEqualNode(pParamNode, pColNode)) { - if (funcType != FUNCTION_TYPE_LAST) { - nodesListMakeAppend(&pLastRowCols, nodesCloneNode(pColNode)); } } } } - } - if (FUNCTION_TYPE_LAST == funcType) { - nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt); - nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1)); - } - if (pFunc->hasPk) { - nodesListMakeAppend(&cxt.pOtherCols, nodesListGetNode(pFunc->pParameterList, LIST_LENGTH(pFunc->pParameterList) - 1)); - } - if (FUNCTION_TYPE_LAST_ROW == funcType && adjLastRowTsColName) { - SNode* pParamNodeTs = nodesListGetNode(pFunc->pParameterList, 1); - strncpy(((SColumnNode*)pParamNodeTs)->colName, tsColName, TSDB_COL_NAME_LEN); - } - } else { - pNode = nodesListGetNode(pFunc->pParameterList, 0); - nodesListMakeAppend(&cxt.pOtherCols, pNode); + if (pFunc->hasPk) { + nodesListMakeAppend(&cxt.pOtherCols, nodesListGetNode(pFunc->pParameterList, LIST_LENGTH(pFunc->pParameterList) - 1)); + } + } else { + pNode = nodesListGetNode(pFunc->pParameterList, 0); + nodesListMakeAppend(&cxt.pOtherCols, pNode); - if (FUNCTION_TYPE_SELECT_VALUE == funcType) { - if (nodeType(pNode) == QUERY_NODE_COLUMN) { - SColumnNode* pCol = (SColumnNode*)pNode; - if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pPKTsCol = pCol; - } else { - pNonPKCol = pCol; + if (FUNCTION_TYPE_SELECT_VALUE == funcType) { + if (nodeType(pNode) == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)pNode; + if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pPKTsCol = pCol; + } else { + pNonPKCol = pCol; + } } } } + WHERE_NEXT; } } From ff32797e05de6d20dd87981bfe3992fde4dbfa42 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 1 Jul 2024 16:58:39 +0800 Subject: [PATCH 3/3] adj second param for last_row --- source/libs/planner/src/planOptimizer.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 92b6c2bc73..c32c75386c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -4169,7 +4169,6 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic } } } - WHERE_NEXT; } }