diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5b889171b3..befd6afce9 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -415,6 +415,7 @@ typedef struct SSelectStmt { int32_t returnRows; // EFuncReturnRows ETimeLineMode timeLineCurMode; ETimeLineMode timeLineResMode; + bool timeLineFromOrderBy; bool isEmptyResult; bool isSubquery; bool hasAggFuncs; @@ -453,6 +454,7 @@ typedef struct SSetOperator { char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; ETimeLineMode timeLineResMode; + bool timeLineFromOrderBy; bool joinContains; } SSetOperator; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1a51620856..84d3f734fe 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -845,6 +845,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(isEmptyResult); COPY_SCALAR_FIELD(timeLineResMode); + COPY_SCALAR_FIELD(timeLineFromOrderBy); COPY_SCALAR_FIELD(timeLineCurMode); COPY_SCALAR_FIELD(hasAggFuncs); COPY_SCALAR_FIELD(hasRepeatScanFuncs); @@ -862,6 +863,8 @@ static int32_t setOperatorCopy(const SSetOperator* pSrc, SSetOperator* pDst) { COPY_CHAR_ARRAY_FIELD(stmtName); COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(timeLineResMode); + COPY_SCALAR_FIELD(timeLineFromOrderBy); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 79b37c2950..c9a05a348c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -959,7 +959,8 @@ static bool isTimeLineQuery(SNode* pStmt) { return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineCurMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineCurMode); } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { - return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; + return (TIME_LINE_MULTI == ((SSetOperator*)pStmt)->timeLineResMode) || + (TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode); } else { return false; } @@ -1000,18 +1001,64 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) { return false; } +SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) { + SNodeList* pPartitionList = NULL; + SNode* pNode = NULL; + if (pOrderList->length <= nodesNum) { + return NULL; + } + + pNode = nodesListGetNode(pOrderList, nodesNum); + SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode; + if (!isPrimaryKeyImpl(pOrder->pExpr)) { + return NULL; + } + + for (int32_t i = 0; i < nodesNum; ++i) { + pNode = nodesListGetNode(pOrderList, i); + pOrder = (SOrderByExprNode*)pNode; + nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr)); + } + + return pPartitionList; +} + + static bool isTimeLineAlignedQuery(SNode* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt; if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return false; } - if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { - return false; + if (QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (pSelect->pPartitionByList) { + if (!pSub->timeLineFromOrderBy && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { + return true; + } + if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) { + SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length); + bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); + nodesDestroyList(pPartitionList); + + if (match) { + return true; + } + } + } } - SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery; - if (pSelect->pPartitionByList && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { - return true; + if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) { + SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length); + bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); + nodesDestroyList(pPartitionList); + + if (match) { + return true; + } + } } + return false; } @@ -6025,9 +6072,19 @@ static void resetResultTimeline(SSelectStmt* pSelect) { if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder)) || (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) { pSelect->timeLineResMode = TIME_LINE_GLOBAL; - } else { - pSelect->timeLineResMode = TIME_LINE_NONE; + return; + } else if (pSelect->pOrderByList->length > 1) { + for (int32_t i = 1; i < pSelect->pOrderByList->length; ++i) { + pOrder = ((SOrderByExprNode*)nodesListGetNode(pSelect->pOrderByList, i))->pExpr; + if (isPrimaryKeyImpl(pOrder)) { + pSelect->timeLineResMode = TIME_LINE_MULTI; + pSelect->timeLineFromOrderBy = true; + return; + } + } } + + pSelect->timeLineResMode = TIME_LINE_NONE; } static int32_t replaceOrderByAliasForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -6180,16 +6237,13 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS } snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft); - if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight)) { - SColumnNode* pLCol = (SColumnNode*)pLeft; - SColumnNode* pRCol = (SColumnNode*)pRight; + bool isLeftPrimTs = isPrimaryKeyImpl(pLeft); + bool isRightPrimTs = isPrimaryKeyImpl(pRight); + + if (isLeftPrimTs && isRightPrimTs) { SColumnNode* pFCol = (SColumnNode*)pProj; - if (pLCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID && pRCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - if (pLCol->isPrimTs && pRCol->isPrimTs) { - pFCol->isPrimTs = true; - } - } + pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + pFCol->isPrimTs = true; } if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) { return TSDB_CODE_OUT_OF_MEMORY; @@ -6225,9 +6279,19 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS SNode* pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, 0))->pExpr; if (isPrimaryKeyImpl(pOrder)) { pSetOperator->timeLineResMode = TIME_LINE_GLOBAL; - } else { - pSetOperator->timeLineResMode = TIME_LINE_NONE; + return code; + } else if (pSetOperator->pOrderByList->length > 1) { + for (int32_t i = 1; i < pSetOperator->pOrderByList->length; ++i) { + pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, i))->pExpr; + if (isPrimaryKeyImpl(pOrder)) { + pSetOperator->timeLineResMode = TIME_LINE_MULTI; + pSetOperator->timeLineFromOrderBy = true; + return code; + } + } } + + pSetOperator->timeLineResMode = TIME_LINE_NONE; } return code; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 342389562a..866d7a152a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -994,6 +994,7 @@ ,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py #tsim test +,,y,script,./test.sh -f tsim/query/timeline.sim ,,y,script,./test.sh -f tsim/join/join.sim ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim ,,y,script,./test.sh -f tsim/parser/where.sim diff --git a/tests/script/tsim/query/timeline.sim b/tests/script/tsim/query/timeline.sim new file mode 100644 index 0000000000..743a6f1a9f --- /dev/null +++ b/tests/script/tsim/query/timeline.sim @@ -0,0 +1,51 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; + +sql CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256)); +sql CREATE TABLE demo_1 USING demo (deviceid) TAGS ('1'); +sql CREATE TABLE demo_2 USING demo (deviceid) TAGS ('2'); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 1.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 01:00:00.001', 2.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 02:00:00.002', 3.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-05 03:00:00.003', 4.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 5.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 01:00:00.001', 6.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 02:00:00.002', 7.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-29 03:00:00.003', 8.0); + +sql_error select diff(faev) from ((select ts, faev from demo union all select ts, faev from demo)); +sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts); +sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev from demo union all select _ts + 1s, faev from demo order by faev, _ts) partition by faev; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by deviceid, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev; + +sql_error select diff(faev) from (select _ts, faev from demo); +sql_error select diff(faev) from (select _ts, faev from demo order by faev, _ts); +sql select diff(faev) from (select _ts, faev from demo order by faev, _ts) partition by faev; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts) partition by deviceid; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by deviceid, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev; + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY ts, deviceid) PARTITION by deviceid; + + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid; +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY ts, deviceid) PARTITION by deviceid; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT