feat: the unique function and tail function are rewritten as the corresponding clauses

include/libs/nodes/nodes.h

@@ -102,6 +102,7 @@ typedef enum ENodeType {
   QUERY_NODE_EXPLAIN_OPTIONS,
   QUERY_NODE_STREAM_OPTIONS,
   QUERY_NODE_LEFT_VALUE,
+  QUERY_NODE_COLUMN_REF,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,

include/libs/nodes/querynodes.h

@@ -65,11 +65,16 @@ typedef struct SColumnNode {
   char        tableName[TSDB_TABLE_NAME_LEN];
   char        tableAlias[TSDB_TABLE_NAME_LEN];
   char        colName[TSDB_COL_NAME_LEN];
-  SNode*      pProjectRef;
+  // SNode*      pProjectRef;
   int16_t dataBlockId;
   int16_t slotId;
 } SColumnNode;
 
+typedef struct SColumnRefNode {
+  ENodeType type;
+  char      colName[TSDB_COL_NAME_LEN];
+} SColumnRefNode;
+
 typedef struct STargetNode {
   ENodeType type;
   int16_t   dataBlockId;

source/libs/nodes/src/nodesUtilFuncs.c

@@ -88,6 +88,8 @@ SNode* nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SStreamOptions));
     case QUERY_NODE_LEFT_VALUE:
       return makeNode(type, sizeof(SLeftValueNode));
+    case QUERY_NODE_COLUMN_REF:
+      return makeNode(type, sizeof(SColumnDefNode));
     case QUERY_NODE_SET_OPERATOR:
       return makeNode(type, sizeof(SSetOperator));
     case QUERY_NODE_SELECT_STMT:

source/libs/parser/src/parAstCreater.c

@@ -648,6 +648,8 @@ SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrder
   CHECK_PARSER_STATUS(pCxt);
   if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
     ((SSelectStmt*)pStmt)->pOrderByList = pOrderByList;
+  } else {
+    ((SSetOperator*)pStmt)->pOrderByList = pOrderByList;
   }
   return pStmt;
 }

source/libs/parser/src/parTranslater.c

@@ -36,6 +36,7 @@ typedef struct STranslateContext {
   int32_t          currLevel;
   ESqlClause       currClause;
   SSelectStmt*     pCurrSelectStmt;
+  SSetOperator*    pCurrSetOperator;
   SCmdMsgInfo*     pCmdMsg;
   SHashObj*        pDbs;
   SHashObj*        pTables;
@@ -432,7 +433,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p
 static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SColumnNode** pColRef) {
   SColumnNode* pCol = *pColRef;
 
-  pCol->pProjectRef = (SNode*)pExpr;
+  // pCol->pProjectRef = (SNode*)pExpr;
   if (NULL == pExpr->pAssociation) {
     pExpr->pAssociation = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
   }
@@ -613,17 +614,36 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod
   return DEAL_RES_CONTINUE;
 }
 
-static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** pCol) {
+static SNodeList* getProjectListFromCxt(STranslateContext* pCxt) {
-  SNodeList* pProjectionList = pCxt->pCurrSelectStmt->pProjectionList;
+  if (NULL != pCxt->pCurrSelectStmt) {
+    return pCxt->pCurrSelectStmt->pProjectionList;
+  } else if (NULL != pCxt->pCurrSetOperator) {
+    return pCxt->pCurrSetOperator->pProjectionList;
+  } else {
+    return NULL;
+  }
+}
+
+static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** pCol, bool* pFound) {
+  SNodeList* pProjectionList = getProjectListFromCxt(pCxt);
   SNode*     pNode;
   FOREACH(pNode, pProjectionList) {
     SExprNode* pExpr = (SExprNode*)pNode;
     if (0 == strcmp((*pCol)->colName, pExpr->aliasName)) {
-      setColumnInfoByExpr(NULL, pExpr, pCol);
+      SColumnRefNode* pColRef = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF);
-      return true;
+      if (NULL == pColRef) {
+        pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
+        return DEAL_RES_ERROR;
+      }
+      strcpy(pColRef->colName, pExpr->aliasName);
+      nodesDestroyNode(*(SNode**)pCol);
+      *(SNode**)pCol = (SNode*)pColRef;
+      *pFound = true;
+      return DEAL_RES_CONTINUE;
     }
   }
-  return false;
+  *pFound = false;
+  return DEAL_RES_CONTINUE;
 }
 
 static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
@@ -638,9 +658,11 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
   } else {
     bool found = false;
     if (SQL_CLAUSE_ORDER_BY == pCxt->currClause) {
-      found = translateColumnUseAlias(pCxt, pCol);
+      res = translateColumnUseAlias(pCxt, pCol, &found);
+    }
+    if (DEAL_RES_ERROR != res && !found) {
+      res = translateColumnWithoutPrefix(pCxt, pCol);
     }
-    res = (found ? DEAL_RES_CONTINUE : translateColumnWithoutPrefix(pCxt, pCol));
   }
   return res;
 }
@@ -1803,11 +1825,11 @@ static int32_t translateOrderByPosition(STranslateContext* pCxt, SNodeList* pPro
       } else if (0 == pos || pos > LIST_LENGTH(pProjectionList)) {
         return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT);
       } else {
-        SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+        SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF);
         if (NULL == pCol) {
           return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY);
         }
-        setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos - 1), &pCol);
+        strcpy(pCol->colName, ((SExprNode*)nodesListGetNode(pProjectionList, pos - 1))->aliasName);
         ((SOrderByExprNode*)pNode)->pExpr = (SNode*)pCol;
         nodesDestroyNode(pExpr);
       }
@@ -1822,9 +1844,7 @@ static int32_t translateOrderByPosition(STranslateContext* pCxt, SNodeList* pPro
 static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
   bool    other;
   int32_t code = translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other);
-  if (TSDB_CODE_SUCCESS != code) {
+  if (TSDB_CODE_SUCCESS == code) {
-    return code;
-  }
     if (!other) {
       return TSDB_CODE_SUCCESS;
     }
@@ -1833,6 +1853,7 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
     if (TSDB_CODE_SUCCESS == code) {
       code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList);
     }
+  }
   return code;
 }
 
@@ -2290,6 +2311,39 @@ static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
   return code;
 }
 
+typedef struct SReplaceOrderByAliasCxt {
+  STranslateContext* pTranslateCxt;
+  SNodeList*         pProjectionList;
+} SReplaceOrderByAliasCxt;
+
+static EDealRes replaceOrderByAliasImpl(SNode** pNode, void* pContext) {
+  SReplaceOrderByAliasCxt* pCxt = pContext;
+  if (QUERY_NODE_COLUMN_REF == nodeType(*pNode)) {
+    SNodeList* pProjectionList = pCxt->pProjectionList;
+    SNode*     pProject = NULL;
+    FOREACH(pProject, pProjectionList) {
+      SExprNode* pExpr = (SExprNode*)pProject;
+      if (0 == strcmp(((SColumnRefNode*)*pNode)->colName, pExpr->aliasName)) {
+        SNode* pNew = nodesCloneNode(pProject);
+        if (NULL == pNew) {
+          pCxt->pTranslateCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
+          return DEAL_RES_ERROR;
+        }
+        nodesDestroyNode(*pNode);
+        *pNode = pNew;
+        return DEAL_RES_CONTINUE;
+      }
+    }
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static int32_t replaceOrderByAlias(STranslateContext* pCxt, SNodeList* pProjectionList, SNodeList* pOrderByList) {
+  SReplaceOrderByAliasCxt cxt = {.pTranslateCxt = pCxt, .pProjectionList = pProjectionList};
+  nodesRewriteExprsPostOrder(pOrderByList, replaceOrderByAliasImpl, &cxt);
+  return pCxt->errCode;
+}
+
 static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
   pCxt->pCurrSelectStmt = pSelect;
   int32_t code = translateFrom(pCxt, pSelect->pFromTable);
@@ -2324,12 +2378,15 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (TSDB_CODE_SUCCESS == code) {
     code = rewriteUniqueStmt(pCxt, pSelect);
   }
-  if (TSDB_CODE_SUCCESS == code) {
+  // if (TSDB_CODE_SUCCESS == code) {
-    code = rewriteTailStmt(pCxt, pSelect);
+  //   code = rewriteTailStmt(pCxt, pSelect);
-  }
+  // }
   if (TSDB_CODE_SUCCESS == code) {
     code = rewriteTimelineFunc(pCxt, pSelect);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = replaceOrderByAlias(pCxt, pSelect->pProjectionList, pSelect->pOrderByList);
+  }
   return code;
 }
 
@@ -2366,7 +2423,7 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* pSetOperator) {
+static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pSetOperator) {
   SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft);
   SNodeList* pRightProjections = getProjectList(pSetOperator->pRight);
   if (LIST_LENGTH(pLeftProjections) != LIST_LENGTH(pRightProjections)) {
@@ -2401,6 +2458,23 @@ static uint8_t calcSetOperatorPrecision(SSetOperator* pSetOperator) {
   return calcPrecision(getStmtPrecision(pSetOperator->pLeft), getStmtPrecision(pSetOperator->pRight));
 }
 
+static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pSetOperator) {
+  bool    other;
+  int32_t code = translateOrderByPosition(pCxt, pSetOperator->pProjectionList, pSetOperator->pOrderByList, &other);
+  if (TSDB_CODE_SUCCESS == code) {
+    if (other) {
+      pCxt->currClause = SQL_CLAUSE_ORDER_BY;
+      pCxt->pCurrSelectStmt = NULL;
+      pCxt->pCurrSetOperator = pSetOperator;
+      code = translateExprList(pCxt, pSetOperator->pOrderByList);
+    }
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = replaceOrderByAlias(pCxt, pSetOperator->pProjectionList, pSetOperator->pOrderByList);
+  }
+  return code;
+}
+
 static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetOperator) {
   int32_t code = translateQuery(pCxt, pSetOperator->pLeft);
   if (TSDB_CODE_SUCCESS == code) {
@@ -2411,7 +2485,10 @@ static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetO
   }
   if (TSDB_CODE_SUCCESS == code) {
     pSetOperator->precision = calcSetOperatorPrecision(pSetOperator);
-    code = translateSetOperatorImpl(pCxt, pSetOperator);
+    code = translateSetOperProject(pCxt, pSetOperator);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateSetOperOrderBy(pCxt, pSetOperator);
   }
   return code;
 }

source/libs/parser/test/parSelectTest.cpp

@@ -362,6 +362,8 @@ TEST_F(ParserSelectTest, setOperator) {
   run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)");
 
   run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
+
+  run("SELECT c1, c2 FROM t1 UNION ALL SELECT c1 as a, c2 as b FROM t1 ORDER BY c1");
 }
 
 TEST_F(ParserSelectTest, informationSchema) {

source/libs/planner/test/planBasicTest.cpp

@@ -62,6 +62,8 @@ TEST_F(PlanBasicTest, uniqueFunc) {
   run("SELECT UNIQUE(c2 + 10) FROM t1 WHERE c1 > 10");
 
   run("SELECT UNIQUE(c2 + 10), ts, c2 FROM t1 WHERE c1 > 10");
+
+  run("SELECT UNIQUE(c1) a FROM t1 ORDER BY a");
 }
 
 TEST_F(PlanBasicTest, tailFunc) {

source/libs/planner/test/planOrderByTest.cpp

@@ -27,6 +27,8 @@ TEST_F(PlanOrderByTest, basic) {
   run("SELECT c1 FROM t1 ORDER BY c1");
   // ORDER BY key is not in the projection list
   run("SELECT c1 FROM t1 ORDER BY c2");
+
+  run("SELECT c1 + 10 AS a FROM t1 ORDER BY a");
 }
 
 TEST_F(PlanOrderByTest, expr) {
@@ -41,6 +43,12 @@ TEST_F(PlanOrderByTest, nullsOrder) {
   run("SELECT * FROM t1 ORDER BY c1 DESC NULLS FIRST");
 }
 
+TEST_F(PlanOrderByTest, withGroupBy) {
+  useDb("root", "test");
+
+  run("SELECT SUM(c1) AS a FROM t1 GROUP BY c2 ORDER BY a");
+}
+
 TEST_F(PlanOrderByTest, stable) {
   useDb("root", "test");
 

source/libs/planner/test/planSetOpTest.cpp

@@ -23,9 +23,9 @@ class PlanSetOpTest : public PlannerTestBase {};
 TEST_F(PlanSetOpTest, unionAll) {
   useDb("root", "test");
 
-  // sql 1: single UNION ALL operator
+  // single UNION ALL operator
   run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20");
-  // sql 2: multi UNION ALL operator
+  // multi UNION ALL operator
   run("SELECT c1, c2 FROM t1 WHERE c1 > 10 "
       "UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20 "
       "UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 30");
@@ -46,6 +46,14 @@ TEST_F(PlanSetOpTest, unionAllWithSubquery) {
   run("SELECT ts FROM (SELECT ts FROM st1) UNION ALL SELECT ts FROM (SELECT ts FROM st1)");
 }
 
+TEST_F(PlanSetOpTest, unionAllWithOrderBy) {
+  useDb("root", "test");
+
+  run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20 ORDER BY c1");
+
+  run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20 ORDER BY 1");
+}
+
 TEST_F(PlanSetOpTest, union) {
   useDb("root", "test");
 

tests/system-test/2-query/tail.py

@@ -188,8 +188,8 @@ class TDTestCase:
 
     def check_tail_table(self , tbname , col_name , tail_rows , offset):
         tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}"
-        #equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts"
+        equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts"
-        equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}"
+        #equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}"
         tdSql.query(tail_sql)
         tail_result = tdSql.queryResult
 
@@ -294,19 +294,19 @@ class TDTestCase:
         tdSql.checkData(0, 0, None)
         tdSql.checkData(1, 0, None)
 
-        tdSql.query("select tail(c1,3,2) from ct4 where c1 >2 ")
+        tdSql.query("select tail(c1,3,2) from ct4 where c1 >2 order by 1")
         tdSql.checkData(0, 0, 5)
         tdSql.checkData(1, 0, 6)
         tdSql.checkData(2, 0, 7)
 
-        tdSql.query("select tail(c1,2,1) from ct4  where c2 between 0  and   99999")
+        tdSql.query("select tail(c1,2,1) from ct4  where c2 between 0  and   99999 order by 1")
         tdSql.checkData(0, 0, 1)
         tdSql.checkData(1, 0, 2)
 
         # tail with union all 
         tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct1")
         tdSql.checkRows(15)
-        tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct2")
+        tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct2 order by 1")
         tdSql.checkRows(2)
         tdSql.checkData(0, 0, 0)
         tdSql.checkData(1, 0, 1)
@@ -335,7 +335,7 @@ class TDTestCase:
             tdSql.execute(f" insert into ttb1 values({ts_value} , {i})")
             tdSql.execute(f" insert into ttb2 values({ts_value} , {i})")
 
-        tdSql.query("select tail(tb2.num,3,2)  from tb1, tb2 where tb1.ts=tb2.ts ")
+        tdSql.query("select tail(tb2.num,3,2)  from tb1, tb2 where tb1.ts=tb2.ts order by 1 desc")
         tdSql.checkRows(3)
         tdSql.checkData(0,0,7)
         tdSql.checkData(1,0,6)
@@ -343,7 +343,7 @@ class TDTestCase:
 
         # nest query
         # tdSql.query("select tail(c1,2) from (select c1 from ct1)")
-        tdSql.query("select c1 from (select tail(c1,2) c1 from ct4)")
+        tdSql.query("select c1 from (select tail(c1,2) c1 from ct4) order by 1 nulls first")
         tdSql.checkRows(2)
         tdSql.checkData(0, 0, None)
         tdSql.checkData(1, 0, 0)

tests/system-test/2-query/unique.py

@@ -289,13 +289,13 @@ class TDTestCase:
         tdSql.query("select unique(c1) from ct4 where c1 is null")
         tdSql.checkData(0, 0, None)
 
-        tdSql.query("select unique(c1) from ct4 where c1 >2")
+        tdSql.query("select unique(c1) from ct4 where c1 >2 order by 1")
-        tdSql.checkData(0, 0, 8)
+        tdSql.checkData(0, 0, 3)
-        tdSql.checkData(1, 0, 7)
+        tdSql.checkData(1, 0, 4)
-        tdSql.checkData(2, 0, 6)
+        tdSql.checkData(2, 0, 5)
-        tdSql.checkData(5, 0, 3)
+        tdSql.checkData(5, 0, 8)
 
-        tdSql.query("select unique(c1) from ct4  where c2 between 0  and   99999")
+        tdSql.query("select unique(c1) from ct4  where c2 between 0  and   99999 order by 1 desc")
         tdSql.checkData(0, 0, 8)
         tdSql.checkData(1, 0, 7)
         tdSql.checkData(2, 0, 6)
@@ -336,23 +336,23 @@ class TDTestCase:
             tdSql.execute(f" insert into ttb1 values({ts_value} , {i})")
             tdSql.execute(f" insert into ttb2 values({ts_value} , {i})")
 
-        tdSql.query("select unique(tb2.num)  from tb1, tb2 where tb1.ts=tb2.ts ")
+        tdSql.query("select unique(tb2.num)  from tb1, tb2 where tb1.ts=tb2.ts order by 1")
         tdSql.checkRows(10)
         tdSql.checkData(0,0,0)
         tdSql.checkData(1,0,1)
         tdSql.checkData(2,0,2)
         tdSql.checkData(9,0,9)
 
-        tdSql.query("select unique(tb2.num)  from tb1, tb2 where tb1.ts=tb2.ts union all select unique(tb1.num)  from tb1, tb2 where tb1.ts=tb2.ts ")
+        tdSql.query("select unique(tb2.num)  from tb1, tb2 where tb1.ts=tb2.ts union all select unique(tb1.num)  from tb1, tb2 where tb1.ts=tb2.ts order by 1")
         tdSql.checkRows(20)
         tdSql.checkData(0,0,0)
-        tdSql.checkData(1,0,1)
+        tdSql.checkData(2,0,1)
-        tdSql.checkData(2,0,2)
+        tdSql.checkData(4,0,2)
-        tdSql.checkData(9,0,9)
+        tdSql.checkData(18,0,9)
 
         # nest query
         # tdSql.query("select unique(c1) from (select c1 from ct1)")
-        tdSql.query("select c1 from (select unique(c1) c1 from ct4)")
+        tdSql.query("select c1 from (select unique(c1) c1 from ct4) order by 1 desc nulls first")
         tdSql.checkRows(10)
         tdSql.checkData(0, 0, None)
         tdSql.checkData(1, 0, 8)
@@ -367,7 +367,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, 45)
         tdSql.checkData(1, 0, 45)
 
-        tdSql.query("select 1-abs(c1) from (select unique(c1) c1 from ct4)")
+        tdSql.query("select 1-abs(c1) from (select unique(c1) c1 from ct4) order by 1 nulls first")
         tdSql.checkRows(10)
         tdSql.checkData(0, 0, None)
         tdSql.checkData(1, 0, -7.000000000)
@@ -422,7 +422,7 @@ class TDTestCase:
                 f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
             )
         
-        tdSql.query("select unique(c2) from sub1_bound")
+        tdSql.query("select unique(c2) from sub1_bound order by 1 desc")
         tdSql.checkRows(5)
         tdSql.checkData(0,0,9223372036854775807)