Merge pull request #12064 from taosdata/feature/3.0_wxy

feat: rewrite timeline function

include/libs/nodes/querynodes.h

@@ -231,6 +231,7 @@ typedef struct SSelectStmt {
   uint8_t     precision;
   bool        isEmptyResult;
   bool        hasAggFuncs;
+  bool        isTimeOrderQuery;
 } SSelectStmt;
 
 typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;

source/libs/function/src/builtins.c

@@ -221,7 +221,7 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE };
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
   return TSDB_CODE_SUCCESS;
 }
 
@@ -256,8 +256,7 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  pFunc->node.resType =
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
-      (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
   return TSDB_CODE_SUCCESS;
 }
 
@@ -439,6 +438,7 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   return TSDB_CODE_SUCCESS;
 }
 
+// clang-format off
 const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "count",
@@ -568,7 +568,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "first",
     .type = FUNCTION_TYPE_FIRST,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
     .translateFunc = translateFirstLast,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -578,7 +578,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "last",
     .type = FUNCTION_TYPE_LAST,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
     .translateFunc = translateFirstLast,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -588,7 +588,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "diff",
     .type = FUNCTION_TYPE_DIFF,
-    .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC,
+    .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
     .translateFunc = translateInOutNum,
     .getEnvFunc   = getDiffFuncEnv,
     .initFunc     = diffFunctionSetup,
@@ -976,5 +976,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .finalizeFunc = NULL
   }
 };
+// clang-format on
 
 const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFuncDefinition));

source/libs/function/src/functionMgt.c

@@ -21,11 +21,8 @@
 #include "taos.h"
 #include "taoserror.h"
 #include "thash.h"
-#include "builtins.h"
-#include "catalog.h"
 #include "tudf.h"
 
-
 typedef struct SFuncMgtService {
   SHashObj* pFuncNameHashTable;
 } SFuncMgtService;
@@ -148,6 +145,8 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG
 
 bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); }
 
+bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); }
+
 bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); }
 
 bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); }

source/libs/parser/src/parAstCreater.c

@@ -586,6 +586,7 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr
   select->pProjectionList = pProjectionList;
   select->pFromTable = pTable;
   sprintf(select->stmtName, "%p", select);
+  select->isTimeOrderQuery = true;
   return (SNode*)select;
 }
 

source/libs/parser/src/parTranslater.c

@@ -319,7 +319,7 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol
   pCol->node.resType = pExpr->resType;
 }
 
-static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) {
+static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) {
   if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
     const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta;
     int32_t           nums =
@@ -643,6 +643,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc)
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING);
     }
     pCxt->pCurrStmt->hasAggFuncs = true;
+    pCxt->pCurrStmt->isTimeOrderQuery = false;
   }
 
   return DEAL_RES_CONTINUE;
@@ -939,7 +940,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) {
   size_t  nums = taosArrayGetSize(pTables);
   for (size_t i = 0; i < nums; ++i) {
     STableNode* pTable = taosArrayGetP(pTables, i);
-    int32_t     code = createColumnNodeByTable(pCxt, pTable, *pCols);
+    int32_t     code = createColumnsByTable(pCxt, pTable, *pCols);
     if (TSDB_CODE_SUCCESS != code) {
       return code;
     }
@@ -1011,7 +1012,7 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SN
     }
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = createColumnNodeByTable(pCxt, pTable, *pOutput);
+    code = createColumnsByTable(pCxt, pTable, *pOutput);
   }
   return code;
 }
@@ -1093,52 +1094,33 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod
   return code;
 }
 
-static bool isCountStar(SNode* pNode) {
-  if (QUERY_NODE_FUNCTION != nodeType(pNode) || 1 != LIST_LENGTH(((SFunctionNode*)pNode)->pParameterList)) {
-    return false;
-  }
-  SNode* pPara = nodesListGetNode(((SFunctionNode*)pNode)->pParameterList, 0);
-  return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*"));
-}
-
-static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) {
-  SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0);
-  STableNode*  pTable = NULL;
-  int32_t      code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable);
-  if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
-    setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol);
-  }
-  return code;
-}
-
 static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (NULL == pSelect->pProjectionList) {  // select * ...
     return createAllColumns(pCxt, &pSelect->pProjectionList);
   } else {
     SNode* pNode = NULL;
     WHERE_EACH(pNode, pSelect->pProjectionList) {
+      int32_t code = TSDB_CODE_SUCCESS;
       if (isMultiResFunc(pNode)) {
         SNodeList* pFuncs = NULL;
-        if (TSDB_CODE_SUCCESS != createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs)) {
+        code = createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs);
-          return TSDB_CODE_OUT_OF_MEMORY;
+        if (TSDB_CODE_SUCCESS == code) {
+          INSERT_LIST(pSelect->pProjectionList, pFuncs);
+          ERASE_NODE(pSelect->pProjectionList);
+          continue;
         }
-        INSERT_LIST(pSelect->pProjectionList, pFuncs);
-        ERASE_NODE(pSelect->pProjectionList);
-        continue;
       } else if (isTableStar(pNode)) {
         SNodeList* pCols = NULL;
-        if (TSDB_CODE_SUCCESS != createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols)) {
+        code = createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols);
-          return TSDB_CODE_OUT_OF_MEMORY;
+        if (TSDB_CODE_SUCCESS == code) {
-        }
+          INSERT_LIST(pSelect->pProjectionList, pCols);
-        INSERT_LIST(pSelect->pProjectionList, pCols);
+          ERASE_NODE(pSelect->pProjectionList);
-        ERASE_NODE(pSelect->pProjectionList);
+          continue;
-        continue;
-      } else if (isCountStar(pNode)) {
-        int32_t code = rewriteCountStar(pCxt, (SFunctionNode*)pNode);
-        if (TSDB_CODE_SUCCESS != code) {
-          return code;
         }
       }
+      if (TSDB_CODE_SUCCESS != code) {
+        return code;
+      }
       WHERE_NEXT;
     }
   }
@@ -1254,12 +1236,14 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (NULL != pSelect->pGroupByList && NULL != pSelect->pWindow) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST);
   }
-  pCxt->currClause = SQL_CLAUSE_GROUP_BY;
+  if (NULL != pSelect->pGroupByList) {
-  return translateExprList(pCxt, pSelect->pGroupByList);
+    pCxt->currClause = SQL_CLAUSE_GROUP_BY;
+    pSelect->isTimeOrderQuery = false;
+    return translateExprList(pCxt, pSelect->pGroupByList);
+  }
+  return TSDB_CODE_SUCCESS;
 }
 
-static bool isValTimeUnit(char unit) { return ('n' == unit || 'y' == unit); }
-
 static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) {
   int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd');
   switch (unit) {
@@ -1286,7 +1270,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
   uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision;
 
   SValueNode* pInter = (SValueNode*)pInterval->pInterval;
-  bool        valInter = isValTimeUnit(pInter->unit);
+  bool        valInter = TIME_IS_VAR_DURATION(pInter->unit);
   if (pInter->datum.i <= 0 ||
       (!valInter && convertTimePrecision(pInter->datum.i, precision, TSDB_TIME_PRECISION_MICRO) < tsMinIntervalTime)) {
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime);
@@ -1300,7 +1284,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
     if (pInter->unit == 'n' && pOffset->unit == 'y') {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT);
     }
-    bool fixed = !isValTimeUnit(pOffset->unit) && !valInter;
+    bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter;
     if ((fixed && pOffset->datum.i >= pInter->datum.i) ||
         (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >=
                        getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) {
@@ -1312,7 +1296,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
     const static int32_t INTERVAL_SLIDING_FACTOR = 100;
 
     SValueNode* pSliding = (SValueNode*)pInterval->pSliding;
-    if (pInter->unit == 'n' || pInter->unit == 'y') {
+    if (TIME_IS_VAR_DURATION(pSliding->unit)) {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT);
     }
     if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) ||
@@ -1419,6 +1403,78 @@ static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) {
   return TSDB_CODE_SUCCESS;
 }
 
+static bool isCountStar(SFunctionNode* pFunc) {
+  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return false;
+  }
+  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
+  return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*"));
+}
+
+// count(*) is rewritten as count(ts) for scannning optimization
+static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) {
+  SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0);
+  STableNode*  pTable = NULL;
+  int32_t      code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable);
+  if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
+    setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol);
+  }
+  return code;
+}
+
+static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) {
+  SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
+  if (NULL == pCol) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
+    setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol);
+  } else {
+    // todo
+  }
+  *pPrimaryKey = (SNode*)pCol;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) {
+  STableNode* pTable = NULL;
+  int32_t     code = findTable(pCxt, NULL, &pTable);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createPrimaryKeyColByTable(pCxt, pTable, pPrimaryKey);
+  }
+  return code;
+}
+
+static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
+  SNode*  pPrimaryKey = NULL;
+  int32_t code = createPrimaryKeyCol(pCxt, &pPrimaryKey);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey);
+  }
+  return code;
+}
+
+EDealRes rewriteFuncForSelectImpl(SNode* pNode, void* pContext) {
+  if (QUERY_NODE_FUNCTION == nodeType(pNode)) {
+    STranslateContext* pCxt = pContext;
+    SFunctionNode*     pFunc = (SFunctionNode*)pNode;
+    if (isCountStar(pFunc)) {
+      pCxt->errCode = rewriteCountStar(pCxt, pFunc);
+    } else if (fmIsTimelineFunc(pFunc->funcId)) {
+      pCxt->errCode = rewriteTimelineFunc(pCxt, pFunc);
+    }
+    if (TSDB_CODE_SUCCESS != pCxt->errCode) {
+      return DEAL_RES_ERROR;
+    }
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static int32_t rewriteFuncForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
+  nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteFuncForSelectImpl, pCxt);
+  return pCxt->errCode;
+}
+
 static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
   pCxt->pCurrStmt = pSelect;
   int32_t code = translateFrom(pCxt, pSelect);
@@ -1449,6 +1505,9 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (TSDB_CODE_SUCCESS == code) {
     code = checkLimit(pCxt, pSelect);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteFuncForSelect(pCxt, pSelect);
+  }
   return code;
 }
 
@@ -1703,13 +1762,17 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
     return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION);
   }
 
-  SNode* pNode = NULL;
+  SNode* pRetention = NULL;
-  FOREACH(pNode, pRetentions) {
+  FOREACH(pRetention, pRetentions) {
-    SNode* pVal = NULL;
+    SNode* pNode = NULL;
-    FOREACH(pVal, ((SNodeListNode*)pNode)->pNodeList) {
+    FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) {
-      if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pVal)) {
+      SValueNode* pVal = (SValueNode*)pNode;
+      if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
         return pCxt->errCode;
       }
+      if (!TIME_IS_VAR_DURATION(pVal->unit)) {
+        pVal->datum.i = convertTimeFromPrecisionToUnit(pVal->datum.i, pVal->node.resType.precision, pVal->unit);
+      }
     }
   }
 

source/libs/parser/test/parSelectTest.cpp

@@ -80,6 +80,18 @@ TEST_F(ParserSelectTest, multiResFunc) {
   run("select last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) from st1s1 t1, st1s2 t2 where t1.ts = t2.ts");
 }
 
+TEST_F(ParserSelectTest, timelineFunc) {
+  useDb("root", "test");
+
+  run("select last(*), first(*) from t1");
+
+  run("select last(*), first(*) from t1 group by c1");
+
+  run("select last(*), first(*) from t1 interval(10s)");
+
+  run("select diff(c1) from t1");
+}
+
 TEST_F(ParserSelectTest, clause) {
   useDb("root", "test");
 

source/libs/planner/test/planBasicTest.cpp

@@ -23,9 +23,9 @@ class PlanBasicTest : public PlannerTestBase {};
 TEST_F(PlanBasicTest, select) {
   useDb("root", "test");
 
-  // run("select * from t1");
+  run("select * from t1");
-  // run("select 1 from t1");
+  run("select 1 from t1");
-  // run("select * from st1");
+  run("select * from st1");
   run("select 1 from st1");
 }
 
@@ -41,3 +41,9 @@ TEST_F(PlanBasicTest, join) {
   run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts");
   run("select t1.c1, t2.c2 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts");
 }
+
+TEST_F(PlanBasicTest, func) {
+  useDb("root", "test");
+
+  run("select diff(c1) from t1");
+}

source/libs/planner/test/planGroupByTest.cpp

@@ -42,3 +42,11 @@ TEST_F(PlanGroupByTest, withOrderBy) {
   // order by alias of aggfunc
   // run("select count(*), sum(c1) a from t1 order by a");
 }
+
+TEST_F(PlanGroupByTest, aggFunc) {
+  useDb("root", "test");
+
+  run("select last(*), first(*) from t1");
+
+  run("select last(*), first(*) from t1 group by c1");
+}