Merge pull request #23222 from taosdata/feat/TS-4072

feat: support interval syntax of quoted duration and default unit

source/common/src/ttime.c

@@ -665,6 +665,9 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
   if (*unit == 'n' || *unit == 'y') {
     return 0;
   }
+  if(isdigit(*unit)) {
+    *unit = getPrecisionUnit(timePrecision);
+  }
 
   return getDuration(*duration, *unit, duration, timePrecision);
 }

source/libs/parser/inc/sql.y

@@ -1119,19 +1119,24 @@ partition_item(A) ::= expr_or_subquery(B) column_alias(C).
 partition_item(A) ::= expr_or_subquery(B) AS column_alias(C).                     { A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
 
 twindow_clause_opt(A) ::= .                                                       { A = NULL; }
-twindow_clause_opt(A) ::=
+twindow_clause_opt(A) ::= SESSION NK_LP column_reference(B) NK_COMMA
-  SESSION NK_LP column_reference(B) NK_COMMA duration_literal(C) NK_RP.           { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); }
+  interval_sliding_duration_literal(C) NK_RP.                                     { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); }
 twindow_clause_opt(A) ::= STATE_WINDOW NK_LP expr_or_subquery(B) NK_RP.           { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); }
+twindow_clause_opt(A) ::= INTERVAL NK_LP interval_sliding_duration_literal(B)
+  NK_RP sliding_opt(C) fill_opt(D).                                               { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); }
 twindow_clause_opt(A) ::=
-  INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D).            { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); }
+  INTERVAL NK_LP interval_sliding_duration_literal(B) NK_COMMA
-twindow_clause_opt(A) ::=
+  interval_sliding_duration_literal(C) NK_RP
-  INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP
   sliding_opt(D) fill_opt(E).                                                     { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), D, E); }
 twindow_clause_opt(A) ::=
   EVENT_WINDOW START WITH search_condition(B) END WITH search_condition(C).       { A = createEventWindowNode(pCxt, B, C); }
 
 sliding_opt(A) ::= .                                                              { A = NULL; }
-sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP.                       { A = releaseRawExprNode(pCxt, B); }
+sliding_opt(A) ::= SLIDING NK_LP interval_sliding_duration_literal(B) NK_RP.      { A = releaseRawExprNode(pCxt, B); }
+
+interval_sliding_duration_literal(A) ::= NK_VARIABLE(B).                          { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
+interval_sliding_duration_literal(A) ::= NK_STRING(B).                            { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
+interval_sliding_duration_literal(A) ::= NK_INTEGER(B).                           { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
 
 fill_opt(A) ::= .                                                                 { A = NULL; }
 fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP.                                    { A = createFillNode(pCxt, B, NULL); }

source/libs/parser/src/parAstCreater.c

@@ -499,7 +499,40 @@ SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral)
   CHECK_PARSER_STATUS(pCxt);
   SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
   CHECK_OUT_OF_MEM(val);
-  val->literal = strndup(pLiteral->z, pLiteral->n);
+  if (pLiteral->type == TK_NK_STRING) {
+    // like '100s' or "100d"
+    // check format: ^[0-9]+[smwbauhdny]$'
+    if (pLiteral->n < 4) {
+      pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, pLiteral->z);
+      return NULL;
+    }
+    char unit = pLiteral->z[pLiteral->n - 2];
+    switch (unit) {
+      case 'a':
+      case 'b':
+      case 'd':
+      case 'h':
+      case 'm':
+      case 's':
+      case 'u':
+      case 'w':
+      case 'y':
+      case 'n':
+        break;
+      default:
+        pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, pLiteral->z);
+        return NULL;
+    }
+    for (uint32_t i = 1; i < pLiteral->n - 2; ++i) {
+      if (!isdigit(pLiteral->z[i])) {
+        pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, pLiteral->z);
+        return NULL;
+      }
+    }
+    val->literal = strndup(pLiteral->z + 1, pLiteral->n - 2);
+  } else {
+    val->literal = strndup(pLiteral->z, pLiteral->n);
+  }
   CHECK_OUT_OF_MEM(val->literal);
   val->isDuration = true;
   val->translate = false;

source/libs/planner/src/planSpliter.c

@@ -379,7 +379,7 @@ static int32_t stbSplRewriteFuns(const SNodeList* pFuncs, SNodeList** pPartialFu
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) {
+static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex, uint8_t precision) {
   int32_t index = 0;
   SNode*  pFunc = NULL;
   FOREACH(pFunc, pFuncs) {
@@ -400,6 +400,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) {
   int32_t len = snprintf(name, sizeof(name) - 1, "%s.%" PRId64 "", pWStart->functionName, pointer);
   taosCreateMD5Hash(name, len);
   strncpy(pWStart->node.aliasName, name, TSDB_COL_NAME_LEN - 1);
+  pWStart->node.resType.precision = precision;
 
   int32_t code = fmGetFuncInfo(pWStart, NULL, 0);
   if (TSDB_CODE_SUCCESS == code) {
@@ -466,7 +467,7 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic
   int32_t index = 0;
   int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs);
   if (TSDB_CODE_SUCCESS == code) {
-    code = stbSplAppendWStart(pPartWin->pFuncs, &index);
+    code = stbSplAppendWStart(pPartWin->pFuncs, &index, ((SColumnNode*)pMergeWindow->pTspk)->node.resType.precision);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = createColumnByRewriteExprs(pPartWin->pFuncs, &pPartWin->node.pTargets);

tests/parallel_test/cases.task

@@ -49,6 +49,10 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_limit_opt.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_unit.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_unit.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_unit.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_unit.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 2

tests/system-test/2-query/interval_unit.py

@@ -0,0 +1,197 @@
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+# from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 4
+        self.ctbNum     = 10
+        self.rowsPerTbl = 10000
+        self.duraion = '1h'
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d', precision: str='ms'):
+        if dropFlag == 1:
+            tsql.execute("drop database if exists %s"%(dbName), 1)
+
+        tsql.execute("create database if not exists %s vgroups %d replica %d duration %s precision '%s'"%(dbName, vgroups, replica, duration, precision), 1)
+        tdLog.debug("complete to create database %s"%(dbName))
+        return
+
+    def create_stable(self,tsql, paraDict):
+        colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
+        tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
+        sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString)
+        tdLog.debug("%s"%(sqlString))
+        tsql.execute(sqlString, 1)
+        return
+
+    def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0):
+        for i in range(ctbNum):
+            sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \
+                    (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx)
+            tsql.execute(sqlString, 1)
+
+        tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
+        return
+
+    def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep):
+        tdLog.debug("start to insert data ............")
+        tsql.execute("use %s" %dbName, 1)
+        pre_insert = "insert into "
+        sql = pre_insert
+
+        for i in range(ctbNum):
+            rowsBatched = 0
+            sql += " %s%d values "%(ctbPrefix,i)
+            for j in range(rowsPerTbl):
+                if (i < ctbNum/2):
+                    sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10)
+                else:
+                    sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10)
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
+                    tsql.execute(sql, 1)
+                    rowsBatched = 0
+                    if j < rowsPerTbl - 1:
+                        sql = "insert into %s%d values " %(ctbPrefix,i)
+                    else:
+                        sql = "insert into "
+        if sql != pre_insert:
+            tsql.execute(sql, 1)
+        tdLog.debug("insert data ............ [OK]")
+        return
+
+    def prepare_db_for_interval_unit_test(self, dbname: str, precision: str = 'ms', startTs = 1537146000000):
+        self.create_database(tdSql, dbname, precision=precision)
+        paraDict = {'dbName':     dbname,
+                    'dropFlag':   1,
+                    'vgroups':    2,
+                    'stbName':    'meters',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},
+                                    {'type': 'BIGINT', 'count':1},
+                                    {'type': 'FLOAT', 'count':1},
+                                    {'type': 'DOUBLE', 'count':1},
+                                    {'type': 'smallint', 'count':1},
+                                    {'type': 'tinyint', 'count':1},
+                                    {'type': 'bool', 'count':1},
+                                    {'type': 'binary', 'len':10, 'count':1},
+                                    {'type': 'nchar', 'len':10, 'count':1} ],
+                    'tagSchema':   [{'type': 'INT', 'count':1},
+                                    {'type': 'nchar', 'len':20, 'count':1},
+                                    {'type': 'binary', 'len':20, 'count':1},
+                                    {'type': 'BIGINT', 'count':1},
+                                    {'type': 'smallint', 'count':1},
+                                    {'type': 'DOUBLE', 'count':1}],
+                    'ctbPrefix':  't',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     100,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   3000,
+                    'startTs':    startTs,
+                    'tsStep':     100}
+
+        self.create_stable(tdSql, paraDict)
+
+        self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \
+                stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\
+                ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"])
+        self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\
+                ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\
+                rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\
+                startTs=paraDict["startTs"],tsStep=paraDict["tsStep"])
+
+    def prepare_for_interval_unit_test(self):
+        self.prepare_db_for_interval_unit_test('msdb', 'ms', startTs=1600000000000)
+        self.prepare_db_for_interval_unit_test('usdb', 'us', startTs=1600000000000000)
+        self.prepare_db_for_interval_unit_test('nsdb', 'ns', startTs=1600000000000000000)
+
+    def check_explain_res_has_row(self, plan_str_expect: str, rows):
+        plan_found = False
+        for row in rows:
+            if str(row).find(plan_str_expect) >= 0:
+                tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row)))
+                plan_found = True
+                break
+        if not plan_found:
+            tdLog.exit("plan: %s not found in res: [%s]" % (plan_str_expect, str(rows)))
+
+    def explain_and_assert_res(self, sql, expect):
+        tdSql.query(sql)
+        res = tdSql.queryResult
+        self.check_explain_res_has_row(expect, res)
+
+    def test_interval_normal_cases(self):
+        tdSql.execute('use msdb')
+        sql_template = 'explain verbose true select count(*), min(c1) from meters interval(%s) sliding(%s)'
+
+        fail_durations = ["'1 s'", "'11ns'", "'11ms'", "'11+2s'",
+        "' 112s '", '"100s "', "' 112 s '", "'112a20s'", "'s'",
+        '"1 s"', '"1"', '"100"', '"1y"', '1y', '"1s""', "'12s'12s'", "'12s\""]
+
+        for dura in fail_durations:
+            tdSql.error(sql_template % (dura, dura))
+
+        msdb_success_durations = ['1s', '"1s"', '"1000000a"', "'100d'",
+                             "'10m'", '"10h"', '1000']
+
+        for dura in msdb_success_durations:
+            tdSql.query(sql_template % (dura, dura), queryTimes=1)
+            unit = ''
+            if dura.isdigit():
+                unit = 'a'
+            self.check_explain_res_has_row('interval=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult)
+            self.check_explain_res_has_row('sliding=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult)
+
+        usdb_success_durations = ['"1s"', "'10s'", '"10d"', '"1000000u"', '1000000']
+        for dura in usdb_success_durations:
+            tdSql.execute("use usdb")
+            tdSql.query(sql_template % (dura, dura), queryTimes=1)
+            unit = ''
+            if dura.isdigit():
+                unit = 'u'
+            self.check_explain_res_has_row('interval=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult)
+            self.check_explain_res_has_row('sliding=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult)
+
+        nsdb_success_durations = ['"1s"', "'10s'", '"10d"', '"1000000u"', '"1000000000b"', '1000000000']
+        for dura in usdb_success_durations:
+            tdSql.execute("use nsdb")
+            tdSql.query(sql_template % (dura, dura), queryTimes=1)
+            unit = ''
+            if dura.isdigit():
+                unit = 'b'
+            self.check_explain_res_has_row('interval=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult)
+            self.check_explain_res_has_row('sliding=' + dura.lstrip('\'"').rstrip('\'"') + unit, tdSql.queryResult)
+
+    def test_interval_unit_feature(self):
+        self.prepare_for_interval_unit_test()
+        self.test_interval_normal_cases()
+
+    def run(self):
+        self.test_interval_unit_feature()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())