From d472c28142f2083241aa6586a3c7a551eb1d7db1 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Thu, 19 May 2022 22:08:06 +0800
Subject: [PATCH 01/33] update table meta based on query sversion

---
 include/common/tmsg.h                 |  3 +++
 include/libs/qcom/query.h             |  6 ++++++
 source/client/src/clientImpl.c        | 18 +++++++++++++++++-
 source/libs/executor/src/executor.c   | 12 ++++++++++--
 source/libs/qworker/inc/qworkerInt.h  |  1 +
 source/libs/qworker/inc/qworkerMsg.h  |  2 +-
 source/libs/qworker/src/qworker.c     | 23 +++++++++++++++++------
 source/libs/qworker/src/qworkerMsg.c  |  7 ++++++-
 source/libs/scheduler/src/scheduler.c | 25 +++++++++++++++++++++++++
 9 files changed, 86 insertions(+), 11 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index addb84046c..5b1871dec6 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -1352,6 +1352,9 @@ typedef struct {
 
 typedef struct {
   int32_t code;
+  char tbFName[TSDB_TABLE_FNAME_LEN];
+  int32_t sversion;
+  int32_t tversion;
 } SResReadyRsp;
 
 typedef struct {
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index daf008108b..521fb78566 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -57,6 +57,12 @@ typedef struct SIndexMeta {
 
 } SIndexMeta;
 
+typedef struct STbVerInfo {
+  char tbFName[TSDB_TABLE_FNAME_LEN];
+  int32_t sversion;
+  int32_t tversion;
+} STbVerInfo;
+
 /*
  * ASSERT(sizeof(SCTableMeta) == 24)
  * ASSERT(tableType == TSDB_CHILD_TABLE)
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 7693d26d3f..2b3bbd21f3 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -344,7 +344,23 @@ int32_t validateSversion(SRequestObj* pRequest, void* res) {
       taosArrayPush(pArray, &tbSver);
     }
   } else if (TDMT_VND_QUERY == pRequest->type) {
+    SArray* pTbArray = (SArray*)res;
+    int32_t tbNum = taosArrayGetSize(pTbArray);
+    if (tbNum <= 0) {
+      return TSDB_CODE_SUCCESS;
+    }
 
+    pArray = taosArrayInit(tbNum, sizeof(STbSVersion));
+    if (NULL == pArray) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+
+    for (int32_t i = 0; i < tbNum; ++i) {
+      STbVerInfo* tbInfo = taosArrayGet(pTbArray, i);
+      STbSVersion tbSver = {.tbFName = tbInfo->tbFName, .sver = tbInfo->sversion};
+      taosArrayPush(pArray, &tbSver);
+    }
   }
 
   SCatalog* pCatalog = NULL;
@@ -369,7 +385,7 @@ void freeRequestRes(SRequestObj* pRequest, void* res) {
   if (TDMT_VND_SUBMIT == pRequest->type) {
     tFreeSSubmitRsp((SSubmitRsp*)res);
   } else if (TDMT_VND_QUERY == pRequest->type) {
-
+    taosArrayDestroy((SArray *)res);
   }
 }
 
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index fa840e1cd6..a6251e047a 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -177,8 +177,16 @@ int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tab
 
   *sversion = pTaskInfo->schemaVer.sversion;
   *tversion = pTaskInfo->schemaVer.tversion;
-  strcpy(dbName, pTaskInfo->schemaVer.dbname);
-  strcpy(tableName, pTaskInfo->schemaVer.tablename);
+  if (pTaskInfo->schemaVer.dbname) {
+    strcpy(dbName, pTaskInfo->schemaVer.dbname);
+  } else {
+    dbName[0] = 0;
+  }
+  if (pTaskInfo->schemaVer.tablename) {
+    strcpy(tableName, pTaskInfo->schemaVer.tablename);
+  } else {
+    tableName[0] = 0;
+  }
 
   return 0;
 }
\ No newline at end of file
diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h
index bcc17a9ae9..511327658f 100644
--- a/source/libs/qworker/inc/qworkerInt.h
+++ b/source/libs/qworker/inc/qworkerInt.h
@@ -131,6 +131,7 @@ typedef struct SQWTaskCtx {
   void     *taskHandle;
   void     *sinkHandle;
   SSubplan *plan;
+  STbVerInfo tbInfo;
 } SQWTaskCtx;
 
 typedef struct SQWSchStatus {
diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h
index 93994c8287..6453cff700 100644
--- a/source/libs/qworker/inc/qworkerMsg.h
+++ b/source/libs/qworker/inc/qworkerMsg.h
@@ -36,7 +36,7 @@ int32_t qwBuildAndSendFetchRsp(SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, i
                                int32_t code);
 void    qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete);
 int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn);
-int32_t qwBuildAndSendReadyRsp(SRpcHandleInfo *pConn, int32_t code);
+int32_t qwBuildAndSendReadyRsp(SRpcHandleInfo *pConn, int32_t code, STbVerInfo* tbInfo);
 int32_t qwBuildAndSendQueryRsp(SRpcHandleInfo *pConn, int32_t code);
 int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execInfo, int32_t num);
 void    qwFreeFetchRsp(void *msg);
diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c
index db63c71d11..102cf9aa0a 100644
--- a/source/libs/qworker/src/qworker.c
+++ b/source/libs/qworker/src/qworker.c
@@ -718,6 +718,16 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void
   return TSDB_CODE_SUCCESS;
 }
 
+
+void qwSaveTbVersionInfo(qTaskInfo_t       pTaskInfo, SQWTaskCtx *ctx) {
+  char dbFName[TSDB_DB_FNAME_LEN];
+  char tbName[TSDB_TABLE_NAME_LEN];
+  
+  qGetQueriedTableSchemaVersion(pTaskInfo, dbFName, tbName, &ctx->tbInfo.sversion, &ctx->tbInfo.tversion);
+
+  sprintf(ctx->tbInfo.tbFName, "%s.%s", dbFName, tbName);
+}
+
 int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) {
   int32_t         code = 0;
   SQWTaskCtx     *ctx = NULL;
@@ -899,6 +909,11 @@ _return:
     qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PARTIAL_SUCCEED);
   }
 
+  if (readyConnection) {
+    qwBuildAndSendReadyRsp(readyConnection, code, ctx ? &ctx->tbInfo : NULL);
+    QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", readyConnection->handle, code, tstrerror(code));
+  }
+
   if (ctx) {
     QW_UPDATE_RSP_CODE(ctx, code);
 
@@ -910,11 +925,6 @@ _return:
     qwReleaseTaskCtx(mgmt, ctx);
   }
 
-  if (readyConnection) {
-    qwBuildAndSendReadyRsp(readyConnection, code);
-    QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", readyConnection->handle, code, tstrerror(code));
-  }
-
   if (code) {
     qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAILED);
   }
@@ -975,6 +985,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex
   atomic_store_ptr(&ctx->sinkHandle, sinkHandle);
 
   if (pTaskInfo && sinkHandle) {
+    qwSaveTbVersionInfo(pTaskInfo, ctx);
     QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL));
   }
 
@@ -1047,7 +1058,7 @@ _return:
   }
 
   if (needRsp) {
-    qwBuildAndSendReadyRsp(&qwMsg->connInfo, code);
+    qwBuildAndSendReadyRsp(&qwMsg->connInfo, code, NULL);
     QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   }
 
diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c
index d502d952f3..d66b8db2bd 100644
--- a/source/libs/qworker/src/qworkerMsg.c
+++ b/source/libs/qworker/src/qworkerMsg.c
@@ -63,9 +63,14 @@ int32_t qwBuildAndSendQueryRsp(SRpcHandleInfo *pConn, int32_t code) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendReadyRsp(SRpcHandleInfo *pConn, int32_t code) {
+int32_t qwBuildAndSendReadyRsp(SRpcHandleInfo *pConn, int32_t code, STbVerInfo* tbInfo) {
   SResReadyRsp *pRsp = (SResReadyRsp *)rpcMallocCont(sizeof(SResReadyRsp));
   pRsp->code = code;
+  if (tbInfo) {
+    strcpy(pRsp->tbFName, tbInfo->tbFName);
+    pRsp->sversion = tbInfo->sversion;
+    pRsp->tversion = tbInfo->tversion;
+  }
 
   SRpcMsg rpcRsp = {
       .msgType = TDMT_VND_RES_READY_RSP,
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index 9354c1a875..7f79122e0e 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -1070,6 +1070,27 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t schSaveJobQueryRes(SSchJob *pJob, SResReadyRsp *rsp) {
+  if (rsp->tbFName[0]) {
+    if (NULL == pJob->resData) {
+      pJob->resData = taosArrayInit(pJob->taskNum, sizeof(STbVerInfo));
+      if (NULL == pJob->resData) {
+        SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+      }
+    }
+
+    STbVerInfo tbInfo;
+    strcpy(tbInfo.tbFName, rsp->tbFName);
+    tbInfo.sversion = rsp->sversion;
+    tbInfo.tversion = rsp->tversion;
+
+    taosArrayPush((SArray *)pJob->resData, &tbInfo);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 // Note: no more task error processing, handled in function internal
 int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize,
                              int32_t rspCode) {
@@ -1225,6 +1246,10 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
         SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
       }
       SCH_ERR_JRET(rsp->code);
+      pJob->resType = SCH_RES_TYPE_QUERY;
+
+      SCH_ERR_JRET(schSaveJobQueryRes(pJob, rsp));
+      
       SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask));
 
       break;

From 9a022bca606879b13e734c944e080b31af5bc377 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Thu, 19 May 2022 23:54:44 +0800
Subject: [PATCH 02/33] feat: add case for update without multi-version

---
 source/dnode/vnode/src/tsdb/tsdbRead.c |  11 +-
 tests/script/tsim/insert/update0.sim   | 155 +++++++++++++++++++++++++
 2 files changed, 163 insertions(+), 3 deletions(-)
 create mode 100644 tests/script/tsim/insert/update0.sim

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 90adba6f4d..6640090085 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -2030,8 +2030,14 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
         }
 #endif
         if (TD_SUPPORT_UPDATE(pCfg->update)) {
+          if (lastKeyAppend != key) {
+            lastKeyAppend = key;
+            if (lastKeyAppend != TSKEY_INITIAL_VAL) {
+              ++curRow;
+            }
+          }
+          // load data from file firstly
           numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos);
-          lastKeyAppend = key;
 
           if (rv1 != TD_ROW_SVER(row1)) {
             rv1 = TD_ROW_SVER(row1);
@@ -2041,7 +2047,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           }
 
           // still assign data into current row
-          mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+          numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
                              pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
 
           if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -2053,7 +2059,6 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           cur->mixBlock = true;
 
           moveToNextRowInMem(pCheckInfo);
-          ++curRow;
 
           pos += step;
         } else {
diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim
new file mode 100644
index 0000000000..c34a08c79d
--- /dev/null
+++ b/tests/script/tsim/insert/update0.sim
@@ -0,0 +1,155 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+print =============== create database
+sql create database d0 keep 365000d,365000d,365000d
+sql use d0
+
+print =============== create super table and register rsma
+sql create table if not exists stb (ts timestamp, c1 int) tags (city binary(20),district binary(20)) rollup(min) file_factor 0.1 delay 2;
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct1 using stb tags("BeiJing", "ChaoYang")
+sql create table ct2 using stb tags("BeiJing", "HaiDian")
+
+sql show tables
+if $rows != 2 then 
+  return -1
+endi
+
+print =============== step3-1 insert records into ct1
+sql insert into ct1 values('2022-05-03 16:59:00.010', 10);
+sql insert into ct1 values('2022-05-03 16:59:00.011', 11);
+sql insert into ct1 values('2022-05-03 16:59:00.016', 16);
+sql insert into ct1 values('2022-05-03 16:59:00.016', 17);
+sql insert into ct1 values('2022-05-03 16:59:00.020', 20);
+sql insert into ct1 values('2022-05-03 16:59:00.016', 18);
+sql insert into ct1 values('2022-05-03 16:59:00.021', 21);
+sql insert into ct1 values('2022-05-03 16:59:00.022', 22);
+
+print =============== step3-1 query records from ct1 from memory
+sql select * from ct1;
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+print $data30 $data31
+print $data40 $data41
+print $data50 $data51
+
+if $rows != 6 then 
+  print rows $rows != 6
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data21 != 18 then
+  print data21 $data21 != 18
+  return -1
+endi
+
+if $data51 != 22 then
+  print data51 $data51 != 22
+  return -1
+endi
+
+print =============== step3-1 insert records into ct2
+sql insert into ct2 values('2022-03-02 16:59:00.010', 1),('2022-03-02 16:59:00.010',11),('2022-04-01 16:59:00.011',2),('2022-04-01 16:59:00.011',5),('2022-03-06 16:59:00.013',7);
+sql insert into ct2 values('2022-03-02 16:59:00.010', 3),('2022-03-02 16:59:00.010',33),('2022-04-01 16:59:00.011',4),('2022-04-01 16:59:00.011',6),('2022-03-06 16:59:00.013',8);
+sql insert into ct2 values('2022-03-02 16:59:00.010', 103),('2022-03-02 16:59:00.010',303),('2022-04-01 16:59:00.011',40),('2022-04-01 16:59:00.011',60),('2022-03-06 16:59:00.013',80);
+
+print =============== step3-1 query records from ct2 from memory
+sql select * from ct2;
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+
+if $rows != 3 then 
+  print rows $rows != 3
+  return -1
+endi
+
+if $data01 != 103 then
+  print data01 $data01 != 103
+  return -1
+endi
+
+if $data11 != 80 then
+  print data11 $data11 != 80
+  return -1
+endi
+
+if $data21 != 40 then
+  print data21 $data21 != 40
+  return -1
+endi
+
+#==================== reboot to trigger commit data to file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+print =============== step3-2 query records from ct1 from file
+sql select * from ct1;
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+print $data30 $data31
+print $data40 $data41
+print $data50 $data51
+
+if $rows != 6 then 
+  print rows $rows != 6
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data21 != 18 then
+  print data21 $data21 != 18
+  return -1
+endi
+
+if $data51 != 22 then
+  print data51 $data51 != 22
+  return -1
+endi
+
+print =============== step3-2 query records from ct2 from file
+sql select * from ct2;
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+
+if $rows != 3 then 
+  print rows $rows != 3
+  return -1
+endi
+
+if $data01 != 103 then
+  print data01 $data01 != 103
+  return -1
+endi
+
+if $data11 != 80 then
+  print data11 $data11 != 80
+  return -1
+endi
+
+if $data21 != 40 then
+  print data21 $data21 != 40
+  return -1
+endi
\ No newline at end of file

From 4d50a1da76179fa17d0418702ff036d3cd734008 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Fri, 20 May 2022 09:49:47 +0800
Subject: [PATCH 03/33] test:modify testcase of muti-inserting data

---
 .../sync/threeReplica1VgElectWihtInsert.sim   |   9 +-
 .../1-insert/insertWithMoreVgroup.py          | 105 ++++++++----------
 2 files changed, 49 insertions(+), 65 deletions(-)

diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim
index f6996f1291..f568008a82 100644
--- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim
+++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim
@@ -71,7 +71,7 @@ sql create database db replica $replica vgroups $vgroups
 $loop_cnt = 0
 check_db_ready:
 $loop_cnt = $loop_cnt + 1
-sleep 200
+sleep 20
 if $loop_cnt == 10 then
   print ====> db not ready!
 	return -1
@@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat
 if $rows != 3 then
   return -1
 endi
-if $data(db)[19] != nostrict then
+if $data(db)[19] != ready then
   goto check_db_ready
 endi
 
@@ -93,7 +93,7 @@ $loop_cnt = 0
 check_vg_ready:
 $loop_cnt = $loop_cnt + 1
 sleep 200
-if $loop_cnt == 40 then
+if $loop_cnt == 300 then
   print ====> vgroups not ready!
 	return -1
 endi
@@ -175,6 +175,7 @@ while $i < $tbNum
 endw
 
 $totalTblNum = $tbNum * 2
+print  ====>totalTblNum:$totalTblNum
 sql show tables
 if $rows != $totalTblNum then 
   return -1
@@ -226,7 +227,7 @@ $loop_cnt = 0
 check_vg_ready_2:
 $loop_cnt = $loop_cnt + 1
 sleep 200
-if $loop_cnt == 10 then
+if $loop_cnt == 300 then
   print ====> vgroups switch fail!!!
 	return -1
 endi
diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py
index d8720e8045..ede33e7aa7 100644
--- a/tests/system-test/1-insert/insertWithMoreVgroup.py
+++ b/tests/system-test/1-insert/insertWithMoreVgroup.py
@@ -30,9 +30,12 @@ class TDTestCase:
     #
     # --------------- main frame -------------------
     #
-    clientCfgDict = {'queryproxy': '1'}
+    clientCfgDict = {'queryproxy': '1','debugFlag': 135}
     clientCfgDict["queryproxy"] = '2'
+    clientCfgDict["debugFlag"] = 143
+
     updatecfgDict = {'clientCfg': {}}
+    updatecfgDict = {'debugFlag': 143}
     updatecfgDict["clientCfg"]  = clientCfgDict
     def caseDescription(self):
         '''
@@ -146,41 +149,39 @@ class TDTestCase:
 
 
     # insert data
-    def insert_data(self, host, dbname, stbname, ts_start,rowCount):
+    def insert_data(self, host, dbname, stbname, chilCount, ts_start, rowCount):
         buildPath = self.getBuildPath()
         config = buildPath+ "../sim/dnode1/cfg/"
-        
         tsql=self.newcur(host,config)
         tdLog.debug("ready to inser data")
-
         tsql.execute("use %s" %dbname)
         pre_insert = "insert into "
         sql = pre_insert
-        tcount=int(tcount)
-        allRows=tcount*rowCount
+        chilCount=int(chilCount)
+        allRows=chilCount*rowCount
         tdLog.debug("doing insert data into stable-index:%s rows:%d ..."%(stbname, allRows))
         exeStartTime=time.time()
-        for i in range(0,tcount):
+        for i in range(0,chilCount):
             sql += " %s_%d values "%(stbname,i)
             for j in range(rowCount):
                 sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j)
-                if j >0 and j%5000 == 0:
+                if j >0 and j%4000 == 0:
                     # print(sql)
-                    tdSql.execute(sql)
+                    tsql.execute(sql)
                     sql = "insert into %s_%d values " %(stbname,i)
         # end sql        
         if sql != pre_insert:
-            # print(sql)
-            tdSql.execute(sql)
+            print(sql)
+            print(len(sql))
+            tsql.execute(sql)
         exeEndTime=time.time()
         spendTime=exeEndTime-exeStartTime
         speedInsert=allRows/spendTime
-        # tdLog.debug("spent %.2fs to INSERT  %d rows , insert rate is  %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert))
-
-        tdLog.debug("INSERT TABLE DATA ............ [OK]")
+        tdLog.debug("spent %.2fs to INSERT  %d rows into %s , insert rate is  %.2f rows/s... [OK]"% (spendTime,allRows,stbname,speedInsert))
+        # tdLog.debug("INSERT TABLE DATA ............ [OK]")
         return
 
-    def mutiThread_insert_data(self, host, dbname, stbname, threadNumbers, ts_start, tcountStart,tcountStop,rowCount):
+    def mutiThread_insert_data(self, host, dbname, stbname, threadNumbers, chilCount, ts_start, childrowcount):
         buildPath = self.getBuildPath()
         config = buildPath+ "../sim/dnode1/cfg/"
         
@@ -188,42 +189,11 @@ class TDTestCase:
         tdLog.debug("ready to inser data")
 
         tsql.execute("use %s" %dbname)
-        pre_insert = "insert into "
-        sql = pre_insert
-        tcount=tcountStop-tcountStart
-        allRows=tcount*rowCount
-        tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows))
-        exeStartTime=time.time()
-        for i in range(tcountStart,tcountStop):
-            sql += " %s_%d values "%(stbname,i)
-            for j in range(rowCount):
-                sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j)
-                if j >0 and j%5000 == 0:
-                    # print(sql)
-                    tdSql.execute(sql)
-                    sql = "insert into %s_%d values " %(stbname,i)
-        # end sql        
-        if sql != pre_insert:
-            # print(sql)
-            tdSql.execute(sql)
-        exeEndTime=time.time()
-        spendTime=exeEndTime-exeStartTime
-        speedInsert=allRows/spendTime
-        # tdLog.debug("spent %.2fs to INSERT  %d rows , insert rate is  %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert))
-
-        tdLog.debug("INSERT TABLE DATA ............ [OK]")
-
-
-        buildPath = self.getBuildPath()
-        config = buildPath+ "../sim/dnode1/cfg/"
-        
-        tsql=self.newcur(host,config)
-        tsql.execute("use %s" %dbname)
-        count=int(count)
+        chilCount=int(chilCount)
         threads = []
         for i in range(threadNumbers):
-            tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i))
-            threads.append(thd.Thread(target=self.create_tables, args=(host, dbname, stbname+"%d"%i, count,))) 
+            # tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i))
+            threads.append(thd.Thread(target=self.insert_data, args=(host, dbname, stbname+"%d"%i, chilCount, ts_start, childrowcount,))) 
         start_time = time.time()
         for tr in threads:
             tr.start()
@@ -231,8 +201,18 @@ class TDTestCase:
             tr.join()
         end_time = time.time()
         spendTime=end_time-start_time
-        speedCreate=count/spendTime
-        tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate))
+        tableCounts=threadNumbers*chilCount
+        stableRows=chilCount*childrowcount
+        allRows=stableRows*threadNumbers
+        speedInsert=allRows/spendTime
+
+        for i in range(threadNumbers):
+            tdSql.execute("use %s" %dbname)
+            tdSql.query("select count(*) from %s%d"%(stbname,i))
+            tdSql.checkData(0,0,stableRows)
+        tdLog.debug("spent %.2fs to insert %d rows  into %d stable and %d table,  speed is %.2f table/s... [OK]"% (spendTime,allRows,threadNumbers,tableCounts,speedInsert))
+        tdLog.debug("INSERT TABLE DATA ............ [OK]")
+
         return
 
 
@@ -288,7 +268,10 @@ class TDTestCase:
     def test_case1(self):
         tdLog.debug("-----create database and muti-thread create tables test------- ")
         #host,dbname,stbname,vgroups,threadNumbers,tcountStart,tcountStop
-        self.mutiThread_create_tables(host="localhost",dbname="db2",stbname="stb2", vgroups=1, threadNumbers=5, count=10000)
+        #host, dbname, stbname, threadNumbers, chilCount, ts_start, childrowcount
+        self.mutiThread_create_tables(host="localhost",dbname="db",stbname="stb", vgroups=1, threadNumbers=5, count=50)
+        self.mutiThread_insert_data(host="localhost",dbname="db",stbname="stb", threadNumbers=5,chilCount=50,ts_start=self.ts,childrowcount=10)
+
         return 
 
     # test case2 base:insert data
@@ -366,17 +349,17 @@ class TDTestCase:
     # run case   
     def run(self):
 
-        # #  test base case
-        # self.test_case1()
-        # tdLog.debug(" LIMIT test_case1 ............ [OK]")
+        # create database and tables。
+        self.test_case1()
+        tdLog.debug(" LIMIT test_case1 ............ [OK]")
 
-        # test case
-        # self.test_case2()
-        # tdLog.debug(" LIMIT test_case2 ............ [OK]")
+    #    # taosBenchmark ： create database and table 
+    #     self.test_case2()
+    #     tdLog.debug(" LIMIT test_case2 ............ [OK]")
 
-        # test case
-        self.test_case3()
-        tdLog.debug(" LIMIT test_case3 ............ [OK]")
+        # # taosBenchmark：create database/table and insert data
+        # self.test_case3()
+        # tdLog.debug(" LIMIT test_case3 ............ [OK]")
 
 
         # # test qnode

From 09fa500d049b965e09e631dcb364cb88b9529eb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”>
Date: Fri, 20 May 2022 14:05:38 +0800
Subject: [PATCH 04/33] feat(query): add nested query function

---
 tests/system-test/2-query/nestedQuery.py | 617 +++++++++++++++--------
 1 file changed, 407 insertions(+), 210 deletions(-)

diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py
index 11f156c7a4..8c61451756 100755
--- a/tests/system-test/2-query/nestedQuery.py
+++ b/tests/system-test/2-query/nestedQuery.py
@@ -244,7 +244,7 @@ class TDTestCase:
 
         q_u_or_where = ['(t1.q_binary like \'binary%\'  or t1.q_binary = \'0\'  or t2.q_binary like \'binary%\'  or t2.q_binary = \'0\' )' , 
         '(t1.q_nchar like \'nchar%\' or t1.q_nchar = \'0\' or t2.q_nchar like \'nchar%\' or t2.q_nchar = \'0\' )' , '(t1.q_bool = true or  t1.q_bool = false or t2.q_bool = true or  t2.q_bool = false)' , 
-        '(t1.q_bool in (0 , 1) or t2.q_bool in (0 , 1)' , 't1.q_bool in ( true , false) or t2.q_bool in ( true , false))' , '(t1.q_bool = 0 or t1.q_bool = 1 or t2.q_bool = 0 or t2.q_bool = 1)' ,
+        '(t1.q_bool in (0 , 1) or t2.q_bool in (0 , 1))' , '(t1.q_bool in ( true , false) or t2.q_bool in ( true , false))' , '(t1.q_bool = 0 or t1.q_bool = 1 or t2.q_bool = 0 or t2.q_bool = 1)' ,
         '(t1.q_bigint between  -9223372036854775807 and 9223372036854775807 or t2.q_bigint between  -9223372036854775807 and 9223372036854775807)',
         '(t1.q_int between -2147483647 and 2147483647 or t2.q_int between -2147483647 and 2147483647)',
         '(t1.q_smallint between -32767 and 32767 or t2.q_smallint between -32767 and 32767)', 
@@ -315,7 +315,63 @@ class TDTestCase:
  
         group_where = ['group by tbname , loc' , 'group by tbname', 'group by tbname, t_bigint', 'group by tbname,t_int', 'group by tbname, t_smallint', 'group by tbname,t_tinyint', 
                     'group by tbname,t_float', 'group by tbname,t_double' , 'group by tbname,t_binary', 'group by tbname,t_nchar', 'group by tbname,t_bool' ,'group by tbname ,loc ,t_bigint',
-                    'group by tbname,t_binary ,t_nchar ,t_bool' , 'group by tbname,t_int ,t_smallint ,t_tinyint' , 'group by tbname,t_float ,t_double ' ]
+                    'group by tbname,t_binary ,t_nchar ,t_bool' , 'group by tbname,t_int ,t_smallint ,t_tinyint' , 'group by tbname,t_float ,t_double ' ,
+                    'PARTITION BY tbname , loc' , 'PARTITION BY tbname', 'PARTITION BY tbname, t_bigint', 'PARTITION BY tbname,t_int', 'PARTITION BY tbname, t_smallint', 'PARTITION BY tbname,t_tinyint', 
+                    'PARTITION BY tbname,t_float', 'PARTITION BY tbname,t_double' , 'PARTITION BY tbname,t_binary', 'PARTITION BY tbname,t_nchar', 'PARTITION BY tbname,t_bool' ,'PARTITION BY tbname ,loc ,t_bigint',
+                    'PARTITION BY tbname,t_binary ,t_nchar ,t_bool' , 'PARTITION BY tbname,t_int ,t_smallint ,t_tinyint' , 'PARTITION BY tbname,t_float ,t_double ']
+        group_where_j = ['group by  t1.loc' , 'group by t1.t_bigint', 'group by t1.t_int', 'group by t1.t_smallint', 'group by t1.t_tinyint', 
+                    'group by t1.t_float', 'group by t1.t_double' , 'group by t1.t_binary', 'group by t1.t_nchar', 'group by t1.t_bool' ,'group by t1.loc ,t1.t_bigint',
+                    'group by t1.t_binary ,t1.t_nchar ,t1.t_bool' , 'group by t1.t_int ,t1.t_smallint ,t1.t_tinyint' , 'group by t1.t_float ,t1.t_double ' ,
+                    'PARTITION BY t1.loc' , 'PARTITION by t1.t_bigint', 'PARTITION by t1.t_int', 'PARTITION by t1.t_smallint', 'PARTITION by t1.t_tinyint', 
+                    'PARTITION by t1.t_float', 'PARTITION by t1.t_double' , 'PARTITION by t1.t_binary', 'PARTITION by t1.t_nchar', 'PARTITION by t1.t_bool' ,'PARTITION BY t1.loc ,t1.t_bigint',
+                    'PARTITION by t1.t_binary ,t1.t_nchar ,t1.t_bool' , 'PARTITION by t1.t_int ,t1.t_smallint ,t1.t_tinyint' , 'PARTITION by t1.t_float ,t1.t_double ',
+                    'group by  t2.loc' , 'group by t2.t_bigint', 'group by t2.t_int', 'group by t2.t_smallint', 'group by t2.t_tinyint', 
+                    'group by t2.t_float', 'group by t2.t_double' , 'group by t2.t_binary', 'group by t2.t_nchar', 'group by t2.t_bool' ,'group by t2.loc ,t2.t_bigint',
+                    'group by t2.t_binary ,t2.t_nchar ,t2.t_bool' , 'group by t2.t_int ,t2.t_smallint ,t2.t_tinyint' , 'group by t2.t_float ,t2.t_double ' ,
+                    'PARTITION BY t2.loc' , 'PARTITION by t2.t_bigint', 'PARTITION by t2.t_int', 'PARTITION by t2.t_smallint', 'PARTITION by t2.t_tinyint', 
+                    'PARTITION by t2.t_float', 'PARTITION by t2.t_double' , 'PARTITION by t2.t_binary', 'PARTITION by t2.t_nchar', 'PARTITION by t2.t_bool' ,'PARTITION BY t2.loc ,t2.t_bigint',
+                    'PARTITION by t2.t_binary ,t2.t_nchar ,t2.t_bool' , 'PARTITION by t2.t_int ,t2.t_smallint ,t2.t_tinyint' , 'PARTITION by t2.t_float ,t2.t_double ']        
+ 
+        partiton_where = ['PARTITION BY tbname , loc' , 'PARTITION BY tbname', 'PARTITION BY tbname, t_bigint', 'PARTITION BY tbname,t_int', 'PARTITION BY tbname, t_smallint', 'PARTITION BY tbname,t_tinyint', 
+                    'PARTITION BY tbname,t_float', 'PARTITION BY tbname,t_double' , 'PARTITION BY tbname,t_binary', 'PARTITION BY tbname,t_nchar', 'PARTITION BY tbname,t_bool' ,'PARTITION BY tbname ,loc ,t_bigint',
+                    'PARTITION BY tbname,t_binary ,t_nchar ,t_bool' , 'PARTITION BY tbname,t_int ,t_smallint ,t_tinyint' , 'PARTITION BY tbname,t_float ,t_double ']
+        partiton_where_j = ['PARTITION BY t1.loc' , 'PARTITION by t1.t_bigint', 'PARTITION by t1.t_int', 'PARTITION by t1.t_smallint', 'PARTITION by t1.t_tinyint', 
+                    'PARTITION by t1.t_float', 'PARTITION by t1.t_double' , 'PARTITION by t1.t_binary', 'PARTITION by t1.t_nchar', 'PARTITION by t1.t_bool' ,'PARTITION BY t1.loc ,t1.t_bigint',
+                    'PARTITION by t1.t_binary ,t1.t_nchar ,t1.t_bool' , 'PARTITION by t1.t_int ,t1.t_smallint ,t1.t_tinyint' , 'PARTITION by t1.t_float ,t1.t_double ',
+                    'PARTITION BY t2.loc' , 'PARTITION by t2.t_bigint', 'PARTITION by t2.t_int', 'PARTITION by t2.t_smallint', 'PARTITION by t2.t_tinyint', 
+                    'PARTITION by t2.t_float', 'PARTITION by t2.t_double' , 'PARTITION by t2.t_binary', 'PARTITION by t2.t_nchar', 'PARTITION by t2.t_bool' ,'PARTITION BY t2.loc ,t2.t_bigint',
+                    'PARTITION by t2.t_binary ,t2.t_nchar ,t2.t_bool' , 'PARTITION by t2.t_int ,t2.t_smallint ,t2.t_tinyint' , 'PARTITION by t2.t_float ,t2.t_double '] 
+
+ 
+        group_where_regular = ['group by tbname ' , 'group by tbname', 'group by tbname, q_bigint', 'group by tbname,q_int', 'group by tbname, q_smallint', 'group by tbname,q_tinyint', 
+                    'group by tbname,q_float', 'group by tbname,q_double' , 'group by tbname,q_binary', 'group by tbname,q_nchar', 'group by tbname,q_bool' ,'group by tbname ,q_bigint',
+                    'group by tbname,q_binary ,q_nchar ,q_bool' , 'group by tbname,q_int ,q_smallint ,q_tinyint' , 'group by tbname,q_float ,q_double ' ,
+                    'PARTITION BY tbname ' , 'PARTITION BY tbname', 'PARTITION BY tbname, q_bigint', 'PARTITION BY tbname,q_int', 'PARTITION BY tbname, q_smallint', 'PARTITION BY tbname,q_tinyint', 
+                    'PARTITION BY tbname,q_float', 'PARTITION BY tbname,q_double' , 'PARTITION BY tbname,q_binary', 'PARTITION BY tbname,q_nchar', 'PARTITION BY tbname,q_bool' ,'PARTITION BY tbname ,q_bigint',
+                    'PARTITION BY tbname,q_binary ,q_nchar ,q_bool' , 'PARTITION BY tbname,q_int ,q_smallint ,q_tinyint' , 'PARTITION BY tbname,q_float ,q_double ']
+        group_where_regular_j = ['group by t1.q_bigint', 'group by t1.q_int', 'group by t1.q_smallint', 'group by t1.q_tinyint', 
+                    'group by t1.q_float', 'group by t1.q_double' , 'group by t1.q_binary', 'group by t1.q_nchar', 'group by t1.q_bool' ,'group by t1.q_bigint',
+                    'group by t1.q_binary ,t1.q_nchar ,t1.q_bool' , 'group by t1.q_int ,t1.q_smallint ,t1.q_tinyint' , 'group by t1.q_float ,t1.q_double ' ,
+                    'PARTITION by t1.q_bigint', 'PARTITION by t1.q_int', 'PARTITION by t1.q_smallint', 'PARTITION by t1.q_tinyint', 
+                    'PARTITION by t1.q_float', 'PARTITION by t1.q_double' , 'PARTITION by t1.q_binary', 'PARTITION by t1.q_nchar', 'PARTITION by t1.q_bool' ,'PARTITION BY t1.q_bigint',
+                    'PARTITION by t1.q_binary ,t1.q_nchar ,t1.q_bool' , 'PARTITION by t1.q_int ,t1.q_smallint ,t1.q_tinyint' , 'PARTITION by t1.q_float ,t1.q_double ',
+                    'group by t2.q_bigint', 'group by t2.q_int', 'group by t2.q_smallint', 'group by t2.q_tinyint', 
+                    'group by t2.q_float', 'group by t2.q_double' , 'group by t2.q_binary', 'group by t2.q_nchar', 'group by t2.q_bool' ,'group by t2.q_bigint',
+                    'group by t2.q_binary ,t2.q_nchar ,t2.q_bool' , 'group by t2.q_int ,t2.q_smallint ,t2.q_tinyint' , 'group by t2.q_float ,t2.q_double ' ,
+                    'PARTITION by t2.q_bigint', 'PARTITION by t2.q_int', 'PARTITION by t2.q_smallint', 'PARTITION by t2.q_tinyint', 
+                    'PARTITION by t2.q_float', 'PARTITION by t2.q_double' , 'PARTITION by t2.q_binary', 'PARTITION by t2.q_nchar', 'PARTITION by t2.q_bool' ,'PARTITION BY t2.q_bigint',
+                    'PARTITION by t2.q_binary ,t2.q_nchar ,t2.q_bool' , 'PARTITION by t2.q_int ,t2.q_smallint ,t2.q_tinyint' , 'PARTITION by t2.q_float ,t2.q_double ']   
+ 
+        partiton_where_regular = ['PARTITION BY tbname ' , 'PARTITION BY tbname', 'PARTITION BY tbname, q_bigint', 'PARTITION BY tbname,q_int', 'PARTITION BY tbname, q_smallint', 'PARTITION BY tbname,q_tinyint', 
+                    'PARTITION BY tbname,q_float', 'PARTITION BY tbname,q_double' , 'PARTITION BY tbname,q_binary', 'PARTITION BY tbname,q_nchar', 'PARTITION BY tbname,q_bool' ,'PARTITION BY tbname ,q_bigint',
+                    'PARTITION BY tbname,q_binary ,q_nchar ,q_bool' , 'PARTITION BY tbname,q_int ,q_smallint ,q_tinyint' , 'PARTITION BY tbname,q_float ,q_double ']
+        partiton_where_regular_j = ['PARTITION by t1.q_bigint', 'PARTITION by t1.q_int', 'PARTITION by t1.q_smallint', 'PARTITION by t1.q_tinyint', 
+                    'PARTITION by t1.q_float', 'PARTITION by t1.q_double' , 'PARTITION by t1.q_binary', 'PARTITION by t1.q_nchar', 'PARTITION by t1.q_bool' ,'PARTITION BY t1.q_bigint',
+                    'PARTITION by t1.q_binary ,t1.q_nchar ,t1.q_bool' , 'PARTITION by t1.q_int ,t1.q_smallint ,t1.q_tinyint' , 'PARTITION by t1.q_float ,t1.q_double ',
+                    'PARTITION by t2.q_bigint', 'PARTITION by t2.q_int', 'PARTITION by t2.q_smallint', 'PARTITION by t2.q_tinyint', 
+                    'PARTITION by t2.q_float', 'PARTITION by t2.q_double' , 'PARTITION by t2.q_binary', 'PARTITION by t2.q_nchar', 'PARTITION by t2.q_bool' ,'PARTITION BY t2.q_bigint',
+                    'PARTITION by t2.q_binary ,t2.q_nchar ,t2.q_bool' , 'PARTITION by t2.q_int ,t2.q_smallint ,t2.q_tinyint' , 'PARTITION by t2.q_float ,t2.q_double ']           
+                
         having_support = ['having count(q_int) > 0','having count(q_bigint) > 0','having count(q_smallint) > 0','having count(q_tinyint) > 0','having count(q_float) > 0','having count(q_double) > 0','having count(q_bool) > 0',
                     'having avg(q_int) > 0','having avg(q_bigint) > 0','having avg(q_smallint) > 0','having avg(q_tinyint) > 0','having avg(q_float) > 0','having avg(q_double) > 0',
                     'having sum(q_int) > 0','having sum(q_bigint) > 0','having sum(q_smallint) > 0','having sum(q_tinyint) > 0','having sum(q_float) > 0','having sum(q_double) > 0',
@@ -334,6 +390,18 @@ class TDTestCase:
                     'having PERCENTILE(q_int,10) > 0','having PERCENTILE(q_bigint,10) > 0','having PERCENTILE(q_smallint,10) > 0','having PERCENTILE(q_tinyint,10) > 0','having PERCENTILE(q_float,10) > 0','having PERCENTILE(q_double,10) > 0']
         having_tagnot_support = ['having LAST_ROW(q_int) > 0','having LAST_ROW(q_bigint) > 0','having LAST_ROW(q_smallint) > 0','having LAST_ROW(q_tinyint) > 0','having LAST_ROW(q_float) > 0','having LAST_ROW(q_double) > 0']    
 
+        having_support_j = ['having count(t1.q_int) > 0','having count(t1.q_bigint) > 0','having count(t1.q_smallint) > 0','having count(t1.q_tinyint) > 0','having count(t1.q_float) > 0','having count(t1.q_double) > 0','having count(t1.q_bool) > 0',
+                    'having avg(t1.q_int) > 0','having avg(t1.q_bigint) > 0','having avg(t1.q_smallint) > 0','having avg(t1.q_tinyint) > 0','having avg(t1.q_float) > 0','having avg(t1.q_double) > 0',
+                    'having sum(t1.q_int) > 0','having sum(t1.q_bigint) > 0','having sum(t1.q_smallint) > 0','having sum(t1.q_tinyint) > 0','having sum(t1.q_float) > 0','having sum(t1.q_double) > 0',
+                    'having STDDEV(t1.q_int) > 0','having STDDEV(t1.q_bigint) > 0','having STDDEV(t1.q_smallint) > 0','having STDDEV(t1.q_tinyint) > 0','having STDDEV(t1.q_float) > 0','having STDDEV(t1.q_double) > 0',
+                    'having TWA(t1.q_int) > 0','having  TWA(t1.q_bigint) > 0','having  TWA(t1.q_smallint) > 0','having  TWA(t1.q_tinyint) > 0','having  TWA(t1.q_float) > 0','having  TWA(t1.q_double) > 0',
+                    'having IRATE(t1.q_int) > 0','having IRATE(t1.q_bigint) > 0','having IRATE(t1.q_smallint) > 0','having IRATE(t1.q_tinyint) > 0','having IRATE(t1.q_float) > 0','having IRATE(t1.q_double) > 0',
+                    'having MIN(t1.q_int) > 0','having MIN(t1.q_bigint) > 0','having MIN(t1.q_smallint) > 0','having MIN(t1.q_tinyint) > 0','having MIN(t1.q_float) > 0','having MIN(t1.q_double) > 0',
+                    'having MAX(t1.q_int) > 0','having MAX(t1.q_bigint) > 0','having MAX(t1.q_smallint) > 0','having MAX(t1.q_tinyint) > 0','having MAX(t1.q_float) > 0','having MAX(t1.q_double) > 0',
+                    'having FIRST(t1.q_int) > 0','having FIRST(t1.q_bigint) > 0','having FIRST(t1.q_smallint) > 0','having FIRST(t1.q_tinyint) > 0','having FIRST(t1.q_float) > 0','having FIRST(t1.q_double) > 0',
+                    'having LAST(t1.q_int) > 0','having LAST(t1.q_bigint) > 0','having LAST(t1.q_smallint) > 0','having LAST(t1.q_tinyint) > 0','having LAST(t1.q_float) > 0','having LAST(t1.q_double) > 0',
+                    'having APERCENTILE(t1.q_int,10) > 0','having APERCENTILE(t1.q_bigint,10) > 0','having APERCENTILE(t1.q_smallint,10) > 0','having APERCENTILE(t1.q_tinyint,10) > 0','having APERCENTILE(t1.q_float,10) > 0','having APERCENTILE(t1.q_double,10) > 0']
+        
         # limit offset where
         limit_where = ['limit 1 offset 1' , 'limit 1' , 'limit 2 offset 1' , 'limit 2', 'limit 12 offset 1' , 'limit 20', 'limit 20 offset 10' , 'limit 200']
         limit1_where = ['limit 1 offset 1' , 'limit 1' ]
@@ -375,7 +443,17 @@ class TDTestCase:
                     'last_row(q_int)' ,  'last_row(q_bigint)' , 'last_row(q_smallint)' , 'last_row(q_tinyint)' , 'last_row(q_float)' , 
                     'last_row(q_double)' , 'last_row(q_bool)' ,'last_row(q_binary)' ,'last_row(q_nchar)' ,'last_row(q_ts)']
         
-
+        calc_select_not_support_ts = ['first(q_int)' , 'first(q_bigint)' , 'first(q_smallint)' , 'first(q_tinyint)' , 'first(q_float)' ,'first(q_double)' ,'first(q_binary)' ,'first(q_nchar)' ,'first(q_bool)' ,'first(q_ts)' ,
+                    'last(q_int)' ,  'last(q_bigint)' , 'last(q_smallint)'  , 'last(q_tinyint)' , 'last(q_float)'  ,'last(q_double)' , 'last(q_binary)' ,'last(q_nchar)' ,'last(q_bool)' ,'last(q_ts)' ,
+                    'last_row(q_int)' ,  'last_row(q_bigint)' , 'last_row(q_smallint)' , 'last_row(q_tinyint)' , 'last_row(q_float)' , 
+                    'last_row(q_double)' , 'last_row(q_bool)' ,'last_row(q_binary)' ,'last_row(q_nchar)' ,'last_row(q_ts)',
+                    'apercentile(q_int,20)' ,  'apercentile(q_bigint,20)'  ,'apercentile(q_smallint,20)'  ,'apercentile(q_tinyint,20)' ,'apercentile(q_float,20)'  ,'apercentile(q_double,20)']  
+              
+        calc_select_support_ts = ['bottom(q_int,20)' , 'bottom(q_bigint,20)' , 'bottom(q_smallint,20)' , 'bottom(q_tinyint,20)' ,'bottom(q_float,20)' , 'bottom(q_double,20)' ,
+                    'top(q_int,20)' , 'top(q_bigint,20)' , 'top(q_smallint,20)' ,'top(q_tinyint,20)' ,'top(q_float,20)' ,'top(q_double,20)' ,
+                    'min(q_int)' , 'min(q_bigint)' , 'min(q_smallint)' , 'min(q_tinyint)' , 'min(q_float)' ,'min(q_double)' ,
+                    'max(q_int)' ,  'max(q_bigint)' , 'max(q_smallint)' , 'max(q_tinyint)' ,'max(q_float)' ,'max(q_double)'  ]
+        
         calc_select_regular = [ 'PERCENTILE(q_int,10)' ,'PERCENTILE(q_bigint,20)' , 'PERCENTILE(q_smallint,30)' ,'PERCENTILE(q_tinyint,40)' ,'PERCENTILE(q_float,50)' ,'PERCENTILE(q_double,60)']
 
                     
@@ -392,6 +470,23 @@ class TDTestCase:
                     'first(t2.q_int)' , 'first(t2.q_bigint)' , 'first(t2.q_smallint)' , 'first(t2.q_tinyint)' , 'first(t2.q_float)' ,'first(t2.q_double)' ,'first(t2.q_binary)' ,'first(t2.q_nchar)' ,'first(t2.q_bool)' ,'first(t2.q_ts)' ,
                     'last(t2.q_int)' ,  'last(t2.q_bigint)' , 'last(t2.q_smallint)'  , 'last(t2.q_tinyint)' , 'last(t2.q_float)'  ,'last(t2.q_double)' , 'last(t2.q_binary)' ,'last(t2.q_nchar)' ,'last(t2.q_bool)' ,'last(t2.q_ts)']
 
+        calc_select_in_support_ts_j = ['bottom(t1.q_int,20)' , 'bottom(t1.q_bigint,20)' , 'bottom(t1.q_smallint,20)' , 'bottom(t1.q_tinyint,20)' ,'bottom(t1.q_float,20)' , 'bottom(t1.q_double,20)' ,
+                    'top(t1.q_int,20)' , 'top(t1.q_bigint,20)' , 'top(t1.q_smallint,20)' ,'top(t1.q_tinyint,20)' ,'top(t1.q_float,20)' ,'top(t1.q_double,20)' ,
+                    'min(t1.q_int)' , 'min(t1.q_bigint)' , 'min(t1.q_smallint)' , 'min(t1.q_tinyint)' , 'min(t1.q_float)' ,'min(t1.q_double)' ,
+                    'max(t1.q_int)' ,  'max(t1.q_bigint)' , 'max(t1.q_smallint)' , 'max(t1.q_tinyint)' ,'max(t1.q_float)' ,'max(t1.q_double)' ,
+                    'bottom(t2.q_int,20)' , 'bottom(t2.q_bigint,20)' , 'bottom(t2.q_smallint,20)' , 'bottom(t2.q_tinyint,20)' ,'bottom(t2.q_float,20)' , 'bottom(t2.q_double,20)' ,
+                    'top(t2.q_int,20)' , 'top(t2.q_bigint,20)' , 'top(t2.q_smallint,20)' ,'top(t2.q_tinyint,20)' ,'top(t2.q_float,20)' ,'top(t2.q_double,20)' ,
+                    'min(t2.q_int)' , 'min(t2.q_bigint)' , 'min(t2.q_smallint)' , 'min(t2.q_tinyint)' , 'min(t2.q_float)' ,'min(t2.q_double)' ,
+                    'max(t2.q_int)' ,  'max(t2.q_bigint)' , 'max(t2.q_smallint)' , 'max(t2.q_tinyint)' ,'max(t2.q_float)' ,'max(t2.q_double)' ,
+                    ]
+
+        calc_select_in_not_support_ts_j = ['apercentile(t1.q_int,20)' ,  'apercentile(t1.q_bigint,20)'  ,'apercentile(t1.q_smallint,20)'  ,'apercentile(t1.q_tinyint,20)' ,'apercentile(t1.q_float,20)'  ,'apercentile(t1.q_double,20)' ,
+                    'last_row(t1.q_int)' ,  'last_row(t1.q_bigint)' , 'last_row(t1.q_smallint)' , 'last_row(t1.q_tinyint)' , 'last_row(t1.q_float)' , 
+                    'last_row(t1.q_double)' , 'last_row(t1.q_bool)' ,'last_row(t1.q_binary)' ,'last_row(t1.q_nchar)' ,'last_row(t1.q_ts)' ,
+                    'apercentile(t2.q_int,20)' ,  'apercentile(t2.q_bigint,20)'  ,'apercentile(t2.q_smallint,20)'  ,'apercentile(t2.q_tinyint,20)' ,'apercentile(t2.q_float,20)'  ,'apercentile(t2.q_double,20)' ,
+                    'last_row(t2.q_int)' ,  'last_row(t2.q_bigint)' , 'last_row(t2.q_smallint)' , 'last_row(t2.q_tinyint)' , 'last_row(t2.q_float)' , 
+                    'last_row(t2.q_double)' , 'last_row(t2.q_bool)' ,'last_row(t2.q_binary)' ,'last_row(t2.q_nchar)' ,'last_row(t2.q_ts)']
+
         calc_select_in_j = ['min(t1.q_int)' , 'min(t1.q_bigint)' , 'min(t1.q_smallint)' , 'min(t1.q_tinyint)' , 'min(t1.q_float)' ,'min(t1.q_double)' ,
                     'max(t1.q_int)' ,  'max(t1.q_bigint)' , 'max(t1.q_smallint)' , 'max(t1.q_tinyint)' ,'max(t1.q_float)' ,'max(t1.q_double)' ,
                     'apercentile(t1.q_int,20)' ,  'apercentile(t1.q_bigint,20)'  ,'apercentile(t1.q_smallint,20)'  ,'apercentile(t1.q_tinyint,20)' ,'apercentile(t1.q_float,20)'  ,'apercentile(t1.q_double,20)' ,
@@ -401,8 +496,7 @@ class TDTestCase:
                     'max(t2.q_int)' ,  'max(t2.q_bigint)' , 'max(t2.q_smallint)' , 'max(t2.q_tinyint)' ,'max(t2.q_float)' ,'max(t2.q_double)' ,
                     'apercentile(t2.q_int,20)' ,  'apercentile(t2.q_bigint,20)'  ,'apercentile(t2.q_smallint,20)'  ,'apercentile(t2.q_tinyint,20)' ,'apercentile(t2.q_float,20)'  ,'apercentile(t2.q_double,20)' ,
                     'last_row(t2.q_int)' ,  'last_row(t2.q_bigint)' , 'last_row(t2.q_smallint)' , 'last_row(t2.q_tinyint)' , 'last_row(t2.q_float)' , 
-                    'last_row(t2.q_double)' , 'last_row(t2.q_bool)' ,'last_row(t2.q_binary)' ,'last_row(t2.q_nchar)' ,'last_row(t2.q_ts)']
-        
+                    'last_row(t2.q_double)' , 'last_row(t2.q_bool)' ,'last_row(t2.q_binary)' ,'last_row(t2.q_nchar)' ,'last_row(t2.q_ts)']        
         calc_select_all_j = calc_select_in_ts_j + calc_select_in_j
 
         calc_select_regular_j = [ 'PERCENTILE(t1.q_int,10)' ,'PERCENTILE(t1.q_bigint,20)' , 'PERCENTILE(t1.q_smallint,30)' ,'PERCENTILE(t1.q_tinyint,40)' ,'PERCENTILE(t1.q_float,50)' ,'PERCENTILE(t1.q_double,60)' ,
@@ -465,6 +559,8 @@ class TDTestCase:
         calc_calculate_all = ['SPREAD(ts)'  , 'SPREAD(q_ts)'  , 'SPREAD(q_int)' ,'SPREAD(q_bigint)' , 'SPREAD(q_smallint)' ,'SPREAD(q_tinyint)' ,'SPREAD(q_float)' ,'SPREAD(q_double)' , 
                      '(SPREAD(q_int) + SPREAD(q_bigint))' , '(SPREAD(q_smallint) - SPREAD(q_float))', '(SPREAD(q_double) * SPREAD(q_tinyint))' , '(SPREAD(q_double) / SPREAD(q_float))']
         calc_calculate_regular = ['DIFF(q_int)' ,'DIFF(q_bigint)' , 'DIFF(q_smallint)' ,'DIFF(q_tinyint)' ,'DIFF(q_float)' ,'DIFF(q_double)' ,
+                                  'DIFF(q_int,0)' ,'DIFF(q_bigint,0)' , 'DIFF(q_smallint,0)' ,'DIFF(q_tinyint,0)' ,'DIFF(q_float,0)' ,'DIFF(q_double,0)' ,
+                                  'DIFF(q_int,1)' ,'DIFF(q_bigint,1)' , 'DIFF(q_smallint,1)' ,'DIFF(q_tinyint,1)' ,'DIFF(q_float,1)' ,'DIFF(q_double,1)' ,
                     'DERIVATIVE(q_int,15s,0)' , 'DERIVATIVE(q_bigint,10s,1)' , 'DERIVATIVE(q_smallint,20s,0)' ,'DERIVATIVE(q_tinyint,10s,1)' ,'DERIVATIVE(q_float,6s,0)' ,'DERIVATIVE(q_double,3s,1)' ]
         calc_calculate_groupbytbname = calc_calculate_regular
         
@@ -475,8 +571,12 @@ class TDTestCase:
                     '(SPREAD(t2.q_int) + SPREAD(t2.q_bigint))' , '(SPREAD(t2.q_smallint) - SPREAD(t2.q_float))', '(SPREAD(t2.q_double) * SPREAD(t2.q_tinyint))' , '(SPREAD(t2.q_double) / SPREAD(t2.q_tinyint))',
                     '(SPREAD(t1.q_int) + SPREAD(t1.q_smallint))' , '(SPREAD(t2.q_smallint) - SPREAD(t2.q_float))', '(SPREAD(t1.q_double) * SPREAD(t1.q_tinyint))' , '(SPREAD(t1.q_double) / SPREAD(t1.q_float))']
         calc_calculate_regular_j = ['DIFF(t1.q_int)' ,'DIFF(t1.q_bigint)' , 'DIFF(t1.q_smallint)' ,'DIFF(t1.q_tinyint)' ,'DIFF(t1.q_float)' ,'DIFF(t1.q_double)' ,
+                    'DIFF(t1.q_int,0)' ,'DIFF(t1.q_bigint,0)' , 'DIFF(t1.q_smallint,0)' ,'DIFF(t1.q_tinyint,0)' ,'DIFF(t1.q_float,0)' ,'DIFF(t1.q_double,0)' ,
+                    'DIFF(t1.q_int,1)' ,'DIFF(t1.q_bigint,1)' , 'DIFF(t1.q_smallint,1)' ,'DIFF(t1.q_tinyint,1)' ,'DIFF(t1.q_float,1)' ,'DIFF(t1.q_double,1)' ,
                     'DERIVATIVE(t1.q_int,15s,0)' , 'DERIVATIVE(t1.q_bigint,10s,1)' , 'DERIVATIVE(t1.q_smallint,20s,0)' ,'DERIVATIVE(t1.q_tinyint,10s,1)' ,'DERIVATIVE(t1.q_float,6s,0)' ,'DERIVATIVE(t1.q_double,3s,1)' ,
                     'DIFF(t2.q_int)' ,'DIFF(t2.q_bigint)' , 'DIFF(t2.q_smallint)' ,'DIFF(t2.q_tinyint)' ,'DIFF(t2.q_float)' ,'DIFF(t2.q_double)' ,
+                    'DIFF(t2.q_int,0)' ,'DIFF(t2.q_bigint,0)' , 'DIFF(t2.q_smallint,0)' ,'DIFF(t2.q_tinyint,0)' ,'DIFF(t2.q_float,0)' ,'DIFF(t2.q_double,0)' ,
+                    'DIFF(t2.q_int,1)' ,'DIFF(t2.q_bigint,1)' , 'DIFF(t2.q_smallint,1)' ,'DIFF(t2.q_tinyint,1)' ,'DIFF(t2.q_float,1)' ,'DIFF(t2.q_double,1)' ,
                     'DERIVATIVE(t2.q_int,15s,0)' , 'DERIVATIVE(t2.q_bigint,10s,1)' , 'DERIVATIVE(t2.q_smallint,20s,0)' ,'DERIVATIVE(t2.q_tinyint,10s,1)' ,'DERIVATIVE(t2.q_float,6s,0)' ,'DERIVATIVE(t2.q_double,3s,1)' ]
         calc_calculate_groupbytbname_j = calc_calculate_regular_j
 
@@ -503,7 +603,7 @@ class TDTestCase:
             tdSql.checkRows(100)
             
         #1 outer union not support        
-        self.dropandcreateDB_random("%s" %db, 1)
+        #self.dropandcreateDB_random("%s" %db, 1)
         tdSql.query("select 1-2 from stable_1;")
         for i in range(self.fornum):
             #sql = "select  ts , * from  ( select  "
@@ -528,7 +628,7 @@ class TDTestCase:
             tdSql.query(sql)
             tdSql.checkRows(100)
                    
-        self.dropandcreateDB_random("%s" %db, 1)
+        #self.dropandcreateDB_random("%s" %db, 1)
         tdSql.query("select 1-2 from stable_1;")
         for i in range(self.fornum):
             #sql = "select  ts , * from  ( select  "
@@ -725,8 +825,7 @@ class TDTestCase:
             sql += "ts from stable_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(order_where)
-            sql += ")"
-            sql += " %s "  % random.choice(unionall_or_union)
+            sql += ") union "
             sql += "select  ts  from  ( select  "
             sql += "%s, " % random.choice(s_r_select)
             sql += "%s, " % random.choice(q_select)
@@ -736,10 +835,29 @@ class TDTestCase:
             sql += ")"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.error(sql)
-            #TD-15610 tdSql.query(sql)
-            # tdSql.checkRows(100)
-
+            tdSql.query(sql)
+            tdSql.checkRows(200)
+        for i in range(self.fornum):
+            #sql = "select  ts , * from  ( select  "
+            sql = "select  ts  from  ( select  "
+            sql += "%s, " % random.choice(s_r_select)
+            sql += "%s, " % random.choice(q_select)
+            sql += "ts from stable_1 where "
+            sql += "%s " % random.choice(q_where)
+            sql += "%s " % random.choice(order_where)
+            sql += ") union all "
+            sql += "select  ts  from  ( select  "
+            sql += "%s, " % random.choice(s_r_select)
+            sql += "%s, " % random.choice(q_select)
+            sql += "ts from stable_2 where "
+            sql += "%s " % random.choice(q_where)
+            sql += "%s " % random.choice(order_where)
+            sql += ")"
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            tdSql.query(sql)
+            tdSql.checkRows(400)
+            
         #3 inter union not support
         tdSql.query("select 3-4 from stable_1;")
         for i in range(self.fornum):
@@ -761,7 +879,7 @@ class TDTestCase:
             tdLog.info(len(sql))      
             tdSql.error(sql)
 
-        #join:TD-6020\TD-6155 select * from (select column form stable1，stable2 where  t1.ts=t2.ts and <\>\in\and\or order by)
+        #join:select * from (select column form stable1，stable2 where  t1.ts=t2.ts and <\>\in\and\or order by)
         tdSql.query("select 3-5 from stable_1;")
         for i in range(self.fornum):
             sql = "select  *  from  ( select  t1.ts ,"
@@ -874,7 +992,20 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            tdSql.error(sql)
+            tdSql.error(sql)  #distinct 和 order by 不能混合使用
+        tdSql.query("select 7-1 from stable_1;")
+        for i in range(self.fornum):
+            sql = "select  * from  ( select  "
+            sql += "%s " % random.choice(dq_select)
+            sql += "  from stable_1 where "
+            sql += "%s " % random.choice(qt_where)
+            #sql += "%s " % random.choice(order_desc_where)
+            sql += "%s " % random.choice([limit_where[0] , limit_where[1]] )
+            sql += ") ;"
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            tdSql.query(sql)
+            tdSql.checkRows(1)
 
         #calc_select,TWA/Diff/Derivative/Irate are not allowed to apply to super table directly
         #8 select * from (select ts,calc form ragular_table  where <\>\in\and\or order by   )
@@ -883,7 +1014,7 @@ class TDTestCase:
         tdSql.query("select 8-1 from stable_1;")
         for i in range(self.fornum):
             sql = "select  *  from  ( select  ts ,"
-            sql += "%s " % random.choice(calc_select_in_ts) 
+            sql += "%s " % random.choice(calc_select_support_ts) 
             sql += "from regular_table_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(order_where)
@@ -891,12 +1022,36 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            tdSql.query(sql) # 聚合函数不在可以和ts一起使用了 DB error: Not a single-group group function 
+        tdSql.query("select 8-1 from stable_1;")
+        for i in range(self.fornum):
+            sql = "select  *  from  ( select  "
+            sql += "%s " % random.choice(calc_select_not_support_ts) 
+            sql += "from regular_table_1 where "
+            sql += "%s " % random.choice(q_where)
+            sql += "%s " % random.choice(limit1_where)
+            sql += ") ;"
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            #TD-15651 tdSql.query(sql) # 聚合函数不在可以和ts一起使用了 DB error: Not a single-group group function 
+            
+        for i in range(self.fornum):
+            sql = "select  *  from  ( select  "
+            sql += "%s " % random.choice(calc_select_in_ts) 
+            sql += "from regular_table_1 where "
+            sql += "%s " % random.choice(q_where)
+            #sql += "%s " % random.choice(order_where)
+            sql += "%s " % random.choice(limit1_where)
+            sql += ") ;"
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            tdSql.query(sql)
+            ##top返回结果有问题nest.sql tdSql.checkRows(1)
 
         tdSql.query("select 8-2 from stable_1;")
         for i in range(self.fornum):
-            sql = "select  *  from  ( select  "
-            sql += "%s " % random.choice(calc_select_in_ts_j) 
+            sql = "select  *  from  ( select t1.ts, "
+            sql += "%s " % random.choice(calc_select_in_support_ts_j) 
             sql += "from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and "
             sql += "%s " % random.choice(q_u_where)
             sql += "%s " % random.choice(order_u_where)
@@ -904,28 +1059,36 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
-        
-        tdSql.query("select 8-3 from stable_1;")
+            tdSql.query(sql)# 聚合函数不在可以和ts一起使用了 DB error: Not a single-group group function 
         for i in range(self.fornum):
             sql = "select  *  from  ( select  "
-            sql += "%s " % random.choice(calc_select_in_ts_j) 
+            sql += "%s " % random.choice(calc_select_in_not_support_ts_j) 
             sql += "from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and "
-            sql += "%s " % random.choice(q_u_or_where)
-            sql += "%s " % random.choice(order_u_where)
+            sql += "%s " % random.choice(q_u_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.error(sql)
-            #tdSql.query(sql)
+            #TD-15651 tdSql.query(sql) 
+            ##top返回结果有问题 tdSql.checkRows(1)      
 
         #9 select * from (select ts,calc form stable  where <\>\in\and\or order by   )      
         # self.dropandcreateDB_random("%s" %db, 1)
         tdSql.query("select 9-1 from stable_1;")
+        for i in range(self.fornum):
+            sql = "select  *  from  ( select  "
+            sql += "%s " % random.choice(calc_select_not_support_ts) 
+            sql += "from stable_1 where "
+            sql += "%s " % random.choice(qt_where)
+            sql += "%s " % random.choice(limit1_where)
+            sql += ") ;"
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            #TD-15651 tdSql.query(sql)
+        tdSql.query("select 9-2 from stable_1;")
         for i in range(self.fornum):
             sql = "select  *  from  ( select  ts ,"
-            sql += "%s " % random.choice(calc_select_in_ts) 
+            sql += "%s " % random.choice(calc_select_support_ts) 
             sql += "from stable_1 where "
             sql += "%s " % random.choice(qt_where)
             sql += "%s " % random.choice(order_where)
@@ -933,35 +1096,33 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
-
-        tdSql.query("select 9-2 from stable_1;")
-        for i in range(self.fornum):
-            sql = "select  *  from  ( select  "
-            sql += "%s " % random.choice(calc_select_in_ts_j) 
-            sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and  "
-            sql += "%s " % random.choice(t_join_where)
-            sql += "%s " % random.choice(order_u_where)
-            sql += "%s " % random.choice(limit1_where)
-            sql += ") ;"
-            tdLog.info(sql) 
-            tdLog.info(len(sql))      
-            #tdSql.query(sql)
-
+            tdSql.query(sql)
+            
         tdSql.query("select 9-3 from stable_1;")
         for i in range(self.fornum):
             sql = "select  *  from  ( select  "
-            sql += "%s " % random.choice(calc_select_in_ts_j) 
+            sql += "%s " % random.choice(calc_select_in_not_support_ts_j) 
             sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and  "
-            sql += "%s " % random.choice(qt_u_or_where)
+            sql += "%s " % random.choice(t_join_where)
+            sql += " and %s " % random.choice(qt_u_or_where)
+            sql += "%s " % random.choice(limit1_where)
+            sql += ") ;"
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            #TD-15651 tdSql.query(sql)
+        tdSql.query("select 9-4 from stable_1;")
+        for i in range(self.fornum):
+            sql = "select  *  from  ( select  t1.ts,"
+            sql += "%s " % random.choice(calc_select_in_support_ts_j) 
+            sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and  "
+            sql += "%s " % random.choice(t_join_where)
+            sql += " and %s " % random.choice(qt_u_or_where)
             sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.error(sql)
-
-            #functions or others can not be mixed up ,calc out select not use with ts
+            tdSql.query(sql)
         
         #10 select calc from (select * form regualr_table  where <\>\in\and\or order by   )       
         tdSql.query("select 10-1 from stable_1;")
@@ -975,8 +1136,8 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # TD-15503 tdSql.query(sql)
-            # tdSql.checkRows(1)
+            tdSql.query(sql)
+            tdSql.checkRows(1)
         
         #10-1 select calc from (select * form regualr_table  where <\>\in\and\or order by   )
         # rsDn = self.restartDnodes()        
@@ -986,44 +1147,47 @@ class TDTestCase:
         for i in range(self.fornum):
             sql = "select  " 
             sql += "%s " % random.choice(calc_select_all) 
-            sql += "as calc10_1 from ( select * from regular_table_1 where "
+            sql += "as calc10_2 from ( select * from regular_table_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(order_desc_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # tdSql.query(sql)
+            #TD-15651 tdSql.query(sql)
             # tdSql.checkRows(1)
 
         #10-2 select calc from (select * form regualr_tables  where <\>\in\and\or order by   )        
         tdSql.query("select 10-3 from stable_1;")
         for i in range(self.fornum):
             sql = "select  " 
-            sql += "%s as calc10_1 " % random.choice(calc_select_all) 
+            sql += "%s as calc10_3 " % random.choice(calc_select_all) 
             sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_where)
+            sql += " and %s " % random.choice(q_u_or_where)
             sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s ;" % random.choice(limit_u_where)
+            sql += "%s ;" % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15651  tdSql.query(sql)
 
         tdSql.query("select 10-4 from stable_1;")
         for i in range(self.fornum):
             sql = "select  " 
-            sql += "%s as calc10_1 " % random.choice(calc_select_all) 
+            sql += "%s as calc10_4 " % random.choice(calc_select_all) 
             sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_or_where)
+            sql += " and %s " % random.choice(q_u_or_where)
             sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s ;" % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.error(sql)
+            #TD-15651  tdSql.query(sql)
+            # tdSql.checkRows(1)
 
         #11 select calc from (select * form stable  where <\>\in\and\or order by limit  )        
         tdSql.query("select 11-1 from stable_1;")
@@ -1037,8 +1201,8 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # tdSql.query(sql)
-            # tdSql.checkRows(1)
+            tdSql.query(sql)
+            tdSql.checkRows(1)
 
         #11-1 select calc from (select * form stable  where <\>\in\and\or order by limit  )        
         tdSql.query("select 11-2 from stable_1;")
@@ -1052,8 +1216,8 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # tdSql.query(sql)
-            # tdSql.checkRows(1)
+            #TD-15651 tdSql.query(sql)
+            #不好计算结果 tdSql.checkRows(1)
  
         #11-2 select calc from (select * form stables  where <\>\in\and\or order by limit  )
         tdSql.query("select 11-3 from stable_1;")
@@ -1068,7 +1232,7 @@ class TDTestCase:
             sql += "%s ;" % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15493 tdSql.query(sql)
+            #TD-15651 tdSql.query(sql)
 
         tdSql.query("select 11-4 from stable_1;")
         for i in range(self.fornum):
@@ -1082,7 +1246,8 @@ class TDTestCase:
             sql += "%s ;" % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.error(sql)
+            tdLog.info(len(sql))      
+            #TD-15651 tdSql.query(sql)
 
         #12 select calc-diff from (select * form regualr_table  where <\>\in\and\or order by limit  )        
         ##self.dropandcreateDB_random("%s" %db, 1)
@@ -1097,7 +1262,7 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # tdSql.query(sql)
+            ##目前derivative不支持  tdSql.query(sql)
             # tdSql.checkRows(1)
 
         tdSql.query("select 12-2 from stable_1;")
@@ -1111,8 +1276,8 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
-            #tdSql.checkRows(1)
+            #目前derivative不支持 tdSql.query(sql)
+            # tdSql.checkRows(1)
 
         tdSql.query("select 12-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -1125,7 +1290,7 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.error(sql)
+            #目前derivative不支持 tdSql.query(sql)
 
         #12-1 select calc-diff from (select * form stable  where <\>\in\and\or order by limit  )
         tdSql.query("select 12-3 from stable_1;")
@@ -1142,7 +1307,7 @@ class TDTestCase:
             sql += " ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #目前derivative不支持 tdSql.query(sql)
 
         tdSql.query("select 12-4 from stable_1;")
         #join query does not support group by
@@ -1151,14 +1316,14 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_calculate_regular_j) 
             sql += "  from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(t_join_where)
-            sql += "%s " % random.choice(group_where)
+            sql += "%s " % random.choice(group_where_j)
             sql += ") "
-            sql += "%s " % random.choice(order_desc_where)
+            #sql += "%s " % random.choice(order_desc_where)
             sql += "%s " % random.choice([limit_where[2] , limit_where[3]] )
             sql += " ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            tdSql.error(sql)
+            #tdSql.query(sql) 目前de函数不支持，另外看看需要不需要将group by和pari by分开
 
         tdSql.query("select 12-5 from stable_1;")
         #join query does not support group by
@@ -1167,14 +1332,14 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_calculate_regular_j) 
             sql += "  from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(qt_u_or_where)
-            sql += "%s " % random.choice(group_where)
+            sql += "%s " % random.choice(group_where_j)
             sql += ") "
             sql += "%s " % random.choice(order_desc_where)
             sql += "%s " % random.choice([limit_where[2] , limit_where[3]] )
             sql += " ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            tdSql.error(sql)
+            #derivative not support  tdSql.query(sql)
         
 
         #13 select calc-diff as diffns from (select * form stable  where <\>\in\and\or order by limit  )
@@ -1189,7 +1354,7 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.error(sql)
+            #derivative not support tdSql.query(sql)
 
         #14 select * from (select calc_aggregate_alls as agg from stable  where <\>\in\and\or group by order by slimit soffset )
         # TD-5955 select   * from  ( select count (q_double)  from stable_1 where t_bool = true or  t_bool = false group by loc order by ts asc slimit 1 ) ;       
@@ -1207,8 +1372,8 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
-            #tdSql.checkRows(1)
+            #TD-15678 tdSql.query(sql)
+            # tdSql.checkRows(1)
 
         # error group by in out query
         tdSql.query("select 14-2 from stable_1;")
@@ -1226,10 +1391,11 @@ class TDTestCase:
             sql += ") "
             sql += "%s " % random.choice(group_where)
             tdLog.info(sql) 
-            tdLog.info(len(sql))      
-           # tdSql.error(sql)   
+            tdLog.info(len(sql))
+            #TD-15678 tdSql.query(sql)
+            # tdSql.checkRows(1) 
 
-        #14-2 TD-6426 select * from (select calc_aggregate_all_js as agg from stables  where <\>\in\and\or group by order by slimit soffset )
+        #14-2 select * from (select calc_aggregate_all_js as agg from stables  where <\>\in\and\or group by order by slimit soffset )
         tdSql.query("select 14-3 from stable_1;")
         for i in range(self.fornum):
             sql = "select   * from  ( select "
@@ -1238,13 +1404,13 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_aggregate_all_j) 
             sql += " as calc14_3 from stable_1  t1, stable_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(t_join_where)
-            sql += "%s " % random.choice(order_u_where)
+            sql += "%s " % random.choice(partiton_where_j)
             sql += "%s " % random.choice(slimit1_where)
             sql += ") "
             sql += "%s ;" % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 14-4 from stable_1;")
         for i in range(self.fornum):
@@ -1254,13 +1420,13 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_aggregate_all_j) 
             sql += " as calc14_3 from stable_1  t1, stable_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(qt_u_or_where)
-            sql += "%s " % random.choice(order_u_where)
+            sql += "%s " % random.choice(partiton_where_j)
             sql += "%s " % random.choice(slimit1_where)
             sql += ") "
             sql += "%s ;" % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.error(sql)
+            tdSql.query(sql)
 
         #15 TD-6320 select * from (select calc_aggregate_regulars as agg from regular_table  where <\>\in\and\or  order by slimit soffset )
         tdSql.query("select 15-1 from stable_1;")
@@ -1271,11 +1437,11 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_aggregate_regular) 
             sql += " as calc15_3 from regular_table_1 where "
             sql += "%s " % random.choice(q_where)
-            sql += "%s " % random.choice(order_desc_where)
+            sql += "%s " % random.choice(group_where_regular)     
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # tdSql.query(sql)
+            #tdSql.query(sql) #Invalid function name: twa'
             # tdSql.checkRows(1)
                 
         tdSql.query("select 15-2 from stable_1;")
@@ -1286,13 +1452,13 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_aggregate_regular_j) 
             sql += " as calc15_3 from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_where)
-            sql += "%s " % random.choice(order_u_where)            
+            sql += "%s " % random.choice(group_where_regular_j)            
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s ;" % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #tdSql.query(sql) #Invalid function name: twa'
 
         tdSql.query("select 15-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -1302,13 +1468,13 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_aggregate_regular_j) 
             sql += " as calc15_3 from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_or_where)
-            sql += "%s " % random.choice(order_u_where)            
+            sql += "%s " % random.choice(group_where_regular_j)            
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s ;" % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.error(sql)
+            #tdSql.query(sql) #Invalid function name: twa'
 
         rsDn = self.restartDnodes()
         tdSql.query("select 15-3 from stable_1;")
@@ -1327,7 +1493,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #tdSql.query(sql) #Invalid function name: twa',可能还的去掉order by
 
         tdSql.query("select 15-4 from stable_1;")
         for i in range(self.fornum):
@@ -1337,15 +1503,15 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_aggregate_groupbytbname_j) 
             sql += " as calc15_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and   "
             sql += "%s " % random.choice(t_join_where)
-            sql += "%s " % random.choice(group_where)
-            sql += "%s " % random.choice(having_support)
-            sql += "%s " % random.choice(orders_desc_where)
+            sql += "%s " % random.choice(group_where_j)
+            sql += "%s " % random.choice(having_support_j)
+            #sql += "%s " % random.choice(orders_desc_where)
             sql += ") "
             sql += "order by  calc15_1  " 
             sql += "%s " % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.error(sql)
+            #tdSql.query(sql)  #'Invalid function name: irate'
 
         tdSql.query("select 15-4.2 from stable_1;")
         for i in range(self.fornum):
@@ -1355,15 +1521,15 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_aggregate_groupbytbname_j) 
             sql += " as calc15_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and   "
             sql += "%s " % random.choice(qt_u_or_where)
-            sql += "%s " % random.choice(group_where)
-            sql += "%s " % random.choice(having_support)
+            sql += "%s " % random.choice(group_where_j)
+            sql += "%s " % random.choice(having_support_j)
             sql += "%s " % random.choice(orders_desc_where)
             sql += ") "
             sql += "order by  calc15_1  " 
             sql += "%s " % random.choice(limit_u_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.error(sql)
+            #TD-15678 #tdSql.query(sql)
 
         tdSql.query("select 15-5 from stable_1;")
         for i in range(self.fornum):
@@ -1373,13 +1539,13 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_aggregate_groupbytbname) 
             sql += " as calc15_3 from stable_1 where "
             sql += "%s " % random.choice(q_where)
-            sql += "%s " % random.choice(orders_desc_where)
+            sql += "%s " % random.choice(group_where)
             sql += ") "
             sql += "order by calc15_1  " 
             sql += "%s " % random.choice(limit_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.error(sql)
+            #tdSql.query(sql) #'Invalid function name: irate'
 
         #16 select * from (select calc_aggregate_regulars as agg from regular_table  where <\>\in\and\or  order by limit offset )       
         #self.dropandcreateDB_random("%s" %db, 1)
@@ -1392,14 +1558,13 @@ class TDTestCase:
             sql += "  from stable_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(group_where)
-            sql += "%s " % random.choice(having_support)
+            #sql += "%s " % random.choice(having_support)having和 partition不能混合使用
             sql += ") "
             sql += "order by calc16_0  " 
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # TD-15497 tdSql.query(sql)
-            # tdSql.checkRows(1)
+            #TD-15651 tdSql.query(sql)
 
         tdSql.query("select 16-2 from stable_1;")
         for i in range(self.fornum):
@@ -1414,7 +1579,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15493 tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 16-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -1428,7 +1593,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            tdSql.query(sql)
         
         tdSql.query("select 16-3 from stable_1;")
         for i in range(self.fornum):
@@ -1440,8 +1605,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # tdSql.query(sql)
-            # tdSql.checkRows(1)
+            #tdSql.query(sql)#Invalid function name: derivative'
 
         tdSql.query("select 16-4 from stable_1;")
         for i in range(self.fornum):
@@ -1453,8 +1617,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # tdSql.query(sql)
-            # tdSql.checkRows(1)
+            #tdSql.query(sql)#Invalid function name: derivative'
 
         tdSql.query("select 16-4.2 from stable_1;")
         for i in range(self.fornum):
@@ -1466,7 +1629,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.error(sql)
+            #tdSql.query(sql)#Invalid function name: derivative'
         
         tdSql.query("select 16-5 from stable_1;")
         for i in range(self.fornum):
@@ -1477,13 +1640,13 @@ class TDTestCase:
             sql += "  from stable_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(group_where)
-            sql += "%s " % random.choice(having_support)
+            #sql += "%s " % random.choice(having_support)
             sql += ") "
             sql += "order by calc16_1  " 
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.error(sql)
+           # tdSql.query(sql)
         
         tdSql.query("select 16-6 from stable_1;")
         for i in range(self.fornum):
@@ -1496,8 +1659,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            # tdSql.query(sql)
-            # tdSql.checkRows(1)
+            #Invalid function name: derivative'  tdSql.query(sql)
 
         tdSql.query("select 16-7 from stable_1;")
         for i in range(self.fornum):
@@ -1509,7 +1671,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #Invalid function name: derivative' tdSql.query(sql)
 
         tdSql.query("select 16-8 from stable_1;")
         for i in range(self.fornum):
@@ -1521,7 +1683,7 @@ class TDTestCase:
             sql += "%s " % random.choice(limit1_where)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #Invalid function name: derivative' tdSql.query(sql)
 
         #17 select apercentile from (select calc_aggregate_alls form regualr_table or stable  where <\>\in\and\or interval_sliding group by having order by limit offset  )interval_sliding
         #self.dropandcreateDB_random("%s" %db, 1)
@@ -1534,16 +1696,16 @@ class TDTestCase:
             sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) 
             sql += " from stable_1 where "
             sql += "%s " % random.choice(qt_where)
+            sql += "%s " % random.choice(partiton_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(group_where)
-            sql += "%s " % random.choice(having_support)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(having_support)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15721 tdSql.query(sql)
         
         tdSql.query("select 17-2 from stable_1;")
         for i in range(self.fornum):
@@ -1555,13 +1717,13 @@ class TDTestCase:
             sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and "
             sql += "%s " % random.choice(t_join_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #Column ambiguously defined: ts tdSql.query(sql)
+            #TD-15721 tdSql.query(sql)
 
         tdSql.query("select 17-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -1573,15 +1735,15 @@ class TDTestCase:
             sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and "
             sql += "%s " % random.choice(qt_u_or_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #TD-15721 tdSql.query(sql)
 
-        rsDn = self.restartDnodes()
+        self.restartDnodes()
         tdSql.query("select 17-3 from stable_1;")
         for i in range(self.fornum):
             #this is having_tagnot_support , because tag-select cannot mix with last_row...
@@ -1590,16 +1752,16 @@ class TDTestCase:
             sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) 
             sql += " from stable_1 where "
             sql += "%s " % random.choice(q_where)
+            sql += "%s " % random.choice(partiton_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(group_where)
-            sql += "%s " % random.choice(having_tagnot_support)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(having_tagnot_support)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15721 tdSql.query(sql)
 
         tdSql.query("select 17-4 from stable_1;")
         for i in range(self.fornum):
@@ -1610,13 +1772,13 @@ class TDTestCase:
             sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(t_join_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15721 tdSql.query(sql)
 
         tdSql.query("select 17-4.2 from stable_1;")
         for i in range(self.fornum):
@@ -1627,13 +1789,13 @@ class TDTestCase:
             sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(qt_u_or_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-         #   tdSql.error(sql)
+            #TD-15721 tdSql.query(sql)
 
         tdSql.query("select 17-5 from stable_1;")
         for i in range(self.fornum):
@@ -1643,16 +1805,16 @@ class TDTestCase:
             sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) 
             sql += " from stable_1 where "
             sql += "%s " % random.choice(qt_where)
+            sql += "%s " % random.choice(partiton_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(group_where)
-            sql += "%s " % random.choice(having_not_support)
-            sql += "%s " % random.choice(order_where)
+            # sql += "%s " % random.choice(having_not_support)
+            # sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.error(sql)
+            #TD-15721 tdSql.query(sql)
        
         tdSql.query("select 17-6 from stable_1;")
         for i in range(self.fornum):
@@ -1662,13 +1824,13 @@ class TDTestCase:
             sql += " from stable_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15721 tdSql.query(sql)
 
         tdSql.query("select 17-7 from stable_1;")
         for i in range(self.fornum):
@@ -1678,13 +1840,13 @@ class TDTestCase:
             sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and "
             sql += "%s " % random.choice(q_u_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15721 tdSql.query(sql)
 
         tdSql.query("select 17-7.2 from stable_1;")
         for i in range(self.fornum):
@@ -1694,13 +1856,13 @@ class TDTestCase:
             sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and "
             sql += "%s " % random.choice(q_u_or_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #TD-15721 tdSql.query(sql)
 
         self.restartDnodes()
         tdSql.query("select 17-8 from stable_1;")
@@ -1711,13 +1873,13 @@ class TDTestCase:
             sql += " from regular_table_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15721 tdSql.query(sql)
        
         tdSql.query("select 17-9 from stable_1;")
         for i in range(self.fornum):
@@ -1727,13 +1889,13 @@ class TDTestCase:
             sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+           #TD-15721  tdSql.query(sql)
 
         tdSql.query("select 17-10 from stable_1;")
         for i in range(self.fornum):
@@ -1743,13 +1905,13 @@ class TDTestCase:
             sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_or_where)
             sql += "%s " % random.choice(interval_sliding)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #TD-15721 tdSql.query(sql)
 
         #18 select apercentile from (select calc_aggregate_alls form regualr_table or stable  where <\>\in\and\or session order by  limit )interval_sliding
         tdSql.query("select 18-1 from stable_1;")
@@ -1760,14 +1922,14 @@ class TDTestCase:
             sql += " from regular_table_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(session_where)
-            sql += "%s " % random.choice(fill_where)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(fill_where)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+           #TD-15721  tdSql.query(sql)
                
         tdSql.query("select 18-2 from stable_1;")
         for i in range(self.fornum):
@@ -1777,13 +1939,13 @@ class TDTestCase:
             sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and   "
             sql += "%s " % random.choice(q_u_where)
             sql += "%s " % random.choice(session_u_where)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15721  tdSql.query(sql)
 
         tdSql.query("select 18-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -1793,13 +1955,13 @@ class TDTestCase:
             sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and   "
             sql += "%s " % random.choice(q_u_or_where)
             sql += "%s " % random.choice(session_u_where)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #TD-15721  tdSql.query(sql)
 
         self.restartDnodes()
         tdSql.query("select 18-3 from stable_1;")
@@ -1810,14 +1972,14 @@ class TDTestCase:
             sql += " from stable_1_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(session_where)
-            sql += "%s " % random.choice(fill_where)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(fill_where)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #TD-15721  tdSql.query(sql)
 
         tdSql.query("select 18-4 from stable_1;")
         for i in range(self.fornum):
@@ -1827,13 +1989,13 @@ class TDTestCase:
             sql += " from stable_1_1 t1, regular_table_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_where)
             sql += "%s " % random.choice(session_u_where)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.query(sql)
+            #TD-15721  tdSql.query(sql)
 
         tdSql.query("select 18-4.2 from stable_1;")
         for i in range(self.fornum):
@@ -1843,13 +2005,13 @@ class TDTestCase:
             sql += " from stable_1_1 t1, regular_table_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_or_where)
             sql += "%s " % random.choice(session_u_where)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            tdSql.error(sql)
+            #TD-15721  tdSql.query(sql)
 
         tdSql.query("select 18-5 from stable_1;")
         for i in range(self.fornum):
@@ -1859,14 +2021,14 @@ class TDTestCase:
             sql += " from stable_1 where "
             sql += "%s " % random.choice(q_where)
             sql += "%s " % random.choice(session_where)
-            sql += "%s " % random.choice(fill_where)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(fill_where)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           # tdSql.error(sql)
+            #TD-15721  tdSql.query(sql)
 
         tdSql.query("select 18-6 from stable_1;")
         for i in range(self.fornum):
@@ -1876,13 +2038,13 @@ class TDTestCase:
             sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and   "
             sql += "%s " % random.choice(t_join_where)
             sql += "%s " % random.choice(session_u_where)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-         #   tdSql.error(sql)
+            #TD-15721  tdSql.query(sql)
         
         tdSql.query("select 18-7 from stable_1;")
         for i in range(self.fornum):
@@ -1892,13 +2054,13 @@ class TDTestCase:
             sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and   "
             sql += "%s " % random.choice(qt_u_or_where)
             sql += "%s " % random.choice(session_u_where)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #TD-15721  tdSql.query(sql)
 
         #19 select apercentile from (select calc_aggregate_alls form regualr_table or stable  where <\>\in\and\or session order by  limit )interval_sliding       
         #self.dropandcreateDB_random("%s" %db, 1)
@@ -1910,15 +2072,14 @@ class TDTestCase:
             sql += " from regular_table_1 where "
             sql += "%s " % random.choice(q_where)        
             sql += "%s " % random.choice(state_window)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15545 tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 19-2 from stable_1;")
-        #TD-6435 state_window not support join
         for i in range(self.fornum):
             sql = "select   apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d  from  ( select  " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000))
             sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) 
@@ -1926,12 +2087,12 @@ class TDTestCase:
             sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and   "
             sql += "%s " % random.choice(q_u_where)        
             sql += "%s " % random.choice(state_u_window)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 19-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -1941,12 +2102,12 @@ class TDTestCase:
             sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and   "
             sql += "%s " % random.choice(q_u_or_where)        
             sql += "%s " % random.choice(state_u_window)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            tdSql.query(sql)
         
         tdSql.query("select 19-3 from stable_1;")
         for i in range(self.fornum):
@@ -1956,12 +2117,12 @@ class TDTestCase:
             sql += " from stable_1_1 where "
             sql += "%s " % random.choice(q_where)        
             sql += "%s " % random.choice(state_window)
-            sql += "%s " % random.choice(order_where)
+            #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #不支持order by tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 19-4 from stable_1;")
         for i in range(self.fornum):
@@ -1971,12 +2132,12 @@ class TDTestCase:
             sql += " from stable_1_1  t1, stable_1_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_where)        
             #sql += "%s " % random.choice(state_window)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 19-4.2 from stable_1;")
         for i in range(self.fornum):
@@ -1985,12 +2146,12 @@ class TDTestCase:
             sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) 
             sql += " from stable_1_1  t1, stable_1_2 t2 where t1.ts = t2.ts and  "
             sql += "%s " % random.choice(q_u_or_where)  
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-         #   tdSql.error(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 19-5 from stable_1;")
         for i in range(self.fornum):
@@ -2006,7 +2167,7 @@ class TDTestCase:
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-        #    tdSql.error(sql)
+            tdSql.error(sql)   #'STATE_WINDOW not support for super table query'
        
         tdSql.query("select 19-6 from stable_1;")
         for i in range(self.fornum):
@@ -2016,13 +2177,13 @@ class TDTestCase:
             sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and "
             sql += "%s " % random.choice(q_u_where)        
             #sql += "%s " % random.choice(state_window)
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-         #   tdSql.error(sql)
+            #TD-15721 tdSql.query(sql)
 
         tdSql.query("select 19-7 from stable_1;")
         for i in range(self.fornum):
@@ -2031,13 +2192,13 @@ class TDTestCase:
             sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) 
             sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and "
             sql += "%s " % random.choice(qt_u_or_where)    
-            sql += "%s " % random.choice(order_u_where)
+            #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
             sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #TD-15721 tdSql.query(sql)
 
         #20 select * from (select calc_select_fills form regualr_table or stable  where <\>\in\and\or fill_where group by  order by limit offset  )        
         #self.dropandcreateDB_random("%s" %db, 1)
@@ -2056,11 +2217,10 @@ class TDTestCase:
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #interp不支持 tdSql.query(sql)
 
         rsDn = self.restartDnodes()
         tdSql.query("select 20-2 from stable_1;")
-        #TD-6438
         for i in range(self.fornum):
             sql = "select   * from  ( select  " 
             sql += "%s , " % random.choice(calc_select_fill_j)
@@ -2074,7 +2234,7 @@ class TDTestCase:
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #interp不支持 tdSql.query(sql)
 
         tdSql.query("select 20-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -2090,7 +2250,7 @@ class TDTestCase:
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql)
+            #interp不支持 tdSql.query(sql)
 
         tdSql.query("select 20-3 from stable_1;")
         for i in range(self.fornum):
@@ -2122,7 +2282,7 @@ class TDTestCase:
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            #interp不支持 tdSql.query(sql)
 
         tdSql.query("select 20-4.2 from stable_1;")
         for i in range(self.fornum):
@@ -2139,7 +2299,7 @@ class TDTestCase:
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-         #   tdSql.error(sql)
+         ##interp不支持    tdSql.error(sql)
                
         tdSql.query("select 20-5 from stable_1;")
         for i in range(self.fornum):
@@ -2155,7 +2315,7 @@ class TDTestCase:
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            ##interp不支持 tdSql.query(sql)
 
         tdSql.query("select 20-6 from stable_1;")
         for i in range(self.fornum):
@@ -2171,40 +2331,77 @@ class TDTestCase:
             sql += ") "
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            ##interp不支持 tdSql.query(sql)
 
-        # error
         #1 select * from (select * from (select * form regular_table  where <\>\in\and\or order by limit  ))
         tdSql.query("select 1-1 from stable_1;")
-        for i in range(self.fornum):
-            sql = "select  * , ts from  ( select * from ( select  "
+        for i in range(self.fornum):            
+            # sql_start = "select  *  from ( "
+            # sql_end = ")"
+            for_num = random.randint(1, 15);
+            sql = "select  *  from ("  * for_num  
+            sql += "select  *  from  ( select * from ( select  "
             sql += "%s, " % random.choice(s_r_select) 
             sql += "%s, " % random.choice(q_select)  
             sql += "ts from regular_table_1 where "
             sql += "%s " % random.choice(q_where)
-            sql += ")) ;"
+            sql += ")) "
+            sql += ")"  * for_num     
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-         #   tdSql.error(sql)  
-
+            tdSql.query(sql)  
+            
+        for i in range(self.fornum):            
+            # sql_start = "select  *  from ( "
+            # sql_end = ")"
+            for_num = random.randint(1, 10);
+            sql = "select  ts  from ("  * for_num  
+            sql += "select  *  from  ( select * from ( select  "
+            sql += "%s, " % random.choice(s_r_select) 
+            sql += "%s, " % random.choice(q_select)  
+            sql += "ts from regular_table_1 where "
+            sql += "%s " % random.choice(q_where)
+            sql += ")) "
+            sql += ")"  * for_num     
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            tdSql.query(sql)  
+            
         #2 select * from (select * from (select * form stable  where <\>\in\and\or order by limit  ))
         tdSql.query("select 2-1 from stable_1;")
         for i in range(self.fornum):
-            sql = "select  * , ts from  ( select * from ( select  "
+            for_num = random.randint(1, 15);
+            sql = "select  *  from ("  * for_num  
+            sql += "select  *  from  ( select * from ( select  "
             sql += "%s, " % random.choice(s_s_select) 
             sql += "%s, " % random.choice(qt_select) 
             sql += "ts from stable_1 where "
             sql += "%s " % random.choice(q_where)
-            sql += ")) ;"
+            sql += ")) "
+            sql += ")"  * for_num  
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-          #  tdSql.error(sql) 
-             
+            tdSql.query(sql) 
+        tdSql.query("select 2-1 from stable_1;")
+        for i in range(self.fornum):
+            for_num = random.randint(1, 10);
+            sql = "select  ts  from ("  * for_num  
+            sql += "select  *  from  ( select * from ( select  "
+            sql += "%s, " % random.choice(s_s_select) 
+            sql += "%s, " % random.choice(qt_select) 
+            sql += "ts from stable_1 where "
+            sql += "%s " % random.choice(q_where)
+            sql += ")) "
+            sql += ")"  * for_num  
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            tdSql.query(sql) 
+                         
         #3 select ts ,calc from  (select * form stable  where <\>\in\and\or order by limit  )       
         #self.dropandcreateDB_random("%s" %db, 1)
         tdSql.query("select 3-1 from stable_1;")
         for i in range(self.fornum):
-            sql = "select   ts , "
+            sql = "select   "
             sql += "%s " % random.choice(calc_calculate_regular) 
             sql += " from ( select * from stable_1 where "
             sql += "%s " % random.choice(qt_where)
@@ -2213,7 +2410,7 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-         #   tdSql.error(sql)   
+            #'Invalid function name: derivative'  tdSql.query(sql)   
 
         #4 select * from  (select calc form stable  where <\>\in\and\or order by limit  )
         tdSql.query("select 4-1 from stable_1;")
@@ -2222,12 +2419,12 @@ class TDTestCase:
             sql += "%s " % random.choice(calc_select_in_ts) 
             sql += "from stable_1 where "
             sql += "%s " % random.choice(qt_where)
-            sql += "%s " % random.choice(order_desc_where)
+            #sql += "%s " % random.choice(order_desc_where)
             sql += "%s " % random.choice(limit_where)
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #tdSql.query(sql)
+            tdSql.query(sql)
             
         #5 select ts ,tbname from  (select * form stable  where <\>\in\and\or order by limit  )
         tdSql.query("select 5-1 from stable_1;")
@@ -2243,7 +2440,7 @@ class TDTestCase:
             sql += ") ;"
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-         #   tdSql.error(sql)  
+            tdSql.error(sql) 
 
         #special sql
         tdSql.query("select 6-1 from stable_1;")

From 344237071edd706c9057988a04cf9fe56f59babd Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 20 May 2022 15:37:39 +0800
Subject: [PATCH 05/33] fix user auth failed issue

---
 source/client/src/clientHb.c          | 14 ++++++++++----
 source/dnode/mnode/impl/src/mndUser.c |  2 ++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c
index fc39e80c1e..5d6241d4f4 100644
--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@@ -45,7 +45,7 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC
     catalogUpdateUserAuthInfo(pCatalog, rsp);
   }
 
-  tFreeSUserAuthBatchRsp(&batchRsp);
+  taosArrayDestroy(batchRsp.pArray);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -285,6 +285,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
     int64_t     *rid = pIter;
     SRequestObj *pRequest = acquireRequest(*rid);
     if (NULL == pRequest) {
+      pIter = taosHashIterate(pObj->pRequests, pIter);
       continue;
     }
 
@@ -544,7 +545,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
     }
 
     taosArrayPush(pBatchReq->reqs, pOneReq);
-    hbClearClientHbReq(pOneReq);
+    //hbClearClientHbReq(pOneReq);
 
     pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
   }
@@ -565,6 +566,11 @@ void hbClearReqInfo(SAppHbMgr *pAppHbMgr) {
     tFreeReqKvHash(pOneReq->info);
     taosHashClear(pOneReq->info);
 
+    if (pOneReq->query) {
+      taosArrayDestroy(pOneReq->query->queryDesc);
+      taosMemoryFreeClear(pOneReq->query);
+    }
+
     pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
   }
 }
@@ -745,13 +751,13 @@ int hbMgrInit() {
   hbMgrInitHandle();
 
   // init backgroud thread
-  /*hbCreateThread();*/
+  hbCreateThread();
 
   return 0;
 }
 
 void hbMgrCleanUp() {
-  // hbStopThread();
+  hbStopThread();
 
   // destroy all appHbMgr
   int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0);
diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c
index b59175d86c..f2a42e3083 100644
--- a/source/dnode/mnode/impl/src/mndUser.c
+++ b/source/dnode/mnode/impl/src/mndUser.c
@@ -233,6 +233,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
   mTrace("user:%s, perform update action, old row:%p new row:%p", pOld->user, pOld, pNew);
   taosWLockLatch(&pOld->lock);
   pOld->updateTime = pNew->updateTime;
+  pOld->authVersion = pNew->authVersion;
   memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
   TSWAP(pOld->readDbs, pNew->readDbs);
   TSWAP(pOld->writeDbs, pNew->writeDbs);
@@ -765,6 +766,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_
       continue;
     }
 
+    pUsers[i].version = ntohl(pUsers[i].version);
     if (pUser->authVersion <= pUsers[i].version) {
       mndReleaseUser(pMnode, pUser);
       continue;

From afda7c637bdc82f48ec089b383125e49b8dc8212 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Fri, 20 May 2022 15:58:25 +0800
Subject: [PATCH 06/33] fix: tq mem leak

---
 source/dnode/mgmt/mgmt_mnode/src/mmWorker.c |  4 +-
 source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 22 +++++------
 source/dnode/vnode/src/tq/tq.c              | 43 +++++++++------------
 source/libs/qworker/src/qworkerMsg.c        | 26 ++++++-------
 source/libs/transport/src/trans.c           |  2 +
 source/libs/transport/src/transComm.c       |  1 +
 source/libs/transport/src/transSrv.c        |  2 +
 7 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
index cf76d3a167..c4314a57b1 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
@@ -126,7 +126,9 @@ int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   return mmPutRpcMsgToWorker(&pMgmt->readWorker, pMsg);
 }
 
-int32_t mmPutRpcMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg); }
+int32_t mmPutRpcMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
+  return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg);
+}
 
 int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
   SSingleWorkerCfg qCfg = {
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
index eec6bb3fb4..6632480aae 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
@@ -72,11 +72,10 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   if (code != 0) {
     if (terrno != 0) code = terrno;
     vmSendRsp(pMsg, code);
-
-    dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
   }
+  dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
+  rpcFreeCont(pMsg->pCont);
+  taosFreeQitem(pMsg);
 }
 
 static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
@@ -87,11 +86,10 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   if (code != 0) {
     if (terrno != 0) code = terrno;
     vmSendRsp(pMsg, code);
-
-    dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
   }
+  dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
+  rpcFreeCont(pMsg->pCont);
+  taosFreeQitem(pMsg);
 }
 
 static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
@@ -149,7 +147,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
 static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
-  SRpcMsg   *pMsg = NULL;
+  SRpcMsg *  pMsg = NULL;
   SRpcMsg    rsp;
 
   for (int32_t i = 0; i < numOfMsgs; ++i) {
@@ -190,7 +188,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
 static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
-  SRpcMsg   *pMsg = NULL;
+  SRpcMsg *  pMsg = NULL;
 
   for (int32_t i = 0; i < numOfMsgs; ++i) {
     taosGetQitem(qall, (void **)&pMsg);
@@ -216,7 +214,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
 
 static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
-  SRpcMsg   *pMsg = NULL;
+  SRpcMsg *  pMsg = NULL;
 
   for (int32_t i = 0; i < numOfMsgs; ++i) {
     taosGetQitem(qall, (void **)&pMsg);
@@ -235,7 +233,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 }
 
 static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
-  SRpcMsg  *pRpc = pMsg;
+  SRpcMsg * pRpc = pMsg;
   SMsgHead *pHead = pRpc->pCont;
   int32_t   code = 0;
 
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 9361c0e6d2..4a3b47c0d1 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -129,7 +129,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
   return 0;
 }
 
-int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
+int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver, SRpcHandleInfo handleInfo) {
   if (msgType != TDMT_VND_SUBMIT) return 0;
   void*       pIter = NULL;
   STqExec*    pExec = NULL;
@@ -239,10 +239,9 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
 
     void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
     tEncodeSMqDataBlkRsp(&abuf, &rsp);
-    pMsg->pCont = buf;
-    pMsg->contLen = tlen;
-    pMsg->code = 0;
-    tmsgSendRsp(pMsg);
+
+    SRpcMsg resp = {.info = handleInfo, .pCont = buf, .contLen = tlen, .code = 0};
+    tmsgSendRsp(&resp);
 
     atomic_store_ptr(&pExec->pushHandle.handle, NULL);
     taosWUnLockLatch(&pExec->pushHandle.lock);
@@ -407,9 +406,9 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
       pTopic->buffer.output[j].status = 0;
       STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
       SReadHandle    handle = {
-             .reader = pReadHandle,
-             .meta = pTq->pVnode->pMeta,
-             .pMsgCb = &pTq->pVnode->msgCb,
+          .reader = pReadHandle,
+          .meta = pTq->pVnode->pMeta,
+          .pMsgCb = &pTq->pVnode->msgCb,
       };
       pTopic->buffer.output[j].pReadHandle = pReadHandle;
       pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
@@ -663,10 +662,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
   tEncodeSMqDataBlkRsp(&abuf, &rsp);
-  pMsg->pCont = buf;
-  pMsg->contLen = tlen;
-  pMsg->code = 0;
-  tmsgSendRsp(pMsg);
+
+  SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0};
+  tmsgSendRsp(&resp);
 
   tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
           TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
@@ -845,12 +843,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       /*rsp.pBlockData = pRes;*/
 
       /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/
-      pMsg->pCont = buf;
-      pMsg->contLen = msgLen;
-      pMsg->code = 0;
+      SRpcMsg resp = {.info = pMsg->info, pCont = buf, .contLen = msgLen, .code = 0};
       tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
              pHead->msgType, consumerId, pReq->epoch);
-      tmsgSendRsp(pMsg);
+      tmsgSendRsp(&resp);
       taosMemoryFree(pHead);
       return 0;
     } else {
@@ -878,10 +874,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
   tEncodeSMqPollRspV2(&abuf, &rspV2);
-  pMsg->pCont = buf;
-  pMsg->contLen = tlen;
-  pMsg->code = 0;
-  tmsgSendRsp(pMsg);
+
+  SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0};
+  tmsgSendRsp(&resp);
   tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
          pReq->epoch);
   /*}*/
@@ -990,10 +985,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
     for (int32_t i = 0; i < parallel; i++) {
       STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
       SReadHandle    handle = {
-             .reader = pStreamReader,
-             .meta = pTq->pVnode->pMeta,
-             .pMsgCb = &pTq->pVnode->msgCb,
-             .vnode = pTq->pVnode,
+          .reader = pStreamReader,
+          .meta = pTq->pVnode->pMeta,
+          .pMsgCb = &pTq->pVnode->msgCb,
+          .vnode = pTq->pVnode,
       };
       pTask->exec.runners[i].inputHandle = pStreamReader;
       pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c
index 60270d3e06..562e550bdc 100644
--- a/source/libs/qworker/src/qworkerMsg.c
+++ b/source/libs/qworker/src/qworkerMsg.c
@@ -47,7 +47,7 @@ int32_t qwBuildAndSendQueryRsp(SRpcHandleInfo *pConn, int32_t code) {
   SQueryTableRsp rsp = {.code = code};
 
   int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp);
-  void   *msg = rpcMallocCont(contLen);
+  void *  msg = rpcMallocCont(contLen);
   tSerializeSQueryTableRsp(msg, contLen, &rsp);
 
   SRpcMsg rpcRsp = {
@@ -85,7 +85,7 @@ int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execIn
   SExplainRsp rsp = {.numOfPlans = num, .subplanInfo = execInfo};
 
   int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp);
-  void   *pRsp = rpcMallocCont(contLen);
+  void *  pRsp = rpcMallocCont(contLen);
   tSerializeSExplainRsp(pRsp, contLen, &rsp);
 
   SRpcMsg rpcRsp = {
@@ -104,7 +104,7 @@ int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execIn
 
 int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
   int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus);
-  void   *pRsp = rpcMallocCont(contLen);
+  void *  pRsp = rpcMallocCont(contLen);
   tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
 
   SRpcMsg rpcRsp = {
@@ -212,7 +212,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
   showRsp.tableMeta.numOfColumns = cols;
 
   int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp);
-  void   *pBuf = rpcMallocCont(bufLen);
+  void *  pBuf = rpcMallocCont(bufLen);
   tSerializeSShowRsp(pBuf, bufLen, &showRsp);
 
   SRpcMsg rpcMsg = {
@@ -341,7 +341,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
 
   int32_t       code = 0;
   SSubQueryMsg *msg = pMsg->pCont;
-  SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
+  SQWorker *    mgmt = (SQWorker *)qWorkerMgmt;
 
   if (NULL == msg || pMsg->contLen <= sizeof(*msg)) {
     QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
@@ -361,7 +361,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   int64_t  rId = msg->refId;
 
   SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info};
-  char  *sql = strndup(msg->msg, msg->sqlLen);
+  char * sql = strndup(msg->msg, msg->sqlLen);
   QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, sql);
   taosMemoryFreeClear(sql);
 
@@ -378,8 +378,8 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   bool               queryDone = false;
   SQueryContinueReq *msg = (SQueryContinueReq *)pMsg->pCont;
   bool               needStop = false;
-  SQWTaskCtx        *handles = NULL;
-  SQWorker      *mgmt = (SQWorker *)qWorkerMgmt;
+  SQWTaskCtx *       handles = NULL;
+  SQWorker *         mgmt = (SQWorker *)qWorkerMgmt;
 
   if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
     QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
@@ -407,7 +407,7 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
 
-  SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
+  SQWorker *    mgmt = (SQWorker *)qWorkerMgmt;
   SResReadyReq *msg = pMsg->pCont;
   if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
     QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
@@ -467,7 +467,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   }
 
   SResFetchReq *msg = pMsg->pCont;
-  SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
+  SQWorker *    mgmt = (SQWorker *)qWorkerMgmt;
 
   if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
     QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
@@ -505,7 +505,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
 
-  SQWorker   *mgmt = (SQWorker *)qWorkerMgmt;
+  SQWorker *      mgmt = (SQWorker *)qWorkerMgmt;
   int32_t         code = 0;
   STaskCancelReq *msg = pMsg->pCont;
   if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
@@ -542,7 +542,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
 
   int32_t       code = 0;
   STaskDropReq *msg = pMsg->pCont;
-  SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
+  SQWorker *    mgmt = (SQWorker *)qWorkerMgmt;
 
   if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
     QW_ELOG("invalid task drop msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
@@ -581,7 +581,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
 
   int32_t         code = 0;
   SSchedulerHbReq req = {0};
-  SQWorker   *mgmt = (SQWorker *)qWorkerMgmt;
+  SQWorker *      mgmt = (SQWorker *)qWorkerMgmt;
 
   if (NULL == pMsg->pCont) {
     QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen);
diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c
index 5c01034f35..5627dbfbf5 100644
--- a/source/libs/transport/src/trans.c
+++ b/source/libs/transport/src/trans.c
@@ -94,7 +94,9 @@ void rpcFreeCont(void* cont) {
   if (cont == NULL) {
     return;
   }
+
   taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD);
+  tTrace("free mem: %p", (char*)cont - TRANS_MSG_OVERHEAD);
 }
 void* rpcReallocCont(void* ptr, int contLen) {
   if (ptr == NULL) {
diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index 98e9e67ede..d5c76ccbf2 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -133,6 +133,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) {
   } else {
     p->cap = p->total;
     p->buf = taosMemoryRealloc(p->buf, p->cap);
+    tTrace("internal malloc mem: %p", p->buf);
 
     uvBuf->base = p->buf + p->len;
     uvBuf->len = p->cap - p->len;
diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c
index 84adeb4bf6..da83a6f37f 100644
--- a/source/libs/transport/src/transSrv.c
+++ b/source/libs/transport/src/transSrv.c
@@ -469,6 +469,8 @@ static void uvStartSendResp(SSrvMsg* smsg) {
 
   if (pConn->broken == true) {
     // persist by
+    transFreeMsg(smsg->msg.pCont);
+    taosMemoryFree(smsg);
     transUnrefSrvHandle(pConn);
     return;
   }

From bda392bbe6e9374eabda4ee162febf307a7208b9 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 20 May 2022 16:19:32 +0800
Subject: [PATCH 07/33] fix: deal with error in schemaless

---
 source/client/src/clientSml.c  | 48 ++++++++++------------------------
 source/client/test/smlTest.cpp | 19 +++++---------
 2 files changed, 20 insertions(+), 47 deletions(-)

diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 5b5071f79e..04dae5bfe1 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -58,13 +58,9 @@ for (int i = 1; i < keyLen; ++i) {      \
 #define IS_INVALID_COL_LEN(len)   ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN)
 #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN)
 
-#define OTD_MAX_FIELDS_NUM      2
 #define OTD_JSON_SUB_FIELDS_NUM 2
 #define OTD_JSON_FIELDS_NUM     4
 
-#define OTD_TIMESTAMP_COLUMN_NAME "ts"
-#define OTD_METRIC_VALUE_COLUMN_NAME "value"
-
 #define TS              "_ts"
 #define TS_LEN          3
 #define TAG             "_tag"
@@ -731,24 +727,24 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) {
   double ts = tsInt64;
   switch (type) {
     case TSDB_TIME_PRECISION_HOURS:
-      ts *= (3600 * 1e9);
-      tsInt64 *= (3600 * 1e9);
+      ts *= NANOSECOND_PER_HOUR;
+      tsInt64 *= NANOSECOND_PER_HOUR;
       break;
     case TSDB_TIME_PRECISION_MINUTES:
-      ts *= (60 * 1e9);
-      tsInt64 *= (60 * 1e9);
+      ts *= NANOSECOND_PER_MINUTE;
+      tsInt64 *= NANOSECOND_PER_MINUTE;
       break;
     case TSDB_TIME_PRECISION_SECONDS:
-      ts *= (1e9);
-      tsInt64 *= (1e9);
+      ts *= NANOSECOND_PER_SEC;
+      tsInt64 *= NANOSECOND_PER_SEC;
       break;
     case TSDB_TIME_PRECISION_MILLI:
-      ts *= (1e6);
-      tsInt64 *= (1e6);
+      ts *= NANOSECOND_PER_MSEC;
+      tsInt64 *= NANOSECOND_PER_MSEC;
       break;
     case TSDB_TIME_PRECISION_MICRO:
-      ts *= (1e3);
-      tsInt64 *= (1e3);
+      ts *= NANOSECOND_PER_USEC;
+      tsInt64 *= NANOSECOND_PER_USEC;
       break;
     case TSDB_TIME_PRECISION_NANO:
       break;
@@ -762,23 +758,6 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) {
   return tsInt64;
 }
 
-static int64_t smlGetTimeNow(int8_t precision) {
-  switch (precision) {
-    case TSDB_TIME_PRECISION_HOURS:
-      return taosGetTimestampMs()/1000/3600;
-    case TSDB_TIME_PRECISION_MINUTES:
-      return taosGetTimestampMs()/1000/60;
-    case TSDB_TIME_PRECISION_SECONDS:
-      return taosGetTimestampMs()/1000;
-    case TSDB_TIME_PRECISION_MILLI:
-    case TSDB_TIME_PRECISION_MICRO:
-    case TSDB_TIME_PRECISION_NANO:
-      return taosGetTimestamp(precision);
-    default:
-      ASSERT(0);
-  }
-}
-
 static int8_t smlGetTsTypeByLen(int32_t len) {
   if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
     return TSDB_TIME_PRECISION_SECONDS;
@@ -810,14 +789,15 @@ static int8_t smlGetTsTypeByPrecision(int8_t precision) {
 }
 
 static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){
+  if(len == 0){
+    return taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
+  }
+
   int8_t tsType = smlGetTsTypeByPrecision(info->precision);
   if (tsType == -1) {
     smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL);
     return -1;
   }
-  if(len == 0){
-    return smlGetTimeNow(tsType);
-  }
 
   int64_t ts = smlGetTimeValue(data, len, tsType);
   if(ts == -1){
diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp
index eeed9dc952..c7935b351c 100644
--- a/source/client/test/smlTest.cpp
+++ b/source/client/test/smlTest.cpp
@@ -1203,24 +1203,17 @@ TEST(testCase, sml_TD15662_Test) {
   SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
-  SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
+  SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
   ASSERT_NE(info, nullptr);
 
   const char *sql[] = {
-      "iyyyje,id=iyyyje_41943_1303,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000",
+      "hetrey,id=sub_table_0123456,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64",
   };
   int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0]));
   ASSERT_EQ(ret, 0);
 
-  // case 1
-  TAOS_RES *res = taos_query(taos, "select * from t_a5615048edae55218a22a149edebdc82");
-  ASSERT_NE(res, nullptr);
-
-  TAOS_ROW row = taos_fetch_row(res);
-  int64_t ts = *(int64_t*)row[0];
-  ASSERT_EQ(ts, 1626006833639000000);
-
-  taos_free_result(res);
+  destroyRequest(request);
+  smlDestroyInfo(info);
 }
 
 TEST(testCase, sml_TD15735_Test) {
@@ -1262,11 +1255,11 @@ TEST(testCase, sml_TD15742_Test) {
   SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
-  SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
+  SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
   ASSERT_NE(info, nullptr);
 
   const char *sql[] = {
-      "zgzbix 1626006833641 False id=zgzbix_992_38861 t0=t t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\"",
+      "test_ms,t0=t c0=f 1626006833641",
   };
   int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
   ASSERT_EQ(ret, 0);

From 28b7f9599ed10d5a119b2f35c285f6b3a38b72a7 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Fri, 20 May 2022 16:20:15 +0800
Subject: [PATCH 08/33] fix(os): disable win compile tests

---
 cmake/cmake.options | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/cmake/cmake.options b/cmake/cmake.options
index d83ab49fd5..c77b580c17 100644
--- a/cmake/cmake.options
+++ b/cmake/cmake.options
@@ -46,6 +46,18 @@ IF(${TD_WINDOWS})
             ON
         )
 
+    option(
+        BUILD_TEST 
+        "If build unit tests using googletest" 
+        OFF
+    )
+ELSE ()
+
+    option(
+        BUILD_TEST 
+        "If build unit tests using googletest" 
+        ON
+    )
 ENDIF ()
 
 option(
@@ -54,12 +66,6 @@ option(
     OFF
     )
 
-option(
-    BUILD_TEST 
-    "If build unit tests using googletest" 
-    ON
-)
-
 option(
     BUILD_WITH_LEVELDB 
     "If build with leveldb" 

From 3c0a1bc7eea1407b56fab95b1cea21c4cc4cbd35 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Fri, 20 May 2022 16:20:27 +0800
Subject: [PATCH 09/33] add valgrind.log to dnodes.py

---
 tests/pytest/util/dnodes.py                        |  4 ++--
 tests/system-test/1-insert/insertWithMoreVgroup.py | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py
index c78efb4e8d..952aca70cf 100644
--- a/tests/pytest/util/dnodes.py
+++ b/tests/pytest/util/dnodes.py
@@ -264,7 +264,7 @@ class TDDnode:
             cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
                 binPath, self.cfgDir)
         else:
-            valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
+            valgrindCmdline = "valgrind --log-file=\"valgrind.log\"  --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
 
             cmd = "nohup %s %s -c %s 2>&1 & " % (
                 valgrindCmdline, binPath, self.cfgDir)
@@ -325,7 +325,7 @@ class TDDnode:
             cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
                 binPath, self.cfgDir)
         else:
-            valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
+            valgrindCmdline = "valgrind  --log-file=\"valgrind.log\"  --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
 
             cmd = "nohup %s %s -c %s 2>&1 & " % (
                 valgrindCmdline, binPath, self.cfgDir)
diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py
index ede33e7aa7..f0f35831db 100644
--- a/tests/system-test/1-insert/insertWithMoreVgroup.py
+++ b/tests/system-test/1-insert/insertWithMoreVgroup.py
@@ -119,7 +119,7 @@ class TDTestCase:
         # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
         return
 
-    def mutiThread_create_tables(self,host,dbname,stbname,vgroups,threadNumbers,count):
+    def mutiThread_create_tables(self,host,dbname,stbname,vgroups,threadNumbers,childrowcount):
         buildPath = self.getBuildPath()
         config = buildPath+ "../sim/dnode1/cfg/"
         
@@ -128,7 +128,7 @@ class TDTestCase:
         tsql.execute("drop database if exists %s"%dbname)
         tsql.execute("create database %s vgroups %d"%(dbname,vgroups))
         tsql.execute("use %s" %dbname)
-        count=int(count)
+        count=int(childrowcount)
         threads = []
         for i in range(threadNumbers):
             tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i))
@@ -140,7 +140,7 @@ class TDTestCase:
             tr.join()
         end_time = time.time()
         spendTime=end_time-start_time
-        speedCreate=count/spendTime
+        speedCreate=threadNumbers*count/spendTime
         tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate))
         
         return
@@ -171,7 +171,7 @@ class TDTestCase:
                     sql = "insert into %s_%d values " %(stbname,i)
         # end sql        
         if sql != pre_insert:
-            print(sql)
+            # print(sql)
             print(len(sql))
             tsql.execute(sql)
         exeEndTime=time.time()
@@ -269,7 +269,7 @@ class TDTestCase:
         tdLog.debug("-----create database and muti-thread create tables test------- ")
         #host,dbname,stbname,vgroups,threadNumbers,tcountStart,tcountStop
         #host, dbname, stbname, threadNumbers, chilCount, ts_start, childrowcount
-        self.mutiThread_create_tables(host="localhost",dbname="db",stbname="stb", vgroups=1, threadNumbers=5, count=50)
+        self.mutiThread_create_tables(host="localhost",dbname="db",stbname="stb", vgroups=1, threadNumbers=5, childrowcount=50)
         self.mutiThread_insert_data(host="localhost",dbname="db",stbname="stb", threadNumbers=5,chilCount=50,ts_start=self.ts,childrowcount=10)
 
         return 

From 7913887204e21f903bfa1087edf62af709833bd6 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 20 May 2022 16:33:04 +0800
Subject: [PATCH 10/33] fix: deal with error in schemaless

---
 source/client/src/clientSml.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 04dae5bfe1..e884748fff 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -724,6 +724,9 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) {
   if(value + len != endPtr){
     return -1;
   }
+  if(tsInt64 == 0){
+    return taosGetTimestampNs();
+  }
   double ts = tsInt64;
   switch (type) {
     case TSDB_TIME_PRECISION_HOURS:
@@ -751,7 +754,7 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) {
     default:
       ASSERT(0);
   }
-  if(ts >= (double)INT64_MAX || ts <= 0){
+  if(ts >= (double)INT64_MAX || ts < 0){
     return -1;
   }
 
@@ -1599,7 +1602,8 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) {
       smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
       return TSDB_CODE_TSC_INVALID_TIME_STAMP;
     }
-    if(timeDouble <= 0){
+
+    if(timeDouble < 0){
       return TSDB_CODE_TSC_INVALID_TIME_STAMP;
     }
     uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble);
@@ -1617,7 +1621,9 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) {
         return TSDB_CODE_TSC_INVALID_TIME_STAMP;
       }
       tsVal = timeDouble;
-    } else {
+    } else if(timeDouble == 0){
+      tsVal = taosGetTimestampNs();
+    }else {
       return TSDB_CODE_TSC_INVALID_TIME_STAMP;
     }
   } else if (cJSON_IsObject(timestamp)) {

From 4a786f1ad0bb009b706362ce6ede82818d34a9cf Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Fri, 20 May 2022 16:48:56 +0800
Subject: [PATCH 11/33] fix: tq mem leak

---
 source/dnode/vnode/src/vnd/vnodeSvr.c | 30 +++++++++++++++------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index dfd78d9dca..466c300d97 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -162,7 +162,7 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
 
 int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
   vTrace("message in fetch queue is processing");
-  char   *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+  char *  msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
   int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
   switch (pMsg->msgType) {
     case TDMT_VND_FETCH:
@@ -184,8 +184,12 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
     case TDMT_VND_TASK_PIPE_EXEC:
     case TDMT_VND_TASK_MERGE_EXEC:
       return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0);
-    case TDMT_VND_STREAM_TRIGGER:
-      return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0);
+    case TDMT_VND_STREAM_TRIGGER: {
+      // refactor, avoid double free
+      int code = tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0);
+      pMsg->pCont = NULL;
+      return code;
+    }
     case TDMT_VND_QUERY_HEARTBEAT:
       return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg);
     default:
@@ -328,12 +332,12 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
   SDecoder           decoder = {0};
   int                rcode = 0;
   SVCreateTbBatchReq req = {0};
-  SVCreateTbReq     *pCreateReq;
+  SVCreateTbReq *    pCreateReq;
   SVCreateTbBatchRsp rsp = {0};
   SVCreateTbRsp      cRsp = {0};
   char               tbName[TSDB_TABLE_FNAME_LEN];
-  STbUidStore       *pStore = NULL;
-  SArray            *tbUids = NULL;
+  STbUidStore *      pStore = NULL;
+  SArray *           tbUids = NULL;
 
   pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
   pRsp->code = TSDB_CODE_SUCCESS;
@@ -517,7 +521,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in
   SDecoder         decoder = {0};
   SEncoder         encoder = {0};
   int              ret;
-  SArray          *tbUids = NULL;
+  SArray *         tbUids = NULL;
 
   pRsp->msgType = TDMT_VND_DROP_TABLE_RSP;
   pRsp->pCont = NULL;
@@ -572,9 +576,9 @@ _exit:
 
 static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) {
   SSubmitBlkIter blkIter = {0};
-  STSchema      *pSchema = NULL;
+  STSchema *     pSchema = NULL;
   tb_uid_t       suid = 0;
-  STSRow        *row = NULL;
+  STSRow *       row = NULL;
 
   tInitSubmitBlkIter(msgIter, pBlock, &blkIter);
   if (blkIter.row == NULL) return 0;
@@ -605,8 +609,8 @@ static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSub
 static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) {
   ASSERT(pMsg != NULL);
   SSubmitMsgIter msgIter = {0};
-  SMeta         *pMeta = pVnode->pMeta;
-  SSubmitBlk    *pBlock = NULL;
+  SMeta *        pMeta = pVnode->pMeta;
+  SSubmitBlk *   pBlock = NULL;
 
   if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
   while (true) {
@@ -620,10 +624,10 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char
 }
 
 static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  SSubmitReq    *pSubmitReq = (SSubmitReq *)pReq;
+  SSubmitReq *   pSubmitReq = (SSubmitReq *)pReq;
   SSubmitRsp     submitRsp = {0};
   SSubmitMsgIter msgIter = {0};
-  SSubmitBlk    *pBlock;
+  SSubmitBlk *   pBlock;
   SSubmitRsp     rsp = {0};
   SVCreateTbReq  createTbReq = {0};
   SDecoder       decoder = {0};

From 5e819fbdb3c00a5704c1b84cf24c91c8fea4288d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 20 May 2022 16:50:39 +0800
Subject: [PATCH 12/33] fix(query): assign the group id in the ssdatablock when
 generating grouped results.

---
 source/libs/executor/src/executorimpl.c | 32 ++++++++++++-------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 37aeb367f5..750554e828 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -2062,15 +2062,7 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p
   pAggInfo->groupId = groupId;
 }
 
-/**
- * For interval query of both super table and table, copy the data in ascending order, since the output results are
- * ordered in SWindowResutl already. While handling the group by query for both table and super table,
- * all group result are completed already.
- *
- * @param pQInfo
- * @param result
- */
-int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
+int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
                            int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) {
   int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
   int32_t start = pGroupResInfo->index;
@@ -2087,6 +2079,15 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn
       continue;
     }
 
+    if (pBlock->info.groupId == 0) {
+      pBlock->info.groupId = pPos->groupId;
+    } else {
+      // current value belongs to different group, it can't be packed into one datablock
+      if (pBlock->info.groupId != pPos->groupId) {
+        break;
+      }
+    }
+
     if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
       break;
     }
@@ -2100,9 +2101,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn
       if (pCtx[j].fpSet.finalize) {
         int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
         if (TAOS_FAILED(code)) {
-          qError("%s build result data block error, code %s", GET_TASKID(taskInfo), tstrerror(code));
-          taskInfo->code = code;
-          longjmp(taskInfo->env, code);
+          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
+          longjmp(pTaskInfo->env, code);
         }
       } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
         // do nothing, todo refactor
@@ -2124,7 +2124,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn
     }
   }
 
-  // qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv));
+  qDebug("%s result generated, rows:%d, groupId:%"PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.groupId);
   blockDataUpdateTsWindow(pBlock);
   return 0;
 }
@@ -2145,10 +2145,9 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
     return;
   }
 
+  // clear the existed group id
+  pBlock->info.groupId = 0;
   doCopyToSDataBlock(pTaskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, rowCellOffset, pCtx, numOfExprs);
-
-  // add condition (pBlock->info.rows >= 1) just to runtime happy
-  blockDataUpdateTsWindow(pBlock);
 }
 
 static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo,
@@ -3656,7 +3655,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
     doSetOperatorCompleted(pOperator);
   }
 
-  doSetOperatorCompleted(pOperator);
   return (blockDataGetNumOfRows(pInfo->pRes) != 0) ? pInfo->pRes : NULL;
 }
 

From fa84d0585bf5bb01a7d6d6f14f710dfe39d79827 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 20 May 2022 17:07:53 +0800
Subject: [PATCH 13/33] enh(query): enable remove the candidate table ids in
 the stream scanner.

---
 source/dnode/vnode/inc/vnode.h      |  2 +
 source/dnode/vnode/src/tq/tqRead.c  | 11 +++++
 source/libs/executor/src/executor.c | 70 ++++++++++++++++-------------
 3 files changed, 53 insertions(+), 30 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index ecb022426b..b870c406ec 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -126,6 +126,8 @@ STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta);
 void    tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList);
 int     tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
 int     tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
+int     tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList);
+
 int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReadHandle *pHandle);
 int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 8fbd1e24e1..6ab2b0d917 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -235,3 +235,14 @@ int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
 
   return 0;
 }
+
+int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) {
+  ASSERT(pHandle->tbIdHash != NULL);
+
+  for(int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
+    int64_t* pKey = (int64_t*) taosArrayGet(tbUidList, i);
+    taosHashRemove(pHandle->tbIdHash, pKey, sizeof(int64_t));
+  }
+
+  return 0;
+}
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index fa840e1cd6..6d308d7221 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -125,6 +125,33 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
   return pTaskInfo;
 }
 
+static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo, const SArray* tableIdList) {
+  SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
+
+  // let's discard the tables those are not created according to the queried super table.
+  SMetaReader mr = {0};
+  metaReaderInit(&mr, pScanInfo->readHandle.meta, 0);
+  for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) {
+    int64_t* id = (int64_t*)taosArrayGet(tableIdList, i);
+
+    int32_t code = metaGetTableEntryByUid(&mr, *id);
+    if (code != TSDB_CODE_SUCCESS) {
+      qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
+      continue;
+    }
+
+    ASSERT(mr.me.type == TSDB_CHILD_TABLE);
+    if (mr.me.ctbEntry.suid != pScanInfo->tableUid) {
+      continue;
+    }
+
+    taosArrayPush(qa, id);
+  }
+
+  metaReaderClear(&mr);
+  return qa;
+}
+
 int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
 
@@ -134,41 +161,24 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
     pInfo = pInfo->pDownstream[0];
   }
 
+  int32_t code = 0;
   SStreamBlockScanInfo* pScanInfo = pInfo->info;
-  if (isAdd) {
-    SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
-
-    SMetaReader mr = {0};
-    metaReaderInit(&mr, pScanInfo->readHandle.meta, 0);
-    for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) {
-      int64_t* id = (int64_t*)taosArrayGet(tableIdList, i);
-
-      int32_t code = metaGetTableEntryByUid(&mr, *id);
-      if (code != TSDB_CODE_SUCCESS) {
-        qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
-        continue;
-      }
-
-      ASSERT(mr.me.type == TSDB_CHILD_TABLE);
-      if (mr.me.ctbEntry.suid != pScanInfo->tableUid) {
-        continue;
-      }
-
-      taosArrayPush(qa, id);
-    }
-
-    metaReaderClear(&mr);
+  if (isAdd) {  // add new table id
+    SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList);
 
     qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
-    int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-  } else {
-    assert(0);
+    code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa);
+    taosArrayDestroy(qa);
+
+  } else {  // remove the table id in current list
+    SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList);
+
+    qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList));
+    code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, tableIdList);
+    taosArrayDestroy(qa);
   }
 
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion) {

From e9607a9e25c7dfdf906b6a8203360038aa3c6f14 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Fri, 20 May 2022 17:24:10 +0800
Subject: [PATCH 14/33] feat: add cases to merge dup data in file and mem

---
 source/dnode/vnode/src/tsdb/tsdbRead.c |  2 +-
 tests/script/tsim/insert/update0.sim   | 83 ++++++++++++++++++++++++--
 2 files changed, 80 insertions(+), 5 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index b9ef72edc5..f0aa4d2ac6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -2077,10 +2077,10 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
 #endif
         if (TD_SUPPORT_UPDATE(pCfg->update)) {
           if (lastKeyAppend != key) {
-            lastKeyAppend = key;
             if (lastKeyAppend != TSKEY_INITIAL_VAL) {
               ++curRow;
             }
+            lastKeyAppend = key;
           }
           // load data from file firstly
           numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos);
diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim
index c34a08c79d..89eecaf860 100644
--- a/tests/script/tsim/insert/update0.sim
+++ b/tests/script/tsim/insert/update0.sim
@@ -35,7 +35,7 @@ sql insert into ct1 values('2022-05-03 16:59:00.016', 18);
 sql insert into ct1 values('2022-05-03 16:59:00.021', 21);
 sql insert into ct1 values('2022-05-03 16:59:00.022', 22);
 
-print =============== step3-1 query records from ct1 from memory
+print =============== step3-1 query records of ct1 from memory
 sql select * from ct1;
 print $data00 $data01
 print $data10 $data11
@@ -69,7 +69,7 @@ sql insert into ct2 values('2022-03-02 16:59:00.010', 1),('2022-03-02 16:59:00.0
 sql insert into ct2 values('2022-03-02 16:59:00.010', 3),('2022-03-02 16:59:00.010',33),('2022-04-01 16:59:00.011',4),('2022-04-01 16:59:00.011',6),('2022-03-06 16:59:00.013',8);
 sql insert into ct2 values('2022-03-02 16:59:00.010', 103),('2022-03-02 16:59:00.010',303),('2022-04-01 16:59:00.011',40),('2022-04-01 16:59:00.011',60),('2022-03-06 16:59:00.013',80);
 
-print =============== step3-1 query records from ct2 from memory
+print =============== step3-1 query records of ct2 from memory
 sql select * from ct2;
 print $data00 $data01
 print $data10 $data11
@@ -99,7 +99,7 @@ endi
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start
 
-print =============== step3-2 query records from ct1 from file
+print =============== step3-2 query records of ct1 from file
 sql select * from ct1;
 print $data00 $data01
 print $data10 $data11
@@ -128,7 +128,7 @@ if $data51 != 22 then
   return -1
 endi
 
-print =============== step3-2 query records from ct2 from file
+print =============== step3-2 query records of ct2 from file
 sql select * from ct2;
 print $data00 $data01
 print $data10 $data11
@@ -152,4 +152,79 @@ endi
 if $data21 != 40 then
   print data21 $data21 != 40
   return -1
+endi
+
+print =============== step3-3 query records of ct1 from memory and file(merge)
+sql insert into ct1 values('2022-05-03 16:59:00.010', 100);
+sql insert into ct1 values('2022-05-03 16:59:00.022', 200);
+sql insert into ct1 values('2022-05-03 16:59:00.016', 160);
+
+sql select * from ct1;
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+print $data30 $data31
+print $data40 $data41
+print $data50 $data51
+
+if $rows != 6 then 
+  print rows $rows != 6
+  return -1
+endi
+
+if $data01 != 100 then
+  print data01 $data01 != 100
+  return -1
+endi
+
+if $data21 != 160 then
+  print data21 $data21 != 160
+  return -1
+endi
+
+if $data51 != 200 then
+  print data51 $data51 != 200
+  return -1
+endi
+
+print =============== step3-3 query records of ct2 from memory and file(merge)
+sql insert into ct2(ts) values('2022-04-02 16:59:00.016');
+sql insert into ct2 values('2022-03-06 16:59:00.013', NULL);
+sql insert into ct2 values('2022-03-01 16:59:00.016', 10);
+sql insert into ct2(ts) values('2022-04-01 16:59:00.011');
+sql select * from ct2;
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+print $data30 $data31
+print $data40 $data41
+
+if $rows != 5 then 
+  print rows $rows != 5
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data11 != 103 then
+  print data11 $data11 != 103
+  return -1
+endi
+
+if $data21 != NULL then
+  print data21 $data21 != NULL
+  return -1
+endi
+
+if $data31 != 40 then
+  print data31 $data31 != 40
+  return -1
+endi
+
+if $data41 != NULL then
+  print data41 $data41 != NULL
+  return -1
 endi
\ No newline at end of file

From 9fb4bafb37d57316d338258ed0de3866cdd7ac5d Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 20 May 2022 17:36:42 +0800
Subject: [PATCH 15/33] feat: dump sdbdata to file

---
 source/dnode/mgmt/exe/dmMain.c                |  3 +-
 tests/test/c/CMakeLists.txt                   | 17 ++++-
 .../test/c/{create_table.c => createTable.c}  |  0
 tests/test/c/sdbDump.c                        | 70 +++++++++++++++++++
 4 files changed, 88 insertions(+), 2 deletions(-)
 rename tests/test/c/{create_table.c => createTable.c} (100%)
 create mode 100644 tests/test/c/sdbDump.c

diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c
index 6a03ff1bba..1a04e83f81 100644
--- a/source/dnode/mgmt/exe/dmMain.c
+++ b/source/dnode/mgmt/exe/dmMain.c
@@ -188,13 +188,14 @@ int main(int argc, char const *argv[]) {
   }
 
   if (dmInitLog() != 0) {
-    printf("failed to start since init log error");
+    printf("failed to start since init log error\n");
     taosCleanupArgs();
     return -1;
   }
 
   if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
     dError("failed to start since read config error");
+    taosCloseLog();
     taosCleanupArgs();
     return -1;
   }
diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index 41e9382379..8ddfbd1655 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -1,6 +1,7 @@
-add_executable(create_table create_table.c)
+add_executable(create_table createTable.c)
 add_executable(tmq_demo tmqDemo.c)
 add_executable(tmq_sim tmqSim.c)
+add_executable(sdbDump sdbDump.c)
 target_link_libraries(
     create_table
     PUBLIC taos_static
@@ -22,3 +23,17 @@ target_link_libraries(
     PUBLIC common
     PUBLIC os
 )
+target_link_libraries(
+    sdbDump
+    PUBLIC dnode
+    PUBLIC mnode
+    PUBLIC sdb
+    PUBLIC os
+)
+target_include_directories(
+    sdbDump
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode"
+    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc"
+    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc"
+    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc"
+)
\ No newline at end of file
diff --git a/tests/test/c/create_table.c b/tests/test/c/createTable.c
similarity index 100%
rename from tests/test/c/create_table.c
rename to tests/test/c/createTable.c
diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c
new file mode 100644
index 0000000000..e9e0df24ee
--- /dev/null
+++ b/tests/test/c/sdbDump.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmMgmt.h"
+#include "tconfig.h"
+
+int32_t main(int32_t argc, char *argv[]) {
+  char datafile[PATH_MAX] = {0};
+
+  for (int32_t i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("config file path overflow");
+          return -1;
+        }
+        tstrncpy(configDir, argv[i], PATH_MAX);
+      } else {
+        printf("'-c' requires a parameter, default is %s\n", configDir);
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-f") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("file path overflow");
+          return -1;
+        }
+        tstrncpy(datafile, argv[i], PATH_MAX);
+      } else {
+        printf("'-f' requires a parameter, default is %s\n", configDir);
+        return -1;
+      }
+    }
+  }
+
+  if (taosCreateLog("dumplog", 1, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
+    printf("failed to dump since init log error\n");
+    return -1;
+  }
+
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
+    uError("failed to dump since read config error");
+    taosCloseLog();
+    return -1;
+  }
+
+  if (datafile[0] == 0) {
+    snprintf(datafile, sizeof(datafile), "%s%sdata%smnode%sdata%ssdb.data", tsDataDir, TD_DIRSEP, TD_DIRSEP, TD_DIRSEP,
+             TD_DIRSEP);
+  }
+
+  dInfo("dump %s to sdb.json", datafile);
+
+  taosCleanupCfg();
+  taosCloseLog();
+  return 0;
+}

From 90f8acf16850f150c0010e0ab643a74931c82090 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Fri, 20 May 2022 17:49:32 +0800
Subject: [PATCH 16/33] fix: tq mem leak

---
 source/libs/transport/src/transComm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index d5c76ccbf2..7014cc481f 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -133,7 +133,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) {
   } else {
     p->cap = p->total;
     p->buf = taosMemoryRealloc(p->buf, p->cap);
-    tTrace("internal malloc mem: %p", p->buf);
+    tTrace("internal malloc mem: %p, size: %d", p->buf, p->cap);
 
     uvBuf->base = p->buf + p->len;
     uvBuf->len = p->cap - p->len;

From a353b1a1fbf34fb120ad28f012d398a74094360f Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 20 May 2022 18:00:05 +0800
Subject: [PATCH 17/33] refresh meta

---
 source/client/src/clientImpl.c          |  2 +
 source/libs/catalog/src/catalog.c       | 54 +++++++++++++++++--------
 source/libs/executor/src/executorimpl.c | 11 ++---
 source/libs/qcom/src/queryUtil.c        |  2 +-
 source/libs/qworker/src/qworker.c       |  3 +-
 source/libs/scheduler/src/scheduler.c   | 28 ++++++++-----
 6 files changed, 62 insertions(+), 38 deletions(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index a32d4110d9..905472dc71 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -299,6 +299,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
       schedulerFreeJob(pRequest->body.queryJob);
     }
 
+    *pRes = res.res;
+
     pRequest->code = code;
     terrno = code;
     return pRequest->code;
diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c
index 23957d1a6b..d5301e3624 100644
--- a/source/libs/catalog/src/catalog.c
+++ b/source/libs/catalog/src/catalog.c
@@ -2885,7 +2885,7 @@ _return:
 
 
 
-int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver) {
+int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver, int32_t *tbType, uint64_t *suid, char* stbName) {
   *sver = -1;
   
   if (NULL == pCtg->dbCache) {
@@ -2903,14 +2903,12 @@ int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t*
     return TSDB_CODE_SUCCESS;
   }
   
-  int32_t tbType = 0;
-  uint64_t suid = 0;
   CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
   STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname));
   if (tbMeta) {
-    tbType = tbMeta->tableType;
-    suid = tbMeta->suid;
-    if (tbType != TSDB_CHILD_TABLE) {
+    *tbType = tbMeta->tableType;
+    *suid = tbMeta->suid;
+    if (*tbType != TSDB_CHILD_TABLE) {
       *sver = tbMeta->sversion;
     }
   }
@@ -2921,44 +2919,49 @@ int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t*
     return TSDB_CODE_SUCCESS;
   }
 
-  if (tbType != TSDB_CHILD_TABLE) {
+  if (*tbType != TSDB_CHILD_TABLE) {
     ctgReleaseDBCache(pCtg, dbCache);
-    ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname);
+    ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname);
     
     return TSDB_CODE_SUCCESS;
   }
 
-  ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, suid);
+  ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid);
   
   CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock);
   
-  STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &suid, sizeof(suid));
+  STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid));
   if (NULL == stbMeta || NULL == *stbMeta) {
     CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
     ctgReleaseDBCache(pCtg, dbCache);
-    ctgDebug("stb not in stbCache, suid:%"PRIx64, suid);
+    ctgDebug("stb not in stbCache, suid:%"PRIx64, *suid);
     return TSDB_CODE_SUCCESS;
   }
 
-  if ((*stbMeta)->suid != suid) {    
+  if ((*stbMeta)->suid != *suid) {
     CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
     ctgReleaseDBCache(pCtg, dbCache);
-    ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, suid, (*stbMeta)->suid);
+    ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, *suid, (*stbMeta)->suid);
     CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
   }
 
+  size_t nameLen = 0;
+  char* name = taosHashGetKey(*stbMeta, &nameLen);
+  
+  strncpy(stbName, name, nameLen);
+  stbName[nameLen] = 0;
+
   *sver = (*stbMeta)->sversion;
 
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock);
 
   ctgReleaseDBCache(pCtg, dbCache);
 
-  ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname);
+  ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname);
   
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) {
   CTG_API_ENTER();
 
@@ -2977,9 +2980,26 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm
       continue;
     }
 
-    ctgGetTbSverFromCache(pCtg, &name, &sver);
+    int32_t tbType = 0;
+    uint64_t suid = 0;
+    char stbName[TSDB_TABLE_FNAME_LEN];
+    ctgGetTbSverFromCache(pCtg, &name, &sver, &tbType, &suid, stbName);
     if (sver >= 0 && sver < pTb->sver) {
-      catalogRemoveTableMeta(pCtg, &name); //TODO REMOVE STB FROM CACHE
+      switch (tbType) {
+        case TSDB_CHILD_TABLE: {
+          SName stb = name;
+          strcpy(stb.tname, stbName);
+          catalogRemoveTableMeta(pCtg, &stb);        
+          break;
+        }
+        case TSDB_SUPER_TABLE:
+        case TSDB_NORMAL_TABLE:
+          catalogRemoveTableMeta(pCtg, &name);
+          break;
+        default:
+          ctgError("ignore table type %d", tbType);
+          break;
+      }
     }
   }
 
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 37aeb367f5..7c9ea1a5c5 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4576,10 +4576,11 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
   return pExprs;
 }
 
-static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model) {
+static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model, char* dbFName) {
   SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
   setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
 
+  pTaskInfo->schemaVer.dbname = strdup(dbFName);
   pTaskInfo->cost.created = taosGetTimestampMs();
   pTaskInfo->id.queryId = queryId;
   pTaskInfo->execModel = model;
@@ -4994,16 +4995,10 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
     return NULL;
   }
 
-  const char* tname = pTaskInfo->schemaVer.tablename;
   for (int32_t i = 0; i < numOfCols; ++i) {
     STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i);
     SColumnNode* pColNode = (SColumnNode*)pNode->pExpr;
 
-    if (tname != NULL && (pTaskInfo->schemaVer.dbname == NULL) &&
-        strncmp(pColNode->tableName, tname, tListLen(pColNode->tableName)) == 0) {
-      pTaskInfo->schemaVer.dbname = strdup(pColNode->dbName);
-    }
-
     SColMatchInfo c = {0};
     c.output = true;
     c.colId = pColNode->colId;
@@ -5099,7 +5094,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
   uint64_t queryId = pPlan->id.queryId;
 
   int32_t code = TSDB_CODE_SUCCESS;
-  *pTaskInfo = createExecTaskInfo(queryId, taskId, model);
+  *pTaskInfo = createExecTaskInfo(queryId, taskId, model, pPlan->dbFName);
   if (*pTaskInfo == NULL) {
     code = TSDB_CODE_QRY_OUT_OF_MEMORY;
     goto _complete;
diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c
index 58886d706a..f4ba2fca81 100644
--- a/source/libs/qcom/src/queryUtil.c
+++ b/source/libs/qcom/src/queryUtil.c
@@ -58,7 +58,7 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen
 
     // 3. valid column names
     for (int32_t j = i + 1; j < numOfCols; ++j) {
-      if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
+      if (strncmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
         return false;
       }
     }
diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c
index 102cf9aa0a..cb7ff08fa3 100644
--- a/source/libs/qworker/src/qworker.c
+++ b/source/libs/qworker/src/qworker.c
@@ -1161,8 +1161,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
   QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput));
 
   if (NULL == rsp) {
-    atomic_store_ptr(&ctx->dataConnInfo.handle, qwMsg->connInfo.handle);
-    atomic_store_ptr(&ctx->dataConnInfo.ahandle, qwMsg->connInfo.ahandle);
+    ctx->dataConnInfo = qwMsg->connInfo;
 
     QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH);
   } else {
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index 7f79122e0e..899748ec45 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -2565,24 +2565,32 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in
     SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
 
+  int32_t code = 0;
+  
+  *pJob = 0;
+  
   if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) {
     SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true));
   } else {
-    SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, true));
+    SCH_ERR_JRET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, true));
   }
 
-  SSchJob *job = schAcquireJob(*pJob);
+_return:
 
-  pRes->code = atomic_load_32(&job->errCode);
-  pRes->numOfRows = job->resNumOfRows;
-  if (SCH_RES_TYPE_QUERY == job->resType) {
-    pRes->res = job->resData;
-    job->resData = NULL;
+  if (*pJob) {
+    SSchJob *job = schAcquireJob(*pJob);
+
+    pRes->code = atomic_load_32(&job->errCode);
+    pRes->numOfRows = job->resNumOfRows;
+    if (SCH_RES_TYPE_QUERY == job->resType) {
+      pRes->res = job->resData;
+      job->resData = NULL;
+    }
+
+    schReleaseJob(*pJob);
   }
 
-  schReleaseJob(*pJob);
-
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pDag, const char *sql, int64_t *pJob) {

From 7feeea974c7e0bdaeecacff2906cbc0c570a5ce5 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Fri, 20 May 2022 18:22:40 +0800
Subject: [PATCH 18/33] feat: add scenarios that trigger metadata refresh

---
 include/libs/qcom/query.h              |  8 ++-
 source/client/src/clientImpl.c         | 93 ++++++++++++--------------
 source/libs/parser/src/parInsert.c     | 75 ++++++++++++---------
 source/libs/parser/src/parTranslater.c | 16 ++++-
 4 files changed, 108 insertions(+), 84 deletions(-)

diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index daf008108b..f4ccc05208 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -182,8 +182,10 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define SET_META_TYPE_TABLE(t)      (t) = META_TYPE_TABLE
 #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
 
-#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \
-  ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
+#define NEED_CLIENT_RM_TBLMETA_ERROR(_code)                                                   \
+  ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST ||       \
+   (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \
+   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED)
 #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
   ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
 #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
@@ -194,7 +196,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define NEED_SCHEDULER_RETRY_ERROR(_code) \
   ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
 
-#define REQUEST_MAX_TRY_TIMES 5
+#define REQUEST_MAX_TRY_TIMES 1
 
 #define qFatal(...)                                                                           \
   do {                                                                                        \
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 1dda4c3024..f493f02cd6 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -13,18 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "cJSON.h"
 #include "clientInt.h"
 #include "clientLog.h"
 #include "command.h"
 #include "scheduler.h"
 #include "tdatablock.h"
+#include "tdataformat.h"
 #include "tdef.h"
 #include "tglobal.h"
 #include "tmsgtype.h"
 #include "tpagedbuf.h"
 #include "tref.h"
-#include "cJSON.h"
-#include "tdataformat.h"
 
 static int32_t       initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
 static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
@@ -189,7 +189,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
       setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
       setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision);
     }
-
+  }
+  if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) {
     TSWAP(pRequest->dbList, (*pQuery)->pDbList);
     TSWAP(pRequest->tableList, (*pQuery)->pTableList);
   }
@@ -293,7 +294,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
 
   SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
   int32_t      code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr,
-                                  pRequest->metric.start, &res);
+                                       pRequest->metric.start, &res);
   if (code != TSDB_CODE_SUCCESS) {
     if (pRequest->body.queryJob != 0) {
       schedulerFreeJob(pRequest->body.queryJob);
@@ -483,7 +484,8 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
   int32_t      retryNum = 0;
   int32_t      code = 0;
 
-  while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
+  do {
+    destroyRequest(pRequest);
     pRequest = launchQuery(pTscObj, sql, sqlLen);
     if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
       break;
@@ -494,9 +496,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
       pRequest->code = code;
       break;
     }
-
-    destroyRequest(pRequest);
-  }
+  } while (retryNum++ < REQUEST_MAX_TRY_TIMES);
 
   return pRequest;
 }
@@ -805,21 +805,20 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-static char* parseTagDatatoJson(void *p){
-  char* string = NULL;
-  cJSON *json = cJSON_CreateObject();
-  if (json == NULL)
-  {
+static char* parseTagDatatoJson(void* p) {
+  char*  string = NULL;
+  cJSON* json = cJSON_CreateObject();
+  if (json == NULL) {
     goto end;
   }
 
   int16_t nCols = kvRowNCols(p);
-  char tagJsonKey[256] = {0};
+  char    tagJsonKey[256] = {0};
   for (int j = 0; j < nCols; ++j) {
-    SColIdx * pColIdx = kvRowColIdxAt(p, j);
-    char* val = (char*)(kvRowColVal(p, pColIdx));
-    if (j == 0){
-      if(*val == TSDB_DATA_TYPE_NULL){
+    SColIdx* pColIdx = kvRowColIdxAt(p, j);
+    char*    val = (char*)(kvRowColVal(p, pColIdx));
+    if (j == 0) {
+      if (*val == TSDB_DATA_TYPE_NULL) {
         string = taosMemoryCalloc(1, 8);
         sprintf(varDataVal(string), "%s", TSDB_DATA_NULL_STR_L);
         varDataSetLen(string, strlen(varDataVal(string)));
@@ -834,19 +833,18 @@ static char* parseTagDatatoJson(void *p){
     // json value
     val += varDataTLen(val);
     char* realData = POINTER_SHIFT(val, CHAR_BYTES);
-    char type = *val;
-    if(type == TSDB_DATA_TYPE_NULL) {
+    char  type = *val;
+    if (type == TSDB_DATA_TYPE_NULL) {
       cJSON* value = cJSON_CreateNull();
-      if (value == NULL)
-      {
+      if (value == NULL) {
         goto end;
       }
       cJSON_AddItemToObject(json, tagJsonKey, value);
-    }else if(type == TSDB_DATA_TYPE_NCHAR) {
+    } else if (type == TSDB_DATA_TYPE_NCHAR) {
       cJSON* value = NULL;
-      if (varDataLen(realData) > 0){
-        char *tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1);
-        int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(realData), varDataLen(realData), tagJsonValue);
+      if (varDataLen(realData) > 0) {
+        char*   tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1);
+        int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(realData), varDataLen(realData), tagJsonValue);
         if (length < 0) {
           tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, val);
           taosMemoryFree(tagJsonValue);
@@ -854,45 +852,41 @@ static char* parseTagDatatoJson(void *p){
         }
         value = cJSON_CreateString(tagJsonValue);
         taosMemoryFree(tagJsonValue);
-        if (value == NULL)
-        {
+        if (value == NULL) {
           goto end;
         }
-      }else if(varDataLen(realData) == 0){
+      } else if (varDataLen(realData) == 0) {
         value = cJSON_CreateString("");
-      }else{
+      } else {
         ASSERT(0);
       }
 
       cJSON_AddItemToObject(json, tagJsonKey, value);
-    }else if(type == TSDB_DATA_TYPE_DOUBLE){
+    } else if (type == TSDB_DATA_TYPE_DOUBLE) {
       double jsonVd = *(double*)(realData);
       cJSON* value = cJSON_CreateNumber(jsonVd);
-      if (value == NULL)
-      {
+      if (value == NULL) {
         goto end;
       }
       cJSON_AddItemToObject(json, tagJsonKey, value);
-//    }else if(type == TSDB_DATA_TYPE_BIGINT){
-//      int64_t jsonVd = *(int64_t*)(realData);
-//      cJSON* value = cJSON_CreateNumber((double)jsonVd);
-//      if (value == NULL)
-//      {
-//        goto end;
-//      }
-//      cJSON_AddItemToObject(json, tagJsonKey, value);
-    }else if (type == TSDB_DATA_TYPE_BOOL) {
-      char jsonVd = *(char*)(realData);
+      //    }else if(type == TSDB_DATA_TYPE_BIGINT){
+      //      int64_t jsonVd = *(int64_t*)(realData);
+      //      cJSON* value = cJSON_CreateNumber((double)jsonVd);
+      //      if (value == NULL)
+      //      {
+      //        goto end;
+      //      }
+      //      cJSON_AddItemToObject(json, tagJsonKey, value);
+    } else if (type == TSDB_DATA_TYPE_BOOL) {
+      char   jsonVd = *(char*)(realData);
       cJSON* value = cJSON_CreateBool(jsonVd);
-      if (value == NULL)
-      {
+      if (value == NULL) {
         goto end;
       }
       cJSON_AddItemToObject(json, tagJsonKey, value);
-    }else{
+    } else {
       ASSERT(0);
     }
-
   }
   string = cJSON_PrintUnformatted(json);
 end:
@@ -930,7 +924,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
 
       pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
       pResultInfo->row[i] = pResultInfo->pCol[i].pData;
-    }else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) {
+    } else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) {
       char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
       if (p == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
@@ -943,7 +937,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
         if (pCol->offset[j] != -1) {
           char* pStart = pCol->offset[j] + pCol->pData;
 
-
           int32_t jsonInnerType = *pStart;
           char*   jsonInnerData = pStart + CHAR_BYTES;
           char    dst[TSDB_MAX_JSON_TAG_LEN] = {0};
@@ -951,7 +944,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
             sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
             varDataSetLen(dst, strlen(varDataVal(dst)));
           } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) {
-            char *jsonString = parseTagDatatoJson(jsonInnerData);
+            char* jsonString = parseTagDatatoJson(jsonInnerData);
             STR_TO_VARSTR(dst, jsonString);
             taosMemoryFree(jsonString);
           } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) {  // value -> "value"
diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index d1b4a745b9..b452950624 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -41,6 +41,13 @@
     sToken = tStrGetToken(pSql, &index, false);  \
   } while (0)
 
+#define NEXT_VALID_TOKEN(pSql, sToken)        \
+  do {                                        \
+    sToken.n = tGetToken(pSql, &sToken.type); \
+    sToken.z = pSql;                          \
+    pSql += sToken.n;                         \
+  } while (TK_NK_SPACE == sToken.type)
+
 typedef struct SInsertParseContext {
   SParseContext*     pComCxt;             // input
   char*              pSql;                // input
@@ -482,9 +489,11 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
           return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
         }
       } else if (pToken->type == TK_NK_INTEGER) {
-        return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param);
+        return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes,
+                    param);
       } else if (pToken->type == TK_NK_FLOAT) {
-        return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param);
+        return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes,
+                    param);
       } else {
         return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
       }
@@ -685,7 +694,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
       isOrdered = false;
     }
     if (index < 0) {
-      return buildSyntaxErrMsg(&pCxt->msg, "invalid column/tag name", sToken.z);
+      return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, sToken.z);
     }
     if (pColList->cols[index].valStat == VAL_STAT_HAS) {
       return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", sToken.z);
@@ -895,8 +904,10 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tb
     return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z);
   }
   CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, name->tname));
-  NEXT_TOKEN(pCxt->pSql, sToken);
-  if (TK_NK_RP != sToken.type) {
+  NEXT_VALID_TOKEN(pCxt->pSql, sToken);
+  if (TK_NK_COMMA == sToken.type) {
+    return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_TAGS_NOT_MATCHED);
+  } else if (TK_NK_RP != sToken.type) {
     return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z);
   }
 
@@ -996,8 +1007,10 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo
       pDataBlock->size += extendedRowSize;  // len;
     }
 
-    NEXT_TOKEN(pCxt->pSql, sToken);
-    if (TK_NK_RP != sToken.type) {
+    NEXT_VALID_TOKEN(pCxt->pSql, sToken);
+    if (TK_NK_COMMA == sToken.type) {
+      return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
+    } else if (TK_NK_RP != sToken.type) {
       return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z);
     }
 
@@ -1057,10 +1070,10 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) {
 //       VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
 //   [...];
 static int32_t parseInsertBody(SInsertParseContext* pCxt) {
-  int32_t tbNum = 0;
-  char tbFName[TSDB_TABLE_FNAME_LEN];
-  bool autoCreateTbl = false;
-  STableMeta *pMeta = NULL;
+  int32_t     tbNum = 0;
+  char        tbFName[TSDB_TABLE_FNAME_LEN];
+  bool        autoCreateTbl = false;
+  STableMeta* pMeta = NULL;
 
   // for each table
   while (1) {
@@ -1121,7 +1134,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
                                     &dataBuf, NULL, &pCxt->createTblReq));
     pMeta = pCxt->pTableMeta;
     pCxt->pTableMeta = NULL;
-    
+
     if (TK_NK_LP == sToken.type) {
       // pSql -> field1_name, ...)
       CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pMeta)));
@@ -1160,7 +1173,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     memcpy(tags, &pCxt->tags, sizeof(pCxt->tags));
-    (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj);
+    (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj,
+                                pCxt->pTableBlockHashObj);
 
     memset(&pCxt->tags, 0, sizeof(pCxt->tags));
     pCxt->pVgroupsHashObj = NULL;
@@ -1231,14 +1245,14 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
-  
+
   context.pOutput->payloadType = PAYLOAD_TYPE_KV;
 
   int32_t code = skipInsertInto(&context);
   if (TSDB_CODE_SUCCESS == code) {
     code = parseInsertBody(&context);
   }
-  if (TSDB_CODE_SUCCESS == code) {
+  if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) {
     SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL);
     while (NULL != pTable) {
       taosArrayPush((*pQuery)->pTableList, pTable);
@@ -1579,9 +1593,9 @@ typedef struct SmlExecTableHandle {
 } SmlExecTableHandle;
 
 typedef struct SmlExecHandle {
-  SHashObj*           pBlockHash;
-  SmlExecTableHandle  tableExecHandle;
-  SQuery              *pQuery;
+  SHashObj*          pBlockHash;
+  SmlExecTableHandle tableExecHandle;
+  SQuery*            pQuery;
 } SSmlExecHandle;
 
 static void smlDestroyTableHandle(void* pHandle) {
@@ -1673,9 +1687,9 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD
     SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1];  // colId starts with 1
     param.schema = pTagSchema;
     SSmlKv* kv = taosArrayGetP(cols, i);
-    if(IS_VAR_DATA_TYPE(kv->type)){
+    if (IS_VAR_DATA_TYPE(kv->type)) {
       KvRowAppend(msg, kv->value, kv->length, &param);
-    }else{
+    } else {
       KvRowAppend(msg, &(kv->value), kv->length, &param);
     }
   }
@@ -1688,13 +1702,13 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols, bool format,
-                    STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen) {
+int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
+                    char* tableName, char* msgBuf, int16_t msgBufLen) {
   SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
 
   SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle;
-  smlDestroyTableHandle(&smlHandle->tableExecHandle);   // free for each table
-  SSchema*        pTagsSchema = getTableTagSchema(pTableMeta);
+  smlDestroyTableHandle(&smlHandle->tableExecHandle);  // free for each table
+  SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
   setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta));
   int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema);
   if (ret != TSDB_CODE_SUCCESS) {
@@ -1702,7 +1716,8 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols
     return ret;
   }
   SKVRow row = NULL;
-  ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema, &row, &pBuf);
+  ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema,
+                       &row, &pBuf);
   if (ret != TSDB_CODE_SUCCESS) {
     return ret;
   }
@@ -1733,7 +1748,7 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols
   initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo);
 
   int32_t rowNum = taosArrayGetSize(cols);
-  if(rowNum <= 0) {
+  if (rowNum <= 0) {
     return buildInvalidOperationMsg(&pBuf, "cols size <= 0");
   }
   ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum);
@@ -1744,9 +1759,9 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols
   for (int32_t r = 0; r < rowNum; ++r) {
     STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size);  // skip the SSubmitBlk header
     tdSRowResetBuf(pBuilder, row);
-    void *rowData = taosArrayGetP(cols, r);
+    void*  rowData = taosArrayGetP(cols, r);
     size_t rowDataSize = 0;
-    if(format){
+    if (format) {
       rowDataSize = taosArrayGetSize(rowData);
     }
 
@@ -1781,9 +1796,9 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols
           kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
         }
 
-        if(IS_VAR_DATA_TYPE(kv->type)){
+        if (IS_VAR_DATA_TYPE(kv->type)) {
           MemRowAppend(&pBuf, kv->value, colLen, &param);
-        }else{
+        } else {
           MemRowAppend(&pBuf, &(kv->value), colLen, &param);
         }
       }
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index dbb29699fc..f6d53dd15a 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -120,6 +120,20 @@ static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const
   return getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta);
 }
 
+static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName,
+                                   STableMeta** pMeta) {
+  SParseContext* pParCxt = pCxt->pParseCxt;
+  SName          name;
+  toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name);
+  int32_t code =
+      catalogRefreshGetTableMeta(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, &name, pMeta, false);
+  if (TSDB_CODE_SUCCESS != code) {
+    parserError("catalogRefreshGetTableMeta error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pDbName,
+                pTableName);
+  }
+  return code;
+}
+
 static int32_t getTableDistVgInfo(STranslateContext* pCxt, const SName* pName, SArray** pVgInfo) {
   SParseContext* pParCxt = pCxt->pParseCxt;
   int32_t        code = collectUseDatabase(pName, pCxt->pDbs);
@@ -3201,7 +3215,7 @@ static int32_t translateExplain(STranslateContext* pCxt, SExplainStmt* pStmt) {
 }
 
 static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) {
-  return getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta);
+  return refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta);
 }
 
 static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt) {

From 7391a756f645f42146c4aade1ac8a1dd3a6b7c2a Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Fri, 20 May 2022 18:53:21 +0800
Subject: [PATCH 19/33] refactor(stream)

---
 include/common/tmsg.h                       |  12 ---
 include/common/tmsgdef.h                    |   4 +
 include/libs/stream/tstream.h               |  38 +++++++-
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c |   4 +
 source/dnode/vnode/src/inc/vnodeInt.h       |  14 ++-
 source/dnode/vnode/src/tq/tq.c              | 100 ++++++++++++++++++--
 source/dnode/vnode/src/vnd/vnodeSvr.c       |  16 ++++
 source/libs/stream/src/tstream.c            |  22 +++--
 8 files changed, 174 insertions(+), 36 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index addb84046c..d9087f59c6 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2589,18 +2589,6 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
   taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp);
 }
 
-typedef struct {
-  int64_t streamId;
-  int32_t taskId;
-  int32_t sourceVg;
-  int64_t sourceVer;
-  SArray* data;  // SArray<SSDataBlock>
-} SStreamDispatchReq;
-
-typedef struct {
-  int8_t inputStatus;
-} SStreamDispatchRsp;
-
 #define TD_AUTO_CREATE_TABLE 0x1
 typedef struct {
   int64_t        suid;
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index 93b2e75360..455898585a 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -200,6 +200,10 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_TASK_WRITE_EXEC, "vnode-task-write-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
 
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_RUN, "vnode-stream-task-run", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_DISPATCH, "vnode-stream-task-dispatch", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_RECOVER, "vnode-stream-task-recover", NULL, NULL)
+
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index bca947b84c..1604749af8 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -107,7 +107,7 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit)
   if (ref == 0) {
     taosMemoryFree(pDataSubmit->data);
     taosMemoryFree(pDataSubmit->dataRef);
-    taosFreeQitem(pDataSubmit);
+    // taosFreeQitem(pDataSubmit);
   }
 }
 
@@ -286,6 +286,36 @@ typedef struct {
   int32_t  taskId;
 } SStreamTaskRunReq;
 
+typedef struct {
+  int64_t streamId;
+  int32_t taskId;
+  int32_t sourceTaskId;
+  int32_t sourceVg;
+#if 0
+  int64_t sourceVer;
+#endif
+  SArray* data;  // SArray<SSDataBlock>
+} SStreamDispatchReq;
+
+typedef struct {
+  int64_t streamId;
+  int32_t taskId;
+  int8_t  inputStatus;
+} SStreamDispatchRsp;
+
+typedef struct {
+  int64_t streamId;
+  int32_t taskId;
+  int32_t sourceTaskId;
+  int32_t sourceVg;
+} SStreamTaskRecoverReq;
+
+typedef struct {
+  int64_t streamId;
+  int32_t taskId;
+  int8_t  inputStatus;
+} SStreamTaskRecoverRsp;
+
 int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input);
 int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input);
 int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
@@ -296,6 +326,12 @@ int32_t streamTaskRun(SStreamTask* pTask);
 
 int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
 
+int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb);
+int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
+int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp);
+int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
+int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 3da3d90ae1..f28209f982 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -314,6 +314,10 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_MERGE_EXEC, vmPutNodeMsgToMergeQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_WRITE_EXEC, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RUN, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DISPATCH, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RECOVER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER;
+
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 1c89649313..23825e6f4a 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -121,10 +121,18 @@ int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
 int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen);
-int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
-int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
-int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
+int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
+#if 0
+int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
+int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
+#endif
+int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* data);
+int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
 
 // sma
 int32_t smaOpen(SVnode* pVnode);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 9361c0e6d2..a8f9e8db00 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -105,12 +105,11 @@ static void tdSRowDemo() {
 }
 
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
-  void*    pIter = NULL;
-  STqExec* pExec = NULL;
+  void* pIter = NULL;
   while (1) {
     pIter = taosHashIterate(pTq->execs, pIter);
     if (pIter == NULL) break;
-    pExec = (STqExec*)pIter;
+    STqExec* pExec = (STqExec*)pIter;
     if (pExec->subType == TOPIC_SUB_TYPE__DB) {
       if (!isAdd) {
         int32_t sz = taosArrayGetSize(tbUidList);
@@ -275,6 +274,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
   }
   memcpy(data, msg, msgLen);
 
+  tqProcessStreamTriggerNew(pTq, data);
+
+#if 0
   SRpcMsg req = {
       .msgType = TDMT_VND_STREAM_TRIGGER,
       .pCont = data,
@@ -282,6 +284,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
   };
 
   tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req);
+#endif
 
   return 0;
 }
@@ -980,12 +983,24 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
 }
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
+  pTask->status = TASK_STATUS__IDLE;
+  pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
+  pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
+
+  pTask->inputQ = taosOpenQueue();
+  pTask->outputQ = taosOpenQueue();
+  pTask->inputQAll = taosAllocateQall();
+  pTask->outputQAll = taosAllocateQall();
+
+  if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL)
+    goto FAIL;
+
   if (pTask->execType != TASK_EXEC__NONE) {
     // expand runners
     pTask->exec.numOfRunners = parallel;
     pTask->exec.runners = taosMemoryCalloc(parallel, sizeof(SStreamRunner));
     if (pTask->exec.runners == NULL) {
-      return -1;
+      goto FAIL;
     }
     for (int32_t i = 0; i < parallel; i++) {
       STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
@@ -1007,6 +1022,13 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
   }
 
   return 0;
+FAIL:
+  if (pTask->inputQ) taosCloseQueue(pTask->inputQ);
+  if (pTask->outputQ) taosCloseQueue(pTask->outputQ);
+  if (pTask->inputQAll) taosFreeQall(pTask->inputQAll);
+  if (pTask->outputQAll) taosFreeQall(pTask->outputQAll);
+  if (pTask) taosMemoryFree(pTask);
+  return -1;
 }
 
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
@@ -1058,6 +1080,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo
   return 0;
 }
 
+#if 0
 int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* data) {
   SStreamDataSubmit* pSubmit = NULL;
 
@@ -1108,6 +1131,7 @@ FAIL:
   }
   return -1;
 }
+#endif
 
 int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId) {
   SStreamTaskExecReq req;
@@ -1125,25 +1149,28 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId)
   return 0;
 }
 
-int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
+int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* pReq) {
   void* pIter = NULL;
   bool  failed = false;
 
   SStreamDataSubmit* pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
   if (pSubmit == NULL) {
     failed = true;
+    goto SET_TASK_FAIL;
   }
   pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t));
   if (pSubmit->dataRef == NULL) {
     failed = true;
+    goto SET_TASK_FAIL;
   }
 
-  pSubmit->type = STREAM_DATA_TYPE_SUBMIT_BLOCK;
-  pSubmit->sourceVer = ver;
-  pSubmit->sourceVg = pTq->pVnode->config.vgId;
+  pSubmit->type = STREAM_INPUT__DATA_SUBMIT;
+  /*pSubmit->sourceVer = ver;*/
+  /*pSubmit->sourceVg = pTq->pVnode->config.vgId;*/
   pSubmit->data = pReq;
   *pSubmit->dataRef = 1;
 
+SET_TASK_FAIL:
   while (1) {
     pIter = taosHashIterate(pTq->pStreamTasks, pIter);
     if (pIter == NULL) break;
@@ -1162,7 +1189,18 @@ int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
 
       int8_t execStatus = atomic_load_8(&pTask->status);
       if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) {
-        // TODO dispatch task launch msg to fetch queue
+        SStreamTaskRunReq* pRunReq = taosMemoryMalloc(sizeof(SStreamTaskRunReq));
+        if (pRunReq == NULL) continue;
+        // TODO: do we need htonl?
+        pRunReq->head.vgId = pTq->pVnode->config.vgId;
+        pRunReq->streamId = pTask->streamId;
+        pRunReq->taskId = pTask->taskId;
+        SRpcMsg msg = {
+            .msgType = TDMT_VND_TASK_RUN,
+            .pCont = pRunReq,
+            .contLen = sizeof(SStreamTaskRunReq),
+        };
+        tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &msg);
       }
 
     } else {
@@ -1174,11 +1212,53 @@ int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
     streamDataSubmitRefDec(pSubmit);
     return 0;
   } else {
+    if (pSubmit) {
+      if (pSubmit->dataRef) {
+        taosMemoryFree(pSubmit->dataRef);
+      }
+      taosFreeQitem(pSubmit);
+    }
     return -1;
   }
 }
 
-int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) {
+int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   //
+  SStreamTaskRunReq* pReq = pMsg->pCont;
+  int32_t            taskId = pReq->taskId;
+  SStreamTask*       pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
+  streamTaskProcessRunReq(pTask, &pTq->pVnode->msgCb);
+  return 0;
+}
+
+int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) {
+  SStreamDispatchReq* pReq = pMsg->pCont;
+  int32_t             taskId = pReq->taskId;
+  SStreamTask*        pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
+  streamTaskProcessDispatchReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg);
+  return 0;
+}
+
+int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) {
+  SStreamTaskRecoverReq* pReq = pMsg->pCont;
+  int32_t                taskId = pReq->taskId;
+  SStreamTask*           pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
+  streamTaskProcessRecoverReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg);
+  return 0;
+}
+
+int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
+  SStreamDispatchRsp* pRsp = pMsg->pCont;
+  int32_t             taskId = pRsp->taskId;
+  SStreamTask*        pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
+  streamTaskProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp);
+  return 0;
+}
+
+int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) {
+  SStreamTaskRecoverRsp* pRsp = pMsg->pCont;
+  int32_t                taskId = pRsp->taskId;
+  SStreamTask*           pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
+  streamTaskProcessRecoverRsp(pTask, pRsp);
   return 0;
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index dfd78d9dca..b6600b5d0b 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -106,11 +106,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
       }
     } break;
+#if 0
     case TDMT_VND_TASK_WRITE_EXEC: {
       if (tqProcessTaskExec(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead),
                             0) < 0) {
       }
     } break;
+#endif
     case TDMT_VND_ALTER_VNODE:
       break;
     default:
@@ -181,11 +183,25 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
       return vnodeGetTableMeta(pVnode, pMsg);
     case TDMT_VND_CONSUME:
       return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId);
+
+    case TDMT_VND_TASK_RUN:
+      return tqProcessTaskRunReq(pVnode->pTq, pMsg);
+    case TDMT_VND_TASK_DISPATCH:
+      return tqProcessTaskDispatchReq(pVnode->pTq, pMsg);
+    case TDMT_VND_TASK_RECOVER:
+      return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);
+    case TDMT_VND_TASK_DISPATCH_RSP:
+      return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg);
+    case TDMT_VND_TASK_RECOVER_RSP:
+      return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);
+
+#if 0
     case TDMT_VND_TASK_PIPE_EXEC:
     case TDMT_VND_TASK_MERGE_EXEC:
       return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0);
     case TDMT_VND_STREAM_TRIGGER:
       return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0);
+#endif
     case TDMT_VND_QUERY_HEARTBEAT:
       return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg);
     default:
diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c
index 38b6f2b0e2..66a661481e 100644
--- a/source/libs/stream/src/tstream.c
+++ b/source/libs/stream/src/tstream.c
@@ -68,7 +68,7 @@ static int32_t streamBuildExecMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMs
 
     // get groupId, compute hash value
     uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));
-    //
+
     // get node
     // TODO: optimize search process
     SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
@@ -152,13 +152,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
 
   // exec
   while (1) {
-    SSDataBlock* output;
+    SSDataBlock* output = NULL;
     uint64_t     ts = 0;
     if (qExecTask(exec, &output, &ts) < 0) {
       ASSERT(false);
     }
     if (output == NULL) break;
-    taosArrayPush(pRes, &output);
+    taosArrayPush(pRes, output);
   }
 
   // destroy
@@ -189,7 +189,7 @@ int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) {
         taosFreeQitem(data);
 
         if (taosArrayGetSize(pRes) != 0) {
-          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
           resQ->type = STREAM_INPUT__DATA_BLOCK;
           resQ->blocks = pRes;
           taosWriteQitem(pTask->outputQ, resQ);
@@ -209,7 +209,7 @@ int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) {
         taosFreeQitem(data);
 
         if (taosArrayGetSize(pRes) != 0) {
-          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
           resQ->type = STREAM_INPUT__DATA_BLOCK;
           resQ->blocks = pRes;
           taosWriteQitem(pTask->outputQ, resQ);
@@ -231,7 +231,7 @@ int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) {
         taosFreeQitem(data);
 
         if (taosArrayGetSize(pRes) != 0) {
-          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
           resQ->type = STREAM_INPUT__DATA_BLOCK;
           resQ->blocks = pRes;
           taosWriteQitem(pTask->outputQ, resQ);
@@ -253,7 +253,7 @@ int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) {
         taosFreeQitem(data);
 
         if (taosArrayGetSize(pRes) != 0) {
-          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+          SStreamDataBlock* resQ = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
           resQ->type = STREAM_INPUT__DATA_BLOCK;
           resQ->blocks = pRes;
           taosWriteQitem(pTask->outputQ, resQ);
@@ -392,12 +392,14 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
   // 1.2 enqueue
   pBlock->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
   pBlock->sourceVg = pReq->sourceVg;
-  pBlock->sourceVer = pReq->sourceVer;
+  /*pBlock->sourceVer = pReq->sourceVer;*/
   taosWriteQitem(pTask->inputQ, pBlock);
 
   // 1.3 rsp by input status
   SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp));
   pCont->inputStatus = status;
+  pCont->streamId = pReq->streamId;
+  pCont->taskId = pReq->sourceTaskId;
   pRsp->pCont = pCont;
   pRsp->contLen = sizeof(SStreamDispatchRsp);
   tmsgSendRsp(pRsp);
@@ -439,12 +441,12 @@ int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) {
   return 0;
 }
 
-int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) {
+int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) {
   //
   return 0;
 }
 
-int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, char* msg) {
+int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) {
   //
   return 0;
 }

From 0b8d3b056d5f130ddaf00be4b3394ef9469fd5ae Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Fri, 20 May 2022 18:58:17 +0800
Subject: [PATCH 20/33] fix: bad merge

---
 source/dnode/vnode/src/vnd/vnodeSvr.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index c09b80af75..297b518ac7 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -184,8 +184,11 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
     case TDMT_VND_CONSUME:
       return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId);
 
-    case TDMT_VND_TASK_RUN:
-      return tqProcessTaskRunReq(pVnode->pTq, pMsg);
+    case TDMT_VND_TASK_RUN: {
+      int32_t code = tqProcessTaskRunReq(pVnode->pTq, pMsg);
+      pMsg->pCont = NULL;
+      return code;
+    }
     case TDMT_VND_TASK_DISPATCH:
       return tqProcessTaskDispatchReq(pVnode->pTq, pMsg);
     case TDMT_VND_TASK_RECOVER:

From 3d5073e7854a396a0210fd1f834b966e63f6c712 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 20 May 2022 19:25:46 +0800
Subject: [PATCH 21/33] fix res issue

---
 source/libs/scheduler/inc/schedulerInt.h |  8 +-----
 source/libs/scheduler/src/scheduler.c    | 32 +++++++++++-------------
 2 files changed, 16 insertions(+), 24 deletions(-)

diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h
index 5a6fcee759..be92de774b 100644
--- a/source/libs/scheduler/inc/schedulerInt.h
+++ b/source/libs/scheduler/inc/schedulerInt.h
@@ -39,12 +39,6 @@ enum {
   SCH_WRITE,
 };
 
-typedef enum {
-  SCH_RES_TYPE_QUERY,
-  SCH_RES_TYPE_FETCH,
-} SCH_RES_TYPE;
-
-
 typedef struct SSchTrans {
   void *transInst;
   void *transHandle;
@@ -197,7 +191,7 @@ typedef struct SSchJob {
   int32_t          errCode;
   SArray          *errList;    // SArray<SQueryErrorInfo>
   SRWLatch         resLock;
-  SCH_RES_TYPE     resType;
+  void            *queryRes;
   void            *resData;         //TODO free it or not
   int32_t          resNumOfRows;
   const char      *sql;
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index 899748ec45..dcd87557aa 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -1058,8 +1058,6 @@ _return:
 int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) {
   SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed);
 
-  pJob->resType = SCH_RES_TYPE_FETCH;
-
   atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows));
   atomic_store_ptr(&pJob->resData, pRsp);
 
@@ -1072,9 +1070,9 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs
 
 int32_t schSaveJobQueryRes(SSchJob *pJob, SResReadyRsp *rsp) {
   if (rsp->tbFName[0]) {
-    if (NULL == pJob->resData) {
-      pJob->resData = taosArrayInit(pJob->taskNum, sizeof(STbVerInfo));
-      if (NULL == pJob->resData) {
+    if (NULL == pJob->queryRes) {
+      pJob->queryRes = taosArrayInit(pJob->taskNum, sizeof(STbVerInfo));
+      if (NULL == pJob->queryRes) {
         SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
       }
     }
@@ -1084,7 +1082,7 @@ int32_t schSaveJobQueryRes(SSchJob *pJob, SResReadyRsp *rsp) {
     tbInfo.sversion = rsp->sversion;
     tbInfo.tversion = rsp->tversion;
 
-    taosArrayPush((SArray *)pJob->resData, &tbInfo);
+    taosArrayPush((SArray *)pJob->queryRes, &tbInfo);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -1201,10 +1199,9 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
         atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows);
         SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows);
 
-        pJob->resType = SCH_RES_TYPE_QUERY;
         SCH_LOCK(SCH_WRITE, &pJob->resLock);
-        if (pJob->resData) {
-          SSubmitRsp *sum = pJob->resData;
+        if (pJob->queryRes) {
+          SSubmitRsp *sum = pJob->queryRes;
           sum->affectedRows += rsp->affectedRows;
           sum->nBlocks += rsp->nBlocks;
           sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks));
@@ -1212,7 +1209,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
           taosMemoryFree(rsp->pBlocks);
           taosMemoryFree(rsp);
         } else {
-          pJob->resData = rsp;
+          pJob->queryRes = rsp;
         }
         SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
       }
@@ -1246,7 +1243,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
         SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
       }
       SCH_ERR_JRET(rsp->code);
-      pJob->resType = SCH_RES_TYPE_QUERY;
 
       SCH_ERR_JRET(schSaveJobQueryRes(pJob, rsp));
       
@@ -2424,6 +2420,12 @@ void schFreeJobImpl(void *job) {
 
   qExplainFreeCtx(pJob->explainCtx);
 
+  if (SCH_IS_QUERY_JOB(pJob)) {
+    taosArrayDestroy((SArray *)pJob->queryRes);
+  } else {
+    tFreeSSubmitRsp((SSubmitRsp*)pJob->queryRes);
+  }
+
   taosMemoryFreeClear(pJob->resData);
   taosMemoryFreeClear(pJob);
 
@@ -2486,8 +2488,6 @@ int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDa
 
   SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData));
 
-  pJob->resType = SCH_RES_TYPE_FETCH;
-
   int64_t refId = taosAddRef(schMgmt.jobRef, pJob);
   if (refId < 0) {
     SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno));
@@ -2582,10 +2582,8 @@ _return:
 
     pRes->code = atomic_load_32(&job->errCode);
     pRes->numOfRows = job->resNumOfRows;
-    if (SCH_RES_TYPE_QUERY == job->resType) {
-      pRes->res = job->resData;
-      job->resData = NULL;
-    }
+    pRes->res = job->queryRes;
+    job->queryRes = NULL;
 
     schReleaseJob(*pJob);
   }

From ace05054e7ca3b81ce2f6c065f851712c77c932a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”>
Date: Fri, 20 May 2022 20:26:10 +0800
Subject: [PATCH 22/33] feat(query): add nested query function

---
 tests/system-test/2-query/nestedQuery.py | 209 +++++++++++++++++------
 1 file changed, 155 insertions(+), 54 deletions(-)

diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py
index 8c61451756..871054de3a 100755
--- a/tests/system-test/2-query/nestedQuery.py
+++ b/tests/system-test/2-query/nestedQuery.py
@@ -52,6 +52,60 @@ class TDTestCase:
 
     #     return(conn1,cur1)  
     
+    def data_matrix_equal(self, sql1,row1_s,row1_e,col1_s,col1_e, sql2,row2_s,row2_e,col2_s,col2_e):
+        #  ----row1_start----col1_start----
+        #  - - - - 是一个矩阵内的数据相等- - - 
+        #  - - - - - - - - - - - - - - - - 
+        #  ----row1_end------col1_end------
+        self.sql1 = sql1
+        list1 =[]
+        tdSql.query(sql1)
+        for i1 in range(row1_s-1,row1_e):
+            #print("iiii=%d"%i1)
+            for j1 in range(col1_s-1,col1_e):
+                #print("jjjj=%d"%j1)
+                #print("data=%s" %(tdSql.getData(i1,j1)))
+                list1.append(tdSql.getData(i1,j1))
+        print("=====list1-------list1---=%s" %set(list1))
+        
+        tdSql.execute("reset query cache;")
+        self.sql2 = sql2  
+        list2 =[]
+        tdSql.query(sql2)
+        for i2 in range(row2_s-1,row2_e):
+            #print("iiii222=%d"%i2)
+            for j2 in range(col2_s-1,col2_e):
+                #print("jjjj222=%d"%j2)
+                #print("data=%s" %(tdSql.getData(i2,j2)))
+                list2.append(tdSql.getData(i2,j2))
+        print("=====list2-------list2---=%s" %set(list2)) 
+       
+        if  (list1 == list2) and len(list2)>0:
+            # print(("=====matrix===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2))
+            tdLog.info(("===matrix===sql1:'%s' matrix_result = sql2:'%s' matrix_result") %(sql1,sql2))
+        elif (set(list2)).issubset(set(list1)):
+            # 解决不同子表排列结果乱序
+            # print(("=====list_issubset==matrix2in1-true===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2))
+            tdLog.info(("===matrix_issubset===sql1:'%s' matrix_set_result = sql2:'%s' matrix_set_result") %(sql1,sql2))
+        #elif abs(float(str(list1).replace("]","").replace("[","").replace("e+","")) - float(str(list2).replace("]","").replace("[","").replace("e+",""))) <= 0.0001:
+        elif abs(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace("e+","").replace(", ","").replace("(","").replace(")","").replace("-","")) - float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace("e+","").replace(", ","").replace("(","").replace(")","").replace("-",""))) <= 0.0001:
+            print(("=====matrix_abs+e+===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2))
+            print(("=====matrix_abs+e+replace_after===sql1.list1:'%s',sql2.list2:'%s'") %(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace("e+","").replace(", ","").replace("(","").replace(")","").replace("-","")),float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace("e+","").replace(", ","").replace("(","").replace(")","").replace("-",""))))
+            tdLog.info(("===matrix_abs+e+===sql1:'%s' matrix_result = sql2:'%s' matrix_result") %(sql1,sql2))
+        elif abs(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")) - float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-",""))) <= 0.1:
+            #{datetime.datetime(2021, 8, 27, 1, 46, 40), -441.46841430664057}replace
+            print(("=====matrix_abs+replace===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2))
+            print(("=====matrix_abs+replace_after===sql1.list1:'%s',sql2.list2:'%s'") %(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")),float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-",""))))
+            tdLog.info(("===matrix_abs+replace===sql1:'%s' matrix_result = sql2:'%s' matrix_result") %(sql1,sql2))
+        elif abs(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")) - float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-",""))) <= 0.5:
+            print(("=====matrix_abs===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2))
+            print(("=====matrix_abs===sql1.list1:'%s',sql2.list2:'%s'") %(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")),float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-",""))))
+            tdLog.info(("===matrix_abs======sql1:'%s' matrix_result = sql2:'%s' matrix_result") %(sql1,sql2))
+        else:
+            print(("=====matrix_error===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2))
+            tdLog.info(("sql1:'%s' matrix_result != sql2:'%s' matrix_result") %(sql1,sql2))
+            return tdSql.checkEqual(list1,list2)
+            
     def restartDnodes(self):
         pass
         # tdDnodes.stop(1)
@@ -1702,10 +1756,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            #TD-15719 tdSql.query(sql)
         
         tdSql.query("select 17-2 from stable_1;")
         for i in range(self.fornum):
@@ -1720,10 +1774,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 17-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -1738,10 +1792,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         self.restartDnodes()
         tdSql.query("select 17-3 from stable_1;")
@@ -1758,10 +1812,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            #TD-15770 tdSql.query(sql)
 
         tdSql.query("select 17-4 from stable_1;")
         for i in range(self.fornum):
@@ -1775,10 +1829,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 17-4.2 from stable_1;")
         for i in range(self.fornum):
@@ -1792,10 +1846,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 17-5 from stable_1;")
         for i in range(self.fornum):
@@ -1811,10 +1865,10 @@ class TDTestCase:
             # sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+           #TD-15719 tdSql.query(sql)
        
         tdSql.query("select 17-6 from stable_1;")
         for i in range(self.fornum):
@@ -1827,10 +1881,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            #TD-15770 tdSql.query(sql)
 
         tdSql.query("select 17-7 from stable_1;")
         for i in range(self.fornum):
@@ -1843,10 +1897,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 17-7.2 from stable_1;")
         for i in range(self.fornum):
@@ -1859,10 +1913,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         self.restartDnodes()
         tdSql.query("select 17-8 from stable_1;")
@@ -1876,10 +1930,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
        
         tdSql.query("select 17-9 from stable_1;")
         for i in range(self.fornum):
@@ -1892,10 +1946,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 17-10 from stable_1;")
         for i in range(self.fornum):
@@ -1908,10 +1962,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         #18 select apercentile from (select calc_aggregate_alls form regualr_table or stable  where <\>\in\and\or session order by  limit )interval_sliding
         tdSql.query("select 18-1 from stable_1;")
@@ -1926,10 +1980,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-           #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
                
         tdSql.query("select 18-2 from stable_1;")
         for i in range(self.fornum):
@@ -1942,10 +1996,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 18-2.2 from stable_1;")
         for i in range(self.fornum):
@@ -1958,10 +2012,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
 
         self.restartDnodes()
         tdSql.query("select 18-3 from stable_1;")
@@ -1976,10 +2030,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 18-4 from stable_1;")
         for i in range(self.fornum):
@@ -1992,10 +2046,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 18-4.2 from stable_1;")
         for i in range(self.fornum):
@@ -2008,10 +2062,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 18-5 from stable_1;")
         for i in range(self.fornum):
@@ -2025,10 +2079,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_where)
             sql += "%s " % random.choice(limit1_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721  tdSql.query(sql)
+            #TD-15770 tdSql.query(sql)
 
         tdSql.query("select 18-6 from stable_1;")
         for i in range(self.fornum):
@@ -2041,10 +2095,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
         
         tdSql.query("select 18-7 from stable_1;")
         for i in range(self.fornum):
@@ -2057,10 +2111,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721  tdSql.query(sql)
+            tdSql.query(sql)
 
         #19 select apercentile from (select calc_aggregate_alls form regualr_table or stable  where <\>\in\and\or session order by  limit )interval_sliding       
         #self.dropandcreateDB_random("%s" %db, 1)
@@ -2180,10 +2234,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         tdSql.query("select 19-7 from stable_1;")
         for i in range(self.fornum):
@@ -2195,10 +2249,10 @@ class TDTestCase:
             #sql += "%s " % random.choice(order_u_where)
             sql += "%s " % random.choice(limit_u_where)
             sql += ") "
-            sql += "%s " % random.choice(interval_sliding)
+            #sql += "%s " % random.choice(interval_sliding)
             tdLog.info(sql) 
             tdLog.info(len(sql))      
-            #TD-15721 tdSql.query(sql)
+            tdSql.query(sql)
 
         #20 select * from (select calc_select_fills form regualr_table or stable  where <\>\in\and\or fill_where group by  order by limit offset  )        
         #self.dropandcreateDB_random("%s" %db, 1)
@@ -2351,10 +2405,21 @@ class TDTestCase:
             tdLog.info(len(sql))      
             tdSql.query(sql)  
             
-        for i in range(self.fornum):            
-            # sql_start = "select  *  from ( "
-            # sql_end = ")"
-            for_num = random.randint(1, 10);
+            sql2 =  "select  *  from  ( select * from ( select  "
+            sql2 += "%s, " % random.choice(s_r_select) 
+            sql2 += "%s, " % random.choice(q_select)  
+            sql2 += "ts from regular_table_1 where "
+            sql2 += "%s " % random.choice(q_where)
+            sql2 += ")) "  
+            tdLog.info(sql2) 
+            tdLog.info(len(sql2)) 
+            
+            self.data_matrix_equal('%s' %sql ,1,10,1,1,'%s' %sql2 ,1,10,1,1)
+            self.data_matrix_equal('%s' %sql ,1,10,1,1,'%s' %sql ,1,10,3,3)
+            self.data_matrix_equal('%s' %sql ,1,10,3,3,'%s' %sql2 ,1,10,3,3)
+            
+        for i in range(self.fornum):
+            for_num = random.randint(1, 15);
             sql = "select  ts  from ("  * for_num  
             sql += "select  *  from  ( select * from ( select  "
             sql += "%s, " % random.choice(s_r_select) 
@@ -2367,6 +2432,18 @@ class TDTestCase:
             tdLog.info(len(sql))      
             tdSql.query(sql)  
             
+            sql2 =  "select  *  from  ( select * from ( select  "
+            sql2 += "%s, " % random.choice(s_r_select) 
+            sql2 += "%s, " % random.choice(q_select)  
+            sql2 += "ts from regular_table_1 where "
+            sql2 += "%s " % random.choice(q_where)
+            sql2 += ")) "   
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            tdSql.query(sql) 
+            
+            self.data_matrix_equal('%s' %sql ,1,10,1,1,'%s' %sql2 ,1,10,1,1)
+            
         #2 select * from (select * from (select * form stable  where <\>\in\and\or order by limit  ))
         tdSql.query("select 2-1 from stable_1;")
         for i in range(self.fornum):
@@ -2382,9 +2459,21 @@ class TDTestCase:
             tdLog.info(sql) 
             tdLog.info(len(sql))      
             tdSql.query(sql) 
-        tdSql.query("select 2-1 from stable_1;")
+            
+            sql2 = "select  *  from  ( select * from ( select  "
+            sql2 += "%s, " % random.choice(s_s_select) 
+            sql2 += "%s, " % random.choice(qt_select) 
+            sql2 += "ts from stable_1 where "
+            sql2 += "%s " % random.choice(q_where)
+            sql2 += ")) " 
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            tdSql.query(sql) 
+            
+            self.data_matrix_equal('%s' %sql ,1,10,3,3,'%s' %sql2 ,1,10,3,3)
+            
         for i in range(self.fornum):
-            for_num = random.randint(1, 10);
+            for_num = random.randint(1, 15);
             sql = "select  ts  from ("  * for_num  
             sql += "select  *  from  ( select * from ( select  "
             sql += "%s, " % random.choice(s_s_select) 
@@ -2396,6 +2485,18 @@ class TDTestCase:
             tdLog.info(sql) 
             tdLog.info(len(sql))      
             tdSql.query(sql) 
+            
+            sql2 = "select  ts  from  ( select * from ( select  "
+            sql2 += "%s, " % random.choice(s_s_select) 
+            sql2 += "%s, " % random.choice(qt_select) 
+            sql2 += "ts from stable_1 where "
+            sql2 += "%s " % random.choice(q_where)
+            sql2 += ")) " 
+            tdLog.info(sql) 
+            tdLog.info(len(sql))      
+            tdSql.query(sql) 
+            
+            self.data_matrix_equal('%s' %sql ,1,10,1,1,'%s' %sql2 ,1,10,1,1)
                          
         #3 select ts ,calc from  (select * form stable  where <\>\in\and\or order by limit  )       
         #self.dropandcreateDB_random("%s" %db, 1)

From ae8f1cd5c4e058758f4110e5284dc4d857986fc3 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 20 May 2022 20:45:53 +0800
Subject: [PATCH 23/33] fix hb exit issue

---
 source/client/src/clientHb.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c
index 5d6241d4f4..13ac43bc39 100644
--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@@ -655,6 +655,9 @@ static int32_t hbCreateThread() {
 }
 
 static void hbStopThread() {
+  if (0 == atomic_load_8(&clientHbMgr.inited)) {
+    return;
+  }
   if (atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 0, 1)) {
     tscDebug("hb thread already stopped");
     return;

From 38394c182909ba77b73ac5416c435f11a8dd4bd2 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 20 May 2022 20:47:00 +0800
Subject: [PATCH 24/33] enh: add file ver to sdb

---
 source/dnode/mnode/sdb/inc/sdbInt.h  |  2 ++
 source/dnode/mnode/sdb/src/sdbFile.c | 26 ++++++++++++++++++++++++--
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h
index 23c0f8a01c..c49d6e8fb2 100644
--- a/source/dnode/mnode/sdb/inc/sdbInt.h
+++ b/source/dnode/mnode/sdb/inc/sdbInt.h
@@ -24,12 +24,14 @@
 extern "C" {
 #endif
 
+// clang-format off
 #define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
 #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
 #define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
 #define mInfo(...)  { if (mDebugFlag & DEBUG_INFO)  { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
 #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
 #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
+// clang-format on
 
 typedef struct SSdbRaw {
   int8_t  type;
diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c
index ad1429f667..a391ea8d03 100644
--- a/source/dnode/mnode/sdb/src/sdbFile.c
+++ b/source/dnode/mnode/sdb/src/sdbFile.c
@@ -20,6 +20,7 @@
 
 #define SDB_TABLE_SIZE   24
 #define SDB_RESERVE_SIZE 512
+#define SDB_FILE_VER     1
 
 static int32_t sdbRunDeployFp(SSdb *pSdb) {
   mDebug("start to deploy sdb");
@@ -39,7 +40,22 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
 }
 
 static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
-  int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t));
+  int64_t sver = 0;
+  int32_t ret = taosReadFile(pFile, &sver, sizeof(int64_t));
+  if (ret < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+  if (ret != sizeof(int64_t)) {
+    terrno = TSDB_CODE_FILE_CORRUPTED;
+    return -1;
+  }
+  if (sver != SDB_FILE_VER) {
+    terrno = TSDB_CODE_FILE_CORRUPTED;
+    return -1;
+  }
+
+  ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t));
   if (ret < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -96,6 +112,12 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
 }
 
 static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
+  int64_t sver = SDB_FILE_VER;
+  if (taosWriteFile(pFile, &sver, sizeof(int64_t)) != sizeof(int64_t)) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
   if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -256,7 +278,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
 
     mTrace("write %s to file, total %d rows", sdbTableName(i), sdbGetSize(pSdb, i));
 
-    SHashObj *hash = pSdb->hashObjs[i];
+    SHashObj       *hash = pSdb->hashObjs[i];
     TdThreadRwlock *pLock = &pSdb->locks[i];
     taosThreadRwlockWrlock(pLock);
 

From 9d600dfa661a65ac38d12425f814d6368c14dc01 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 20 May 2022 20:47:56 +0800
Subject: [PATCH 25/33] enh: dump sdb.data to json

---
 tests/test/c/CMakeLists.txt |   5 +-
 tests/test/c/sdbDump.c      | 223 ++++++++++++++++++++++++++++++++++--
 2 files changed, 215 insertions(+), 13 deletions(-)

diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index 8ddfbd1655..428b8e7bd0 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -23,6 +23,8 @@ target_link_libraries(
     PUBLIC common
     PUBLIC os
 )
+
+IF (TD_LINUX)
 target_link_libraries(
     sdbDump
     PUBLIC dnode
@@ -36,4 +38,5 @@ target_include_directories(
     PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc"
     PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc"
     PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc"
-)
\ No newline at end of file
+)
+ENDIF ()
\ No newline at end of file
diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c
index e9e0df24ee..1bdfda8a6f 100644
--- a/tests/test/c/sdbDump.c
+++ b/tests/test/c/sdbDump.c
@@ -15,10 +15,200 @@
 
 #define _DEFAULT_SOURCE
 #include "dmMgmt.h"
+#include "mndInt.h"
+#include "sdbInt.h"
 #include "tconfig.h"
+#include "tjson.h"
 
-int32_t main(int32_t argc, char *argv[]) {
-  char datafile[PATH_MAX] = {0};
+#define TMP_SDB_DATA_DIR  "/tmp/dumpsdb"
+#define TMP_SDB_MNODE_DIR "/tmp/dumpsdb/mnode"
+#define TMP_SDB_FILE      "/tmp/dumpsdb/mnode/data/sdb.data"
+#define TMP_SDB_PATH      "/tmp/dumpsdb/mnode/data"
+
+void reportStartup(const char *name, const char *desc) {}
+
+void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); }
+
+int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
+  terrno = TSDB_CODE_INVALID_PTR;
+  return -1;
+}
+
+char *i642str(int64_t val) {
+  static char str[24] = {0};
+  snprintf(str, sizeof(str), "%" PRId64, val);
+  return str;
+}
+
+void dumpFunc(SSdb *pSdb, SJson *json) {}
+
+void dumpDb(SSdb *pSdb, SJson *json) {}
+
+void dumpStb(SSdb *pSdb, SJson *json) {}
+
+void dumpSma(SSdb *pSdb, SJson *json) {}
+
+void dumpVgroup(SSdb *pSdb, SJson *json) {}
+
+void dumpTopic(SSdb *pSdb, SJson *json) {}
+
+void dumpConsumber(SSdb *pSdb, SJson *json) {}
+
+void dumpSubscribe(SSdb *pSdb, SJson *json) {}
+
+void dumpOffset(SSdb *pSdb, SJson *json) {}
+
+void dumpStream(SSdb *pSdb, SJson *json) {}
+
+void dumpAcct(SSdb *pSdb, SJson *json) {}
+
+void dumpAuth(SSdb *pSdb, SJson *json) {}
+
+void dumpUser(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "users", items);
+
+  while (1) {
+    SUserObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "user", item);
+
+    tjsonAddStringToObject(item, "name", pObj->user);
+    tjsonAddStringToObject(item, "pass", pObj->pass);
+    tjsonAddStringToObject(item, "acct", pObj->acct);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddIntegerToObject(item, "superUser", pObj->superUser);
+    tjsonAddIntegerToObject(item, "authVersion", pObj->authVersion);
+    tjsonAddIntegerToObject(item, "numOfReadDbs", taosHashGetSize(pObj->readDbs));
+    tjsonAddIntegerToObject(item, "numOfWriteDbs", taosHashGetSize(pObj->writeDbs));
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpDnode(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "dnodes", items);
+
+  while (1) {
+    SDnodeObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "dnode", item);
+
+    tjsonAddIntegerToObject(item, "id", pObj->id);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddIntegerToObject(item, "port", pObj->port);
+    tjsonAddStringToObject(item, "fqdn", pObj->fqdn);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpBnode(SSdb *pSdb, SJson *json) {}
+
+void dumpSnode(SSdb *pSdb, SJson *json) {}
+
+void dumpQnode(SSdb *pSdb, SJson *json) {}
+
+void dumpMnode(SSdb *pSdb, SJson *json) {}
+
+void dumpCluster(SSdb *pSdb, SJson *json) {}
+
+void dumpTrans(SSdb *pSdb, SJson *json) {}
+
+void dumpHeader(SSdb *pSdb, SJson *json) {
+  tjsonAddIntegerToObject(json, "sver", 1);
+  tjsonAddStringToObject(json, "curVer", i642str(pSdb->curVer));
+
+  SJson *maxIdsJson = tjsonCreateObject();
+  tjsonAddItemToObject(json, "maxIds", maxIdsJson);
+  for (int32_t i = 0; i < SDB_MAX; ++i) {
+    int64_t maxId = 0;
+    if (i < SDB_MAX) {
+      maxId = pSdb->maxId[i];
+    }
+    tjsonAddStringToObject(maxIdsJson, sdbTableName(i), i642str(maxId));
+  }
+
+  SJson *tableVersJson = tjsonCreateObject();
+  tjsonAddItemToObject(json, "tableVers", tableVersJson);
+  for (int32_t i = 0; i < SDB_MAX; ++i) {
+    int64_t tableVer = 0;
+    if (i < SDB_MAX) {
+      tableVer = pSdb->tableVer[i];
+    }
+    tjsonAddStringToObject(tableVersJson, sdbTableName(i), i642str(tableVer));
+  }
+}
+
+int32_t dumpSdb() {
+  SMsgCb msgCb = {0};
+  msgCb.reportStartupFp = reportStartup;
+  msgCb.sendReqFp = sendReq;
+  msgCb.sendRspFp = sendRsp;
+  msgCb.mgmt = (SMgmtWrapper *)(&msgCb);  // hack
+  tmsgSetDefault(&msgCb);
+  walInit();
+
+  SMnodeOpt opt = {.msgCb = msgCb};
+  SMnode   *pMnode = mndOpen(TMP_SDB_MNODE_DIR, &opt);
+  if (pMnode == NULL) return -1;
+
+  SSdb  *pSdb = pMnode->pSdb;
+  SJson *json = tjsonCreateObject();
+  dumpHeader(pSdb, json);
+  dumpFunc(pSdb, json);
+  dumpDb(pSdb, json);
+  dumpStb(pSdb, json);
+  dumpSma(pSdb, json);
+  dumpVgroup(pSdb, json);
+  dumpTopic(pSdb, json);
+  dumpConsumber(pSdb, json);
+  dumpSubscribe(pSdb, json);
+  dumpOffset(pSdb, json);
+  dumpStream(pSdb, json);
+  dumpAcct(pSdb, json);
+  dumpAuth(pSdb, json);
+  dumpUser(pSdb, json);
+  dumpDnode(pSdb, json);
+  dumpBnode(pSdb, json);
+  dumpSnode(pSdb, json);
+  dumpQnode(pSdb, json);
+  dumpMnode(pSdb, json);
+  dumpCluster(pSdb, json);
+  dumpTrans(pSdb, json);
+
+  char     *pCont = tjsonToString(json);
+  int32_t   contLen = strlen(pCont);
+  char      file[] = "sdb.json";
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+  taosWriteFile(pFile, pCont, contLen);
+  taosWriteFile(pFile, "\n", 1);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  tjsonDelete(json);
+  taosMemoryFree(pCont);
+  taosRemoveDir(TMP_SDB_DATA_DIR);
+  return 0;
+}
+
+int32_t parseArgs(int32_t argc, char *argv[]) {
+  char file[PATH_MAX] = {0};
 
   for (int32_t i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-c") == 0) {
@@ -38,11 +228,15 @@ int32_t main(int32_t argc, char *argv[]) {
           printf("file path overflow");
           return -1;
         }
-        tstrncpy(datafile, argv[i], PATH_MAX);
+        tstrncpy(file, argv[i], PATH_MAX);
       } else {
         printf("'-f' requires a parameter, default is %s\n", configDir);
         return -1;
       }
+    } else {
+      printf("-c Configuration directory. \n");
+      printf("-f Input sdb.data file. \n");
+      return -1;
     }
   }
 
@@ -52,19 +246,24 @@ int32_t main(int32_t argc, char *argv[]) {
   }
 
   if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
-    uError("failed to dump since read config error");
-    taosCloseLog();
+    printf("failed to dump since read config error\n");
     return -1;
   }
 
-  if (datafile[0] == 0) {
-    snprintf(datafile, sizeof(datafile), "%s%sdata%smnode%sdata%ssdb.data", tsDataDir, TD_DIRSEP, TD_DIRSEP, TD_DIRSEP,
-             TD_DIRSEP);
+  if (file[0] == 0) {
+    snprintf(file, PATH_MAX, "%s/mnode/data/sdb.data", tsDataDir);
   }
 
-  dInfo("dump %s to sdb.json", datafile);
-
-  taosCleanupCfg();
-  taosCloseLog();
+  strcpy(tsDataDir, TMP_SDB_DATA_DIR);
+  taosMulMkDir(TMP_SDB_PATH);
+  taosCopyFile(file, TMP_SDB_FILE);
   return 0;
 }
+
+int32_t main(int32_t argc, char *argv[]) {
+  if (parseArgs(argc, argv) != 0) {
+    return -1;
+  }
+
+  return dumpSdb();
+}

From 56fba8eb277518b6bbbe1dea76f2549e1b444167 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 20 May 2022 20:57:45 +0800
Subject: [PATCH 26/33] fix: unit test in json scalar operator

---
 source/libs/scalar/test/scalar/scalarTests.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp
index 627c3c438c..f02bef4ce2 100644
--- a/source/libs/scalar/test/scalar/scalarTests.cpp
+++ b/source/libs/scalar/test/scalar/scalarTests.cpp
@@ -1238,8 +1238,8 @@ TEST(columnTest, json_column_logic_op) {
 
   printf("--------------------json null---------------------\n");
 
-  key = "k3";
-  bool eRes2[len+len1] = {false, false, false, false, false, false, true, false, false, false, false, false, false};
+  key = "k3";   // (null is true) return NULL, so use DBL_MAX represent NULL
+  double eRes2[len+len1] = {false, false, false, false, false, false, true, false, false, DBL_MAX, false, false, false};
   for(int i = 0; i < len; i++){
     makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
   }
@@ -1290,8 +1290,8 @@ TEST(columnTest, json_column_logic_op) {
 
   printf("---------------------json not exist--------------------\n");
 
-  key = "k10";
-  double eRes10[len+len1] = {false, false, false, false, false, false, true, false, false, false, false, false, false};
+  key = "k10";    // (NULL is true) return NULL, so use DBL_MAX represent NULL
+  double eRes10[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false};
   for(int i = 0; i < len; i++){
     makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]);
   }
@@ -3456,7 +3456,7 @@ TEST(ScalarFunctionTest, powFunction_column) {
 
   //TINYINT AND FLOAT
   int8_t param0[] = {2, 3, 4};
-  float  param1[] = {3.0, 3.0, 2.0};
+  float  param1[] = {3.0, 3.0, 3.0};
   scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, 0, rowNum, false);
   pInput[0] = *input[0];
   for (int32_t i = 0; i < rowNum; ++i) {

From 5c553a37f561cdbc014258d9591d1598290a3314 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 20 May 2022 21:01:22 +0800
Subject: [PATCH 27/33] fix: unit test in json scalar operator

---
 source/libs/scalar/test/scalar/CMakeLists.txt  | 8 ++++----
 source/libs/scalar/test/scalar/scalarTests.cpp | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt
index 866297b017..15d1c2cb44 100644
--- a/source/libs/scalar/test/scalar/CMakeLists.txt
+++ b/source/libs/scalar/test/scalar/CMakeLists.txt
@@ -17,7 +17,7 @@ TARGET_INCLUDE_DIRECTORIES(
         PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
         PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
 )
-#add_test(
-#    NAME scalarTest
-#    COMMAND scalarTest
-#)
+add_test(
+    NAME scalarTest
+    COMMAND scalarTest
+)
diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp
index f02bef4ce2..fb67695e89 100644
--- a/source/libs/scalar/test/scalar/scalarTests.cpp
+++ b/source/libs/scalar/test/scalar/scalarTests.cpp
@@ -1239,7 +1239,7 @@ TEST(columnTest, json_column_logic_op) {
   printf("--------------------json null---------------------\n");
 
   key = "k3";   // (null is true) return NULL, so use DBL_MAX represent NULL
-  double eRes2[len+len1] = {false, false, false, false, false, false, true, false, false, DBL_MAX, false, false, false};
+  double eRes2[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false};
   for(int i = 0; i < len; i++){
     makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]);
   }

From 760c39da04675a8b3c5842b574a221ebe39e3a48 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 20 May 2022 21:24:02 +0800
Subject: [PATCH 28/33] feat: dump sdbdata to file

---
 tests/test/c/CMakeLists.txt |   2 +-
 tests/test/c/sdbDump.c      | 177 ++++++++++++++++++++++++++++++++++--
 2 files changed, 172 insertions(+), 7 deletions(-)

diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index 428b8e7bd0..5f6209b725 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -24,7 +24,7 @@ target_link_libraries(
     PUBLIC os
 )
 
-IF (TD_LINUX)
+IF (${TD_WINDOWS})
 target_link_libraries(
     sdbDump
     PUBLIC dnode
diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c
index 1bdfda8a6f..2bc60f777c 100644
--- a/tests/test/c/sdbDump.c
+++ b/tests/test/c/sdbDump.c
@@ -42,9 +42,87 @@ char *i642str(int64_t val) {
 
 void dumpFunc(SSdb *pSdb, SJson *json) {}
 
-void dumpDb(SSdb *pSdb, SJson *json) {}
+void dumpDb(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "dbs", items);
 
-void dumpStb(SSdb *pSdb, SJson *json) {}
+  while (1) {
+    SDbObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "db", item);
+
+    tjsonAddStringToObject(item, "name", pObj->name);
+    tjsonAddStringToObject(item, "acct", pObj->acct);
+    tjsonAddStringToObject(item, "createUser", pObj->createUser);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
+    tjsonAddIntegerToObject(item, "cfgVersion", pObj->cfgVersion);
+    tjsonAddIntegerToObject(item, "vgVersion", pObj->vgVersion);
+    tjsonAddIntegerToObject(item, "numOfVgroups", pObj->cfg.numOfVgroups);
+    tjsonAddIntegerToObject(item, "numOfStables", pObj->cfg.numOfStables);
+    tjsonAddIntegerToObject(item, "buffer", pObj->cfg.buffer);
+    tjsonAddIntegerToObject(item, "pageSize", pObj->cfg.pageSize);
+    tjsonAddIntegerToObject(item, "pages", pObj->cfg.pages);
+    tjsonAddIntegerToObject(item, "daysPerFile", pObj->cfg.daysPerFile);
+    tjsonAddIntegerToObject(item, "daysToKeep0", pObj->cfg.daysToKeep0);
+    tjsonAddIntegerToObject(item, "daysToKeep1", pObj->cfg.daysToKeep1);
+    tjsonAddIntegerToObject(item, "daysToKeep2", pObj->cfg.daysToKeep2);
+    tjsonAddIntegerToObject(item, "minRows", pObj->cfg.minRows);
+    tjsonAddIntegerToObject(item, "maxRows", pObj->cfg.maxRows);
+    tjsonAddIntegerToObject(item, "fsyncPeriod", pObj->cfg.fsyncPeriod);
+    tjsonAddIntegerToObject(item, "walLevel", pObj->cfg.walLevel);
+    tjsonAddIntegerToObject(item, "precision", pObj->cfg.precision);
+    tjsonAddIntegerToObject(item, "compression", pObj->cfg.compression);
+    tjsonAddIntegerToObject(item, "replications", pObj->cfg.replications);
+    tjsonAddIntegerToObject(item, "strict", pObj->cfg.strict);
+    tjsonAddIntegerToObject(item, "cacheLastRow", pObj->cfg.cacheLastRow);
+    tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod);
+    tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpStb(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "stbs", items);
+
+  while (1) {
+    SStbObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "stb", item);
+
+    tjsonAddStringToObject(item, "name", pObj->name);
+    tjsonAddStringToObject(item, "db", pObj->db);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
+    tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
+    tjsonAddIntegerToObject(item, "version", pObj->version);
+    tjsonAddIntegerToObject(item, "tagVer", pObj->tagVer);
+    tjsonAddIntegerToObject(item, "colVer", pObj->colVer);
+    tjsonAddIntegerToObject(item, "nextColId", pObj->nextColId);
+    tjsonAddIntegerToObject(item, "xFilesFactor", pObj->xFilesFactor * 10000);
+    tjsonAddIntegerToObject(item, "delay", pObj->delay);
+    tjsonAddIntegerToObject(item, "ttl", pObj->ttl);
+    tjsonAddIntegerToObject(item, "numOfColumns", pObj->numOfColumns);
+    tjsonAddIntegerToObject(item, "numOfTags", pObj->numOfTags);
+    tjsonAddIntegerToObject(item, "commentLen", pObj->commentLen);
+    tjsonAddIntegerToObject(item, "ast1Len", pObj->ast1Len);
+    tjsonAddIntegerToObject(item, "ast2Len", pObj->ast2Len);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
 
 void dumpSma(SSdb *pSdb, SJson *json) {}
 
@@ -60,7 +138,27 @@ void dumpOffset(SSdb *pSdb, SJson *json) {}
 
 void dumpStream(SSdb *pSdb, SJson *json) {}
 
-void dumpAcct(SSdb *pSdb, SJson *json) {}
+void dumpAcct(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "accts", items);
+
+  while (1) {
+    SAcctObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_ACCT, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "acct", item);
+
+    tjsonAddStringToObject(item, "acct", pObj->acct);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddIntegerToObject(item, "acctId", pObj->acctId);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
 
 void dumpAuth(SSdb *pSdb, SJson *json) {}
 
@@ -120,11 +218,78 @@ void dumpSnode(SSdb *pSdb, SJson *json) {}
 
 void dumpQnode(SSdb *pSdb, SJson *json) {}
 
-void dumpMnode(SSdb *pSdb, SJson *json) {}
+void dumpMnode(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "mnodes", items);
 
-void dumpCluster(SSdb *pSdb, SJson *json) {}
+  while (1) {
+    SMnodeObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
 
-void dumpTrans(SSdb *pSdb, SJson *json) {}
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "mnode", item);
+
+    tjsonAddIntegerToObject(item, "id", pObj->id);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpCluster(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "clusters", items);
+
+  while (1) {
+    SClusterObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "cluster", item);
+
+    tjsonAddStringToObject(item, "id", i642str(pObj->id));
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddStringToObject(item, "name", pObj->name);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpTrans(SSdb *pSdb, SJson *json) {
+    void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "transactions", items);
+
+  while (1) {
+    STrans *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "trans", item);
+
+    tjsonAddIntegerToObject(item, "id", pObj->id);
+    tjsonAddIntegerToObject(item, "stage", pObj->stage);
+    tjsonAddIntegerToObject(item, "policy", pObj->policy);
+    tjsonAddIntegerToObject(item, "type", pObj->type);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
+    tjsonAddStringToObject(item, "dbname", pObj->dbname);
+    tjsonAddIntegerToObject(item, "redoLogNum", taosArrayGetSize(pObj->redoLogs));
+    tjsonAddIntegerToObject(item, "undoLogNum", taosArrayGetSize(pObj->undoLogs));
+    tjsonAddIntegerToObject(item, "commitLogNum", taosArrayGetSize(pObj->commitLogs));
+    tjsonAddIntegerToObject(item, "redoActionNum", taosArrayGetSize(pObj->redoActions));
+    tjsonAddIntegerToObject(item, "undoActionNum", taosArrayGetSize(pObj->undoActions));
+
+    sdbRelease(pSdb, pObj);
+  }
+}
 
 void dumpHeader(SSdb *pSdb, SJson *json) {
   tjsonAddIntegerToObject(json, "sver", 1);

From 440908169de3ed067e69d4a1f5d149d0b1734227 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 20 May 2022 21:35:46 +0800
Subject: [PATCH 29/33] fix hb rpc issue

---
 source/dnode/mnode/impl/src/mndProfile.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c
index 7e99c5583d..cd57e76b2c 100644
--- a/source/dnode/mnode/impl/src/mndProfile.c
+++ b/source/dnode/mnode/impl/src/mndProfile.c
@@ -340,8 +340,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
   if (pHbReq->query) {
     SQueryHbReqBasic *pBasic = pHbReq->query;
 
-    SRpcConnInfo connInfo = {0};
-    rpcGetConnInfo(pMsg->info.handle, &connInfo);
+    SRpcConnInfo connInfo = pMsg->conn;
 
     SConnObj *pConn = mndAcquireConn(pMnode, pBasic->connId);
     if (pConn == NULL) {

From ccbcdbad1968661dcf353a21d47d3fd22ca15704 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 20 May 2022 21:53:56 +0800
Subject: [PATCH 30/33] fix: retry if no enough shm

---
 source/dnode/mgmt/node_mgmt/src/dmProc.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c
index de58366fe6..d044ad3b2e 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmProc.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c
@@ -96,6 +96,11 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg
   const int32_t fullLen = headLen + bodyLen + 8;
   const int64_t handle = (int64_t)pMsg->info.handle;
 
+  if (fullLen > queue->total) {
+    terrno = TSDB_CODE_OUT_OF_RANGE;
+    return -1;
+  }
+
   taosThreadMutexLock(&queue->mutex);
   if (fullLen > queue->avail) {
     taosThreadMutexUnlock(&queue->mutex);
@@ -448,7 +453,7 @@ void dmPutToProcPQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) {
       break;
     }
 
-    if (retry == 10) {
+    if (terrno != TSDB_CODE_OUT_OF_SHM_MEM) {
       pMsg->code = terrno;
       if (pMsg->contLen > 0) {
         rpcFreeCont(pMsg->pCont);

From 9fcf91525e9f5d2dae4e1c30c5b5dfd01556ca70 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 20 May 2022 22:00:43 +0800
Subject: [PATCH 31/33] feat: dump sdbdata to file

---
 tests/test/c/CMakeLists.txt | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index 5f6209b725..cabd302e40 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -24,19 +24,19 @@ target_link_libraries(
     PUBLIC os
 )
 
-IF (${TD_WINDOWS})
-target_link_libraries(
-    sdbDump
-    PUBLIC dnode
-    PUBLIC mnode
-    PUBLIC sdb
-    PUBLIC os
-)
-target_include_directories(
-    sdbDump
-    PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode"
-    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc"
-    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc"
-    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc"
-)
+if(NOT TD_WINDOWS)
+    target_link_libraries(
+        sdbDump
+        PUBLIC dnode
+        PUBLIC mnode
+        PUBLIC sdb
+        PUBLIC os
+    )
+    target_include_directories(
+        sdbDump
+        PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode"
+        PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc"
+        PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc"
+        PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc"
+    )
 ENDIF ()
\ No newline at end of file

From ba92e1e604db537d52a8f48bf4d798c7bdc6ea01 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 20 May 2022 22:30:01 +0800
Subject: [PATCH 32/33] feat: dump sdbdata to file

---
 tests/test/c/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index cabd302e40..964f9fee4a 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -1,7 +1,6 @@
-add_executable(create_table createTable.c)
 add_executable(tmq_demo tmqDemo.c)
 add_executable(tmq_sim tmqSim.c)
-add_executable(sdbDump sdbDump.c)
+add_executable(create_table createTable.c)
 target_link_libraries(
     create_table
     PUBLIC taos_static
@@ -25,6 +24,7 @@ target_link_libraries(
 )
 
 if(NOT TD_WINDOWS)
+    add_executable(sdbDump sdbDump.c)
     target_link_libraries(
         sdbDump
         PUBLIC dnode

From 2964542617f070ecf6a6b6f1fa09ae08503daad2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 20 May 2022 22:45:43 +0800
Subject: [PATCH 33/33] fix(query): ignore the check of qualified table id.

---
 source/libs/executor/src/executor.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 6d308d7221..79b177fc45 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -169,13 +169,9 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
     qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
     code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa);
     taosArrayDestroy(qa);
-
   } else {  // remove the table id in current list
-    SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList);
-
     qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList));
-    code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, tableIdList);
-    taosArrayDestroy(qa);
+    code = tqReadHandleRemoveTbUidList(pScanInfo->streamBlockReader, tableIdList);
   }
 
   return code;