From 006a3c38045ce840fd8eec6d396d46d0cab710d4 Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Fri, 12 Jan 2024 14:14:03 +0800
Subject: [PATCH 01/45] add inc Snapshot copy case

---
 tests/army/community/cluster/incSnapshot.py | 104 ++++++++++++++++++++
 tests/army/frame/autogen.py                 |  61 ++++++------
 tests/army/frame/caseBase.py                |  14 +--
 tests/army/frame/server/cluster.py          |  11 ++-
 tests/army/frame/server/dnodes.py           |   5 +
 tests/army/test.py                          |  21 ++--
 6 files changed, 165 insertions(+), 51 deletions(-)
 create mode 100644 tests/army/community/cluster/incSnapshot.py

diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py
new file mode 100644
index 0000000000..a91e51cda4
--- /dev/null
+++ b/tests/army/community/cluster/incSnapshot.py
@@ -0,0 +1,104 @@
+import taos
+import sys
+import os
+import subprocess
+import glob
+import shutil
+import time
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+from frame.server.dnodes import *
+from frame.server.cluster import *
+
+
+class TDTestCase(TBase):
+
+    def init(self, conn, logSql, replicaVar=3):
+        super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1")
+        self.valgrind = 0
+        self.childtable_count = 10
+        # tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def run(self):
+        tdSql.prepare()
+        autoGen = AutoGen()
+        autoGen.create_db(self.db, 2, 3)
+        autoGen.create_stable(self.stb, 5, 10, 8, 8)
+        autoGen.create_child(self.stb, "d", self.childtable_count)
+        autoGen.insert_data(1)
+        tdSql.execute(f"flush database {self.db}")
+        clusterDnodes.stoptaosd(3)
+        clusterDnodes.stoptaosd(1)
+        clusterDnodes.starttaosd(3)
+        time.sleep(5)
+        clusterDnodes.stoptaosd(2)
+        clusterDnodes.starttaosd(1)
+        time.sleep(5)
+        autoGen.insert_data(10, 1600000000001)
+        # tdSql.execute(f"flush database {self.db}")
+
+        sql = 'show vnodes;'
+        while True:
+            bFinish = True
+            param_list = tdSql.query(sql, row_tag=True)
+            for param in param_list:
+                if param[3] == 'leading' or param[3] == 'following':
+                    bFinish = False
+                    break
+            if bFinish:
+                break
+        self.snapshotAgg()
+
+        clusterDnodes.stopAll()
+        # for i in range(1, 4):
+        #     path = clusterDnodes.getDnodeDir(i)
+        #     dnodesRootDir = os.path.join(path,"data","vnode", "vnode*")
+        #     dirs = glob.glob(dnodesRootDir)
+        #     for dir in dirs:
+        #         if os.path.isdir(dir):
+        #             tdLog.debug("delete dir: %s " % (dnodesRootDir))
+        #             self.remove_directory(os.path.join(dir, "wal"))
+
+        clusterDnodes.starttaosd(1)
+        clusterDnodes.starttaosd(2)
+        clusterDnodes.starttaosd(3)
+
+        time.sleep(3)
+        while True:
+            bFinish = True
+            param_list = tdSql.query(sql, row_tag=True)
+            for param in param_list:
+                if param[3] == 'offline':
+                    tdLog.exit(
+                        "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1]))
+                if param[3] == 'leading' or param[3] == 'following':
+                    bFinish = False
+                    break
+            if bFinish:
+                break
+
+        self.timestamp_step = 1
+        self.insert_rows = 11
+        self.checkInsertCorrect()
+        self.checkAggCorrect()
+
+    def remove_directory(self, directory):
+        try:
+            shutil.rmtree(directory)
+            tdLog.debug("delete dir: %s " % (directory))
+        except OSError as e:
+            tdLog.exit("delete fail dir: %s " % (directory))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py
index 9dca96e7b0..a641deaf00 100644
--- a/tests/army/frame/autogen.py
+++ b/tests/army/frame/autogen.py
@@ -30,7 +30,7 @@ class AutoGen:
 
     #  _columns_sql
     def gen_columns_sql(self, pre, cnt, binary_len, nchar_len):
-        types = [ 
+        types = [
             'timestamp',
             'tinyint',
             'smallint',
@@ -58,33 +58,33 @@ class AutoGen:
                 sqls += ","
             sqls += sql
             metas.append(sel)
-        
-        return metas, sqls;    
+
+        return metas, sqls;
 
     # gen tags data
     def gen_data(self, i, marr):
-        datas = ""   
+        datas = ""
         for c in marr:
             data = ""
-            if c == 0 : # timestamp
+            if c == 0:  # timestamp
                 data = "%d" % (self.ts + i)
-            elif c <= 4 : # small
-                data = "%d"%(i%128)
-            elif c <= 8 : # int
+            elif c <= 4:  # small
+                data = "%d" % (i % 128)
+            elif c <= 8:  # int
                 data = f"{i}"
-            elif c <= 10 : # float
-                data = "%f"%(i+i/1000)
-            elif c <= 11 : # bool
-                data = "%d"%(i%2)
-            elif c == 12 : # binary
+            elif c <= 10:  # float
+                data = "%f" % (i + i / 1000)
+            elif c <= 11:  # bool
+                data = "%d" % (i % 2)
+            elif c == 12:  # binary
                 data = '"' + self.random_string(self.bin_len) + '"'
-            elif c == 13 : # binary
+            elif c == 13:  # binary
                 data = '"' + self.random_string(self.nch_len) + '"'
 
             if datas != "":
                 datas += ","
             datas += data
-        
+
         return datas
 
     # generate specail wide random string
@@ -93,11 +93,11 @@ class AutoGen:
         return ''.join(random.choice(letters) for i in range(count))
 
     # create db
-    def create_db(self, dbname, vgroups = 2, replica = 1):
-        self.dbname  = dbname
+    def create_db(self, dbname, vgroups=2, replica=1):
+        self.dbname = dbname
         tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}')
         tdSql.execute(f'use {dbname}')
-        
+
     # create table or stable
     def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len):
         self.bin_len = binary_len
@@ -109,7 +109,7 @@ class AutoGen:
         sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})"
         tdSql.execute(sql)
 
-    # create child table 
+    # create child table
     def create_child(self, stbname, prename, cnt):
         self.child_cnt = cnt
         self.child_name = prename
@@ -120,7 +120,7 @@ class AutoGen:
 
         tdLog.info(f"create child tables {cnt} ok")
 
-    def insert_data_child(self, child_name, cnt, batch_size, step):        
+    def insert_data_child(self, child_name, cnt, batch_size, step):
         values = ""
         print("insert child data")
         ts = self.ts
@@ -130,7 +130,7 @@ class AutoGen:
             value = self.gen_data(i, self.mcols)
             ts += step
             values += f"({ts},{value}) "
-            if batch_size == 1 or (i > 0 and i % batch_size == 0) :
+            if batch_size == 1 or (i > 0 and i % batch_size == 0):
                 sql = f"insert into {child_name} values {values}"
                 tdSql.execute(sql)
                 values = ""
@@ -138,18 +138,25 @@ class AutoGen:
         # end batch
         if values != "":
             sql = f"insert into {child_name} values {values}"
+            tdLog.info(f" insert child data SQL{sql}")
             tdSql.execute(sql)
             tdLog.info(f" insert data i={i}")
             values = ""
 
-        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")    
+        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt} last_ts={ts}")
+        return ts
 
-    # insert data 
-    def insert_data(self, cnt):
+    # insert data
+    def insert_data(self, cnt, bContinue=False):
+        if not bContinue:
+            self.ts = 1600000000000
+
+        currTs = 1600000000000
         for i in range(self.child_cnt):
             name = f"{self.child_name}{i}"
-            self.insert_data_child(name, cnt, self.batch_size, 1)
+            currTs = self.insert_data_child(name, cnt, self.batch_size, 1)
 
+        self.ts = currTs
         tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
 
     # insert same timestamp to all childs
@@ -158,6 +165,4 @@ class AutoGen:
             name = f"{self.child_name}{i}"
             self.insert_data_child(name, cnt, self.batch_size, 0)
 
-        tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")         
-
-
+        tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")
diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py
index 8d2c1e6d18..863a117cdd 100644
--- a/tests/army/frame/caseBase.py
+++ b/tests/army/frame/caseBase.py
@@ -28,7 +28,7 @@ class TBase:
 #          
 
     # init
-    def init(self, conn, logSql, replicaVar=1):
+    def init(self, conn, logSql, replicaVar=1, db="db", stb="stb", checkColName="ic"):
         # save param
         self.replicaVar = int(replicaVar)
         tdSql.init(conn.cursor(), True)
@@ -40,14 +40,14 @@ class TBase:
         self.mLevelDisk = 0
 
         # test case information
-        self.db     = "db"
-        self.stb    = "stb"
+        self.db     = db
+        self.stb    = stb
 
         # sql 
-        self.sqlSum = f"select sum(ic) from {self.stb}"
-        self.sqlMax = f"select max(ic) from {self.stb}"
-        self.sqlMin = f"select min(ic) from {self.stb}"
-        self.sqlAvg = f"select avg(ic) from {self.stb}"
+        self.sqlSum = f"select sum({checkColName}) from {self.stb}"
+        self.sqlMax = f"select max({checkColName}) from {self.stb}"
+        self.sqlMin = f"select min({checkColName}) from {self.stb}"
+        self.sqlAvg = f"select avg({checkColName}) from {self.stb}"
         self.sqlFirst = f"select first(ts) from {self.stb}"
         self.sqlLast  = f"select last(ts) from {self.stb}"
 
diff --git a/tests/army/frame/server/cluster.py b/tests/army/frame/server/cluster.py
index ade8ac39a2..9be7fe56b1 100644
--- a/tests/army/frame/server/cluster.py
+++ b/tests/army/frame/server/cluster.py
@@ -13,23 +13,24 @@ from frame.common import *
 
 class ClusterDnodes(TDDnodes):
     """rewrite TDDnodes and make MyDdnodes as TDDnodes child class"""
-    def __init__(self ,dnodes_lists):
-
+    def __init__(self):
         super(ClusterDnodes,self).__init__()
-        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
         self.simDeployed = False
         self.testCluster = False
         self.valgrind = 0
         self.killValgrind = 1
+    def init(self, dnodes_lists, deployPath, masterIp):
+        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
+        super(ClusterDnodes, self).init(deployPath, masterIp)
 
+clusterDnodes = ClusterDnodes()
 
 class ConfigureyCluster:
     """This will create defined number of dnodes and create a cluster.
        at the same time, it will return TDDnodes list:  dnodes, """
     hostname = socket.gethostname()
-
     def __init__(self):
-        self.dnodes = []      
+        self.dnodes = []
         self.dnodeNums = 5
         self.independent = True
         self.startPort = 6030
diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py
index 0d40b665dd..b58deeb789 100644
--- a/tests/army/frame/server/dnodes.py
+++ b/tests/army/frame/server/dnodes.py
@@ -899,6 +899,11 @@ class TDDnodes:
         dnodesRootDir = "%s/sim" % (self.path)
         return dnodesRootDir
 
+    def getDnodeDir(self, index):
+        self.check(index)
+        dnodesDir = "%s/sim/dnode%d" % (self.path, index)
+        return dnodesDir
+
     def getSimCfgPath(self):
         return self.sim.getCfgDir()
 
diff --git a/tests/army/test.py b/tests/army/test.py
index a3e28b772d..37fecea370 100644
--- a/tests/army/test.py
+++ b/tests/army/test.py
@@ -405,11 +405,11 @@ if __name__ == "__main__":
         else :
             tdLog.debug("create an cluster  with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
             dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
-            tdDnodes = ClusterDnodes(dnodeslist)
-            tdDnodes.init(deployPath, masterIp)
-            tdDnodes.setTestCluster(testCluster)
-            tdDnodes.setValgrind(valgrind)
-            tdDnodes.stopAll()
+            clusterDnodes.init(dnodeslist, deployPath, masterIp)
+            clusterDnodes.setTestCluster(testCluster)
+            clusterDnodes.setValgrind(valgrind)
+            clusterDnodes.setAsan(asan)
+            clusterDnodes.stopAll()
             for dnode in tdDnodes.dnodes:
                 tdDnodes.deploy(dnode.index, updateCfgDict)
             for dnode in tdDnodes.dnodes:
@@ -591,12 +591,11 @@ if __name__ == "__main__":
             print(independentMnode,"independentMnode valuse")
             # create dnode list
             dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
-            tdDnodes = ClusterDnodes(dnodeslist)
-            tdDnodes.init(deployPath, masterIp)
-            tdDnodes.setTestCluster(testCluster)
-            tdDnodes.setValgrind(valgrind)
-            tdDnodes.setAsan(asan)
-            tdDnodes.stopAll()
+            clusterDnodes.init(dnodeslist, deployPath, masterIp)
+            clusterDnodes.setTestCluster(testCluster)
+            clusterDnodes.setValgrind(valgrind)
+            clusterDnodes.setAsan(asan)
+            clusterDnodes.stopAll()
             for dnode in tdDnodes.dnodes:
                 tdDnodes.deploy(dnode.index,updateCfgDict)
             for dnode in tdDnodes.dnodes:

From 2be6098a2e3e7fdd539ccb6e9e029f14f0fd00cb Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Mon, 15 Jan 2024 14:59:09 +0800
Subject: [PATCH 02/45] add inc Snapshot copy case

---
 tests/army/community/cluster/incSnapshot.py | 58 ++++++++++-----------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py
index a91e51cda4..94013eea70 100644
--- a/tests/army/community/cluster/incSnapshot.py
+++ b/tests/army/community/cluster/incSnapshot.py
@@ -31,45 +31,45 @@ class TDTestCase(TBase):
         autoGen.create_db(self.db, 2, 3)
         autoGen.create_stable(self.stb, 5, 10, 8, 8)
         autoGen.create_child(self.stb, "d", self.childtable_count)
-        autoGen.insert_data(1)
+        autoGen.insert_data(1000)
         tdSql.execute(f"flush database {self.db}")
         clusterDnodes.stoptaosd(3)
-        clusterDnodes.stoptaosd(1)
-        clusterDnodes.starttaosd(3)
-        time.sleep(5)
-        clusterDnodes.stoptaosd(2)
-        clusterDnodes.starttaosd(1)
-        time.sleep(5)
-        autoGen.insert_data(10, 1600000000001)
-        # tdSql.execute(f"flush database {self.db}")
+        # clusterDnodes.stoptaosd(1)
+        # clusterDnodes.starttaosd(3)
+        # time.sleep(5)
+        # clusterDnodes.stoptaosd(2)
+        # clusterDnodes.starttaosd(1)
+        # time.sleep(5)
+        autoGen.insert_data(5000, 1600000000001)
+        tdSql.execute(f"flush database {self.db}")
 
-        sql = 'show vnodes;'
-        while True:
-            bFinish = True
-            param_list = tdSql.query(sql, row_tag=True)
-            for param in param_list:
-                if param[3] == 'leading' or param[3] == 'following':
-                    bFinish = False
-                    break
-            if bFinish:
-                break
+        # sql = 'show vnodes;'
+        # while True:
+        #     bFinish = True
+        #     param_list = tdSql.query(sql, row_tag=True)
+        #     for param in param_list:
+        #         if param[3] == 'leading' or param[3] == 'following':
+        #             bFinish = False
+        #             break
+        #     if bFinish:
+        #         break
         self.snapshotAgg()
-
+        time.sleep(10)
         clusterDnodes.stopAll()
-        # for i in range(1, 4):
-        #     path = clusterDnodes.getDnodeDir(i)
-        #     dnodesRootDir = os.path.join(path,"data","vnode", "vnode*")
-        #     dirs = glob.glob(dnodesRootDir)
-        #     for dir in dirs:
-        #         if os.path.isdir(dir):
-        #             tdLog.debug("delete dir: %s " % (dnodesRootDir))
-        #             self.remove_directory(os.path.join(dir, "wal"))
+        for i in range(1, 4):
+            path = clusterDnodes.getDnodeDir(i)
+            dnodesRootDir = os.path.join(path,"data","vnode", "vnode*")
+            dirs = glob.glob(dnodesRootDir)
+            for dir in dirs:
+                if os.path.isdir(dir):
+                    tdLog.debug("delete dir: %s " % (dnodesRootDir))
+                    self.remove_directory(os.path.join(dir, "wal"))
 
         clusterDnodes.starttaosd(1)
         clusterDnodes.starttaosd(2)
         clusterDnodes.starttaosd(3)
 
-        time.sleep(3)
+        time.sleep(10)
         while True:
             bFinish = True
             param_list = tdSql.query(sql, row_tag=True)

From 764365047d07bc83ffff50c10bac1ad179665fd1 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 16 Jan 2024 09:28:45 +0800
Subject: [PATCH 03/45] fix: stream scan core due to table end index introduced
 in 1 null row for empty group

---
 source/libs/executor/inc/executorInt.h  |  2 ++
 source/libs/executor/src/executor.c     | 10 ++++++----
 source/libs/executor/src/scanoperator.c | 19 ++++++++++++-------
 3 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index c3f47cde9d..2ea23e73f2 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -273,6 +273,8 @@ typedef struct STableScanInfo {
   int32_t         tableStartIndex;    // current group scan start
   int32_t         tableEndIndex;      // current group scan end
   int32_t         currentGroupIndex;  // current group index of groupOffset
+  int32_t         currentGroupId;
+  int32_t         currentTable;  
   int8_t          scanMode;
   int8_t          assignBlockUid;
   uint8_t         countState;     // empty table count state
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index eb84cb0639..1bccc514e4 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1264,6 +1264,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
           STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0);
           uid = pTableInfo->uid;
           ts = INT64_MIN;
+          pScanInfo->currentTable = 0;
           pScanInfo->tableEndIndex = 0;
         } else {
           taosRUnLockLatch(&pTaskInfo->lock);
@@ -1278,16 +1279,17 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       pInfo->pTableScanOp->resultInfo.totalRows = 0;
 
       // start from current accessed position
-      // we cannot start from the pScanInfo->tableEndIndex, since the commit offset may cause the rollback of the start
+      // we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start
       // position, let's find it from the beginning.
       index = tableListFind(pTableListInfo, uid, 0);
       taosRUnLockLatch(&pTaskInfo->lock);
 
       if (index >= 0) {
+        pScanInfo->currentTable = index;
         pScanInfo->tableEndIndex = index;
       } else {
         qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid,
-               numOfTables, pScanInfo->tableEndIndex, id);
+               numOfTables, pScanInfo->currentTable, id);
         terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
         return -1;
       }
@@ -1310,12 +1312,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
         }
 
         qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s",
-               uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id);
+               uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
       } else {
         pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
         pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
         qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 "  table index:%d numOfTable:%d, %s",
-               uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id);
+               uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
       }
 
       // restore the key value
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 3ed5128858..6736d6e137 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -939,7 +939,7 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) {
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
   SStorageAPI*    pAPI = &pTaskInfo->storageAPI;
   int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo);
-  if (pInfo->tableEndIndex + 1 >= numOfTables) {
+  if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
     setOperatorCompleted(pOperator);
     if (pOperator->dynamicTask) {
       taosArrayClear(pInfo->base.pTableListInfo->pTableList);
@@ -978,9 +978,9 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) {
   int32_t        num = 0;
   STableKeyInfo* pList = NULL;
 
-  if (pInfo->tableEndIndex == -1) {
+  if (pInfo->currentGroupId == -1) {
     int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo);
-    if (pInfo->tableEndIndex + 1 == numOfTables) {
+    if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
       setOperatorCompleted(pOperator);
       return NULL;
     }
@@ -1034,7 +1034,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
       T_LONG_JMP(pTaskInfo->env, code);
     }
     if (pOperator->status == OP_EXEC_DONE) {
-      pInfo->tableEndIndex = -1;
+      pInfo->currentGroupId = -1;
       pOperator->status = OP_OPENED;
       SSDataBlock* result = NULL;
       while (true) {
@@ -1059,23 +1059,24 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
       }
 
       // if no data, switch to next table and continue scan
+      pInfo->currentTable++;
       pInfo->tableEndIndex++;
 
       taosRLockLatch(&pTaskInfo->lock);
       numOfTables = tableListGetSize(pInfo->base.pTableListInfo);
 
-      if (pInfo->tableEndIndex >= numOfTables) {
+      if (pInfo->currentTable >= numOfTables) {
         qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo));
         taosRUnLockLatch(&pTaskInfo->lock);
         return NULL;
       }
 
-      tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableEndIndex);
+      tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable);
       taosRUnLockLatch(&pTaskInfo->lock);
 
       pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1);
       qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables,
-             pInfo->tableEndIndex, numOfTables, GET_TASKID(pTaskInfo));
+             pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo));
 
       pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
       pInfo->scanTimes = 0;
@@ -1167,6 +1168,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
+  
+  pInfo->currentGroupId = -1;
 
   pInfo->tableEndIndex = -1;
   pInfo->currentGroupIndex = -1;
@@ -1264,6 +1267,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6
   pTableScanInfo->base.cond.startVersion = 0;
   pTableScanInfo->base.cond.endVersion = ver;
   pTableScanInfo->scanTimes = 0;
+  pTableScanInfo->currentGroupId = -1;
   pTableScanInfo->tableEndIndex = -1;
   pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
   pTableScanInfo->base.dataReader = NULL;
@@ -2167,6 +2171,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     pInfo->pTableScanOp->status = OP_OPENED;
 
     pTSInfo->scanTimes = 0;
+    pTSInfo->currentGroupId = -1;
     pTSInfo->tableEndIndex = -1;
   }
 

From 1119d95a4aa9166f9f06bc4a4a3571c5c7cf4c71 Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Tue, 16 Jan 2024 14:38:53 +0800
Subject: [PATCH 04/45] add inc Snapshot copy case

---
 tests/army/community/cluster/incSnapshot.py | 2 +-
 tests/parallel_test/cases.task              | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py
index 94013eea70..21f67b45fc 100644
--- a/tests/army/community/cluster/incSnapshot.py
+++ b/tests/army/community/cluster/incSnapshot.py
@@ -40,7 +40,7 @@ class TDTestCase(TBase):
         # clusterDnodes.stoptaosd(2)
         # clusterDnodes.starttaosd(1)
         # time.sleep(5)
-        autoGen.insert_data(5000, 1600000000001)
+        autoGen.insert_data(5000, True)
         tdSql.execute(f"flush database {self.db}")
 
         # sql = 'show vnodes;'
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 63745f75ab..5e8921768d 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -11,6 +11,7 @@
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
 
 

From 9a7ef7fa182f3a350060bd55ba63743ae9e7bf15 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 16 Jan 2024 15:08:50 +0800
Subject: [PATCH 05/45] fix: reuse tableListGetGroupList

---
 source/libs/executor/src/executil.c     |  4 ---
 source/libs/executor/src/scanoperator.c | 39 ++++++++++++-------------
 2 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 1da7818c3c..19f33d2420 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -646,10 +646,6 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf
     }
   }
 
-  if (initRemainGroups) {
-    pTableListInfo->numOfOuputGroups = taosHashGetSize(pTableListInfo->remainGroups);
-  }
-
   if (tsTagFilterCache) {
     tableList = taosArrayDup(pTableListInfo->pTableList, NULL);
     pAPI->metaFn.metaPutTbGroupToCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest),
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 6736d6e137..36a7f4c183 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -657,33 +657,17 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData,
 
 
 static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) {
-  pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
+  tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, pKeyInfo, size);
 
-  STableListInfo* pTableListInfo = pInfo->base.pTableListInfo;
-  int32_t numOfTables = tableListGetSize(pTableListInfo);
-  STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pTableListInfo, pInfo->tableStartIndex);
+  pInfo->tableStartIndex = TARRAY_ELEM_IDX(pInfo->base.pTableListInfo->pTableList, *pKeyInfo);
 
-  if (pTableListInfo->oneTableForEachGroup) {
-    pInfo->tableEndIndex = pInfo->tableStartIndex;
-  } else if (pTableListInfo->groupOffset) {
-    pInfo->currentGroupIndex++;
-    if (pInfo->currentGroupIndex + 1 < pTableListInfo->numOfOuputGroups) {
-      pInfo->tableEndIndex = pTableListInfo->groupOffset[pInfo->currentGroupIndex + 1] - 1;
-    } else {
-      pInfo->tableEndIndex = numOfTables - 1;
-    }
-  } else {
-    pInfo->tableEndIndex = numOfTables - 1;
-  }
+  pInfo->tableEndIndex = (pInfo->tableStartIndex + (*size) - 1);
 
   if (!pInfo->needCountEmptyTable) {
     pInfo->countState = TABLE_COUNT_STATE_END;
   } else {
     pInfo->countState = TABLE_COUNT_STATE_SCAN;
   }
-
-  *pKeyInfo = pStart;
-  *size = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;
 }
 
 void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) {
@@ -984,7 +968,8 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) {
       setOperatorCompleted(pOperator);
       return NULL;
     }
-  
+
+
     initNextGroupScan(pInfo, &pList, &num);
     ASSERT(pInfo->base.dataReader == NULL);
   
@@ -1006,16 +991,28 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) {
     if (pOperator->dynamicTask) {
       result->info.id.groupId = result->info.id.uid;
     }
+    uInfo("slzhou 1 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows);
     return result;
   }
 
   while (true) {
     result = startNextGroupScan(pOperator);
     if (result || pOperator->status == OP_EXEC_DONE) {
+      if (result) {
+        uInfo("slzhou 2 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows);
+      }
+      else {
+        uInfo("slzhou 2 null block" );
+      }
       return result;
     }
   }
-
+      if (result) {
+        uInfo("slzhou 3 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows);
+      }
+      else {
+        uInfo("slzhou 3 null block" );
+      }
   return result;
 }
 

From de6b559ab9bd1302bd4fb610868abd0cff524edc Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 16 Jan 2024 15:21:20 +0800
Subject: [PATCH 06/45] fix: remove some uage of table end index

---
 source/libs/executor/inc/executorInt.h  | 1 -
 source/libs/executor/src/executor.c     | 2 --
 source/libs/executor/src/scanoperator.c | 3 ---
 3 files changed, 6 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 2ea23e73f2..007eab40f8 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -272,7 +272,6 @@ typedef struct STableScanInfo {
   SSampleExecInfo sample;  // sample execution info
   int32_t         tableStartIndex;    // current group scan start
   int32_t         tableEndIndex;      // current group scan end
-  int32_t         currentGroupIndex;  // current group index of groupOffset
   int32_t         currentGroupId;
   int32_t         currentTable;  
   int8_t          scanMode;
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 1bccc514e4..e872c87237 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1265,7 +1265,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
           uid = pTableInfo->uid;
           ts = INT64_MIN;
           pScanInfo->currentTable = 0;
-          pScanInfo->tableEndIndex = 0;
         } else {
           taosRUnLockLatch(&pTaskInfo->lock);
           qError("no table in table list, %s", id);
@@ -1286,7 +1285,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 
       if (index >= 0) {
         pScanInfo->currentTable = index;
-        pScanInfo->tableEndIndex = index;
       } else {
         qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid,
                numOfTables, pScanInfo->currentTable, id);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 36a7f4c183..103166e3e7 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1057,7 +1057,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
 
       // if no data, switch to next table and continue scan
       pInfo->currentTable++;
-      pInfo->tableEndIndex++;
 
       taosRLockLatch(&pTaskInfo->lock);
       numOfTables = tableListGetSize(pInfo->base.pTableListInfo);
@@ -1169,7 +1168,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   pInfo->currentGroupId = -1;
 
   pInfo->tableEndIndex = -1;
-  pInfo->currentGroupIndex = -1;
   pInfo->assignBlockUid = pTableScanNode->assignBlockUid;
   pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false;
 
@@ -2169,7 +2167,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
 
     pTSInfo->scanTimes = 0;
     pTSInfo->currentGroupId = -1;
-    pTSInfo->tableEndIndex = -1;
   }
 
   if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) {

From 7db577d664d4ec7d69678210a1f3f178eab511de Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Tue, 16 Jan 2024 15:28:19 +0800
Subject: [PATCH 07/45] add inc Snapshot copy case

---
 tests/army/test.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/army/test.py b/tests/army/test.py
index 37fecea370..62b3b46a61 100644
--- a/tests/army/test.py
+++ b/tests/army/test.py
@@ -410,10 +410,10 @@ if __name__ == "__main__":
             clusterDnodes.setValgrind(valgrind)
             clusterDnodes.setAsan(asan)
             clusterDnodes.stopAll()
-            for dnode in tdDnodes.dnodes:
-                tdDnodes.deploy(dnode.index, updateCfgDict)
-            for dnode in tdDnodes.dnodes:
-                tdDnodes.starttaosd(dnode.index)
+            for dnode in clusterDnodes.dnodes:
+                clusterDnodes.deploy(dnode.index, updateCfgDict)
+            for dnode in clusterDnodes.dnodes:
+                clusterDnodes.starttaosd(dnode.index)
             tdCases.logSql(logSql)
                             
             if restful or websocket:
@@ -596,10 +596,10 @@ if __name__ == "__main__":
             clusterDnodes.setValgrind(valgrind)
             clusterDnodes.setAsan(asan)
             clusterDnodes.stopAll()
-            for dnode in tdDnodes.dnodes:
-                tdDnodes.deploy(dnode.index,updateCfgDict)
-            for dnode in tdDnodes.dnodes:
-                tdDnodes.starttaosd(dnode.index)
+            for dnode in clusterDnodes.dnodes:
+                clusterDnodes.deploy(dnode.index,updateCfgDict)
+            for dnode in clusterDnodes.dnodes:
+                clusterDnodes.starttaosd(dnode.index)
             tdCases.logSql(logSql)
 
             if restful or websocket:

From 24066a7734cabcf4070e61633cba92bed78654cc Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Tue, 16 Jan 2024 15:35:33 +0800
Subject: [PATCH 08/45] add inc Snapshot copy case

---
 tests/army/community/cluster/incSnapshot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py
index 21f67b45fc..2095b6008b 100644
--- a/tests/army/community/cluster/incSnapshot.py
+++ b/tests/army/community/cluster/incSnapshot.py
@@ -68,7 +68,7 @@ class TDTestCase(TBase):
         clusterDnodes.starttaosd(1)
         clusterDnodes.starttaosd(2)
         clusterDnodes.starttaosd(3)
-
+        sql = "show vnodes;"
         time.sleep(10)
         while True:
             bFinish = True

From 022965661a7fcd0fa0a04823105b72aef7daeaad Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Tue, 16 Jan 2024 15:38:34 +0800
Subject: [PATCH 09/45] add inc Snapshot copy case

---
 tests/army/community/cluster/incSnapshot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py
index 2095b6008b..9f50ea52f1 100644
--- a/tests/army/community/cluster/incSnapshot.py
+++ b/tests/army/community/cluster/incSnapshot.py
@@ -84,7 +84,7 @@ class TDTestCase(TBase):
                 break
 
         self.timestamp_step = 1
-        self.insert_rows = 11
+        self.insert_rows = 6000
         self.checkInsertCorrect()
         self.checkAggCorrect()
 

From 8edff6e35593ced01718207dea26d7b4c0688470 Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Tue, 16 Jan 2024 15:51:16 +0800
Subject: [PATCH 10/45] add inc Snapshot copy case

---
 tests/army/frame/autogen.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py
index 1e041f633d..d38d2e2f19 100644
--- a/tests/army/frame/autogen.py
+++ b/tests/army/frame/autogen.py
@@ -162,16 +162,23 @@ class AutoGen:
             tdLog.info(f" insert data i={i}")
             values = ""
 
-        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")    
+        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")
+        return ts
 
-    # insert data 
-    def insert_data(self, cnt):
+    def insert_data(self, cnt, bContinue=False):
+        if not bContinue:
+            self.ts = 1600000000000
+
+        currTs = 1600000000000
         for i in range(self.child_cnt):
             name = f"{self.child_name}{i}"
-            self.insert_data_child(name, cnt, self.batch_size, 1)
+            currTs = self.insert_data_child(name, cnt, self.batch_size, 1)
 
+        self.ts = currTs
         tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
 
+    # insert same timestamp to all childs
+
     # insert same timestamp to all childs
     def insert_samets(self, cnt):
         for i in range(self.child_cnt):

From 449c1631ef04224b67193a6069044d35f63f3317 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 17 Jan 2024 13:34:37 +0800
Subject: [PATCH 11/45] fix: add numOfOutputGroups back when group order scan

---
 source/libs/executor/src/executil.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 19f33d2420..fd34a98eca 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -2155,6 +2155,8 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
       return code;
     }
 
+    if (pScanNode->groupOrderScan) pTableListInfo->numOfOuputGroups = taosArrayGetSize(pTableListInfo->pTableList);
+
     if (groupSort || pScanNode->groupOrderScan) {
       code = sortTableGroup(pTableListInfo);
     }

From eb2899d44b287a0a6e1b6647bf385baa37a10859 Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Wed, 17 Jan 2024 09:50:57 +0000
Subject: [PATCH 12/45] not check mem on arm

---
 tests/parallel_test/cases.task | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 1c2f5ec23c..63ca624d62 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -3,7 +3,13 @@
 #NA,NA,y or n,script,./test.sh -f tsim/user/basic.sim
 
 #unit-test
+
+archOs=$(arch)
+if [[ $archOs =~ "aarch64" ]]; then
+,,n,unit-test,bash test.sh
+else
 ,,y,unit-test,bash test.sh
+fi
 
 #
 # army-test

From f1fb7f5c147d112e20f84a07ca4f40d99e25cd7e Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 18 Jan 2024 08:43:36 +0800
Subject: [PATCH 13/45] fix: change num of output groups when cout empty group
 is needed

---
 source/libs/executor/src/executil.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index fd34a98eca..033cda43f2 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -2138,6 +2138,8 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
       pTableListInfo->numOfOuputGroups = numOfTables;
     } else if (groupByTbname && pScanNode->groupOrderScan){
       pTableListInfo->numOfOuputGroups = numOfTables;
+    } else if (groupByTbname && tsCountAlwaysReturnValue && ((STableScanPhysiNode*)pScanNode)->needCountEmptyTable) {
+      pTableListInfo->numOfOuputGroups = numOfTables;
     } else {
       pTableListInfo->numOfOuputGroups = 1;
     }

From a1f7169b639af0f8852d4d953aa15895903a40ae Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 18 Jan 2024 09:52:04 +0800
Subject: [PATCH 14/45] fix: remove uInfo

---
 source/libs/executor/src/scanoperator.c | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 103166e3e7..09f4f6ac3c 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -991,28 +991,16 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) {
     if (pOperator->dynamicTask) {
       result->info.id.groupId = result->info.id.uid;
     }
-    uInfo("slzhou 1 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows);
     return result;
   }
 
   while (true) {
     result = startNextGroupScan(pOperator);
     if (result || pOperator->status == OP_EXEC_DONE) {
-      if (result) {
-        uInfo("slzhou 2 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows);
-      }
-      else {
-        uInfo("slzhou 2 null block" );
-      }
       return result;
     }
   }
-      if (result) {
-        uInfo("slzhou 3 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows);
-      }
-      else {
-        uInfo("slzhou 3 null block" );
-      }
+
   return result;
 }
 

From 94ab74cc7f3d1738381aecc125347439744b65e6 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Wed, 17 Jan 2024 14:47:54 +0800
Subject: [PATCH 15/45] clone task id

---
 source/libs/executor/src/exchangeoperator.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index 2797cb2d82..140b390913 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -37,7 +37,7 @@ typedef struct SSourceDataInfo {
   int64_t            startTime;
   int32_t            code;
   EX_SOURCE_STATUS   status;
-  const char*        taskId;
+  char*              taskId;
   SArray*            pSrcUidList;
   int32_t            srcOpType;
   bool               tableSeq;
@@ -260,14 +260,16 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const
     return TSDB_CODE_SUCCESS;
   }
 
+  int32_t len = strlen(id);
   for (int32_t i = 0; i < numOfSources; ++i) {
     SSourceDataInfo dataInfo = {0};
     dataInfo.status = EX_SOURCE_DATA_NOT_READY;
-    dataInfo.taskId = id;
+    dataInfo.taskId = taosMemoryCalloc(1, len);
+    memcpy(dataInfo.taskId, id, len);
     dataInfo.index = i;
     SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo);
     if (pDs == NULL) {
-      taosArrayDestroy(pInfo->pSourceDataInfo);
+      taosArrayDestroyEx(pInfo->pSourceDataInfo, freeSourceDataInfo);
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -368,6 +370,7 @@ void freeBlock(void* pParam) {
 void freeSourceDataInfo(void* p) {
   SSourceDataInfo* pInfo = (SSourceDataInfo*)p;
   taosMemoryFreeClear(pInfo->pRsp);
+  taosMemoryFreeClear(pInfo->taskId);
 }
 
 void doDestroyExchangeOperatorInfo(void* param) {
@@ -782,7 +785,10 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic
   if (pIdx->inUseIdx < 0) {
     SSourceDataInfo dataInfo = {0};
     dataInfo.status = EX_SOURCE_DATA_NOT_READY;
-    dataInfo.taskId = GET_TASKID(pOperator->pTaskInfo);
+    char* pTaskId = GET_TASKID(pOperator->pTaskInfo);
+    int32_t len = strlen(pTaskId);
+    dataInfo.taskId = taosMemoryCalloc(1, len);
+    memcpy(dataInfo.taskId, pTaskId, len);
     dataInfo.index = pIdx->srcIdx;
     dataInfo.pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL);
     dataInfo.srcOpType = pBasicParam->srcOpType;

From 1309e56a95eb0a38cd69254276da608c89d7cfe1 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Wed, 17 Jan 2024 16:52:47 +0800
Subject: [PATCH 16/45] use strncpy

---
 source/libs/executor/src/exchangeoperator.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index 140b390913..ac103ebbc1 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -260,12 +260,12 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const
     return TSDB_CODE_SUCCESS;
   }
 
-  int32_t len = strlen(id);
+  int32_t len = strlen(id) + 1;
   for (int32_t i = 0; i < numOfSources; ++i) {
     SSourceDataInfo dataInfo = {0};
     dataInfo.status = EX_SOURCE_DATA_NOT_READY;
     dataInfo.taskId = taosMemoryCalloc(1, len);
-    memcpy(dataInfo.taskId, id, len);
+    strncpy(dataInfo.taskId, id, len);
     dataInfo.index = i;
     SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo);
     if (pDs == NULL) {
@@ -786,9 +786,9 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic
     SSourceDataInfo dataInfo = {0};
     dataInfo.status = EX_SOURCE_DATA_NOT_READY;
     char* pTaskId = GET_TASKID(pOperator->pTaskInfo);
-    int32_t len = strlen(pTaskId);
+    int32_t len = strlen(pTaskId) + 1;
     dataInfo.taskId = taosMemoryCalloc(1, len);
-    memcpy(dataInfo.taskId, pTaskId, len);
+    strncpy(dataInfo.taskId, pTaskId, len);
     dataInfo.index = pIdx->srcIdx;
     dataInfo.pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL);
     dataInfo.srcOpType = pBasicParam->srcOpType;

From 7986951f440e9cbf585d1238ab077881b1b24e7f Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 18 Jan 2024 09:22:39 +0800
Subject: [PATCH 17/45] use taskid of operator

---
 source/libs/executor/inc/executorInt.h      |  1 +
 source/libs/executor/src/exchangeoperator.c | 15 +++++++--------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index c3f47cde9d..72da249f50 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -200,6 +200,7 @@ typedef struct SExchangeInfo {
   uint64_t            self;
   SLimitInfo          limitInfo;
   int64_t             openedTs;  // start exec time stamp, todo: move to SLoadRemoteDataInfo
+  char*               pTaskId;
 } SExchangeInfo;
 
 typedef struct SScanInfo {
diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index ac103ebbc1..06dd43e170 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -37,7 +37,7 @@ typedef struct SSourceDataInfo {
   int64_t            startTime;
   int32_t            code;
   EX_SOURCE_STATUS   status;
-  char*              taskId;
+  const char*        taskId;
   SArray*            pSrcUidList;
   int32_t            srcOpType;
   bool               tableSeq;
@@ -261,11 +261,12 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const
   }
 
   int32_t len = strlen(id) + 1;
+  pInfo->pTaskId = taosMemoryCalloc(1, len);
+  strncpy(pInfo->pTaskId, id, len);
   for (int32_t i = 0; i < numOfSources; ++i) {
     SSourceDataInfo dataInfo = {0};
     dataInfo.status = EX_SOURCE_DATA_NOT_READY;
-    dataInfo.taskId = taosMemoryCalloc(1, len);
-    strncpy(dataInfo.taskId, id, len);
+    dataInfo.taskId = pInfo->pTaskId;
     dataInfo.index = i;
     SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo);
     if (pDs == NULL) {
@@ -370,7 +371,6 @@ void freeBlock(void* pParam) {
 void freeSourceDataInfo(void* p) {
   SSourceDataInfo* pInfo = (SSourceDataInfo*)p;
   taosMemoryFreeClear(pInfo->pRsp);
-  taosMemoryFreeClear(pInfo->taskId);
 }
 
 void doDestroyExchangeOperatorInfo(void* param) {
@@ -386,6 +386,8 @@ void doDestroyExchangeOperatorInfo(void* param) {
   tSimpleHashCleanup(pExInfo->pHashSources);
 
   tsem_destroy(&pExInfo->ready);
+  taosMemoryFreeClear(pExInfo->pTaskId);
+
   taosMemoryFreeClear(param);
 }
 
@@ -785,10 +787,7 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic
   if (pIdx->inUseIdx < 0) {
     SSourceDataInfo dataInfo = {0};
     dataInfo.status = EX_SOURCE_DATA_NOT_READY;
-    char* pTaskId = GET_TASKID(pOperator->pTaskInfo);
-    int32_t len = strlen(pTaskId) + 1;
-    dataInfo.taskId = taosMemoryCalloc(1, len);
-    strncpy(dataInfo.taskId, pTaskId, len);
+    dataInfo.taskId = pExchangeInfo->pTaskId;
     dataInfo.index = pIdx->srcIdx;
     dataInfo.pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL);
     dataInfo.srcOpType = pBasicParam->srcOpType;

From 84810185c733b8ba691ae1496aba7e0794647a70 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Thu, 18 Jan 2024 11:16:18 +0800
Subject: [PATCH 18/45] fix: skip acked msg in snapshotReSend

---
 source/libs/sync/src/syncSnapshot.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c
index 578e6798e0..d9c8bb21ac 100644
--- a/source/libs/sync/src/syncSnapshot.c
+++ b/source/libs/sync/src/syncSnapshot.c
@@ -345,9 +345,9 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
 
   for (int32_t seq = pSndBuf->cursor + 1; seq < pSndBuf->end; ++seq) {
     SyncSnapBlock *pBlk = pSndBuf->entries[seq % pSndBuf->size];
-    ASSERT(pBlk && !pBlk->acked);
+    ASSERT(pBlk);
     int64_t nowMs = taosGetTimestampMs();
-    if (nowMs < pBlk->sendTimeMs + SYNC_SNAP_RESEND_MS) {
+    if (pBlk->acked || nowMs < pBlk->sendTimeMs + SYNC_SNAP_RESEND_MS) {
       continue;
     }
     if (syncSnapSendMsg(pSender, pBlk->seq, pBlk->pBlock, pBlk->blockLen, 0) != 0) {

From 06c9b8974bf3d0eaaa144d5aa20c71dc7727c66b Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Thu, 18 Jan 2024 04:06:25 +0000
Subject: [PATCH 19/45] rebuild stream  backen from locak

---
 source/libs/stream/src/streamBackendRocksdb.c | 33 ++++++++-----------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index c8f944071f..bcf289a019 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -500,34 +500,27 @@ int32_t backendCopyFiles(char* src, char* dst) {
   // return 0;
 }
 int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) {
-  int32_t code = -1;
-  int32_t len = strlen(defaultPath) + 32;
-  char*   tmp = taosMemoryCalloc(1, len);
-  sprintf(tmp, "%s%s", defaultPath, "_tmp");
-
-  if (taosIsDir(tmp)) taosRemoveDir(tmp);
-  if (taosIsDir(defaultPath)) taosRenameFile(defaultPath, tmp);
-
-  if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) {
-    if (taosIsDir(tmp)) {
-      taosRemoveDir(tmp);
-    }
+  int32_t code = 0;
+  if (taosIsDir(defaultPath)) {
+    taosRemoveDir(defaultPath);
     taosMkDir(defaultPath);
+
+    stInfo("succ to clear stream backend %s", defaultPath);
+  }
+  if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) {
     code = backendCopyFiles(chkpPath, defaultPath);
     if (code != 0) {
-      stError("failed to restart stream backend from %s, reason: %s", chkpPath, tstrerror(TAOS_SYSTEM_ERROR(errno)));
+      taosRemoveDir(defaultPath);
+      taosMkDir(defaultPath);
+
+      stError("failed to restart stream backend from %s, reason: %s, start to restart from empty path: %s", chkpPath,
+              tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath);
+      code = 0;
     } else {
       stInfo("start to restart stream backend at checkpoint path: %s", chkpPath);
     }
   }
-  if (code != 0) {
-    if (taosIsDir(defaultPath)) taosRemoveDir(defaultPath);
-    if (taosIsDir(tmp)) taosRenameFile(tmp, defaultPath);
-  } else {
-    taosRemoveDir(tmp);
-  }
 
-  taosMemoryFree(tmp);
   return code;
 }
 

From ea6b39c01e272e784064730b69951263ecbab30d Mon Sep 17 00:00:00 2001
From: Yihao Deng <luomoxyz@126.com>
Date: Thu, 18 Jan 2024 06:53:16 +0000
Subject: [PATCH 20/45] del rsma case

---
 tests/parallel_test/cases.task | 4 +++-
 tests/script/win-test-file     | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 694af127af..acfe0f2a16 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -1245,7 +1245,9 @@ e
 ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
 ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
 ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQueryDelete.sim
-,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
+
+### refactor stream backend, open case after rsma refactored
+#,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
 ,,y,script,./test.sh -f tsim/sync/vnodesnapshot-rsma-test.sim
 ,,n,script,./test.sh -f tsim/valgrind/checkError1.sim
 ,,n,script,./test.sh -f tsim/valgrind/checkError2.sim
diff --git a/tests/script/win-test-file b/tests/script/win-test-file
index 744415c89f..b9f250927f 100644
--- a/tests/script/win-test-file
+++ b/tests/script/win-test-file
@@ -280,7 +280,9 @@
 ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
 ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
 ./test.sh -f tsim/sma/rsmaCreateInsertQueryDelete.sim
-./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
+
+### refactor stream backend, open case after rsma refactored
+#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
 ./test.sh -f tsim/sync/vnodesnapshot-rsma-test.sim
 ./test.sh -f tsim/valgrind/checkError1.sim
 ./test.sh -f tsim/valgrind/checkError2.sim

From 00c8405fc95a48a125e0a62337f37ddd9d0b6d6c Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Thu, 18 Jan 2024 15:10:24 +0800
Subject: [PATCH 21/45] fix: add query_basic.py to task

---
 tests/army/community/query/query_basic.py |   7 +-
 tests/army/frame/sql.py                   | 314 ++++++++++++----------
 tests/parallel_test/cases.task            |   1 +
 3 files changed, 173 insertions(+), 149 deletions(-)

diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py
index e3fdeb0f34..159a70e0d1 100644
--- a/tests/army/community/query/query_basic.py
+++ b/tests/army/community/query/query_basic.py
@@ -52,13 +52,14 @@ class TDTestCase(TBase):
         tdLog.info(f"do query.")
         
         # __group_key
-        sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti;"
+        sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti"
         tdSql.execute(sql)
-        tdSql.checkRows(251)
+        # column index 1 value same with 2
+        tdSql.checkSameColumn(1, 2)
 
         sql = f"select count(*),_group_key(usi) from {self.stb} group by usi;"
         tdSql.execute(sql)
-        tdSql.checkRows(997)
+        tdSql.checkSameColumn(1, 2)
 
         # tail
         sql1 = "select ts,ui from d0 order by ts desc limit 5 offset 2;"
diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py
index 5477f96b3d..a513edfb13 100644
--- a/tests/army/frame/sql.py
+++ b/tests/army/frame/sql.py
@@ -78,6 +78,107 @@ class TDSql:
         self.cursor.execute(s)
         time.sleep(2)
 
+
+    #
+    #  do execute
+    #
+
+    def query(self, sql, row_tag=None, queryTimes=10, count_expected_res=None):
+        self.sql = sql
+        i=1
+        while i <= queryTimes:
+            try:
+                self.cursor.execute(sql)
+                self.res = self.cursor.fetchall()
+                self.queryRows = len(self.res)
+                self.queryCols = len(self.cursor.description)
+
+                if count_expected_res is not None:
+                    counter = 0
+                    while count_expected_res != self.res[0][0]:
+                        self.cursor.execute(sql)
+                        self.res = self.cursor.fetchall()
+                        if counter < queryTimes:
+                            counter += 0.5
+                            time.sleep(0.5)
+                        else:
+                            return False
+                if row_tag:
+                    return self.res
+                return self.queryRows
+            except Exception as e:
+                tdLog.notice("Try to query again, query times: %d "%i)
+                if i == queryTimes:
+                    caller = inspect.getframeinfo(inspect.stack()[1][0])
+                    args = (caller.filename, caller.lineno, sql, repr(e))
+                    tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
+                    raise Exception(repr(e))
+                i+=1
+                time.sleep(1)
+                pass
+
+    def executeTimes(self, sql, times):
+        for i in range(times):
+            try:
+                return self.cursor.execute(sql)
+            except BaseException:
+                time.sleep(1)
+                continue
+
+    def execute(self, sql, queryTimes=30, show=False):
+        self.sql = sql
+        if show:
+            tdLog.info(sql)
+        i=1
+        while i <= queryTimes:
+            try:
+                self.affectedRows = self.cursor.execute(sql)
+                return self.affectedRows
+            except Exception as e:
+                tdLog.notice("Try to execute sql again, query times: %d "%i)
+                if i == queryTimes:
+                    caller = inspect.getframeinfo(inspect.stack()[1][0])
+                    args = (caller.filename, caller.lineno, sql, repr(e))
+                    tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
+                    raise Exception(repr(e))
+                i+=1
+                time.sleep(1)
+                pass
+
+    # execute many sql
+    def executes(self, sqls, queryTimes=30, show=False):
+        for sql in sqls:
+            self.execute(sql, queryTimes, show)
+
+    def waitedQuery(self, sql, expectRows, timeout):
+        tdLog.info("sql: %s, try to retrieve %d rows in %d seconds" % (sql, expectRows, timeout))
+        self.sql = sql
+        try:
+            for i in range(timeout):
+                self.cursor.execute(sql)
+                self.res = self.cursor.fetchall()
+                self.queryRows = len(self.res)
+                self.queryCols = len(self.cursor.description)
+                tdLog.info("sql: %s, try to retrieve %d rows,get %d rows" % (sql, expectRows, self.queryRows))
+                if self.queryRows >= expectRows:
+                    return (self.queryRows, i)
+                time.sleep(1)
+        except Exception as e:
+            caller = inspect.getframeinfo(inspect.stack()[1][0])
+            args = (caller.filename, caller.lineno, sql, repr(e))
+            tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
+            raise Exception(repr(e))
+        return (self.queryRows, timeout)
+
+    def is_err_sql(self, sql):
+        err_flag = True
+        try:
+            self.cursor.execute(sql)
+        except BaseException:
+            err_flag = False
+
+        return False if err_flag else True
+
     def error(self, sql, expectedErrno = None, expectErrInfo = None):
         caller = inspect.getframeinfo(inspect.stack()[1][0])
         expectErrNotOccured = True
@@ -95,7 +196,7 @@ class TDSql:
         else:
             self.queryRows = 0
             self.queryCols = 0
-            self.queryResult = None
+            self.res = None
 
             if expectedErrno != None:
                 if  expectedErrno == self.errno:
@@ -115,49 +216,25 @@ class TDSql:
 
             return self.error_info
 
-    def query(self, sql, row_tag=None, queryTimes=10, count_expected_res=None):
+    #
+    #  get session
+    #
+
+    def getData(self, row, col):
+        self.checkRowCol(row, col)
+        return self.res[row][col]
+
+    def getResult(self, sql):
         self.sql = sql
-        i=1
-        while i <= queryTimes:
-            try:
-                self.cursor.execute(sql)
-                self.queryResult = self.cursor.fetchall()
-                self.queryRows = len(self.queryResult)
-                self.queryCols = len(self.cursor.description)
-
-                if count_expected_res is not None:
-                    counter = 0
-                    while count_expected_res != self.queryResult[0][0]:
-                        self.cursor.execute(sql)
-                        self.queryResult = self.cursor.fetchall()
-                        if counter < queryTimes:
-                            counter += 0.5
-                            time.sleep(0.5)
-                        else:
-                            return False
-                if row_tag:
-                    return self.queryResult
-                return self.queryRows
-            except Exception as e:
-                tdLog.notice("Try to query again, query times: %d "%i)
-                if i == queryTimes:
-                    caller = inspect.getframeinfo(inspect.stack()[1][0])
-                    args = (caller.filename, caller.lineno, sql, repr(e))
-                    tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
-                    raise Exception(repr(e))
-                i+=1
-                time.sleep(1)
-                pass
-
-
-    def is_err_sql(self, sql):
-        err_flag = True
         try:
             self.cursor.execute(sql)
-        except BaseException:
-            err_flag = False
-
-        return False if err_flag else True
+            self.res = self.cursor.fetchall()
+        except Exception as e:
+            caller = inspect.getframeinfo(inspect.stack()[1][0])
+            args = (caller.filename, caller.lineno, sql, repr(e))
+            tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
+            raise Exception(repr(e))
+        return self.res
 
     def getVariable(self, search_attr):
         '''
@@ -193,29 +270,19 @@ class TDSql:
             return col_name_list, col_type_list
         return col_name_list
 
-    def waitedQuery(self, sql, expectRows, timeout):
-        tdLog.info("sql: %s, try to retrieve %d rows in %d seconds" % (sql, expectRows, timeout))
-        self.sql = sql
-        try:
-            for i in range(timeout):
-                self.cursor.execute(sql)
-                self.queryResult = self.cursor.fetchall()
-                self.queryRows = len(self.queryResult)
-                self.queryCols = len(self.cursor.description)
-                tdLog.info("sql: %s, try to retrieve %d rows,get %d rows" % (sql, expectRows, self.queryRows))
-                if self.queryRows >= expectRows:
-                    return (self.queryRows, i)
-                time.sleep(1)
-        except Exception as e:
-            caller = inspect.getframeinfo(inspect.stack()[1][0])
-            args = (caller.filename, caller.lineno, sql, repr(e))
-            tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
-            raise Exception(repr(e))
-        return (self.queryRows, timeout)
-
     def getRows(self):
         return self.queryRows
 
+    # get first value
+    def getFirstValue(self, sql) :
+        self.query(sql)
+        return self.getData(0, 0)
+
+    
+    #
+    #  check session
+    #
+
     def checkRows(self, expectRows):
         if self.queryRows == expectRows:
             tdLog.info("sql:%s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows))
@@ -273,26 +340,26 @@ class TDSql:
       
         self.checkRowCol(row, col)
 
-        if self.queryResult[row][col] != data:
+        if self.res[row][col] != data:
             if self.cursor.istype(col, "TIMESTAMP"):
                 # suppose user want to check nanosecond timestamp if a longer data passed``
                 if isinstance(data,str) :
                     if (len(data) >= 28):
-                        if self.queryResult[row][col] == _parse_ns_timestamp(data):
+                        if self.res[row][col] == _parse_ns_timestamp(data):
                             if(show):
                                tdLog.info("check successfully")
                         else:
                             caller = inspect.getframeinfo(inspect.stack()[1][0])
-                            args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                            args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data)
                             tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
                     else:
-                        if self.queryResult[row][col].astimezone(datetime.timezone.utc) == _parse_datetime(data).astimezone(datetime.timezone.utc):
-                            # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                        if self.res[row][col].astimezone(datetime.timezone.utc) == _parse_datetime(data).astimezone(datetime.timezone.utc):
+                            # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.res[row][col]} == expect:{data}")
                             if(show):
                                tdLog.info("check successfully")
                         else:
                             caller = inspect.getframeinfo(inspect.stack()[1][0])
-                            args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                            args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data)
                             tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
                     return
                 elif isinstance(data,int):
@@ -304,72 +371,72 @@ class TDSql:
                         precision = 'ns'
                     else:
                         caller = inspect.getframeinfo(inspect.stack()[1][0])
-                        args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                        args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data)
                         tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
                         return
                     success = False
                     if precision == 'ms':
-                        dt_obj = self.queryResult[row][col]
+                        dt_obj = self.res[row][col]
                         ts = int(int((dt_obj-datetime.datetime.fromtimestamp(0,dt_obj.tzinfo)).total_seconds())*1000) + int(dt_obj.microsecond/1000)
                         if ts == data:
                             success = True
                     elif precision == 'us':
-                        dt_obj = self.queryResult[row][col]
+                        dt_obj = self.res[row][col]
                         ts = int(int((dt_obj-datetime.datetime.fromtimestamp(0,dt_obj.tzinfo)).total_seconds())*1e6) + int(dt_obj.microsecond)
                         if ts == data:
                             success = True
                     elif precision == 'ns':
-                        if data == self.queryResult[row][col]:
+                        if data == self.res[row][col]:
                             success = True
                     if success:
                         if(show):
                             tdLog.info("check successfully")
                     else:
                         caller = inspect.getframeinfo(inspect.stack()[1][0])
-                        args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                        args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data)
                         tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
                     return
                 elif isinstance(data,datetime.datetime):
-                    dt_obj = self.queryResult[row][col]
+                    dt_obj = self.res[row][col]
                     delt_data = data-datetime.datetime.fromtimestamp(0,data.tzinfo)
-                    delt_result = self.queryResult[row][col] - datetime.datetime.fromtimestamp(0,self.queryResult[row][col].tzinfo)
+                    delt_result = self.res[row][col] - datetime.datetime.fromtimestamp(0,self.res[row][col].tzinfo)
                     if delt_data == delt_result:
                         if(show):
                             tdLog.info("check successfully")
                     else:
                         caller = inspect.getframeinfo(inspect.stack()[1][0])
-                        args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                        args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data)
                         tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
                     return
                 else:
                     caller = inspect.getframeinfo(inspect.stack()[1][0])
-                    args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                    args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data)
                     tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
 
-            if str(self.queryResult[row][col]) == str(data):
-                # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+            if str(self.res[row][col]) == str(data):
+                # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.res[row][col]} == expect:{data}")
                 if(show):
                     tdLog.info("check successfully")
                 return
 
             elif isinstance(data, float):
-                if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001:
-                    # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                if abs(data) >= 1 and abs((self.res[row][col] - data) / data) <= 0.000001:
+                    # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.res[row][col]} == expect:{data}")
                     if(show):
                         tdLog.info("check successfully")
-                elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001:
-                    # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
+                elif abs(data) < 1 and abs(self.res[row][col] - data) <= 0.000001:
+                    # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.res[row][col]} == expect:{data}")
                     if(show):
                         tdLog.info("check successfully")
 
                 else:
                     caller = inspect.getframeinfo(inspect.stack()[1][0])
-                    args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                    args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data)
                     tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
                 return
             else:
                 caller = inspect.getframeinfo(inspect.stack()[1][0])
-                args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
+                args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data)
                 tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
         if(show):         
             tdLog.info("check successfully")
@@ -397,23 +464,23 @@ class TDSql:
     def checkDataNoExit(self, row, col, data):
         if self.checkRowColNoExit(row, col) == False:
             return False
-        if self.queryResult[row][col] != data:
+        if self.res[row][col] != data:
             if self.cursor.istype(col, "TIMESTAMP"):
                 # suppose user want to check nanosecond timestamp if a longer data passed
                 if (len(data) >= 28):
-                    if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data):
+                    if pd.to_datetime(self.res[row][col]) == pd.to_datetime(data):
                         return True
                 else:
-                    if self.queryResult[row][col] == _parse_datetime(data):
+                    if self.res[row][col] == _parse_datetime(data):
                         return True
                 return False
 
-            if str(self.queryResult[row][col]) == str(data):
+            if str(self.res[row][col]) == str(data):
                 return True
             elif isinstance(data, float):
-                if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001:
+                if abs(data) >= 1 and abs((self.res[row][col] - data) / data) <= 0.000001:
                     return True
-                elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001:
+                elif abs(data) < 1 and abs(self.res[row][col] - data) <= 0.000001:
                     return True
                 else:
                     return False
@@ -437,56 +504,6 @@ class TDSql:
         self.query(sql)
         self.checkData(row, col, data)
 
-
-    def getData(self, row, col):
-        self.checkRowCol(row, col)
-        return self.queryResult[row][col]
-
-    def getResult(self, sql):
-        self.sql = sql
-        try:
-            self.cursor.execute(sql)
-            self.queryResult = self.cursor.fetchall()
-        except Exception as e:
-            caller = inspect.getframeinfo(inspect.stack()[1][0])
-            args = (caller.filename, caller.lineno, sql, repr(e))
-            tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
-            raise Exception(repr(e))
-        return self.queryResult
-
-    def executeTimes(self, sql, times):
-        for i in range(times):
-            try:
-                return self.cursor.execute(sql)
-            except BaseException:
-                time.sleep(1)
-                continue
-
-    def execute(self, sql, queryTimes=30, show=False):
-        self.sql = sql
-        if show:
-            tdLog.info(sql)
-        i=1
-        while i <= queryTimes:
-            try:
-                self.affectedRows = self.cursor.execute(sql)
-                return self.affectedRows
-            except Exception as e:
-                tdLog.notice("Try to execute sql again, query times: %d "%i)
-                if i == queryTimes:
-                    caller = inspect.getframeinfo(inspect.stack()[1][0])
-                    args = (caller.filename, caller.lineno, sql, repr(e))
-                    tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
-                    raise Exception(repr(e))
-                i+=1
-                time.sleep(1)
-                pass
-
-    # execute many sql
-    def executes(self, sqls, queryTimes=30, show=False):
-        for sql in sqls:
-            self.execute(sql, queryTimes, show)
-
     def checkAffectedRows(self, expectAffectedRows):
         if self.affectedRows != expectAffectedRows:
             caller = inspect.getframeinfo(inspect.stack()[1][0])
@@ -544,17 +561,22 @@ class TDSql:
     def checkAgg(self, sql, expectCnt):
         self.query(sql)
         self.checkData(0, 0, expectCnt)
-
-    # get first value
-    def getFirstValue(self, sql) :
-        self.query(sql)
-        return self.getData(0, 0)
     
     # expect first value
     def checkFirstValue(self, sql, expect):
         self.query(sql)
         self.checkData(0, 0, expect)
-        
+
+    # colIdx1 value same with colIdx2
+    def checkSameColumn(self, c1, c2):
+        for i in range(self.queryRows):
+            if self.res[i][c1] != self.res[i][c2]:
+                tdLog.exit(f"Not same. row={i} col1={c1} col2={c2}. {self.res[i][c1]}!={self.res[i][c2]}")
+        tdLog.info(f"check {self.queryRows} rows. column{c1} value equal tp column{c2}")
+
+    #
+    # others session
+    #
 
     def get_times(self, time_str, precision="ms"):
         caller = inspect.getframeinfo(inspect.stack()[1][0])
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 694af127af..b0e4b5725a 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -11,6 +11,7 @@
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
 
 

From f7d7ce35155c8b0159f5dc2f3195c90813957d1d Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Thu, 18 Jan 2024 15:33:07 +0800
Subject: [PATCH 22/45] fix: rename all queryResult name

---
 tests/army/frame/caseBase.py       |  4 ++--
 tests/army/frame/common.py         | 28 ++++++++++++++--------------
 tests/army/frame/server/cluster.py |  3 +--
 tests/army/test.py                 | 11 -----------
 4 files changed, 17 insertions(+), 29 deletions(-)

diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py
index fcfcb7d066..b1500b69e1 100644
--- a/tests/army/frame/caseBase.py
+++ b/tests/army/frame/caseBase.py
@@ -193,10 +193,10 @@ class TBase:
 
         # sql
         rows1 = tdSql.query(sql1,queryTimes=2)
-        res1 = copy.deepcopy(tdSql.queryResult)
+        res1 = copy.deepcopy(tdSql.res)
 
         tdSql.query(sql2,queryTimes=2)
-        res2 = tdSql.queryResult
+        res2 = tdSql.res
 
         rowlen1 = len(res1)
         rowlen2 = len(res2)
diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py
index 5cdb3f9f46..d11a0dbb4d 100644
--- a/tests/army/frame/common.py
+++ b/tests/army/frame/common.py
@@ -795,7 +795,7 @@ class TDCom:
     def getOneRow(self, location, containElm):
         res_list = list()
         if 0 <= location < tdSql.queryRows:
-            for row in tdSql.queryResult:
+            for row in tdSql.res:
                 if row[location] == containElm:
                     res_list.append(row)
             return res_list
@@ -943,7 +943,7 @@ class TDCom:
         """drop all streams
         """
         tdSql.query("show streams")
-        stream_name_list = list(map(lambda x: x[0], tdSql.queryResult))
+        stream_name_list = list(map(lambda x: x[0], tdSql.res))
         for stream_name in stream_name_list:
             tdSql.execute(f'drop stream if exists {stream_name};')
 
@@ -962,7 +962,7 @@ class TDCom:
         """drop all databases
         """
         tdSql.query("show databases;")
-        db_list = list(map(lambda x: x[0], tdSql.queryResult))
+        db_list = list(map(lambda x: x[0], tdSql.res))
         for dbname in db_list:
             if dbname not in self.white_list and "telegraf" not in dbname:
                 tdSql.execute(f'drop database if exists `{dbname}`')
@@ -1412,7 +1412,7 @@ class TDCom:
             input_function (str): scalar
         """
         tdSql.query(sql)
-        res = tdSql.queryResult
+        res = tdSql.res
         if input_function in ["acos", "asin", "atan", "cos", "log", "pow", "sin", "sqrt", "tan"]:
             tdSql.checkEqual(res[1][1], "DOUBLE")
             tdSql.checkEqual(res[2][1], "DOUBLE")
@@ -1490,7 +1490,7 @@ class TDCom:
             bigint: bigint-ts
         """
         tdSql.query(f'select cast({str_ts} as bigint)')
-        return tdSql.queryResult[0][0]
+        return tdSql.res[0][0]
 
     def cast_query_data(self, query_data):
         """cast query-result for existed-stb
@@ -1514,7 +1514,7 @@ class TDCom:
                         tdSql.query(f'select cast("{v}" as binary(6))')
                     else:
                         tdSql.query(f'select cast("{v}" as {" ".join(col_tag_type_list[i].strip().split(" ")[1:])})')
-                    query_data_l[i] = tdSql.queryResult[0][0]
+                    query_data_l[i] = tdSql.res[0][0]
                 else:
                     query_data_l[i] = v
             nl.append(tuple(query_data_l))
@@ -1566,9 +1566,9 @@ class TDCom:
         if tag_value_list:
             dvalue = len(self.tag_type_str.split(',')) - defined_tag_count
         tdSql.query(sql1)
-        res1 = tdSql.queryResult
+        res1 = tdSql.res
         tdSql.query(sql2)
-        res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult
+        res2 = self.cast_query_data(tdSql.res) if tag_value_list or use_exist_stb else tdSql.res
         tdSql.sql = sql1
         new_list = list()
         if tag_value_list:
@@ -1601,10 +1601,10 @@ class TDCom:
                 tdLog.info("query retrying ...")
                 new_list = list()
                 tdSql.query(sql1)
-                res1 = tdSql.queryResult
+                res1 = tdSql.res
                 tdSql.query(sql2)
-                # res2 = tdSql.queryResult
-                res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult
+                # res2 = tdSql.res
+                res2 = self.cast_query_data(tdSql.res) if tag_value_list or use_exist_stb else tdSql.res
                 tdSql.sql = sql1
 
                 if tag_value_list:
@@ -1643,10 +1643,10 @@ class TDCom:
                 tdLog.info("query retrying ...")
                 new_list = list()
                 tdSql.query(sql1)
-                res1 = tdSql.queryResult
+                res1 = tdSql.res
                 tdSql.query(sql2)
-                # res2 = tdSql.queryResult
-                res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult
+                # res2 = tdSql.res
+                res2 = self.cast_query_data(tdSql.res) if tag_value_list or use_exist_stb else tdSql.res
                 tdSql.sql = sql1
 
                 if tag_value_list:
diff --git a/tests/army/frame/server/cluster.py b/tests/army/frame/server/cluster.py
index ade8ac39a2..aba2d2e630 100644
--- a/tests/army/frame/server/cluster.py
+++ b/tests/army/frame/server/cluster.py
@@ -86,10 +86,9 @@ class ConfigureyCluster:
         count=0
         while count < 5:
             tdSql.query("select * from information_schema.ins_dnodes")
-            # tdLog.debug(tdSql.queryResult)
             status=0
             for i in range(self.dnodeNums):
-                if tdSql.queryResult[i][4] == "ready":
+                if tdSql.res[i][4] == "ready":
                     status+=1
             # tdLog.debug(status)
             
diff --git a/tests/army/test.py b/tests/army/test.py
index a3e28b772d..a94fefe0be 100644
--- a/tests/army/test.py
+++ b/tests/army/test.py
@@ -560,17 +560,6 @@ if __name__ == "__main__":
                     conn = taosws.connect(f"taosws://root:taosdata@{host}:6041")
                 else:
                     conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath())
-                # tdSql.init(conn.cursor())
-                # tdSql.execute("create qnode on dnode 1")
-                # tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy)
-                # tdSql.query("show local variables;")
-                # for i in range(tdSql.queryRows):
-                #     if tdSql.queryResult[i][0] == "queryPolicy" :
-                #         if int(tdSql.queryResult[i][1]) == int(queryPolicy):
-                #             tdLog.info('alter queryPolicy to %d successfully'%queryPolicy)
-                #         else :
-                #             tdLog.debug(tdSql.queryResult)
-                #             tdLog.exit("alter queryPolicy to  %d failed"%queryPolicy)
 
                 cursor = conn.cursor()
                 cursor.execute("create qnode on dnode 1")

From be3f16b7e4c16077cdf6389ad43a239a4af925b6 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Thu, 18 Jan 2024 15:53:16 +0800
Subject: [PATCH 23/45] fix: taos -n server killed

---
 tests/army/community/cmdline/fullopt.py   | 2 ++
 tests/army/community/query/query_basic.py | 6 +++---
 tests/army/frame/sql.py                   | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/army/community/cmdline/fullopt.py b/tests/army/community/cmdline/fullopt.py
index 99c7095a38..39d1d581ed 100644
--- a/tests/army/community/cmdline/fullopt.py
+++ b/tests/army/community/cmdline/fullopt.py
@@ -119,6 +119,8 @@ class TDTestCase(TBase):
         # stop taosd test taos as server
         sc.dnodeStop(idx)
         etool.exeBinFile("taos", f'-n server', wait=False)
+        time.sleep(3)
+        eos.exe("pkill -9 taos")
 
     # run
     def run(self):
diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py
index 159a70e0d1..90916a1c6c 100644
--- a/tests/army/community/query/query_basic.py
+++ b/tests/army/community/query/query_basic.py
@@ -53,12 +53,12 @@ class TDTestCase(TBase):
         
         # __group_key
         sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti"
-        tdSql.execute(sql)
+        tdSql.query(sql)
         # column index 1 value same with 2
         tdSql.checkSameColumn(1, 2)
 
-        sql = f"select count(*),_group_key(usi) from {self.stb} group by usi;"
-        tdSql.execute(sql)
+        sql = f"select count(*),_group_key(usi),usi from {self.stb} group by usi limit 100;"
+        tdSql.query(sql)
         tdSql.checkSameColumn(1, 2)
 
         # tail
diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py
index a513edfb13..8b7538321d 100644
--- a/tests/army/frame/sql.py
+++ b/tests/army/frame/sql.py
@@ -572,7 +572,7 @@ class TDSql:
         for i in range(self.queryRows):
             if self.res[i][c1] != self.res[i][c2]:
                 tdLog.exit(f"Not same. row={i} col1={c1} col2={c2}. {self.res[i][c1]}!={self.res[i][c2]}")
-        tdLog.info(f"check {self.queryRows} rows. column{c1} value equal tp column{c2}")
+        tdLog.info(f"check {self.queryRows} rows two column value same. column index [{c1},{c2}]")
 
     #
     # others session

From bb715c249d770c2160d7058ab3b5b22ac5e89a13 Mon Sep 17 00:00:00 2001
From: zk66214 <zk662144@163.com>
Date: Thu, 18 Jan 2024 16:58:55 +0800
Subject: [PATCH 24/45] delete duplicated cases

---
 tests/system-test/2-query/orderBy.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/tests/system-test/2-query/orderBy.py b/tests/system-test/2-query/orderBy.py
index dedc57eab3..1cf1478439 100644
--- a/tests/system-test/2-query/orderBy.py
+++ b/tests/system-test/2-query/orderBy.py
@@ -278,19 +278,19 @@ class TDTestCase:
 
     def queryOrderByAgg(self):
 
-        tdSql.query("SELECT COUNT(*) FROM t1 order by COUNT(*)")
+        tdSql.no_error("SELECT COUNT(*) FROM t1 order by COUNT(*)")
 
-        tdSql.query("SELECT COUNT(*) FROM t1 order by last(c2)")
+        tdSql.no_error("SELECT COUNT(*) FROM t1 order by last(c2)")
 
-        tdSql.query("SELECT c1 FROM t1 order by last(ts)")
+        tdSql.no_error("SELECT c1 FROM t1 order by last(ts)")
 
-        tdSql.query("SELECT ts FROM t1 order by last(ts)")
+        tdSql.no_error("SELECT ts FROM t1 order by last(ts)")
 
-        tdSql.query("SELECT last(ts), ts, c1 FROM t1 order by 2")
+        tdSql.no_error("SELECT last(ts), ts, c1 FROM t1 order by 2")
 
-        tdSql.query("SELECT ts, last(ts) FROM t1 order by last(ts)")
+        tdSql.no_error("SELECT ts, last(ts) FROM t1 order by last(ts)")
 
-        tdSql.query(f"SELECT * FROM t1 order by last(ts)")
+        tdSql.no_error(f"SELECT * FROM t1 order by last(ts)")
 
         tdSql.query(f"SELECT last(ts) as t2, ts FROM t1 order by 1")
         tdSql.checkRows(1)
@@ -302,6 +302,18 @@ class TDTestCase:
 
         tdSql.error(f"SELECT last(ts) as t2, ts FROM t1 order by last(t2)")
 
+    def queryOrderByAmbiguousName(self):
+        tdSql.error(sql="select c1 as name, c2 as name, c3 from t1 order by name", expectErrInfo='ambiguous',
+                    fullMatched=False)
+
+        tdSql.error(sql="select c1, c2 as c1, c3 from t1 order by c1", expectErrInfo='ambiguous', fullMatched=False)
+
+        tdSql.error(sql='select last(ts), last(c1) as name ,last(c2) as name,last(c3) from t1 order by name',
+                    expectErrInfo='ambiguous', fullMatched=False)
+
+        tdSql.no_error("select c1 as name, c2 as c1, c3 from t1 order by c1")
+
+        tdSql.no_error('select c1 as name from (select c1, c2 as name from st) order by name')
 
     # run
     def run(self):
@@ -317,6 +329,8 @@ class TDTestCase:
         # agg
         self.queryOrderByAgg()
 
+        # td-28332
+        self.queryOrderByAmbiguousName()
 
     # stop
     def stop(self):

From b83bb8c1507d29bf4e0ef7a2a83e451ec8aa7d16 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Thu, 18 Jan 2024 17:20:31 +0800
Subject: [PATCH 25/45] fix: last_row error

---
 source/dnode/vnode/src/tsdb/tsdbCache.c |  2 +-
 tests/system-test/2-query/last_row.py   | 46 +++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index d86219542f..cc0bf2b774 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -941,7 +941,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
   }
 
   if(lastrowTmpIndexArray != NULL) {
-    mergeLastCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds);
+    mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds);
     for(int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) {
       taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i));
     }
diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py
index 6469b3ea54..a7223db1cd 100644
--- a/tests/system-test/2-query/last_row.py
+++ b/tests/system-test/2-query/last_row.py
@@ -861,11 +861,55 @@ class TDTestCase:
 
         self.support_super_table_test()
 
+    def initLastRowDelayTest(self, dbname="db"):
+        tdSql.execute(f"drop database if exists {dbname} ")
+        create_db_sql = f"create database if not exists {dbname} keep 3650 duration 1000 cachemodel 'NONE' REPLICA 1"
+        tdSql.execute(create_db_sql)
+
+        tdSql.execute(f"use {dbname}")
+        tdSql.execute(f'create stable {dbname}.st(ts timestamp, v_int int, v_float float) TAGS (ctname varchar(32))')
+
+        tdSql.execute(f"create table {dbname}.ct1 using st tags('ct1')")
+        tdSql.execute(f"create table {dbname}.ct2 using st tags('ct2')")
+
+        tdSql.execute(f"insert into {dbname}.st(tbname,ts,v_float, v_int) values('ct1',1630000000000,86,86)")
+        tdSql.execute(f"insert into {dbname}.st(tbname,ts,v_float, v_int) values('ct1',1630000021255,59,59)")
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute(f'select last(*) from {dbname}.st')
+        tdSql.execute(f'select last_row(*) from {dbname}.st')
+        tdSql.execute(f"insert into {dbname}.st(tbname,ts) values('ct1',1630000091255)")
+        tdSql.execute(f'flush database {dbname}')
+        tdSql.execute(f'select last(*) from {dbname}.st')
+        tdSql.execute(f'select last_row(*) from {dbname}.st')
+        tdSql.execute(f'alter database {dbname} cachemodel "both"')
+        tdSql.query(f'select last(*) from {dbname}.st')
+        tdSql.checkData(0 , 1 , 59)
+
+        tdSql.query(f'select last_row(*) from {dbname}.st')
+        tdSql.checkData(0 , 1 , None)
+        tdSql.checkData(0 , 2 , None)
+
+        tdLog.printNoPrefix("========== delay test init success ==============")
+
+    def lastRowDelayTest(self, dbname="db"):
+        tdLog.printNoPrefix("========== delay test start ==============")
+
+        tdSql.execute(f"use {dbname}")
+
+        tdSql.query(f'select last(*) from {dbname}.st')
+        tdSql.checkData(0 , 1 , 59)
+
+        tdSql.query(f'select last_row(*) from {dbname}.st')
+        tdSql.checkData(0 , 1 , None)
+        tdSql.checkData(0 , 2 , None)
+
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         # tdSql.prepare()
 
         tdLog.printNoPrefix("==========step1:create table ==============")
 
+        self.initLastRowDelayTest("DELAYTEST")
+
         # cache_last 0
         self.prepare_datas("'NONE' ")
         self.prepare_tag_datas("'NONE'")
@@ -890,6 +934,8 @@ class TDTestCase:
         self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'BOTH'")
         self.basic_query()
 
+        self.lastRowDelayTest("DELAYTEST")
+
 
     def stop(self):
         tdSql.close()

From cd6026c6a9f34b3b603244355f8e677b0cbf490d Mon Sep 17 00:00:00 2001
From: facetosea <25808407@qq.com>
Date: Thu, 18 Jan 2024 17:31:23 +0800
Subject: [PATCH 26/45] docs: keepColumnName notes

---
 docs/en/14-reference/12-config/index.md | 7 ++++---
 docs/zh/14-reference/12-config/index.md | 7 ++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md
index 65c48f9190..c1abfd3e39 100755
--- a/docs/en/14-reference/12-config/index.md
+++ b/docs/en/14-reference/12-config/index.md
@@ -226,9 +226,10 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo
 | Attribute     | Description                                                                                                     |
 | ------------- | --------------------------------------------------------------------------------------------------------------- |
 | Applicable    | Client only                                                                                                     |
-| Meaning       | When the Last, First, LastRow function is queried, whether the returned column name contains the function name. |
-| Value Range   | 0 means including the function name, 1 means not including the function name.                                   |
-| Default Value | 0                                                                                                               |
+| Meaning       | When the Last, First, and LastRow functions are queried and no alias is specified, the alias is automatically set to the column name (excluding the function name). Therefore, if the order by clause refers to the column name, it will automatically refer to the function corresponding to the column. |
+| Value Range   | 1 means automatically setting the alias to the column name (excluding the function name), 0 means not automatically setting the alias.                                   |
+| Default Value | 0          |
+| Notes | When multiple of the above functions act on the same column at the same time and no alias is specified, if the order by clause refers to the column name, column selection ambiguous will occur because the aliases of multiple columns are the same. |
 
 ## Locale Parameters
 
diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md
index 27a9618107..4d47f0771c 100755
--- a/docs/zh/14-reference/12-config/index.md
+++ b/docs/zh/14-reference/12-config/index.md
@@ -215,9 +215,10 @@ taos -C
 | 属性     | 说明                                                        |
 | -------- | ----------------------------------------------------------- |
 | 适用范围 | 仅客户端适用                                                |
-| 含义     | Last、First、LastRow 函数查询时，返回的列名是否包含函数名。 |
-| 取值范围 | 0 表示包含函数名，1 表示不包含函数名。                      |
-| 缺省值   | 0                                                           |
+| 含义     | Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数 |
+| 取值范围 | 1 表示自动设置别名为列名(不包含函数名), 0 表示不自动设置别名。                      |
+| 缺省值   | 0                                                    |
+| 补充说明 | 当同时出现多个上述函数作用于同一列且未指定别名时，如果 order by 子句引用了该列名，将会因为多列别名相同引发列选择冲突|
 
 ### countAlwaysReturnValue
 

From afbe4d99a8f6ad0517907c233b51f0860578b244 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Thu, 18 Jan 2024 18:16:18 +0800
Subject: [PATCH 27/45] fix: case

---
 tests/system-test/2-query/last_row.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py
index a7223db1cd..0744b3bae5 100644
--- a/tests/system-test/2-query/last_row.py
+++ b/tests/system-test/2-query/last_row.py
@@ -866,11 +866,12 @@ class TDTestCase:
         create_db_sql = f"create database if not exists {dbname} keep 3650 duration 1000 cachemodel 'NONE' REPLICA 1"
         tdSql.execute(create_db_sql)
 
+        time.sleep(3)
         tdSql.execute(f"use {dbname}")
         tdSql.execute(f'create stable {dbname}.st(ts timestamp, v_int int, v_float float) TAGS (ctname varchar(32))')
 
-        tdSql.execute(f"create table {dbname}.ct1 using st tags('ct1')")
-        tdSql.execute(f"create table {dbname}.ct2 using st tags('ct2')")
+        tdSql.execute(f"create table {dbname}.ct1 using {dbname}.st tags('ct1')")
+        tdSql.execute(f"create table {dbname}.ct2 using {dbname}.st tags('ct2')")
 
         tdSql.execute(f"insert into {dbname}.st(tbname,ts,v_float, v_int) values('ct1',1630000000000,86,86)")
         tdSql.execute(f"insert into {dbname}.st(tbname,ts,v_float, v_int) values('ct1',1630000021255,59,59)")

From ff8908dedebfffc72cb798a7f4a46de496d52bdd Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 18 Jan 2024 18:30:21 +0800
Subject: [PATCH 28/45] ignore invalid state

---
 source/libs/stream/src/streamSessionState.c | 4 ++++
 source/libs/stream/src/streamState.c        | 1 +
 tests/script/tsim/stream/partitionby1.sim   | 2 ++
 3 files changed, 7 insertions(+)

diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index 2cb77c60dc..1f991d309f 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -90,6 +90,7 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe
   SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
   memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
   pNewPos->needFree = true;
+  pNewPos->beFlushed = true;
   memcpy(pNewPos->pRowBuff, p, *pVLen);
   taosMemoryFree(p);
   return pNewPos;
@@ -217,6 +218,7 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v
   SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
   memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
   pNewPos->needFree = true;
+  pNewPos->beFlushed = true;
   void*   pBuff = NULL;
   int32_t code = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen);
   if (code != TSDB_CODE_SUCCESS) {
@@ -307,6 +309,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream
     }
     pNewPos = getNewRowPosForWrite(pFileState);
     pNewPos->needFree = true;
+    pNewPos->beFlushed = true;
   }
 
 _end:
@@ -482,6 +485,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void
       SRowBuffPos* pNewPos = getNewRowPosForWrite(pCur->pStreamFileState);
       memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey));
       pNewPos->needFree = true;
+      pNewPos->beFlushed = true;
       memcpy(pNewPos->pRowBuff, pData, *pVLen);
       (*pVal) = pNewPos;
     }
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 19b7359981..ea20f0e2b1 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -698,6 +698,7 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void
       stDebug("===stream===save skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64 ".code:%d", key->win.skey,
               key->win.ekey, key->groupId, code);
     } else {
+      pos->beFlushed = false;
       code = putSessionWinResultBuff(pState->pFileState, value);
     }
   }
diff --git a/tests/script/tsim/stream/partitionby1.sim b/tests/script/tsim/stream/partitionby1.sim
index d92aecb3a6..24c588d410 100644
--- a/tests/script/tsim/stream/partitionby1.sim
+++ b/tests/script/tsim/stream/partitionby1.sim
@@ -13,6 +13,8 @@ sql create table ts3 using st tags(3,2,2);
 sql create table ts4 using st tags(4,2,2);
 sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0  into streamtST1 as select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b)  c4, min(c) c5 from st partition by tbname interval(10s);
 
+sleep 1000
+
 sql insert into ts1 values(1648791213001,1,12,3,1.0);
 sql insert into ts2 values(1648791213001,1,12,3,1.0);
 

From 2365054ada6aa591cf78f4cbae2ea63c280a0b85 Mon Sep 17 00:00:00 2001
From: zk66214 <zk662144@163.com>
Date: Thu, 18 Jan 2024 20:09:03 +0800
Subject: [PATCH 29/45] add common function no_error to TDsql

---
 tests/pytest/util/sql.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index 62af9a1af9..92074161b6 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -97,7 +97,24 @@ class TDSql:
                 i+=1
                 time.sleep(1)
                 pass
-            
+
+    def no_error(self, sql):
+        caller = inspect.getframeinfo(inspect.stack()[1][0])
+        expectErrOccurred = False
+
+        try:
+            self.cursor.execute(sql)
+        except BaseException as e:
+            expectErrOccurred = True
+            self.errno = e.errno
+            error_info = repr(e)
+            self.error_info = ','.join(error_info[error_info.index('(') + 1:-1].split(",")[:-1]).replace("'", "")
+
+        if expectErrOccurred:
+            tdLog.exit("%s(%d) failed: sql:%s, unexpect error '%s' occurred" % (caller.filename, caller.lineno, sql, self.error_info))
+        else:
+            tdLog.info("sql:%s, check passed, no ErrInfo occurred" % (sql))
+
     def error(self, sql, expectedErrno = None, expectErrInfo = None, fullMatched = True):
         caller = inspect.getframeinfo(inspect.stack()[1][0])
         expectErrNotOccured = True
@@ -126,9 +143,9 @@ class TDSql:
 
                 if expectErrInfo != None:
                     if expectErrInfo == self.error_info:
-                        tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo))
+                        tdLog.info("sql:%s, expected ErrInfo '%s' occured" % (sql, expectErrInfo))
                     else:
-                        tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
+                        tdLog.exit("%s(%d) failed: sql:%s, ErrInfo '%s' occured, but not expected ErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
             else:
                 if expectedErrno != None:
                     if expectedErrno in self.errno:
@@ -138,9 +155,9 @@ class TDSql:
 
                 if expectErrInfo != None:
                     if expectErrInfo in self.error_info:
-                        tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo))
+                        tdLog.info("sql:%s, expected ErrInfo '%s' occured" % (sql, expectErrInfo))
                     else:
-                        tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
+                        tdLog.exit("%s(%d) failed: sql:%s, ErrInfo %s occured, but not expected ErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
 
             return self.error_info
 

From ba48115231b9b5fb6413bc5e12f4adeb9053f465 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Fri, 19 Jan 2024 10:24:39 +0800
Subject: [PATCH 30/45] fix: heap user after free

---
 source/client/inc/clientInt.h        |  1 +
 source/client/src/clientEnv.c        |  1 +
 source/client/src/clientHb.c         | 22 +++++++++++++---------
 source/client/src/clientMsgHandler.c | 20 ++++++++++++++++++--
 4 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index a6d5039be7..fd4776664c 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -155,6 +155,7 @@ typedef struct STscObj {
   int8_t         biMode;
   int32_t        acctId;
   uint32_t       connId;
+  int32_t        appHbMgrIdx;
   int64_t        id;         // ref ID returned by taosAddRef
   TdThreadMutex  mutex;      // used to protect the operation on db
   int32_t        numOfReqs;  // number of sqlObj bound to this connection
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index b6c5701915..f1e9e36433 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -283,6 +283,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c
 
   pObj->connType = connType;
   pObj->pAppInfo = pAppInfo;
+  pObj->appHbMgrIdx = pAppInfo->pAppHbMgr->idx;
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN);
 
diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c
index b3d9a9ced5..e076a874d3 100644
--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@@ -30,7 +30,7 @@ typedef struct {
   };
 } SHbParam;
 
-static SClientHbMgr clientHbMgr = {0};
+SClientHbMgr clientHbMgr = {0};
 
 static int32_t hbCreateThread();
 static void    hbStopThread();
@@ -1294,9 +1294,8 @@ void hbMgrCleanUp() {
 
   taosThreadMutexLock(&clientHbMgr.lock);
   appHbMgrCleanup();
-  taosArrayDestroy(clientHbMgr.appHbMgrs);
+  clientHbMgr.appHbMgrs = taosArrayDestroy(clientHbMgr.appHbMgrs);
   taosThreadMutexUnlock(&clientHbMgr.lock);
-  clientHbMgr.appHbMgrs = NULL;
 }
 
 int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, int64_t clusterId) {
@@ -1335,13 +1334,18 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in
 }
 
 void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) {
-  SAppHbMgr    *pAppHbMgr = pTscObj->pAppInfo->pAppHbMgr;
-  SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
-  if (pReq) {
-    tFreeClientHbReq(pReq);
-    taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
-    taosHashRelease(pAppHbMgr->activeInfo, pReq);
+  SClientHbReq *pReq = NULL;
+  taosThreadMutexLock(&clientHbMgr.lock);
+  SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx);
+  if (pAppHbMgr) {
+    pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
+    if (pReq) {
+      tFreeClientHbReq(pReq);
+      taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
+      taosHashRelease(pAppHbMgr->activeInfo, pReq);
+    }
   }
+  taosThreadMutexUnlock(&clientHbMgr.lock);
 
   if (NULL == pReq) {
     return;
diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c
index e0cedb9924..fbef9dfad4 100644
--- a/source/client/src/clientMsgHandler.c
+++ b/source/client/src/clientMsgHandler.c
@@ -26,6 +26,8 @@
 #include "tname.h"
 #include "tversion.h"
 
+extern SClientHbMgr clientHbMgr;
+
 static void setErrno(SRequestObj* pRequest, int32_t code) {
   pRequest->code = code;
   terrno = code;
@@ -63,12 +65,21 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
 
   STscObj* pTscObj = pRequest->pTscObj;
 
-  if (NULL == pTscObj->pAppInfo || NULL == pTscObj->pAppInfo->pAppHbMgr) {
+  if (NULL == pTscObj->pAppInfo) {
     setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
     tsem_post(&pRequest->body.rspSem);
     goto End;
   }
 
+  taosThreadMutexLock(&clientHbMgr.lock);
+  if (NULL == taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx)) {
+    taosThreadMutexUnlock(&clientHbMgr.lock);
+    setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
+    tsem_post(&pRequest->body.rspSem);
+    goto End;
+  }
+  taosThreadMutexUnlock(&clientHbMgr.lock);
+
   SConnectRsp connectRsp = {0};
   if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) {
     code = TSDB_CODE_TSC_INVALID_VERSION;
@@ -142,7 +153,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
   pTscObj->authVer = connectRsp.authVer;
   pTscObj->whiteListInfo.ver = connectRsp.whiteListVer;
 
-  hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType);
+  taosThreadMutexLock(&clientHbMgr.lock);
+  SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx);
+  if (pAppHbMgr) {
+    hbRegisterConn(pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType);
+  }
+  taosThreadMutexUnlock(&clientHbMgr.lock);
 
   tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,
            pTscObj->pAppInfo->numOfConns);

From 1e3f70a1de2c56cf5b9cef0833734e374243de51 Mon Sep 17 00:00:00 2001
From: chenhaoran <haoran920c@163.com>
Date: Fri, 19 Jan 2024 14:04:22 +0800
Subject: [PATCH 31/45] test: add test of  empty string in tmq

---
 tests/system-test/7-tmq/tmqCheckData1.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py
index 1209c2812c..f443ce5b1e 100644
--- a/tests/system-test/7-tmq/tmqCheckData1.py
+++ b/tests/system-test/7-tmq/tmqCheckData1.py
@@ -51,7 +51,8 @@ class TDTestCase:
         tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
         tdLog.info("insert data")
         tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
-
+        tdSql.execute("insert into ctb0  values(now,1,'');")
+        
         tdLog.info("create topics from stb with filter")
         queryString = "select ts,c1,c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName'])
         sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"])

From ff06f9ef4260c92069fa993504da6af00b27bc78 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Fri, 19 Jan 2024 14:31:02 +0800
Subject: [PATCH 32/45] fix: heap user after free

---
 source/client/src/clientMsgHandler.c | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c
index fbef9dfad4..324b99022b 100644
--- a/source/client/src/clientMsgHandler.c
+++ b/source/client/src/clientMsgHandler.c
@@ -66,20 +66,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
   STscObj* pTscObj = pRequest->pTscObj;
 
   if (NULL == pTscObj->pAppInfo) {
-    setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
+    code = TSDB_CODE_TSC_DISCONNECTED;
+    setErrno(pRequest, code);
     tsem_post(&pRequest->body.rspSem);
     goto End;
   }
 
-  taosThreadMutexLock(&clientHbMgr.lock);
-  if (NULL == taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx)) {
-    taosThreadMutexUnlock(&clientHbMgr.lock);
-    setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
-    tsem_post(&pRequest->body.rspSem);
-    goto End;
-  }
-  taosThreadMutexUnlock(&clientHbMgr.lock);
-
   SConnectRsp connectRsp = {0};
   if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) {
     code = TSDB_CODE_TSC_INVALID_VERSION;
@@ -106,7 +98,8 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   if (connectRsp.epSet.numOfEps == 0) {
-    setErrno(pRequest, TSDB_CODE_APP_ERROR);
+    code = TSDB_CODE_APP_ERROR;
+    setErrno(pRequest, code);
     tsem_post(&pRequest->body.rspSem);
     goto End;
   }
@@ -157,6 +150,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
   SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx);
   if (pAppHbMgr) {
     hbRegisterConn(pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType);
+  } else {
+    taosThreadMutexUnlock(&clientHbMgr.lock);
+    code = TSDB_CODE_TSC_DISCONNECTED;
+    setErrno(pRequest, code);
+    tsem_post(&pRequest->body.rspSem);
+    goto End;
   }
   taosThreadMutexUnlock(&clientHbMgr.lock);
 

From bc9dfd8a8325bff3a7fc7d081991218a7d333eea Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Fri, 19 Jan 2024 14:42:02 +0800
Subject: [PATCH 33/45] fix: heap use after free

---
 source/client/src/clientHb.c | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c
index e076a874d3..63a65d7c95 100644
--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@@ -1334,24 +1334,18 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in
 }
 
 void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) {
-  SClientHbReq *pReq = NULL;
   taosThreadMutexLock(&clientHbMgr.lock);
   SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx);
   if (pAppHbMgr) {
-    pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
+    SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
     if (pReq) {
       tFreeClientHbReq(pReq);
       taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
       taosHashRelease(pAppHbMgr->activeInfo, pReq);
+      atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1);
     }
   }
   taosThreadMutexUnlock(&clientHbMgr.lock);
-
-  if (NULL == pReq) {
-    return;
-  }
-
-  atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1);
 }
 
 // set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner

From 547b75977e24a658bf52ae5aa02f968ee4e18aa8 Mon Sep 17 00:00:00 2001
From: facetosea <25808407@qq.com>
Date: Fri, 19 Jan 2024 17:37:43 +0800
Subject: [PATCH 34/45] fix: timezone error on windows

---
 source/os/src/osTime.c | 70 ++++++++++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 27 deletions(-)

diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c
index e506ee603b..9cad1dd6ef 100644
--- a/source/os/src/osTime.c
+++ b/source/os/src/osTime.c
@@ -37,9 +37,6 @@
 // This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC)
 // until 00:00:00 January 1, 1970
 static const uint64_t TIMEEPOCH = ((uint64_t)116444736000000000ULL);
-// This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC)
-// until 00:00:00 January 1, 1900
-static const uint64_t TIMEEPOCH1900 = ((uint64_t)116445024000000000ULL);
 
 /*
  * We do not implement alternate representations. However, we always
@@ -360,6 +357,7 @@ int32_t taosGetTimeOfDay(struct timeval *tv) {
   t.QuadPart -= TIMEEPOCH;
   tv->tv_sec = t.QuadPart / 10000000;
   tv->tv_usec = (t.QuadPart % 10000000) / 10;
+  return 0;
 #else
   return gettimeofday(tv, NULL);
 #endif
@@ -482,33 +480,51 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) {
       sprintf(buf, "NaN");
     }
     return NULL;
-  }
+  } else if (*timep < 0) {
+    SYSTEMTIME ss, s;
+    FILETIME   ff, f;
 
-  SYSTEMTIME    s;
-  FILETIME      f;
-  LARGE_INTEGER offset;
-  struct tm     tm1;
-  time_t        tt = 0;
-  if (localtime_s(&tm1, &tt) != 0) {
-    if (buf != NULL) {
-      sprintf(buf, "NaN");
+    LARGE_INTEGER offset;
+    struct tm     tm1;
+    time_t        tt = 0;
+    if (localtime_s(&tm1, &tt) != 0) {
+      if (buf != NULL) {
+        sprintf(buf, "NaN");
+      }
+      return NULL;
+    }
+    ss.wYear = tm1.tm_year + 1900;
+    ss.wMonth = tm1.tm_mon + 1;
+    ss.wDay = tm1.tm_mday;
+    ss.wHour = tm1.tm_hour;
+    ss.wMinute = tm1.tm_min;
+    ss.wSecond = tm1.tm_sec;
+    ss.wMilliseconds = 0;
+    SystemTimeToFileTime(&ss, &ff);
+    offset.QuadPart = ff.dwHighDateTime;
+    offset.QuadPart <<= 32;
+    offset.QuadPart |= ff.dwLowDateTime;
+    offset.QuadPart += *timep * 10000000;
+    f.dwLowDateTime = offset.QuadPart & 0xffffffff;
+    f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff;
+    FileTimeToSystemTime(&f, &s);
+    result->tm_sec = s.wSecond;
+    result->tm_min = s.wMinute;
+    result->tm_hour = s.wHour;
+    result->tm_mday = s.wDay;
+    result->tm_mon = s.wMonth - 1;
+    result->tm_year = s.wYear - 1900;
+    result->tm_wday = s.wDayOfWeek;
+    result->tm_yday = 0;
+    result->tm_isdst = 0;
+  } else {
+    if (localtime_s(result, timep) != 0) {
+      if (buf != NULL) {
+        sprintf(buf, "NaN");
+      }
+      return NULL;
     }
-    return NULL;
   }
-  offset.QuadPart = TIMEEPOCH1900;
-  offset.QuadPart += *timep * 10000000;
-  f.dwLowDateTime = offset.QuadPart & 0xffffffff;
-  f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff;
-  FileTimeToSystemTime(&f, &s);
-  result->tm_sec = s.wSecond;
-  result->tm_min = s.wMinute;
-  result->tm_hour = s.wHour;
-  result->tm_mday = s.wDay;
-  result->tm_mon = s.wMonth - 1;
-  result->tm_year = s.wYear - 1900;
-  result->tm_wday = s.wDayOfWeek;
-  result->tm_yday = 0;
-  result->tm_isdst = 0;
 #else
   res = localtime_r(timep, result);
   if (res == NULL && buf != NULL) {

From 1e1951e86d0bebf4f6807b4e00c1e9126a8821f8 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Fri, 19 Jan 2024 17:51:55 +0800
Subject: [PATCH 35/45] fix: add distributed total information

---
 tests/army/community/query/query_basic.py | 14 ++++++++++++
 tests/army/frame/caseBase.py              | 27 +++++++++++++++++++++--
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py
index 90916a1c6c..13b37ef253 100644
--- a/tests/army/community/query/query_basic.py
+++ b/tests/army/community/query/query_basic.py
@@ -76,12 +76,26 @@ class TDTestCase(TBase):
         sql2 = "select bi from stb where bi is not null order by bi desc limit 10;"
         self.checkSameResult(sql1, sql2)
 
+        # distributed expect values
+        expects = {
+            "Block_Rows"     : 6*100000,
+            "Total_Tables"   : 6,
+            "Total_Vgroups"  : 3
+        }
+        self.waitTransactionZero()
+        reals = self.getDistributed(self.stb)
+        for k in expects.keys():
+            v = expects[k]
+            if int(reals[k]) != v:
+                tdLog.exit(f"distribute {k} expect: {v} real: {reals[k]}")
+
     # run
     def run(self):
         tdLog.debug(f"start to excute {__file__}")
 
         # insert data
         self.insertData()
+        self.flushDb()
 
         # check insert data correct
         self.checkInsertCorrect()
diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py
index b1500b69e1..5715867faf 100644
--- a/tests/army/frame/caseBase.py
+++ b/tests/army/frame/caseBase.py
@@ -229,9 +229,9 @@ class TBase:
 #
 
     # get vgroups
-    def getVGroup(self, db_name):
+    def getVGroup(self, dbName):
         vgidList = []
-        sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'"
+        sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{dbName}'"
         res = tdSql.getResult(sql)
         rows = len(res)
         for i in range(rows):
@@ -239,6 +239,29 @@ class TBase:
 
         return vgidList
     
+    # get distributed rows
+    def getDistributed(self, tbName):
+        sql = f"show table distributed {tbName}"
+        tdSql.query(sql)
+        dics = {}
+        i = 0
+        for i in range(tdSql.getRows()):
+            row = tdSql.getData(i, 0)
+            #print(row)
+            row = row.replace('[', '').replace(']', '')
+            #print(row)
+            items = row.split(' ')
+            #print(items)
+            for item in items:
+                #print(item)
+                v = item.split('=')
+                #print(v)
+                if len(v) == 2:
+                    dics[v[0]] = v[1]
+            if i > 5:
+                break
+        print(dics)
+        return dics
 
 
 #

From 897d3f5dba8209d80c4e297dc18cbb64c581b76f Mon Sep 17 00:00:00 2001
From: chenhaoran <haoran920c@163.com>
Date: Fri, 19 Jan 2024 17:54:23 +0800
Subject: [PATCH 36/45] test:add special compatibility testcase for code 
 coverage

---
 .../system-test/0-others/com_alltypedata.json |  81 ++++++
 tests/system-test/0-others/compatibility.py   |   7 +
 .../0-others/compatibility_coverage.py        | 275 ++++++++++++++++++
 3 files changed, 363 insertions(+)
 create mode 100644 tests/system-test/0-others/com_alltypedata.json
 create mode 100644 tests/system-test/0-others/compatibility_coverage.py

diff --git a/tests/system-test/0-others/com_alltypedata.json b/tests/system-test/0-others/com_alltypedata.json
new file mode 100644
index 0000000000..0e6d8e3a07
--- /dev/null
+++ b/tests/system-test/0-others/com_alltypedata.json
@@ -0,0 +1,81 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "connection_pool_size": 8,
+    "num_of_records_per_req": 2000,
+    "thread_count": 2,
+    "create_table_thread_count": 10,
+    "result_file": "./insert_res_mix.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "check_sql": "yes",
+    "continue_if_fail": "yes",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "curdb",
+                "drop": "yes",
+                "vgroups": 2,
+                "replica": 1,
+                "precision": "ms",
+                "stt_trigger": 8,
+                "minRows": 100,
+                "maxRows": 4096
+            },
+            "super_tables": [
+                {
+                    "name": "meters",
+                    "child_table_exists": "no",
+                    "childtable_count": 5,
+                    "insert_rows": 100000,
+                    "childtable_prefix": "d",
+                    "insert_mode": "taosc",
+                    "insert_interval": 0,
+                    "timestamp_step": 1000,
+                    "start_timestamp":"2022-09-01 10:00:00",
+                    "disorder_ratio": 60,
+                    "update_ratio": 70,
+                    "delete_ratio": 30,
+                    "disorder_fill_interval": 300,
+                    "update_fill_interval": 25,
+                    "generate_row_rule": 2,
+                    "columns": [
+                        { "type": "bool",        "name": "bc"},
+                        { "type": "float",       "name": "fc",  "max": 1, "min": 0 },
+                        { "type": "double",      "name": "dc",  "max": 1, "min": 0 },
+                        { "type": "tinyint",     "name": "ti",  "max": 100, "min": 0 },
+                        { "type": "smallint",    "name": "si",  "max": 100, "min": 0 },
+                        { "type": "int",         "name": "ic",  "max": 100, "min": 0 },
+                        { "type": "bigint",      "name": "bi",  "max": 100, "min": 0 },
+                        { "type": "utinyint",    "name": "uti", "max": 100, "min": 0 },
+                        { "type": "usmallint",   "name": "usi", "max": 100, "min": 0 },
+                        { "type": "uint",        "name": "ui",  "max": 100, "min": 0 },
+                        { "type": "ubigint",     "name": "ubi", "max": 100, "min": 0 },
+                        { "type": "binary",      "name": "bin", "len": 32},
+                        { "type": "nchar",       "name": "nch", "len": 64}
+                    ],
+                    "tags": [
+                        {
+                            "type": "tinyint",
+                            "name": "groupid",
+                            "max": 10,
+                            "min": 1
+                        },
+                        {
+                            "name": "location",
+                            "type": "binary",
+                            "len": 16,
+                            "values": ["San Francisco", "Los Angles", "San Diego",
+                                "San Jose", "Palo Alto", "Campbell", "Mountain View",
+                                "Sunnyvale", "Santa Clara", "Cupertino"]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py
index d54c676c0d..c936cf1ae4 100644
--- a/tests/system-test/0-others/compatibility.py
+++ b/tests/system-test/0-others/compatibility.py
@@ -152,6 +152,13 @@ class TDTestCase:
         tdLog.info(f" LD_LIBRARY_PATH=/usr/lib  taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y  ")
         os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y  ")
         os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'flush database test '")
+        os.system("LD_LIBRARY_PATH=/usr/lib  taosBenchmark -f 0-others/com_alltypedata.json -y")
+        os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'flush database curdb '")
+        os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'select count(*) from curdb.meters '")
+        os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'select sum(fc) from curdb.meters '")
+        os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'select avg(ic) from curdb.meters '")
+        os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'select min(ui) from curdb.meters '")
+        os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'select max(bi) from curdb.meters '")
 
         # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ")
         # os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ')
diff --git a/tests/system-test/0-others/compatibility_coverage.py b/tests/system-test/0-others/compatibility_coverage.py
new file mode 100644
index 0000000000..7a123739f7
--- /dev/null
+++ b/tests/system-test/0-others/compatibility_coverage.py
@@ -0,0 +1,275 @@
+from urllib.parse import uses_relative
+import taos
+import sys
+import os
+import time
+import platform
+import inspect
+from taos.tmq import Consumer
+
+from pathlib import Path
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+import subprocess
+
+BASEVERSION = "3.0.2.3"
+class TDTestCase:
+    def caseDescription(self):
+        f'''
+        3.0 data compatibility test 
+        case1: basedata version is {BASEVERSION}
+        '''
+        return
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+        self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata;
+                            create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int);
+                            create table deldata.ct1 using deldata.stb1 tags ( 1 );
+                            insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );
+                            select avg(c1) from deldata.ct1;
+                            delete from deldata.stb1;
+                            flush database deldata;
+                            insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );
+                            delete from deldata.ct1;
+                            insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a );
+                            flush database deldata;'''   
+    def checkProcessPid(self,processName):
+        i=0
+        while i<60:
+            print(f"wait stop {processName}")
+            processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1]
+            print(f"times:{i},{processName}-pid:{processPid}")
+            if(processPid == ""):
+                break
+            i += 1
+            sleep(1)
+        else:
+            print(f'this processName is not stoped in 60s')
+
+            
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        self.projPath = projPath
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files or "taosd.exe" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def getCfgPath(self):
+        buildPath = self.getBuildPath()
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            cfgPath = buildPath + "/../sim/dnode1/cfg/"
+        else:
+            cfgPath = buildPath + "/../sim/dnode1/cfg/"
+
+        return cfgPath
+
+    def installTaosd(self,bPath,cPath):
+        # os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp &&  mv {self.getBuildPath()}/build/lib/libtaos.so*  {self.getBuildPath()}/build/lib/temp/ ")
+        # os.system(f" mv {bPath}/build  {bPath}/build_bak ")
+        # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so  {self.getBuildPath()}/build/lib/libtaos.so_bak ")
+        # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1  {self.getBuildPath()}/build/lib/libtaos.so.1_bak ")
+        
+        packagePath = "/usr/local/src/"
+        dataPath = cPath + "/../data/"
+        if platform.system() == "Linux" and platform.machine() == "aarch64":
+            packageName = "TDengine-server-"+  BASEVERSION + "-Linux-arm64.tar.gz"
+        else:
+            packageName = "TDengine-server-"+  BASEVERSION + "-Linux-x64.tar.gz"
+        packageTPath = packageName.split("-Linux-")[0]
+        my_file = Path(f"{packagePath}/{packageName}")
+        if not  my_file.exists():
+            print(f"{packageName} is not exists")
+            tdLog.info(f"cd {packagePath} &&  wget https://www.tdengine.com/assets-download/3.0/{packageName}")
+            os.system(f"cd {packagePath} &&  wget https://www.tdengine.com/assets-download/3.0/{packageName}")
+        else: 
+            print(f"{packageName} has been exists")
+        os.system(f" cd {packagePath} &&  tar xvf  {packageName} && cd {packageTPath} &&  ./install.sh  -e no  " )
+        tdDnodes.stop(1)
+        print(f"start taosd: rm -rf {dataPath}/*  && nohup taosd -c {cPath} & ")
+        os.system(f"rm -rf {dataPath}/*  && nohup taosd -c {cPath} & " )
+        sleep(5)
+
+
+    def buildTaosd(self,bPath):
+        # os.system(f"mv {bPath}/build_bak  {bPath}/build ")
+        os.system(f" cd {bPath} ")
+
+    def is_list_same_as_ordered_list(self,unordered_list, ordered_list):
+        sorted_list = sorted(unordered_list)
+        return sorted_list == ordered_list
+        
+    def run(self):
+        scriptsPath = os.path.dirname(os.path.realpath(__file__))
+        distro_id = distro.id()
+        if distro_id == "alpine":
+            tdLog.info(f"alpine skip compatibility test")
+            return True
+        if platform.system().lower() == 'windows':
+            tdLog.info(f"Windows skip compatibility test")
+            return True
+        bPath = self.getBuildPath()
+        cPath = self.getCfgPath()
+        dbname = "test"
+        stb = f"{dbname}.meters"
+        os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ")
+        tableNumbers=100
+        recordNumbers1=100
+        recordNumbers2=1000
+
+        # tdsqlF=tdCom.newTdSql()
+        # print(tdsqlF)
+        # tdsqlF.query(f"SELECT SERVER_VERSION();")
+        # print(tdsqlF.query(f"SELECT SERVER_VERSION();"))
+        # oldServerVersion=tdsqlF.queryResult[0][0]
+        # tdLog.info(f"Base server version is {oldServerVersion}")
+        # tdsqlF.query(f"SELECT CLIENT_VERSION();")
+        # # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison
+        # oldClientVersion=tdsqlF.queryResult[0][0]
+        # tdLog.info(f"Base client version is {oldClientVersion}")
+        # baseVersion = "3.0.1.8"
+
+        tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{BASEVERSION}")
+        os.system(f"rm -rf {cPath}/../data")
+        print(self.projPath)
+        # this data file is special for coverage test in 192.168.1.96
+        os.system("cp -r  f{self.projPath}/../comp_testdata/data/ {self.projPath}/sim/dnode1")
+        tdDnodes.stop(1) 
+        tdDnodes.start(1) 
+
+
+        tdsql=tdCom.newTdSql()
+        tdsql.query(f"SELECT SERVER_VERSION();")
+        nowServerVersion=tdsql.queryResult[0][0]
+        tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}")
+        tdsql.query(f"select count(*) from {stb}")
+        tdsql.checkData(0,0,tableNumbers*recordNumbers1)
+        # tdsql.query("show streams;")
+        # os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y  ")
+        # tdsql.query("show streams;")
+        # tdsql.query(f"select count(*) from {stb}")
+        # tdsql.checkData(0,0,tableNumbers*recordNumbers2)
+        
+        # checkout db4096
+        tdsql.query("select count(*) from db4096.stb0")
+        tdsql.checkData(0,0,50000)
+        
+        # checkout deleted data
+        tdsql.execute("insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );")
+        tdsql.execute("flush database deldata;")
+        tdsql.query("select avg(c1) from deldata.ct1;")
+
+
+        tdsql=tdCom.newTdSql()
+        tdLog.printNoPrefix("==========step4:verify backticks in taos Sql-TD18542")
+        tdsql.execute("drop database if exists db")
+        tdsql.execute("create database db")
+        tdsql.execute("use db")
+        tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);")
+        tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);")
+        tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);")
+        tdsql.error(" insert into db.`db.ct2` using db.stb1 TAGS(9) values(now(),11);")
+        tdsql.execute("insert into `db`.ct3 using db.stb1 TAGS(3) values(now(),13);")
+        tdsql.query("select * from db.ct3")
+        tdsql.checkData(0,1,13)
+        tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);")
+        tdsql.query("select * from db.ct4")
+        tdsql.checkData(0,1,14)
+
+        #check retentions
+        tdsql=tdCom.newTdSql()
+        tdsql.query("describe  information_schema.ins_databases;")
+        qRows=tdsql.queryRows   
+        comFlag=True
+        j=0
+        while comFlag:
+            for i in  range(qRows) :
+                if tdsql.queryResult[i][0] == "retentions" :
+                    print("parameters include retentions")
+                    comFlag=False
+                    break
+                else :
+                    comFlag=True
+                    j=j+1
+            if j == qRows:
+                print("parameters don't include retentions")
+                caller = inspect.getframeinfo(inspect.stack()[0][0])
+                args = (caller.filename, caller.lineno)
+                tdLog.exit("%s(%d) failed" % args)
+
+        # check stream
+        tdsql.query("show streams;")
+        tdsql.checkRows(0)
+
+        #check TS-3131
+        tdsql.query("select *,tbname from d0.almlog where mcid='m0103';")
+        tdsql.checkRows(6)
+        expectList = [0,3003,20031,20032,20033,30031]
+        resultList = []
+        for i in range(6):
+            resultList.append(tdsql.queryResult[i][3])
+        print(resultList)
+        if self.is_list_same_as_ordered_list(resultList,expectList):
+            print("The unordered list is the same as the ordered list.")
+        else:
+            tdLog.exit("The unordered list is not the same as the ordered list.")
+        tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);")
+        tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);")
+
+
+        # check tmq
+        conn = taos.connect()
+
+        consumer = Consumer(
+            {
+                "group.id": "tg75",
+                "client.id": "124",
+                "td.connect.user": "root",
+                "td.connect.pass": "taosdata",
+                "enable.auto.commit": "true",
+                "experimental.snapshot.enable":  "true",
+            }
+        )
+        consumer.subscribe(["tmq_test_topic"])
+
+        while True:
+            res = consumer.poll(10)
+            if not res:
+                break
+            err = res.error()
+            if err is not None:
+                raise err
+            val = res.value()
+
+            for block in val:
+                print(block.fetchall())
+        tdsql.query("show topics;")
+        tdsql.checkRows(1)
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

From 95975801d52c1e4631b8a39b26285bdea3d148d1 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Sat, 20 Jan 2024 13:42:42 +0800
Subject: [PATCH 37/45] feat: add check max and ts window correctness

---
 tests/army/community/query/query_basic.json |   2 +-
 tests/army/community/query/query_basic.py   | 142 +++++++++++++++++++-
 2 files changed, 140 insertions(+), 4 deletions(-)

diff --git a/tests/army/community/query/query_basic.json b/tests/army/community/query/query_basic.json
index c6a3482cd4..d8d47266f9 100644
--- a/tests/army/community/query/query_basic.json
+++ b/tests/army/community/query/query_basic.json
@@ -32,7 +32,7 @@
                     "childtable_prefix": "d",
                     "insert_mode": "taosc",
                     "timestamp_step": 30000,
-                    "start_timestamp":"2023-10-01 10:00:00",
+                    "start_timestamp":1700000000000,
                     "columns": [
                         { "type": "bool",        "name": "bc"},
                         { "type": "float",       "name": "fc" },
diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py
index 13b37ef253..eb332b8c71 100644
--- a/tests/army/community/query/query_basic.py
+++ b/tests/army/community/query/query_basic.py
@@ -34,23 +34,158 @@ class TDTestCase(TBase):
         "querySmaOptimize": "1"
     }
 
+
     def insertData(self):
         tdLog.info(f"insert data.")
         # taosBenchmark run
         jfile = etool.curFile(__file__, "query_basic.json")
-        etool.benchMark(json=jfile)
+        etool.benchMark(json = jfile)
 
         tdSql.execute(f"use {self.db}")
         tdSql.execute("select database();")
-        # set insert data information
+        # come from query_basic.json
         self.childtable_count = 6
         self.insert_rows      = 100000
         self.timestamp_step   = 30000
+        self.start_timestamp  = 1700000000000
+
+
+    def genTime(self, preCnt, cnt):
+        start = self.start_timestamp + preCnt * self.timestamp_step
+        end = start + self.timestamp_step * cnt
+        return (start, end)
+
+
+    def doWindowQuery(self):
+        pre = f"select count(ts) from {self.stb} "
+        # case1 operator "in" "and" is same
+        cnt = 6000
+        s,e = self.genTime(12000, cnt)
+        sql1 = f"{pre} where ts between {s} and {e} "
+        sql2 = f"{pre} where ts >= {s} and ts <={e} "
+        expectCnt = (cnt + 1) * self.childtable_count
+        tdSql.checkFirstValue(sql1, expectCnt)
+        tdSql.checkFirstValue(sql2, expectCnt)
+
+        # case2 no overloap "or" left
+        cnt1 = 120
+        s1, e1 = self.genTime(4000, cnt1)
+        cnt2 = 3000
+        s2, e2 = self.genTime(10000, cnt2)
+        sql = f"{pre} where (ts >= {s1} and ts < {e1}) or (ts >= {s2} and ts < {e2})"
+        expectCnt = (cnt1 + cnt2) * self.childtable_count
+        tdSql.checkFirstValue(sql, expectCnt)
+
+        # case3 overloap "or" right
+        cnt1 = 300
+        s1, e1 = self.genTime(17000, cnt1)
+        cnt2 = 8000
+        s2, e2 = self.genTime(70000, cnt2)
+        sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})"
+        expectCnt = (cnt1 + cnt2) * self.childtable_count
+        tdSql.checkFirstValue(sql, expectCnt)
+
+        # case4 overloap "or" 
+        cnt1 = 1000
+        s1, e1 = self.genTime(9000, cnt1)
+        cnt2 = 1000
+        s2, e2 = self.genTime(9000 + 500 , cnt2)
+        sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})"
+        expectCnt = (cnt1 + 500) * self.childtable_count # expect=1500
+        tdSql.checkFirstValue(sql, expectCnt)
+
+        # case5 overloap "or" boundary hollow->solid
+        cnt1 = 3000
+        s1, e1 = self.genTime(45000, cnt1)
+        cnt2 = 2000
+        s2, e2 = self.genTime(45000 + cnt1 , cnt2)
+        sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})"
+        expectCnt = (cnt1+cnt2) * self.childtable_count
+        tdSql.checkFirstValue(sql, expectCnt)
+
+        # case6 overloap "or" boundary solid->solid
+        cnt1 = 300
+        s1, e1 = self.genTime(55000, cnt1)
+        cnt2 = 500
+        s2, e2 = self.genTime(55000 + cnt1 , cnt2)
+        sql = f"{pre} where (ts >= {s1} and ts <= {e1}) or (ts >= {s2} and ts <= {e2})"
+        expectCnt = (cnt1+cnt2+1) * self.childtable_count
+        tdSql.checkFirstValue(sql, expectCnt)
+
+        # case7 overloap "and" 
+        cnt1 = 1000
+        s1, e1 = self.genTime(40000, cnt1)
+        cnt2 = 1000
+        s2, e2 = self.genTime(40000 + 500 , cnt2)
+        sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts > {s2} and ts <= {e2})"
+        expectCnt = cnt1/2 * self.childtable_count
+        tdSql.checkFirstValue(sql, expectCnt)
+
+        # case8 overloap "and" boundary hollow->solid solid->hollow
+        cnt1 = 3000
+        s1, e1 = self.genTime(45000, cnt1)
+        cnt2 = 2000
+        s2, e2 = self.genTime(45000 + cnt1 , cnt2)
+        sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts < {e2})"
+        expectCnt = 1 * self.childtable_count
+        tdSql.checkFirstValue(sql, expectCnt)
+
+        # case9 no overloap "and" 
+        cnt1 = 6000
+        s1, e1 = self.genTime(20000, cnt1)
+        cnt2 = 300
+        s2, e2 = self.genTime(70000, cnt2)
+        sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts <= {e2})"
+        expectCnt = 0
+        tdSql.checkFirstValue(sql, expectCnt)
+
+        # case10 cnt1 contain cnt2 and
+        cnt1 = 5000
+        s1, e1 = self.genTime(25000, cnt1)
+        cnt2 = 400
+        s2, e2 = self.genTime(28000, cnt2)
+        sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts < {e2})"
+        expectCnt = cnt2 * self.childtable_count
+        tdSql.checkFirstValue(sql, expectCnt)
+
+
+    def queryMax(self, colname):
+        sql = f"select max({colname}) from {self.stb}"
+        tdSql.query(sql)
+        return tdSql.getData(0, 0)
+
+
+    def checkMax(self):
+        # max for tsdbRetrieveDatablockSMA2 coverage
+        colname = "ui"
+        max = self.queryMax(colname)
+
+        # insert over max
+        sql = f"insert into d0(ts, {colname}) values"
+        for i in range(1, 5):
+            sql += f" (now + {i}s, {max+i})"
+        tdSql.execute(sql)
+        self.flushDb()
+
+        expectMax = max + 4
+        for i in range(1, 5):
+            realMax = self.queryMax(colname)
+            if realMax != expectMax:
+                tdLog.exit(f"Max value not expect. expect:{expectMax} real:{realMax}")
+            sql = f"delete from d0 where ui={expectMax}"
+            tdSql.execute(sql)
+            expectMax -= 1
+
+        self.checkInsertCorrect()    
 
 
     def doQuery(self):
         tdLog.info(f"do query.")
-        
+        self.doWindowQuery()
+
+        # max
+        self.checkMax()
+
         # __group_key
         sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti"
         tdSql.query(sql)
@@ -89,6 +224,7 @@ class TDTestCase(TBase):
             if int(reals[k]) != v:
                 tdLog.exit(f"distribute {k} expect: {v} real: {reals[k]}")
 
+
     # run
     def run(self):
         tdLog.debug(f"start to excute {__file__}")

From e1b0d4a555b383232241a39c83d728886d0ac60d Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Sat, 20 Jan 2024 14:26:34 +0800
Subject: [PATCH 38/45] fix: tweak delete max value

---
 tests/army/community/query/query_basic.py | 9 ++++++++-
 tests/army/frame/sql.py                   | 6 ++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py
index eb332b8c71..575655c2b1 100644
--- a/tests/army/community/query/query_basic.py
+++ b/tests/army/community/query/query_basic.py
@@ -172,7 +172,14 @@ class TDTestCase(TBase):
             realMax = self.queryMax(colname)
             if realMax != expectMax:
                 tdLog.exit(f"Max value not expect. expect:{expectMax} real:{realMax}")
-            sql = f"delete from d0 where ui={expectMax}"
+
+            # query ts list
+            sql = f"select ts from d0 where ui={expectMax}"
+            tdSql.query(sql)
+            tss = tdSql.getColData(0)
+            strts = ",".join(tss)
+            # delete
+            sql = f"delete from d0 where ts in ({strts})"
             tdSql.execute(sql)
             expectMax -= 1
 
diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py
index 8b7538321d..83f1243167 100644
--- a/tests/army/frame/sql.py
+++ b/tests/army/frame/sql.py
@@ -223,6 +223,12 @@ class TDSql:
     def getData(self, row, col):
         self.checkRowCol(row, col)
         return self.res[row][col]
+    
+    def getColData(self, col):
+        colDatas = []
+        for i in range(self.queryRows):
+            colDatas.append(self.res[i][col])
+        return colDatas
 
     def getResult(self, sql):
         self.sql = sql

From 684bc221300e35cf352c8fbd9617d0ac9ccea75c Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Sat, 20 Jan 2024 14:26:41 +0800
Subject: [PATCH 39/45] fix: tweak delete max value

---
 tests/army/community/query/query_basic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py
index 575655c2b1..740d14e11b 100644
--- a/tests/army/community/query/query_basic.py
+++ b/tests/army/community/query/query_basic.py
@@ -177,7 +177,7 @@ class TDTestCase(TBase):
             sql = f"select ts from d0 where ui={expectMax}"
             tdSql.query(sql)
             tss = tdSql.getColData(0)
-            strts = ",".join(tss)
+            strts = ",".join(map(str,tss))
             # delete
             sql = f"delete from d0 where ts in ({strts})"
             tdSql.execute(sql)

From 6cd689ea53714bf8035eb32bb66f61badd6c428c Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Sat, 20 Jan 2024 14:43:00 +0800
Subject: [PATCH 40/45] fix: delete max ok

---
 tests/army/community/query/query_basic.py |  8 ++++----
 tests/army/frame/caseBase.py              | 12 ++++++++++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py
index 740d14e11b..211d3e6ec3 100644
--- a/tests/army/community/query/query_basic.py
+++ b/tests/army/community/query/query_basic.py
@@ -177,10 +177,10 @@ class TDTestCase(TBase):
             sql = f"select ts from d0 where ui={expectMax}"
             tdSql.query(sql)
             tss = tdSql.getColData(0)
-            strts = ",".join(map(str,tss))
-            # delete
-            sql = f"delete from d0 where ts in ({strts})"
-            tdSql.execute(sql)
+            for ts in tss:            
+                # delete
+                sql = f"delete from d0 where ts = '{ts}'"
+                tdSql.execute(sql)
             expectMax -= 1
 
         self.checkInsertCorrect()    
diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py
index 5715867faf..de258e6267 100644
--- a/tests/army/frame/caseBase.py
+++ b/tests/army/frame/caseBase.py
@@ -292,3 +292,15 @@ class TBase:
         if len(lists) == 0:
             tdLog.exit(f"list is empty {tips}")
 
+
+#
+#  str util
+#
+    # covert list to sql format string
+    def listSql(self, lists, sepa = ","):
+        strs = ""
+        for ls in lists:
+            if strs != "":
+                strs += sepa
+            strs += f"'{ls}'"
+        return strs
\ No newline at end of file

From 68b2a06a02289bea1d9e4e8f4d93530eac3627a0 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Sat, 20 Jan 2024 14:52:40 +0800
Subject: [PATCH 41/45] feat: add cquery_basic.json write again

---
 tests/army/community/query/cquery_basic.json | 61 ++++++++++++++++++++
 tests/army/community/query/query_basic.py    |  6 +-
 2 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 tests/army/community/query/cquery_basic.json

diff --git a/tests/army/community/query/cquery_basic.json b/tests/army/community/query/cquery_basic.json
new file mode 100644
index 0000000000..5a57d59d93
--- /dev/null
+++ b/tests/army/community/query/cquery_basic.json
@@ -0,0 +1,61 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "connection_pool_size": 8,
+    "num_of_records_per_req": 4000,
+    "prepared_rand": 10000,
+    "thread_count": 3,
+    "create_table_thread_count": 1,
+    "confirm_parameter_prompt": "no",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "db",
+                "drop": "no",
+                "vgroups": 3,
+                "replica": 3,
+                "duration":"3d",
+                "wal_retention_period": 1,
+                "wal_retention_size": 1,
+                "stt_trigger": 1
+            },
+            "super_tables": [
+                {
+                    "name": "stb",
+                    "child_table_exists": "yes",
+                    "childtable_count": 6,
+                    "insert_rows": 50000,
+                    "childtable_prefix": "d",
+                    "insert_mode": "taosc",
+                    "timestamp_step": 60000,
+                    "start_timestamp":1700000000000,
+                    "columns": [
+                        { "type": "bool",        "name": "bc"},
+                        { "type": "float",       "name": "fc" },
+                        { "type": "double",      "name": "dc"},
+                        { "type": "tinyint",     "name": "ti"},
+                        { "type": "smallint",    "name": "si" },
+                        { "type": "int",         "name": "ic" },
+                        { "type": "bigint",      "name": "bi" },
+                        { "type": "utinyint",    "name": "uti"},
+                        { "type": "usmallint",   "name": "usi"},
+                        { "type": "uint",        "name": "ui" },
+                        { "type": "ubigint",     "name": "ubi"},
+                        { "type": "binary",      "name": "bin", "len": 8},
+                        { "type": "nchar",       "name": "nch", "len": 16}
+                    ],
+                    "tags": [
+                        {"type": "tinyint", "name": "groupid","max": 10,"min": 1},
+                        {"name": "location","type": "binary", "len": 16, "values":
+                           ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}
diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py
index 211d3e6ec3..912974d8ab 100644
--- a/tests/army/community/query/query_basic.py
+++ b/tests/army/community/query/query_basic.py
@@ -49,6 +49,11 @@ class TDTestCase(TBase):
         self.timestamp_step   = 30000
         self.start_timestamp  = 1700000000000
 
+        # write again disorder
+        self.flushDb()
+        jfile = etool.curFile(__file__, "cquery_basic.json")
+        etool.benchMark(json = jfile)
+        
 
     def genTime(self, preCnt, cnt):
         start = self.start_timestamp + preCnt * self.timestamp_step
@@ -238,7 +243,6 @@ class TDTestCase(TBase):
 
         # insert data
         self.insertData()
-        self.flushDb()
 
         # check insert data correct
         self.checkInsertCorrect()

From c2772994e43d7fa08a54c95f6ece2ee5040636e4 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Sat, 20 Jan 2024 15:00:14 +0800
Subject: [PATCH 42/45] fix: add order by desc coverage

---
 tests/army/frame/caseBase.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py
index de258e6267..f562ea1086 100644
--- a/tests/army/frame/caseBase.py
+++ b/tests/army/frame/caseBase.py
@@ -136,7 +136,7 @@ class TBase:
         tdSql.checkAgg(sql, self.childtable_count)
 
         # check step
-        sql = f"select * from (select diff(ts) as dif from {self.stb} partition by tbname) where dif != {self.timestamp_step}"
+        sql = f"select * from (select diff(ts) as dif from {self.stb} partition by tbname order by ts desc) where dif != {self.timestamp_step}"
         tdSql.query(sql)
         tdSql.checkRows(0)
 

From 53b385108c1f0478adc3d18b7536a1980a56091e Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Sun, 21 Jan 2024 18:38:25 +0800
Subject: [PATCH 43/45] add use db

---
 tests/army/community/cluster/incSnapshot.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py
index 9f50ea52f1..b1b53e65bd 100644
--- a/tests/army/community/cluster/incSnapshot.py
+++ b/tests/army/community/cluster/incSnapshot.py
@@ -29,6 +29,7 @@ class TDTestCase(TBase):
         tdSql.prepare()
         autoGen = AutoGen()
         autoGen.create_db(self.db, 2, 3)
+        tdSql.execute(f"use {self.db}")
         autoGen.create_stable(self.stb, 5, 10, 8, 8)
         autoGen.create_child(self.stb, "d", self.childtable_count)
         autoGen.insert_data(1000)

From 091da20ab827e4d1ab4b5733eb7dad6fb24ce8db Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Sun, 21 Jan 2024 20:40:06 +0800
Subject: [PATCH 44/45] add use db

---
 tests/army/frame/autogen.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py
index d38d2e2f19..bbcbf9ad9a 100644
--- a/tests/army/frame/autogen.py
+++ b/tests/army/frame/autogen.py
@@ -163,22 +163,18 @@ class AutoGen:
             values = ""
 
         tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")
-        return ts
+        self.ts = ts
 
     def insert_data(self, cnt, bContinue=False):
         if not bContinue:
             self.ts = 1600000000000
 
-        currTs = 1600000000000
         for i in range(self.child_cnt):
             name = f"{self.child_name}{i}"
-            currTs = self.insert_data_child(name, cnt, self.batch_size, 1)
+            self.insert_data_child(name, cnt, self.batch_size, 1)
 
-        self.ts = currTs
         tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
 
-    # insert same timestamp to all childs
-
     # insert same timestamp to all childs
     def insert_samets(self, cnt):
         for i in range(self.child_cnt):

From 9a8660dd4e30f49e531b9a73210911911da53ad6 Mon Sep 17 00:00:00 2001
From: menshibin <men_shi_bin@163.com>
Date: Sun, 21 Jan 2024 20:43:07 +0800
Subject: [PATCH 45/45] modify insert ts contiune

---
 tests/army/frame/autogen.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py
index bbcbf9ad9a..d1f02e7865 100644
--- a/tests/army/frame/autogen.py
+++ b/tests/army/frame/autogen.py
@@ -163,16 +163,18 @@ class AutoGen:
             values = ""
 
         tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")
-        self.ts = ts
+        return ts
 
     def insert_data(self, cnt, bContinue=False):
         if not bContinue:
             self.ts = 1600000000000
 
+        currTs = 1600000000000
         for i in range(self.child_cnt):
             name = f"{self.child_name}{i}"
-            self.insert_data_child(name, cnt, self.batch_size, 1)
+            currTs = self.insert_data_child(name, cnt, self.batch_size, 1)
 
+        self.ts = currTs
         tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
 
     # insert same timestamp to all childs