Merge pull request #24469 from taosdata/fix/xsren/TD-28204/whereOptimizeConditional

Fix/xsren/td 28204/where optimize conditional

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -4100,9 +4100,9 @@ void tsdbReaderClose2(STsdbReader* pReader) {
   size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
   if (pReader->status.pTableMap != NULL) {
     destroyAllBlockScanInfo(pReader->status.pTableMap);
-    clearBlockScanInfoBuf(&pReader->blockInfoBuf);
     pReader->status.pTableMap = NULL;
   }
+  clearBlockScanInfoBuf(&pReader->blockInfoBuf);
 
   if (pReader->pFileReader != NULL) {
     tsdbDataFileReaderClose(&pReader->pFileReader);

source/libs/parser/src/parTranslater.c

@@ -2743,6 +2743,14 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
+static int32_t checkIsEmptyResult(STranslateContext* pCxt, SSelectStmt* pSelect) {
+  if (pSelect->timeRange.skey > pSelect->timeRange.ekey
+    && !pSelect->hasCountFunc) {
+    pSelect->isEmptyResult = true;
+  }
+    return TSDB_CODE_SUCCESS;
+}
+
 static int32_t resetSelectFuncNumWithoutDup(SSelectStmt* pSelect) {
   if (pSelect->selectFuncNum <= 1) return TSDB_CODE_SUCCESS;
   pSelect->selectFuncNum = 0;
@@ -4416,9 +4424,6 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (TSDB_CODE_SUCCESS == code) {
     code = getQueryTimeRange(pCxt, pSelect->pWhere, &pSelect->timeRange);
   }
-  if (TSDB_CODE_SUCCESS == code && pSelect->timeRange.skey > pSelect->timeRange.ekey) {
-    pSelect->isEmptyResult = true;
-  }
   if (pSelect->pWhere != NULL) {
     setTableVgroupsFromEqualTbnameCond(pCxt, pSelect);
   }
@@ -4606,6 +4611,9 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect
   if (TSDB_CODE_SUCCESS == code) {
     code = translateOrderBy(pCxt, pSelect);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = checkIsEmptyResult(pCxt, pSelect);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     resetSelectFuncNumWithoutDup(pSelect);
     code = checkAggColCoexist(pCxt, pSelect);

tests/pytest/util/sql.py

@@ -88,7 +88,7 @@ class TDSql:
                 self.affectedRows = self.cursor.execute(sql)
                 return self.affectedRows
             except Exception as e:
-                tdLog.notice("Try to execute sql again, query times: %d "%i)
+                tdLog.notice("Try to execute sql again, execute times: %d "%i)
                 if i == queryTimes:
                     caller = inspect.getframeinfo(inspect.stack()[1][0])
                     args = (caller.filename, caller.lineno, sql, repr(e))

tests/system-test/2-query/nestedQueryInterval.py

@@ -1310,7 +1310,7 @@ class TDTestCase:
         ts = ts + 20
         tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;")
         tdSql.checkRows(1)
-        tdSql.checkData(0, 1, 52);
+        tdSql.checkData(0, 1, 52)
         
         
         #stables
@@ -1347,16 +1347,16 @@ class TDTestCase:
         
         tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;")
         tdSql.checkRows(6)
-        tdSql.checkData(0, 1, 162);
+        tdSql.checkData(0, 1, 162)
-        tdSql.checkData(1, 1, 200);
+        tdSql.checkData(1, 1, 200)
         
         tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;")
         tdSql.checkRows(1)
-        tdSql.checkData(0, 1, 62);
+        tdSql.checkData(0, 1, 62)
         
         tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;")
         tdSql.checkRows(1)
-        tdSql.checkData(0, 1, 62);
+        tdSql.checkData(0, 1, 62)
         
         #test special character
 
@@ -1366,25 +1366,25 @@ class TDTestCase:
         
         tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;")
         tdSql.checkRows(7)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 162);
+        tdSql.checkData(1, 1, 162)
-        tdSql.checkData(2, 1, 200);
+        tdSql.checkData(2, 1, 200)
         tdSql.query(f"select count(*) from nested.stable_1 where tbname ='!@!@$$^$' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         
         tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;")
         tdSql.checkRows(2)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 62);
+        tdSql.checkData(1, 1, 62)
         tdSql.query(f"select count(*) from nested.stable_null_data where tbname ='%^$^&^&' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         
         tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;")
         tdSql.checkRows(2)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 62);
+        tdSql.checkData(1, 1, 62)
         tdSql.query(f"select count(*) from nested.stable_null_childtable where tbname ='$^%$%^&' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         
         #test csv
         sql1 = "select tbname,ts,q_int,q_binary from nested.stable_1 >>'%s/stable_1.csv';" %self.testcasePath
@@ -1404,31 +1404,31 @@ class TDTestCase:
         
         tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;")
         tdSql.checkRows(7)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 162);
+        tdSql.checkData(1, 1, 162)
-        tdSql.checkData(2, 1, 200);
+        tdSql.checkData(2, 1, 200)
         tdSql.query(f"select count(*) from nested.stable_1 where tbname ='!@!@$$^$' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         tdSql.query(f"select count(q_bool) from nested.stable_1;")
-        tdSql.checkData(0, 0, 0);
+        tdSql.checkData(0, 0, 0)
         
         tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;")
         tdSql.checkRows(2)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 62);
+        tdSql.checkData(1, 1, 62)
         tdSql.query(f"select count(*) from nested.stable_null_data where tbname ='%^$^&^&' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         tdSql.query(f"select count(q_bool) from nested.stable_null_data;")
-        tdSql.checkData(0, 0, 0);
+        tdSql.checkData(0, 0, 0)
         
         tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;")
         tdSql.checkRows(2)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 62);
+        tdSql.checkData(1, 1, 62)
         tdSql.query(f"select count(*) from nested.stable_null_childtable where tbname ='$^%$%^&' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         tdSql.query(f"select count(q_bool) from nested.stable_null_childtable;")
-        tdSql.checkData(0, 0, 0);
+        tdSql.checkData(0, 0, 0)
         
         
         tdSql.query(f"delete from nested.stable_1;")
@@ -1440,31 +1440,40 @@ class TDTestCase:
         
         tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;")
         tdSql.checkRows(7)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 162);
+        tdSql.checkData(1, 1, 162)
-        tdSql.checkData(2, 1, 200);
+        tdSql.checkData(2, 1, 200)
         tdSql.query(f"select count(*) from nested.stable_1 where tbname ='!@!@$$^$' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         tdSql.query(f"select count(q_bool) from nested.stable_1;")
-        tdSql.checkData(0, 0, 0);
+        tdSql.checkData(0, 0, 0)
         
         tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;")
         tdSql.checkRows(2)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 62);
+        tdSql.checkData(1, 1, 62)
         tdSql.query(f"select count(*) from nested.stable_null_data where tbname ='%^$^&^&' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         tdSql.query(f"select count(q_bool) from nested.stable_null_data;")
-        tdSql.checkData(0, 0, 0);
+        tdSql.checkData(0, 0, 0)
         
         tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;")
         tdSql.checkRows(2)
-        tdSql.checkData(0, 1, 1);
+        tdSql.checkData(0, 1, 1)
-        tdSql.checkData(1, 1, 62);
+        tdSql.checkData(1, 1, 62)
         tdSql.query(f"select count(*) from nested.stable_null_childtable where tbname ='$^%$%^&' ;")
-        tdSql.checkData(0, 0, 1);
+        tdSql.checkData(0, 0, 1)
         tdSql.query(f"select count(q_bool) from nested.stable_null_childtable;")
-        tdSql.checkData(0, 0, 0);
+        tdSql.checkData(0, 0, 0)
+
+        tdSql.query(f"select tbname,count(*) from nested.stable_1 where ts is not null group by tbname order by tbname;")
+        tdSql.checkRows(7)
+        tdSql.query(f"select tbname,count(*) from nested.stable_1 where ts is null group by tbname order by tbname;")
+        tdSql.checkRows(7)
+        tdSql.query(f"select tbname,last(*) from nested.stable_1 where ts is not null group by tbname order by tbname;")
+        tdSql.checkRows(3)
+        tdSql.query(f"select tbname,last(*) from nested.stable_1 where ts is null group by tbname order by tbname;")
+        tdSql.checkRows(0)
         
         #test stable 
 

tests/system-test/7-tmq/tmqVnodeSplit-stb-false.py

@@ -0,0 +1,218 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+from util.cluster import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 1000
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), True)
+
+    def getDataPath(self):
+        selfPath = tdCom.getBuildPath()
+
+        return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*';
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  60,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        tdCom.drop_all_db()
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        return
+
+    def restartAndRemoveWal(self, deleteWal):
+        tdDnodes = cluster.dnodes
+        tdSql.query("select * from information_schema.ins_vnodes")
+        for result in tdSql.queryResult:
+            if result[2] == 'dbt':
+                tdLog.debug("dnode is %d"%(result[0]))
+                dnodeId = result[0]
+                vnodeId = result[1]
+
+                tdDnodes[dnodeId - 1].stoptaosd()
+                time.sleep(1)
+                dataPath = self.getDataPath()
+                dataPath = dataPath%(dnodeId,vnodeId)
+                tdLog.debug("dataPath:%s"%dataPath)
+                if deleteWal:
+                    if os.system('rm -rf ' + dataPath) != 0:
+                        tdLog.exit("rm error")
+
+                tdDnodes[dnodeId - 1].starttaosd()
+                time.sleep(1)
+                break
+        tdLog.debug("restart dnode ok")
+
+    def splitVgroups(self):
+        tdSql.query("select * from information_schema.ins_vnodes")
+        vnodeId = 0
+        for result in tdSql.queryResult:
+            if result[2] == 'dbt':
+                vnodeId = result[1]
+                tdLog.debug("vnode is %d"%(vnodeId))
+                break
+        splitSql = "split vgroup %d" %(vnodeId)
+        tdLog.debug("splitSql:%s"%(splitSql))
+        tdSql.query(splitSql)
+        tdLog.debug("splitSql ok")
+
+    def tmqCase1(self, deleteWal=False):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb1',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  120,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        topicNameList = ['topic1']
+        # expectRowsList = []
+        tmqCom.initConsumerTable()
+
+        tdLog.info("create topics from stb ")
+        queryString = "stable %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        # tdSql.query(queryString)
+        # expectRowsList.append(tdSql.getRows())
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+
+        tdLog.info("create ctb1")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("create ctb2")
+        paraDict2 = paraDict.copy()
+        paraDict2['ctbPrefix'] = "ctb2"
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("insert ctb1 data")
+        pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+
+        tmqCom.getStartConsumeNotifyFromTmqsim()
+        tmqCom.getStartCommitNotifyFromTmqsim()
+
+        #restart dnode & remove wal
+        self.restartAndRemoveWal(deleteWal)
+
+        # split vgroup
+        self.splitVgroups()
+
+
+        tdLog.info("insert ctb2 data")
+        pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2)
+        pInsertThread.join()
+        pInsertThread1.join()
+
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+
+        if expectrowcnt / 2 > resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        # tmqCom.checkFileContent(consumerId, queryString)
+
+        time.sleep(2)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        if deleteWal == True:
+            clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240)   
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def run(self):
+        self.prepareTestEnv()
+        self.tmqCase1(False)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py

@@ -0,0 +1,222 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+from util.cluster import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 1000
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), True)
+
+    def getDataPath(self):
+        selfPath = tdCom.getBuildPath()
+
+        return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*';
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  60,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        tdCom.drop_all_db()
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        return
+
+    def restartAndRemoveWal(self, deleteWal):
+        tdDnodes = cluster.dnodes
+        tdSql.query("select * from information_schema.ins_vnodes")
+        for result in tdSql.queryResult:
+            if result[2] == 'dbt':
+                tdLog.debug("dnode is %d"%(result[0]))
+                dnodeId = result[0]
+                vnodeId = result[1]
+
+                tdDnodes[dnodeId - 1].stoptaosd()
+                time.sleep(1)
+                dataPath = self.getDataPath()
+                dataPath = dataPath%(dnodeId,vnodeId)
+                tdLog.debug("dataPath:%s"%dataPath)
+                if deleteWal:
+                    if os.system('rm -rf ' + dataPath) != 0:
+                        tdLog.exit("rm error")
+
+                tdDnodes[dnodeId - 1].starttaosd()
+                time.sleep(1)
+                break
+        tdLog.debug("restart dnode ok")
+
+    def splitVgroups(self):
+        tdSql.query("select * from information_schema.ins_vnodes")
+        vnodeId = 0
+        for result in tdSql.queryResult:
+            if result[2] == 'dbt':
+                vnodeId = result[1]
+                tdLog.debug("vnode is %d"%(vnodeId))
+                break
+        splitSql = "split vgroup %d" %(vnodeId)
+        tdLog.debug("splitSql:%s"%(splitSql))
+        tdSql.query(splitSql)
+        tdLog.debug("splitSql ok")
+
+    def tmqCase1(self, deleteWal=False):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb1',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  180,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        topicNameList = ['topic1']
+        # expectRowsList = []
+        tmqCom.initConsumerTable()
+
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        # tdSql.query(queryString)
+        # expectRowsList.append(tdSql.getRows())
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+
+        tdLog.info("create ctb1")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("create ctb2")
+        paraDict2 = paraDict.copy()
+        paraDict2['ctbPrefix'] = "ctb2"
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        
+        tdLog.info("insert ctb1 data")
+        pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+
+        tmqCom.getStartConsumeNotifyFromTmqsim()
+        tmqCom.getStartCommitNotifyFromTmqsim()
+
+        #restart dnode & remove wal
+        self.restartAndRemoveWal(deleteWal)
+
+        # split vgroup
+        self.splitVgroups()
+
+
+        tdLog.info("insert ctb2 data")
+        pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2)
+        pInsertThread.join()
+        pInsertThread1.join()
+
+        expectRows = 1
+
+
+        resultList = tmqCom.selectConsumeResult(expectRows)
+
+        if expectrowcnt / 2 > resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        # tmqCom.checkFileContent(consumerId, queryString)
+
+        time.sleep(2)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        if deleteWal == True:
+            clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240)   
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def run(self):
+        self.prepareTestEnv()
+        self.tmqCase1(False)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py

@@ -190,6 +190,8 @@ class TDTestCase:
         pInsertThread1.join()
 
         expectRows = 1
+
+
         resultList = tmqCom.selectConsumeResult(expectRows)
 
         if expectrowcnt / 2 > resultList[0]:
@@ -209,8 +211,6 @@ class TDTestCase:
     def run(self):
         self.prepareTestEnv()
         self.tmqCase1(True)
-        self.prepareTestEnv()
-        self.tmqCase1(False)
 
     def stop(self):
         tdSql.close()

tests/system-test/7-tmq/tmqVnodeSplit-stb.py

@@ -207,8 +207,6 @@ class TDTestCase:
     def run(self):
         self.prepareTestEnv()
         self.tmqCase1(True)
-        self.prepareTestEnv()
-        self.tmqCase1(False)
 
     def stop(self):
         tdSql.close()