[TD-4840]<test>: update testcase of tsdb meta compressed

tests/pytest/tsdb/tsdbComp.py

@@ -64,8 +64,7 @@ class TDTestCase:
         os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath)
         tdSql.execute("drop table if exists db2.stb0") 
         os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath)
-        # query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+
-        # print(query_pid1)
         tdSql.execute("use db2")
         tdSql.execute("drop table if exists stb1_0")
         tdSql.execute("drop table if exists stb1_1")
@@ -109,11 +108,6 @@ class TDTestCase:
         self.rowNum2 = 100
         self.rowNum3 = 20
         self.ts = 1537146000000
-        self.ts1 = 1537146000000000
-        self.ts2 = 1597146000000
-        # tdSql.execute("create table test1 using test tags('beijing', 10)")
-        # tdSql.execute("create table test2 using test tags('tianjing', 20)")
-        # tdSql.execute("create table test3 using test tags('shanghai', 20)")   
         
         for j in range(self.rowNum2):
             tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
@@ -154,7 +148,7 @@ class TDTestCase:
         # delete useless file
         testcaseFilename = os.path.split(__file__)[-1]
         os.system("rm -rf ./insert_res.txt")
-        os.system("rm -rf wal/%s.sql" % testcaseFilename )       
+        os.system("rm -rf tsdb/%s.sql" % testcaseFilename )       
         
         
         

tests/pytest/tsdb/tsdbCompCluster.py

@@ -19,6 +19,8 @@ from util.sql import *
 from util.dnodes import *
 import taos
 import threading
+import subprocess
+from random import choice
 
  
 class TwoClients:
@@ -62,17 +64,15 @@ class TwoClients:
         cur1 = conn1.cursor()
         tdSql.init(cur1, True)
 
-        # new db and insert data
+        # new db ,new super tables , child tables, and insert  data
-        os.system("rm -rf  /var/lib/taos/mnode_bak/")
-        os.system("rm -rf  /var/lib/taos/mnode_temp/")
         tdSql.execute("drop database if exists db2")
-        os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath)
+        os.system("%staosdemo -f tsdb/insertDataDb1.json -y " % binPath)
         tdSql.execute("drop database if exists db1") 
-        os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath)
+        os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath)
         tdSql.execute("drop table if exists db2.stb0") 
-        os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath)
+        os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath)
-        query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+
-        print(query_pid1)
+        # new general tables and modify general tables;
         tdSql.execute("use db2")
         tdSql.execute("drop table if exists stb1_0")
         tdSql.execute("drop table if exists stb1_1")
@@ -87,17 +87,41 @@ class TwoClients:
         tdSql.execute("alter table stb2_0 drop column col1")
         tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
 
-        # stop taosd and compact wal file
+        # create  db utest and modify super tables;
+        dataType= [ "tinyint", "smallint",  "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned",  "bigint unsigned"] 
+        tdSql.execute("drop database if exists utest") 
+        tdSql.execute("create database utest keep 3650")
+        tdSql.execute("use utest")
+        tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+                    col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''')
+        rowNum2= 988
+        for i in range(rowNum2):
+            tdSql.execute("alter table test add column col%d  %s ;" %( i+14, choice(dataType))     ) 
+        rowNum3= 988
+        for i in range(rowNum3):
+            tdSql.execute("alter table test drop column col%d   ;" %( i+14)     )    
+
+        self.rowNum = 1
+        self.rowNum2 = 100
+        self.ts = 1537146000000
+        for j in range(self.rowNum2):
+            tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
+            for i in range(self.rowNum):
+                tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" 
+                        % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
+        # delete child tables;  
+        for j in range(self.rowNum2):    
+            tdSql.execute("drop table if exists test%d" % (j+1))
+                    
+        #restart taosd
         os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
-        sleep(10)
+        sleep(20)
-        os.system("nohup taosd  --compact-mnode-wal  -c /etc/taos & ")
+        print("123")
-        sleep(10)
         os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
         sleep(4)
         tdSql.execute("reset query cache")
         query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
         print(query_pid2)
-        assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath    
 
         # new taos connecting to server
         conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
@@ -105,11 +129,9 @@ class TwoClients:
         cur2 = conn2.cursor()
         tdSql.init(cur2, True)
 
-        # use new wal file to start up tasod 
+
+        # check data correct 
         tdSql.query("show databases")
-        for i in range(tdSql.queryRows):
-            if tdSql.queryResult[i][0]=="db2":
-                assert tdSql.queryResult[i][4]==1 , "replica is wrong"
         tdSql.execute("use db2")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 1)
@@ -123,11 +145,14 @@ class TwoClients:
         tdSql.checkData(0, 0, 2)
         tdSql.query("select * from stb2_0")
         tdSql.checkData(1, 2, 'R')
-        
+        tdSql.execute("use utest")
+        tdSql.query("select count (tbname) from test")
+        tdSql.checkData(0, 0, 1) 
+
         # delete useless file       
         testcaseFilename = os.path.split(__file__)[-1]
         os.system("rm -rf ./insert_res.txt")
-        os.system("rm -rf wal/%s.sql" % testcaseFilename )       
+        os.system("rm -rf tsdb/%s.sql" % testcaseFilename )       
 
 clients = TwoClients()
 clients.initConnection()

tests/pytest/tsdb/tsdbCompClusterReplica2.py

@@ -19,7 +19,8 @@ from util.sql import *
 from util.dnodes import *
 import taos
 import threading
-
+import subprocess
+from random import choice
  
 class TwoClients:
     def initConnection(self):
@@ -62,17 +63,15 @@ class TwoClients:
         cur1 = conn1.cursor()
         tdSql.init(cur1, True)
 
-        # new db and insert data
+        # new db ,new super tables , child tables, and insert  data
-        os.system("rm -rf  /var/lib/taos/mnode_bak/")
-        os.system("rm -rf  /var/lib/taos/mnode_temp/")
         tdSql.execute("drop database if exists db2")
-        os.system("%staosdemo -f wal/insertDataDb1Replica2.json -y " % binPath)
+        os.system("%staosdemo -f tsdb/insertDataDb1Replica2.json -y " % binPath)
         tdSql.execute("drop database if exists db1") 
-        os.system("%staosdemo -f wal/insertDataDb2Replica2.json -y " % binPath)
+        os.system("%staosdemo -f tsdb/insertDataDb2Replica2.json -y " % binPath)
         tdSql.execute("drop table if exists db2.stb0") 
-        os.system("%staosdemo -f wal/insertDataDb2NewstabReplica2.json -y " % binPath)
+        os.system("%staosdemo -f tsdb/insertDataDb2NewstabReplica2.json -y " % binPath)
-        query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+
-        print(query_pid1)
+        # new general tables and modify general tables;  
         tdSql.execute("use db2")
         tdSql.execute("drop table if exists stb1_0")
         tdSql.execute("drop table if exists stb1_1")
@@ -86,19 +85,53 @@ class TwoClients:
         tdSql.execute("alter table stb2_0 add column col2 binary(4)")
         tdSql.execute("alter table stb2_0 drop column col1")
         tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
-        
 
-        # stop taosd and compact wal file
+
-        os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
+        # create  db utest  replica 2 and modify super tables;
-        sleep(10)
+        dataType= [ "tinyint", "smallint",  "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned",  "bigint unsigned"] 
-        os.system("nohup taosd  --compact-mnode-wal  -c /etc/taos & ")
+        tdSql.execute("drop database if exists utest") 
-        sleep(10)
+        tdSql.execute("create database utest keep 3650  replica 2 ")
+        tdSql.execute("use utest")
+        tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+                    col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''')
+        rowNum2= 988
+        for i in range(rowNum2):
+            tdSql.execute("alter table test add column col%d  %s ;" %( i+14, choice(dataType))     ) 
+        rowNum3= 988
+        for i in range(rowNum3):
+            tdSql.execute("alter table test drop column col%d   ;" %( i+14)     )    
+        self.rowNum = 1
+        self.rowNum2 = 100
+        self.ts = 1537146000000
+        for j in range(self.rowNum2):
+            tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
+            for i in range(self.rowNum):
+                tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" 
+                        % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
+        #  delete child tables;  
+        for j in range(self.rowNum2):    
+            tdSql.execute("drop table if exists test%d" % (j+1))
+        
+        # drop dnodes and restart taosd;
+        sleep(3)
+        tdSql.execute(" drop dnode 'chenhaoran02:6030'; ")
+        sleep(20)
+        os.system("rm -rf /var/lib/taos/*")
+        print("clear dnode chenhaoran02'data files")
         os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
-        sleep(4)
+        print("start taosd")
-        tdSql.execute("reset query cache")
+        sleep(10)
-        query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+        tdSql.execute("reset query cache ;")
-        print(query_pid2)
+        tdSql.execute("create dnode chenhaoran02 ;")
-        assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath    
+
+        # # 
+        # os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
+        # sleep(20)
+        # os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
+        # sleep(4)
+        # tdSql.execute("reset query cache")
+        # query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+        # print(query_pid2)
 
         # new taos connecting to server
         conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
@@ -106,11 +139,8 @@ class TwoClients:
         cur2 = conn2.cursor()
         tdSql.init(cur2, True)
 
-        # use new wal file to start up tasod 
+        # check data correct 
         tdSql.query("show databases")
-        for i in range(tdSql.queryRows):
-            if tdSql.queryResult[i][0]=="db2":
-                assert tdSql.queryResult[i][4]==2 , "replica is wrong"
         tdSql.execute("use db2")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 1)
@@ -125,10 +155,14 @@ class TwoClients:
         tdSql.query("select * from stb2_0")
         tdSql.checkData(1, 2, 'R')
 
+        tdSql.execute("use utest")
+        tdSql.query("select count (tbname) from test")
+        tdSql.checkData(0, 0, 1) 
+
         # delete useless file
         testcaseFilename = os.path.split(__file__)[-1]
         os.system("rm -rf ./insert_res.txt")
-        os.system("rm -rf wal/%s.sql" % testcaseFilename )       
+        os.system("rm -rf tsdb/%s.sql" % testcaseFilename )       
    
 clients = TwoClients()
 clients.initConnection()