From 705d6e3eb3d2a70d097d33a1c830cf2d6c4ee3bd Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 28 Jul 2021 16:11:09 +0800 Subject: [PATCH] [TD-4840]: update testcase of tsdb meta compressed --- tests/pytest/tsdb/tsdbComp.py | 10 +-- .../{sdbCompCluster.py => tsdbCompCluster.py} | 63 +++++++++----- ...Replica2.py => tsdbCompClusterReplica2.py} | 84 +++++++++++++------ 3 files changed, 105 insertions(+), 52 deletions(-) rename tests/pytest/tsdb/{sdbCompCluster.py => tsdbCompCluster.py} (65%) rename tests/pytest/tsdb/{sdbCompClusterReplica2.py => tsdbCompClusterReplica2.py} (57%) diff --git a/tests/pytest/tsdb/tsdbComp.py b/tests/pytest/tsdb/tsdbComp.py index b2ea36d239..a46b3280fb 100644 --- a/tests/pytest/tsdb/tsdbComp.py +++ b/tests/pytest/tsdb/tsdbComp.py @@ -64,8 +64,7 @@ class TDTestCase: os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath) tdSql.execute("drop table if exists db2.stb0") os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath) - # query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) - # print(query_pid1) + tdSql.execute("use db2") tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") @@ -109,11 +108,6 @@ class TDTestCase: self.rowNum2 = 100 self.rowNum3 = 20 self.ts = 1537146000000 - self.ts1 = 1537146000000000 - self.ts2 = 1597146000000 - # tdSql.execute("create table test1 using test tags('beijing', 10)") - # tdSql.execute("create table test2 using test tags('tianjing', 20)") - # tdSql.execute("create table test3 using test tags('shanghai', 20)") for j in range(self.rowNum2): tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) ) @@ -154,7 +148,7 @@ class TDTestCase: # delete useless file testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") - os.system("rm -rf wal/%s.sql" % testcaseFilename ) + os.system("rm -rf tsdb/%s.sql" % testcaseFilename ) diff --git a/tests/pytest/tsdb/sdbCompCluster.py b/tests/pytest/tsdb/tsdbCompCluster.py similarity index 65% rename from tests/pytest/tsdb/sdbCompCluster.py rename to tests/pytest/tsdb/tsdbCompCluster.py index 4fa84817ec..39c8b563c5 100644 --- a/tests/pytest/tsdb/sdbCompCluster.py +++ b/tests/pytest/tsdb/tsdbCompCluster.py @@ -19,6 +19,8 @@ from util.sql import * from util.dnodes import * import taos import threading +import subprocess +from random import choice class TwoClients: @@ -62,17 +64,15 @@ class TwoClients: cur1 = conn1.cursor() tdSql.init(cur1, True) - # new db and insert data - os.system("rm -rf /var/lib/taos/mnode_bak/") - os.system("rm -rf /var/lib/taos/mnode_temp/") + # new db ,new super tables , child tables, and insert data tdSql.execute("drop database if exists db2") - os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb1.json -y " % binPath) tdSql.execute("drop database if exists db1") - os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath) tdSql.execute("drop table if exists db2.stb0") - os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) - query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) - print(query_pid1) + os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath) + + # new general tables and modify general tables; tdSql.execute("use db2") tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") @@ -87,17 +87,41 @@ class TwoClients: tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") - # stop taosd and compact wal file + # create db utest and modify super tables; + dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"] + tdSql.execute("drop database if exists utest") + tdSql.execute("create database utest keep 3650") + tdSql.execute("use utest") + tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''') + rowNum2= 988 + for i in range(rowNum2): + tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) ) + rowNum3= 988 + for i in range(rowNum3): + tdSql.execute("alter table test drop column col%d ;" %( i+14) ) + + self.rowNum = 1 + self.rowNum2 = 100 + self.ts = 1537146000000 + for j in range(self.rowNum2): + tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) ) + for i in range(self.rowNum): + tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + # delete child tables; + for j in range(self.rowNum2): + tdSql.execute("drop table if exists test%d" % (j+1)) + + #restart taosd os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") - sleep(10) - os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") - sleep(10) + sleep(20) + print("123") os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") sleep(4) tdSql.execute("reset query cache") query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) print(query_pid2) - assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath # new taos connecting to server conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) @@ -105,11 +129,9 @@ class TwoClients: cur2 = conn2.cursor() tdSql.init(cur2, True) - # use new wal file to start up tasod + + # check data correct tdSql.query("show databases") - for i in range(tdSql.queryRows): - if tdSql.queryResult[i][0]=="db2": - assert tdSql.queryResult[i][4]==1 , "replica is wrong" tdSql.execute("use db2") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 1) @@ -123,11 +145,14 @@ class TwoClients: tdSql.checkData(0, 0, 2) tdSql.query("select * from stb2_0") tdSql.checkData(1, 2, 'R') - + tdSql.execute("use utest") + tdSql.query("select count (tbname) from test") + tdSql.checkData(0, 0, 1) + # delete useless file testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") - os.system("rm -rf wal/%s.sql" % testcaseFilename ) + os.system("rm -rf tsdb/%s.sql" % testcaseFilename ) clients = TwoClients() clients.initConnection() diff --git a/tests/pytest/tsdb/sdbCompClusterReplica2.py b/tests/pytest/tsdb/tsdbCompClusterReplica2.py similarity index 57% rename from tests/pytest/tsdb/sdbCompClusterReplica2.py rename to tests/pytest/tsdb/tsdbCompClusterReplica2.py index 117da8ca2f..9b43b7ce80 100644 --- a/tests/pytest/tsdb/sdbCompClusterReplica2.py +++ b/tests/pytest/tsdb/tsdbCompClusterReplica2.py @@ -19,7 +19,8 @@ from util.sql import * from util.dnodes import * import taos import threading - +import subprocess +from random import choice class TwoClients: def initConnection(self): @@ -62,17 +63,15 @@ class TwoClients: cur1 = conn1.cursor() tdSql.init(cur1, True) - # new db and insert data - os.system("rm -rf /var/lib/taos/mnode_bak/") - os.system("rm -rf /var/lib/taos/mnode_temp/") + # new db ,new super tables , child tables, and insert data tdSql.execute("drop database if exists db2") - os.system("%staosdemo -f wal/insertDataDb1Replica2.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb1Replica2.json -y " % binPath) tdSql.execute("drop database if exists db1") - os.system("%staosdemo -f wal/insertDataDb2Replica2.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb2Replica2.json -y " % binPath) tdSql.execute("drop table if exists db2.stb0") - os.system("%staosdemo -f wal/insertDataDb2NewstabReplica2.json -y " % binPath) - query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) - print(query_pid1) + os.system("%staosdemo -f tsdb/insertDataDb2NewstabReplica2.json -y " % binPath) + + # new general tables and modify general tables; tdSql.execute("use db2") tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") @@ -86,19 +85,53 @@ class TwoClients: tdSql.execute("alter table stb2_0 add column col2 binary(4)") tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") - - # stop taosd and compact wal file - os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") - sleep(10) - os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") - sleep(10) + + # create db utest replica 2 and modify super tables; + dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"] + tdSql.execute("drop database if exists utest") + tdSql.execute("create database utest keep 3650 replica 2 ") + tdSql.execute("use utest") + tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''') + rowNum2= 988 + for i in range(rowNum2): + tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) ) + rowNum3= 988 + for i in range(rowNum3): + tdSql.execute("alter table test drop column col%d ;" %( i+14) ) + self.rowNum = 1 + self.rowNum2 = 100 + self.ts = 1537146000000 + for j in range(self.rowNum2): + tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) ) + for i in range(self.rowNum): + tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + # delete child tables; + for j in range(self.rowNum2): + tdSql.execute("drop table if exists test%d" % (j+1)) + + # drop dnodes and restart taosd; + sleep(3) + tdSql.execute(" drop dnode 'chenhaoran02:6030'; ") + sleep(20) + os.system("rm -rf /var/lib/taos/*") + print("clear dnode chenhaoran02'data files") os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") - sleep(4) - tdSql.execute("reset query cache") - query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) - print(query_pid2) - assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath + print("start taosd") + sleep(10) + tdSql.execute("reset query cache ;") + tdSql.execute("create dnode chenhaoran02 ;") + + # # + # os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") + # sleep(20) + # os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") + # sleep(4) + # tdSql.execute("reset query cache") + # query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + # print(query_pid2) # new taos connecting to server conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) @@ -106,11 +139,8 @@ class TwoClients: cur2 = conn2.cursor() tdSql.init(cur2, True) - # use new wal file to start up tasod + # check data correct tdSql.query("show databases") - for i in range(tdSql.queryRows): - if tdSql.queryResult[i][0]=="db2": - assert tdSql.queryResult[i][4]==2 , "replica is wrong" tdSql.execute("use db2") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 1) @@ -125,10 +155,14 @@ class TwoClients: tdSql.query("select * from stb2_0") tdSql.checkData(1, 2, 'R') + tdSql.execute("use utest") + tdSql.query("select count (tbname) from test") + tdSql.checkData(0, 0, 1) + # delete useless file testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") - os.system("rm -rf wal/%s.sql" % testcaseFilename ) + os.system("rm -rf tsdb/%s.sql" % testcaseFilename ) clients = TwoClients() clients.initConnection()