From 60ba3849789958104a50212f384ef45e5b26e6f9 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 1 Jun 2021 18:16:28 +0800 Subject: [PATCH 1/9] [TD-4290]: add testcase that compact wal file --- tests/pytest/wal/insertDataDb1.json | 87 +++++++++++++++ tests/pytest/wal/insertDataDb2.json | 86 +++++++++++++++ tests/pytest/wal/insertDataDb2Newstab.json | 86 +++++++++++++++ tests/pytest/wal/sdbComp.py | 118 +++++++++++++++++++++ tests/pytest/wal/sdbCompCluster.py | 95 +++++++++++++++++ 5 files changed, 472 insertions(+) create mode 100644 tests/pytest/wal/insertDataDb1.json create mode 100644 tests/pytest/wal/insertDataDb2.json create mode 100644 tests/pytest/wal/insertDataDb2Newstab.json create mode 100644 tests/pytest/wal/sdbComp.py create mode 100644 tests/pytest/wal/sdbCompCluster.py diff --git a/tests/pytest/wal/insertDataDb1.json b/tests/pytest/wal/insertDataDb1.json new file mode 100644 index 0000000000..1dce00a4d5 --- /dev/null +++ b/tests/pytest/wal/insertDataDb1.json @@ -0,0 +1,87 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db1", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 200, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/wal/insertDataDb2.json b/tests/pytest/wal/insertDataDb2.json new file mode 100644 index 0000000000..2cf8af5805 --- /dev/null +++ b/tests/pytest/wal/insertDataDb2.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 2000, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 2, + "childtable_prefix": "stb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 5, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/wal/insertDataDb2Newstab.json b/tests/pytest/wal/insertDataDb2Newstab.json new file mode 100644 index 0000000000..f9d0713385 --- /dev/null +++ b/tests/pytest/wal/insertDataDb2Newstab.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 0, + "childtable_limit": -1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"yes", + "childtable_count": 1, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/wal/sdbComp.py b/tests/pytest/wal/sdbComp.py new file mode 100644 index 0000000000..b645a0e445 --- /dev/null +++ b/tests/pytest/wal/sdbComp.py @@ -0,0 +1,118 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + #new db and insert data + os.system("rm -rf /home/chr/TDengine/sim/dnode1/data/mnode_tmp/") + os.system("rm -rf /home/chr/TDengine/sim/dnode1/data/mnode_bak/") + tdSql.execute("drop database if exists db2") + os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath) + tdSql.execute("drop database if exists db1") + os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath) + tdSql.execute("drop table if exists db2.stb0") + os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) + query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid1) + tdSql.execute("use db2") + tdSql.execute("drop table if exists stb1_0") + tdSql.execute("drop table if exists stb1_1") + tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") + tdSql.execute("alter table db2.stb0 add column col4 int") + tdSql.execute("alter table db2.stb0 drop column col2") + tdSql.execute("alter table db2.stb0 add tag t3 int;") + tdSql.execute("alter table db2.stb0 drop tag t1") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") + tdSql.execute("alter table stb2_0 add column col2 binary(4)") + tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") + + # stop taosd and compact wal file + os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9") + sleep(2) + # os.system("nohup taosd --compact-mnode-wal -c /home/chr/TDengine/sim/dnode1/cfg/ & ") + sleep(5) + os.system("nohup /home/chr/TDengine/debug/build/bin/taosd -c /home/chr/TDengine/sim/dnode1/cfg > /dev/null 2>&1 &") + sleep(4) + tdSql.execute("reset query cache") + query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid2) + + # use new wal file to start up tasod + # conn1 = taos.connect(host="chenhaoran02", user="root", password="taosdata", config="/home/chr/TDengine/sim/dnode1/cfg/" ) + # cur1 = conn1.cursor() + # tdSql.init(cur1, True) + + tdSql.execute("use db2") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count (tbname) from stb1") + tdSql.checkRows(0) + tdSql.query("select count(*) from stb0_0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb2_0") + tdSql.checkData(0, 0, 2) + + + + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf wal/sdbComp.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/wal/sdbCompCluster.py b/tests/pytest/wal/sdbCompCluster.py new file mode 100644 index 0000000000..2cefa0a403 --- /dev/null +++ b/tests/pytest/wal/sdbCompCluster.py @@ -0,0 +1,95 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading + + +class TwoClients: + def initConnection(self): + self.host = "chenhaoran01" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taos/" + self.port =6030 + self.rowNum = 10 + self.ts = 1537146000000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # query data from cluster'db + conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn1) + cur1 = conn1.cursor() + tdSql.init(cur1, True) + # tdSql.init(cur2, True) + os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath) + tdSql.execute("drop database if exists db1") + os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath) + tdSql.execute("drop table if exists db2.stb0") + os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) + tdSql.execute("alter table db2.stb0 add column col4 int") + tdSql.execute("alter table db2.stb0 drop column col2") + os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9") + os.system("nohup taosd --compact-mnode-wal & ") + sleep(5) + os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") + sleep(10) + conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn2) + cur2 = conn2.cursor() + tdSql.init(cur2, True) + tdSql.execute("use db2") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb00_0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 20) + +clients = TwoClients() +clients.initConnection() +# clients.getBuildPath() +clients.run() \ No newline at end of file From 119537014b196c88485c4e54e3895819d47ab0d1 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 2 Jun 2021 17:18:34 +0800 Subject: [PATCH 2/9] [TD-4290]: modify testcase that compact wal file --- tests/pytest/wal/sdbComp.py | 28 ++++++++-------- tests/pytest/wal/sdbCompCluster.py | 51 +++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/tests/pytest/wal/sdbComp.py b/tests/pytest/wal/sdbComp.py index b645a0e445..0d1fac2031 100644 --- a/tests/pytest/wal/sdbComp.py +++ b/tests/pytest/wal/sdbComp.py @@ -11,6 +11,7 @@ # -*- coding: utf-8 -*- +from distutils.log import debug import sys import os import taos @@ -18,6 +19,7 @@ from util.log import * from util.cases import * from util.sql import * from util.dnodes import * +import subprocess class TDTestCase: @@ -27,7 +29,7 @@ class TDTestCase: def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - + if ("community" in selfPath): projPath = selfPath[:selfPath.find("community")] else: @@ -47,11 +49,13 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" + binPath = buildPath+ "/build/bin/" + testPath = buildPath[:buildPath.find("debug")] + #new db and insert data - os.system("rm -rf /home/chr/TDengine/sim/dnode1/data/mnode_tmp/") - os.system("rm -rf /home/chr/TDengine/sim/dnode1/data/mnode_bak/") + os.system("rm -rf %s/sim/dnode1/data/mnode_tmp/" % testPath) + os.system("rm -rf %s/sim/dnode1/data/mnode_bak/" % testPath) tdSql.execute("drop database if exists db2") os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath) tdSql.execute("drop database if exists db1") @@ -73,23 +77,19 @@ class TDTestCase: tdSql.execute("alter table stb2_0 add column col2 binary(4)") tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") - # stop taosd and compact wal file - os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9") - sleep(2) - # os.system("nohup taosd --compact-mnode-wal -c /home/chr/TDengine/sim/dnode1/cfg/ & ") - sleep(5) - os.system("nohup /home/chr/TDengine/debug/build/bin/taosd -c /home/chr/TDengine/sim/dnode1/cfg > /dev/null 2>&1 &") + tdDnodes.stop(1) + sleep(10) + os.system("nohup %s/taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " %(binPath,testPath) ) + # os.system("nohup taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " % testPath ) + # tdDnodes.start(1) + os.system("nohup %s/taosd -c %s/sim/dnode1/cfg > /dev/null 2>&1 &" %(binPath,testPath) ) sleep(4) tdSql.execute("reset query cache") query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) print(query_pid2) # use new wal file to start up tasod - # conn1 = taos.connect(host="chenhaoran02", user="root", password="taosdata", config="/home/chr/TDengine/sim/dnode1/cfg/" ) - # cur1 = conn1.cursor() - # tdSql.init(cur1, True) - tdSql.execute("use db2") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 1) diff --git a/tests/pytest/wal/sdbCompCluster.py b/tests/pytest/wal/sdbCompCluster.py index 2cefa0a403..b4411353d8 100644 --- a/tests/pytest/wal/sdbCompCluster.py +++ b/tests/pytest/wal/sdbCompCluster.py @@ -23,7 +23,7 @@ import threading class TwoClients: def initConnection(self): - self.host = "chenhaoran01" + self.host = "chenhaoran02" self.user = "root" self.password = "taosdata" self.config = "/etc/taos/" @@ -61,34 +61,61 @@ class TwoClients: cur1 = conn1.cursor() tdSql.init(cur1, True) # tdSql.init(cur2, True) + + # new db and insert data + os.system("rm -rf /var/lib/taos/mnode_bak/") + os.system("rm -rf /var/lib/taos/mnode_temp/") + tdSql.execute("drop database if exists db2") os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath) tdSql.execute("drop database if exists db1") os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath) tdSql.execute("drop table if exists db2.stb0") os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) + query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid1) + tdSql.execute("use db2") + tdSql.execute("drop table if exists stb1_0") + tdSql.execute("drop table if exists stb1_1") + tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") tdSql.execute("alter table db2.stb0 add column col4 int") tdSql.execute("alter table db2.stb0 drop column col2") - os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9") - os.system("nohup taosd --compact-mnode-wal & ") - sleep(5) - os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") - sleep(10) + tdSql.execute("alter table db2.stb0 add tag t3 int") + tdSql.execute("alter table db2.stb0 drop tag t1") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") + tdSql.execute("alter table stb2_0 add column col2 binary(4)") + tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") + conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) print(conn2) cur2 = conn2.cursor() tdSql.init(cur2, True) + + # stop taosd and compact wal file + os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9") + sleep(2) + os.system("nohup taosd --compact-mnode-wal -c /etc/taos/taos.cfg & ") + sleep(5) + os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") + sleep(4) + tdSql.execute("reset query cache") + query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid2) + + # use new wal file to start up tasod tdSql.execute("use db2") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 1) tdSql.query("select count (tbname) from stb1") + tdSql.checkRows(0) + tdSql.query("select count(*) from stb0_0") tdSql.checkData(0, 0, 2) - tdSql.query("select count(*) from stb00_0") - tdSql.checkData(0, 0, 1) tdSql.query("select count(*) from stb0") - tdSql.checkData(0, 0, 1) - tdSql.query("select count(*) from stb1") - tdSql.checkData(0, 0, 20) - + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb2_0") + tdSql.checkData(0, 0, 2) + clients = TwoClients() clients.initConnection() # clients.getBuildPath() From 17077cb92d1559b00e559dc4b16ccd2bc2ca8498 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 2 Jun 2021 17:46:48 +0800 Subject: [PATCH 3/9] add sleep when start compacting --- tests/pytest/wal/sdbComp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/wal/sdbComp.py b/tests/pytest/wal/sdbComp.py index 0d1fac2031..aab21419bb 100644 --- a/tests/pytest/wal/sdbComp.py +++ b/tests/pytest/wal/sdbComp.py @@ -82,6 +82,7 @@ class TDTestCase: sleep(10) os.system("nohup %s/taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " %(binPath,testPath) ) # os.system("nohup taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " % testPath ) + sleep(5) # tdDnodes.start(1) os.system("nohup %s/taosd -c %s/sim/dnode1/cfg > /dev/null 2>&1 &" %(binPath,testPath) ) sleep(4) From 8112eeaeb9b318d27e75278a553a10a3e08c4373 Mon Sep 17 00:00:00 2001 From: lichuang Date: Wed, 2 Jun 2021 18:30:33 +0800 Subject: [PATCH 4/9] fix test script bug --- tests/pytest/wal/sdbComp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/wal/sdbComp.py b/tests/pytest/wal/sdbComp.py index 0d1fac2031..e1f1c3e4ff 100644 --- a/tests/pytest/wal/sdbComp.py +++ b/tests/pytest/wal/sdbComp.py @@ -83,7 +83,8 @@ class TDTestCase: os.system("nohup %s/taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " %(binPath,testPath) ) # os.system("nohup taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " % testPath ) # tdDnodes.start(1) - os.system("nohup %s/taosd -c %s/sim/dnode1/cfg > /dev/null 2>&1 &" %(binPath,testPath) ) + #os.system("nohup %s/taosd -c %s/sim/dnode1/cfg > /dev/null 2>&1 &" %(binPath,testPath) ) + tdDnodes.start(1) sleep(4) tdSql.execute("reset query cache") query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) From d01ea3012acf93e306307141b8081ee0407beec1 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 2 Jun 2021 21:24:30 +0800 Subject: [PATCH 5/9] [TD-4290]: modify testcase that compact wal file --- tests/pytest/wal/sdbComp.py | 17 +++++++++++------ tests/pytest/wal/sdbCompCluster.py | 8 +++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/pytest/wal/sdbComp.py b/tests/pytest/wal/sdbComp.py index aab21419bb..e0a43d5e0e 100644 --- a/tests/pytest/wal/sdbComp.py +++ b/tests/pytest/wal/sdbComp.py @@ -44,6 +44,8 @@ class TDTestCase: return buildPath def run(self): + + # set path para buildPath = self.getBuildPath() if (buildPath == ""): tdLog.exit("taosd not found!") @@ -52,7 +54,8 @@ class TDTestCase: binPath = buildPath+ "/build/bin/" testPath = buildPath[:buildPath.find("debug")] - + walFilePath = testPath + "/sim/dnode1/data/mnode_bak/wal/" + #new db and insert data os.system("rm -rf %s/sim/dnode1/data/mnode_tmp/" % testPath) os.system("rm -rf %s/sim/dnode1/data/mnode_bak/" % testPath) @@ -77,20 +80,22 @@ class TDTestCase: tdSql.execute("alter table stb2_0 add column col2 binary(4)") tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") + # stop taosd and compact wal file tdDnodes.stop(1) sleep(10) os.system("nohup %s/taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " %(binPath,testPath) ) - # os.system("nohup taosd --compact-mnode-wal -c %s/sim/dnode1/cfg/ & " % testPath ) sleep(5) - # tdDnodes.start(1) - os.system("nohup %s/taosd -c %s/sim/dnode1/cfg > /dev/null 2>&1 &" %(binPath,testPath) ) - sleep(4) + assert os.path.exists(walFilePath) , "%s is not generated, compact didn't take effect " % walFilePath + + # use new wal file to start taosd + tdDnodes.start(1) + sleep(5) tdSql.execute("reset query cache") query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) print(query_pid2) - # use new wal file to start up tasod + # verify that the data is correct tdSql.execute("use db2") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 1) diff --git a/tests/pytest/wal/sdbCompCluster.py b/tests/pytest/wal/sdbCompCluster.py index b4411353d8..43cf4be3c6 100644 --- a/tests/pytest/wal/sdbCompCluster.py +++ b/tests/pytest/wal/sdbCompCluster.py @@ -54,8 +54,9 @@ class TwoClients: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - - # query data from cluster'db + walFilePath = "/var/lib/taos/mnode_bak/wal/" + + # new taos client conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) print(conn1) cur1 = conn1.cursor() @@ -95,13 +96,14 @@ class TwoClients: # stop taosd and compact wal file os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9") sleep(2) - os.system("nohup taosd --compact-mnode-wal -c /etc/taos/taos.cfg & ") + os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") sleep(5) os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") sleep(4) tdSql.execute("reset query cache") query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) print(query_pid2) + assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath # use new wal file to start up tasod tdSql.execute("use db2") From 1b7ccdcdc45ae542ac8b80c05052366d6bd820f7 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 3 Jun 2021 15:19:37 +0800 Subject: [PATCH 6/9] temp From b9502d4ba6c663ed1370941f3df0d16a49d17e9c Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 3 Jun 2021 15:23:59 +0800 Subject: [PATCH 7/9] temp From 046ec16ac42a2a3572fb85cf2ede5c4e4cdd0e10 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 5 Jun 2021 16:24:30 +0800 Subject: [PATCH 8/9] [TD-4552]: add testcase that compact wal file with replica --- tests/pytest/wal/insertDataDb1Replica2.json | 87 +++++++++++ .../wal/insertDataDb2NewstabReplica2.json | 86 +++++++++++ tests/pytest/wal/insertDataDb2Replica2.json | 86 +++++++++++ tests/pytest/wal/sdbComp.py | 6 +- tests/pytest/wal/sdbCompCluster.py | 31 ++-- tests/pytest/wal/sdbCompClusterReplica2.py | 136 ++++++++++++++++++ 6 files changed, 419 insertions(+), 13 deletions(-) create mode 100644 tests/pytest/wal/insertDataDb1Replica2.json create mode 100644 tests/pytest/wal/insertDataDb2NewstabReplica2.json create mode 100644 tests/pytest/wal/insertDataDb2Replica2.json create mode 100644 tests/pytest/wal/sdbCompClusterReplica2.py diff --git a/tests/pytest/wal/insertDataDb1Replica2.json b/tests/pytest/wal/insertDataDb1Replica2.json new file mode 100644 index 0000000000..fec38bcdec --- /dev/null +++ b/tests/pytest/wal/insertDataDb1Replica2.json @@ -0,0 +1,87 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db1", + "drop": "yes", + "replica": 2, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 200, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/wal/insertDataDb2NewstabReplica2.json b/tests/pytest/wal/insertDataDb2NewstabReplica2.json new file mode 100644 index 0000000000..e052f2850f --- /dev/null +++ b/tests/pytest/wal/insertDataDb2NewstabReplica2.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "no", + "replica": 2, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 0, + "childtable_limit": -1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"yes", + "childtable_count": 1, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/wal/insertDataDb2Replica2.json b/tests/pytest/wal/insertDataDb2Replica2.json new file mode 100644 index 0000000000..121f70956a --- /dev/null +++ b/tests/pytest/wal/insertDataDb2Replica2.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "yes", + "replica": 2, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 2000, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 2, + "childtable_prefix": "stb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 5, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/wal/sdbComp.py b/tests/pytest/wal/sdbComp.py index e0a43d5e0e..c0ac02610f 100644 --- a/tests/pytest/wal/sdbComp.py +++ b/tests/pytest/wal/sdbComp.py @@ -108,10 +108,10 @@ class TDTestCase: tdSql.query("select count(*) from stb2_0") tdSql.checkData(0, 0, 2) - - + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") - os.system("rm -rf wal/sdbComp.py.sql") + os.system("rm -rf wal/%s.sql" % testcaseFilename ) diff --git a/tests/pytest/wal/sdbCompCluster.py b/tests/pytest/wal/sdbCompCluster.py index 43cf4be3c6..4fa84817ec 100644 --- a/tests/pytest/wal/sdbCompCluster.py +++ b/tests/pytest/wal/sdbCompCluster.py @@ -61,7 +61,6 @@ class TwoClients: print(conn1) cur1 = conn1.cursor() tdSql.init(cur1, True) - # tdSql.init(cur2, True) # new db and insert data os.system("rm -rf /var/lib/taos/mnode_bak/") @@ -87,17 +86,12 @@ class TwoClients: tdSql.execute("alter table stb2_0 add column col2 binary(4)") tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") - - conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) - print(conn2) - cur2 = conn2.cursor() - tdSql.init(cur2, True) # stop taosd and compact wal file - os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9") - sleep(2) + os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") + sleep(10) os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") - sleep(5) + sleep(10) os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") sleep(4) tdSql.execute("reset query cache") @@ -105,7 +99,17 @@ class TwoClients: print(query_pid2) assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath + # new taos connecting to server + conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn2) + cur2 = conn2.cursor() + tdSql.init(cur2, True) + # use new wal file to start up tasod + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0]=="db2": + assert tdSql.queryResult[i][4]==1 , "replica is wrong" tdSql.execute("use db2") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 1) @@ -117,7 +121,14 @@ class TwoClients: tdSql.checkData(0, 0, 2) tdSql.query("select count(*) from stb2_0") tdSql.checkData(0, 0, 2) - + tdSql.query("select * from stb2_0") + tdSql.checkData(1, 2, 'R') + + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf wal/%s.sql" % testcaseFilename ) + clients = TwoClients() clients.initConnection() # clients.getBuildPath() diff --git a/tests/pytest/wal/sdbCompClusterReplica2.py b/tests/pytest/wal/sdbCompClusterReplica2.py new file mode 100644 index 0000000000..117da8ca2f --- /dev/null +++ b/tests/pytest/wal/sdbCompClusterReplica2.py @@ -0,0 +1,136 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading + + +class TwoClients: + def initConnection(self): + self.host = "chenhaoran02" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taos/" + self.port =6030 + self.rowNum = 10 + self.ts = 1537146000000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + walFilePath = "/var/lib/taos/mnode_bak/wal/" + + # new taos client + conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn1) + cur1 = conn1.cursor() + tdSql.init(cur1, True) + + # new db and insert data + os.system("rm -rf /var/lib/taos/mnode_bak/") + os.system("rm -rf /var/lib/taos/mnode_temp/") + tdSql.execute("drop database if exists db2") + os.system("%staosdemo -f wal/insertDataDb1Replica2.json -y " % binPath) + tdSql.execute("drop database if exists db1") + os.system("%staosdemo -f wal/insertDataDb2Replica2.json -y " % binPath) + tdSql.execute("drop table if exists db2.stb0") + os.system("%staosdemo -f wal/insertDataDb2NewstabReplica2.json -y " % binPath) + query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid1) + tdSql.execute("use db2") + tdSql.execute("drop table if exists stb1_0") + tdSql.execute("drop table if exists stb1_1") + tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") + tdSql.execute("alter table db2.stb0 add column col4 int") + tdSql.execute("alter table db2.stb0 drop column col2") + tdSql.execute("alter table db2.stb0 add tag t3 int") + tdSql.execute("alter table db2.stb0 drop tag t1") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") + tdSql.execute("alter table stb2_0 add column col2 binary(4)") + tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") + + + # stop taosd and compact wal file + os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") + sleep(10) + os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") + sleep(10) + os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") + sleep(4) + tdSql.execute("reset query cache") + query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid2) + assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath + + # new taos connecting to server + conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn2) + cur2 = conn2.cursor() + tdSql.init(cur2, True) + + # use new wal file to start up tasod + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0]=="db2": + assert tdSql.queryResult[i][4]==2 , "replica is wrong" + tdSql.execute("use db2") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count (tbname) from stb1") + tdSql.checkRows(0) + tdSql.query("select count(*) from stb0_0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb2_0") + tdSql.checkData(0, 0, 2) + tdSql.query("select * from stb2_0") + tdSql.checkData(1, 2, 'R') + + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf wal/%s.sql" % testcaseFilename ) + +clients = TwoClients() +clients.initConnection() +# clients.getBuildPath() +clients.run() \ No newline at end of file From e6293954137a949adf223331061f3fe803bdfc5d Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 5 Jun 2021 18:34:36 +0800 Subject: [PATCH 9/9] add compacting wal log testcase into fulltest.sh --- tests/pytest/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 8fed9aa81c..be920dc1ba 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -283,6 +283,7 @@ python3 ./test.py -f topic/topicQuery.py python3 ./test.py -f update/merge_commit_data-0.py # wal python3 ./test.py -f wal/addOldWalTest.py +python3 ./test.py -f wal/sdbComp.py # function python3 ./test.py -f functions/all_null_value.py