[TD-4552]<test>: add testcase that compact wal file with replica
This commit is contained in:
tomchon
parent
2ff6b5166b
commit
046ec16ac4
|
|
@ -0,0 +1,87 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"thread_count": 4,
|
||||
"thread_count_create_tbl": 4,
|
||||
"result_file": "./insert_res.txt",
|
||||
"confirm_parameter_prompt": "no",
|
||||
"insert_interval": 0,
|
||||
"interlace_rows": 10,
|
||||
"num_of_records_per_req": 1000,
|
||||
"max_sql_len": 1024000,
|
||||
"databases": [{
|
||||
"dbinfo": {
|
||||
"name": "db1",
|
||||
"drop": "yes",
|
||||
"replica": 2,
|
||||
"days": 10,
|
||||
"cache": 50,
|
||||
"blocks": 8,
|
||||
"precision": "ms",
|
||||
"keep": 365,
|
||||
"minRows": 100,
|
||||
"maxRows": 4096,
|
||||
"comp":2,
|
||||
"walLevel":1,
|
||||
"cachelast":0,
|
||||
"quorum":1,
|
||||
"fsync":3000,
|
||||
"update": 0
|
||||
},
|
||||
"super_tables": [{
|
||||
"name": "stb0",
|
||||
"child_table_exists":"no",
|
||||
"childtable_count": 1000,
|
||||
"childtable_prefix": "stb00_",
|
||||
"auto_create_table": "no",
|
||||
"batch_create_tbl_num": 100,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": 100,
|
||||
"childtable_limit": 0,
|
||||
"childtable_offset":0,
|
||||
"interlace_rows": 0,
|
||||
"insert_interval":0,
|
||||
"max_sql_len": 1024000,
|
||||
"disorder_ratio": 0,
|
||||
"disorder_range": 1000,
|
||||
"timestamp_step": 1,
|
||||
"start_timestamp": "2020-10-01 00:00:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"tags_file": "",
|
||||
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
|
||||
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
|
||||
},
|
||||
{
|
||||
"name": "stb1",
|
||||
"child_table_exists":"no",
|
||||
"childtable_count": 1000,
|
||||
"childtable_prefix": "stb01_",
|
||||
"auto_create_table": "no",
|
||||
"batch_create_tbl_num": 10,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": 200,
|
||||
"childtable_limit": 0,
|
||||
"childtable_offset":0,
|
||||
"interlace_rows": 0,
|
||||
"insert_interval":0,
|
||||
"max_sql_len": 1024000,
|
||||
"disorder_ratio": 0,
|
||||
"disorder_range": 1000,
|
||||
"timestamp_step": 1,
|
||||
"start_timestamp": "2020-10-01 00:00:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"tags_file": "",
|
||||
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
|
||||
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
|
||||
}]
|
||||
}]
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"thread_count": 4,
|
||||
"thread_count_create_tbl": 4,
|
||||
"result_file": "./insert_res.txt",
|
||||
"confirm_parameter_prompt": "no",
|
||||
"insert_interval": 0,
|
||||
"interlace_rows": 0,
|
||||
"num_of_records_per_req": 3000,
|
||||
"max_sql_len": 1024000,
|
||||
"databases": [{
|
||||
"dbinfo": {
|
||||
"name": "db2",
|
||||
"drop": "no",
|
||||
"replica": 2,
|
||||
"days": 10,
|
||||
"cache": 50,
|
||||
"blocks": 8,
|
||||
"precision": "ms",
|
||||
"keep": 365,
|
||||
"minRows": 100,
|
||||
"maxRows": 4096,
|
||||
"comp":2,
|
||||
"walLevel":1,
|
||||
"cachelast":0,
|
||||
"quorum":1,
|
||||
"fsync":3000,
|
||||
"update": 0
|
||||
},
|
||||
"super_tables": [{
|
||||
"name": "stb0",
|
||||
"child_table_exists":"no",
|
||||
"childtable_count": 1,
|
||||
"childtable_prefix": "stb0_",
|
||||
"auto_create_table": "no",
|
||||
"batch_create_tbl_num": 100,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": 0,
|
||||
"childtable_limit": -1,
|
||||
"childtable_offset":0,
|
||||
"interlace_rows": 0,
|
||||
"insert_interval":0,
|
||||
"max_sql_len": 1024000,
|
||||
"disorder_ratio": 0,
|
||||
"disorder_range": 1000,
|
||||
"timestamp_step": 1,
|
||||
"start_timestamp": "2020-10-01 00:00:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"tags_file": "",
|
||||
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
|
||||
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
|
||||
},
|
||||
{
|
||||
"name": "stb1",
|
||||
"child_table_exists":"yes",
|
||||
"childtable_count": 1,
|
||||
"childtable_prefix": "stb01_",
|
||||
"auto_create_table": "no",
|
||||
"batch_create_tbl_num": 10,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": 10,
|
||||
"childtable_limit": -1,
|
||||
"childtable_offset":0,
|
||||
"interlace_rows": 0,
|
||||
"insert_interval":0,
|
||||
"max_sql_len": 1024000,
|
||||
"disorder_ratio": 0,
|
||||
"disorder_range": 1000,
|
||||
"timestamp_step": 1,
|
||||
"start_timestamp": "2020-11-01 00:00:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"tags_file": "",
|
||||
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}],
|
||||
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
|
||||
}]
|
||||
}]
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"thread_count": 4,
|
||||
"thread_count_create_tbl": 4,
|
||||
"result_file": "./insert_res.txt",
|
||||
"confirm_parameter_prompt": "no",
|
||||
"insert_interval": 0,
|
||||
"interlace_rows": 0,
|
||||
"num_of_records_per_req": 3000,
|
||||
"max_sql_len": 1024000,
|
||||
"databases": [{
|
||||
"dbinfo": {
|
||||
"name": "db2",
|
||||
"drop": "yes",
|
||||
"replica": 2,
|
||||
"days": 10,
|
||||
"cache": 50,
|
||||
"blocks": 8,
|
||||
"precision": "ms",
|
||||
"keep": 365,
|
||||
"minRows": 100,
|
||||
"maxRows": 4096,
|
||||
"comp":2,
|
||||
"walLevel":1,
|
||||
"cachelast":0,
|
||||
"quorum":1,
|
||||
"fsync":3000,
|
||||
"update": 0
|
||||
},
|
||||
"super_tables": [{
|
||||
"name": "stb0",
|
||||
"child_table_exists":"no",
|
||||
"childtable_count": 2000,
|
||||
"childtable_prefix": "stb0_",
|
||||
"auto_create_table": "no",
|
||||
"batch_create_tbl_num": 100,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": 2000,
|
||||
"childtable_limit": 0,
|
||||
"childtable_offset":0,
|
||||
"interlace_rows": 0,
|
||||
"insert_interval":0,
|
||||
"max_sql_len": 1024000,
|
||||
"disorder_ratio": 0,
|
||||
"disorder_range": 1000,
|
||||
"timestamp_step": 1,
|
||||
"start_timestamp": "2020-10-01 00:00:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"tags_file": "",
|
||||
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
|
||||
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
|
||||
},
|
||||
{
|
||||
"name": "stb1",
|
||||
"child_table_exists":"no",
|
||||
"childtable_count": 2,
|
||||
"childtable_prefix": "stb1_",
|
||||
"auto_create_table": "no",
|
||||
"batch_create_tbl_num": 10,
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": 5,
|
||||
"childtable_limit": 0,
|
||||
"childtable_offset":0,
|
||||
"interlace_rows": 0,
|
||||
"insert_interval":0,
|
||||
"max_sql_len": 1024000,
|
||||
"disorder_ratio": 0,
|
||||
"disorder_range": 1000,
|
||||
"timestamp_step": 1,
|
||||
"start_timestamp": "2020-10-01 00:00:00.000",
|
||||
"sample_format": "csv",
|
||||
"sample_file": "./sample.csv",
|
||||
"tags_file": "",
|
||||
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
|
||||
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
|
||||
}]
|
||||
}]
|
||||
}
|
||||
|
|
@ -108,10 +108,10 @@ class TDTestCase:
|
|||
tdSql.query("select count(*) from stb2_0")
|
||||
tdSql.checkData(0, 0, 2)
|
||||
|
||||
|
||||
|
||||
# delete useless file
|
||||
testcaseFilename = os.path.split(__file__)[-1]
|
||||
os.system("rm -rf ./insert_res.txt")
|
||||
os.system("rm -rf wal/sdbComp.py.sql")
|
||||
os.system("rm -rf wal/%s.sql" % testcaseFilename )
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -61,7 +61,6 @@ class TwoClients:
|
|||
print(conn1)
|
||||
cur1 = conn1.cursor()
|
||||
tdSql.init(cur1, True)
|
||||
# tdSql.init(cur2, True)
|
||||
|
||||
# new db and insert data
|
||||
os.system("rm -rf /var/lib/taos/mnode_bak/")
|
||||
|
|
@ -88,16 +87,11 @@ class TwoClients:
|
|||
tdSql.execute("alter table stb2_0 drop column col1")
|
||||
tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
|
||||
|
||||
conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
|
||||
print(conn2)
|
||||
cur2 = conn2.cursor()
|
||||
tdSql.init(cur2, True)
|
||||
|
||||
# stop taosd and compact wal file
|
||||
os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -9")
|
||||
sleep(2)
|
||||
os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
|
||||
sleep(10)
|
||||
os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ")
|
||||
sleep(5)
|
||||
sleep(10)
|
||||
os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
|
||||
sleep(4)
|
||||
tdSql.execute("reset query cache")
|
||||
|
|
@ -105,7 +99,17 @@ class TwoClients:
|
|||
print(query_pid2)
|
||||
assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath
|
||||
|
||||
# new taos connecting to server
|
||||
conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
|
||||
print(conn2)
|
||||
cur2 = conn2.cursor()
|
||||
tdSql.init(cur2, True)
|
||||
|
||||
# use new wal file to start up tasod
|
||||
tdSql.query("show databases")
|
||||
for i in range(tdSql.queryRows):
|
||||
if tdSql.queryResult[i][0]=="db2":
|
||||
assert tdSql.queryResult[i][4]==1 , "replica is wrong"
|
||||
tdSql.execute("use db2")
|
||||
tdSql.query("select count (tbname) from stb0")
|
||||
tdSql.checkData(0, 0, 1)
|
||||
|
|
@ -117,6 +121,13 @@ class TwoClients:
|
|||
tdSql.checkData(0, 0, 2)
|
||||
tdSql.query("select count(*) from stb2_0")
|
||||
tdSql.checkData(0, 0, 2)
|
||||
tdSql.query("select * from stb2_0")
|
||||
tdSql.checkData(1, 2, 'R')
|
||||
|
||||
# delete useless file
|
||||
testcaseFilename = os.path.split(__file__)[-1]
|
||||
os.system("rm -rf ./insert_res.txt")
|
||||
os.system("rm -rf wal/%s.sql" % testcaseFilename )
|
||||
|
||||
clients = TwoClients()
|
||||
clients.initConnection()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.getcwd())
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.dnodes import *
|
||||
import taos
|
||||
import threading
|
||||
|
||||
|
||||
class TwoClients:
|
||||
def initConnection(self):
|
||||
self.host = "chenhaoran02"
|
||||
self.user = "root"
|
||||
self.password = "taosdata"
|
||||
self.config = "/etc/taos/"
|
||||
self.port =6030
|
||||
self.rowNum = 10
|
||||
self.ts = 1537146000000
|
||||
|
||||
def getBuildPath(self):
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
if ("community" in selfPath):
|
||||
projPath = selfPath[:selfPath.find("community")]
|
||||
else:
|
||||
projPath = selfPath[:selfPath.find("tests")]
|
||||
|
||||
for root, dirs, files in os.walk(projPath):
|
||||
if ("taosd" in files):
|
||||
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||
if ("packaging" not in rootRealPath):
|
||||
buildPath = root[:len(root)-len("/build/bin")]
|
||||
break
|
||||
return buildPath
|
||||
|
||||
def run(self):
|
||||
buildPath = self.getBuildPath()
|
||||
if (buildPath == ""):
|
||||
tdLog.exit("taosd not found!")
|
||||
else:
|
||||
tdLog.info("taosd found in %s" % buildPath)
|
||||
binPath = buildPath+ "/build/bin/"
|
||||
walFilePath = "/var/lib/taos/mnode_bak/wal/"
|
||||
|
||||
# new taos client
|
||||
conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
|
||||
print(conn1)
|
||||
cur1 = conn1.cursor()
|
||||
tdSql.init(cur1, True)
|
||||
|
||||
# new db and insert data
|
||||
os.system("rm -rf /var/lib/taos/mnode_bak/")
|
||||
os.system("rm -rf /var/lib/taos/mnode_temp/")
|
||||
tdSql.execute("drop database if exists db2")
|
||||
os.system("%staosdemo -f wal/insertDataDb1Replica2.json -y " % binPath)
|
||||
tdSql.execute("drop database if exists db1")
|
||||
os.system("%staosdemo -f wal/insertDataDb2Replica2.json -y " % binPath)
|
||||
tdSql.execute("drop table if exists db2.stb0")
|
||||
os.system("%staosdemo -f wal/insertDataDb2NewstabReplica2.json -y " % binPath)
|
||||
query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
|
||||
print(query_pid1)
|
||||
tdSql.execute("use db2")
|
||||
tdSql.execute("drop table if exists stb1_0")
|
||||
tdSql.execute("drop table if exists stb1_1")
|
||||
tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')")
|
||||
tdSql.execute("alter table db2.stb0 add column col4 int")
|
||||
tdSql.execute("alter table db2.stb0 drop column col2")
|
||||
tdSql.execute("alter table db2.stb0 add tag t3 int")
|
||||
tdSql.execute("alter table db2.stb0 drop tag t1")
|
||||
tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ")
|
||||
tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)")
|
||||
tdSql.execute("alter table stb2_0 add column col2 binary(4)")
|
||||
tdSql.execute("alter table stb2_0 drop column col1")
|
||||
tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
|
||||
|
||||
|
||||
# stop taosd and compact wal file
|
||||
os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
|
||||
sleep(10)
|
||||
os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ")
|
||||
sleep(10)
|
||||
os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
|
||||
sleep(4)
|
||||
tdSql.execute("reset query cache")
|
||||
query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
|
||||
print(query_pid2)
|
||||
assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath
|
||||
|
||||
# new taos connecting to server
|
||||
conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
|
||||
print(conn2)
|
||||
cur2 = conn2.cursor()
|
||||
tdSql.init(cur2, True)
|
||||
|
||||
# use new wal file to start up tasod
|
||||
tdSql.query("show databases")
|
||||
for i in range(tdSql.queryRows):
|
||||
if tdSql.queryResult[i][0]=="db2":
|
||||
assert tdSql.queryResult[i][4]==2 , "replica is wrong"
|
||||
tdSql.execute("use db2")
|
||||
tdSql.query("select count (tbname) from stb0")
|
||||
tdSql.checkData(0, 0, 1)
|
||||
tdSql.query("select count (tbname) from stb1")
|
||||
tdSql.checkRows(0)
|
||||
tdSql.query("select count(*) from stb0_0")
|
||||
tdSql.checkData(0, 0, 2)
|
||||
tdSql.query("select count(*) from stb0")
|
||||
tdSql.checkData(0, 0, 2)
|
||||
tdSql.query("select count(*) from stb2_0")
|
||||
tdSql.checkData(0, 0, 2)
|
||||
tdSql.query("select * from stb2_0")
|
||||
tdSql.checkData(1, 2, 'R')
|
||||
|
||||
# delete useless file
|
||||
testcaseFilename = os.path.split(__file__)[-1]
|
||||
os.system("rm -rf ./insert_res.txt")
|
||||
os.system("rm -rf wal/%s.sql" % testcaseFilename )
|
||||
|
||||
clients = TwoClients()
|
||||
clients.initConnection()
|
||||
# clients.getBuildPath()
|
||||
clients.run()
|
||||
Loading…
Reference in New Issue