add inc Snapshot copy case

This commit is contained in:
menshibin 2024-01-12 14:14:03 +08:00
parent bba8532a96
commit 006a3c3804
6 changed files with 165 additions and 51 deletions

View File

@ -0,0 +1,104 @@
import taos
import sys
import os
import subprocess
import glob
import shutil
import time
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
from frame.autogen import *
from frame.server.dnodes import *
from frame.server.cluster import *
class TDTestCase(TBase):
def init(self, conn, logSql, replicaVar=3):
super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1")
self.valgrind = 0
self.childtable_count = 10
# tdSql.init(conn.cursor())
tdSql.init(conn.cursor(), logSql) # output sql.txt file
def run(self):
tdSql.prepare()
autoGen = AutoGen()
autoGen.create_db(self.db, 2, 3)
autoGen.create_stable(self.stb, 5, 10, 8, 8)
autoGen.create_child(self.stb, "d", self.childtable_count)
autoGen.insert_data(1)
tdSql.execute(f"flush database {self.db}")
clusterDnodes.stoptaosd(3)
clusterDnodes.stoptaosd(1)
clusterDnodes.starttaosd(3)
time.sleep(5)
clusterDnodes.stoptaosd(2)
clusterDnodes.starttaosd(1)
time.sleep(5)
autoGen.insert_data(10, 1600000000001)
# tdSql.execute(f"flush database {self.db}")
sql = 'show vnodes;'
while True:
bFinish = True
param_list = tdSql.query(sql, row_tag=True)
for param in param_list:
if param[3] == 'leading' or param[3] == 'following':
bFinish = False
break
if bFinish:
break
self.snapshotAgg()
clusterDnodes.stopAll()
# for i in range(1, 4):
# path = clusterDnodes.getDnodeDir(i)
# dnodesRootDir = os.path.join(path,"data","vnode", "vnode*")
# dirs = glob.glob(dnodesRootDir)
# for dir in dirs:
# if os.path.isdir(dir):
# tdLog.debug("delete dir: %s " % (dnodesRootDir))
# self.remove_directory(os.path.join(dir, "wal"))
clusterDnodes.starttaosd(1)
clusterDnodes.starttaosd(2)
clusterDnodes.starttaosd(3)
time.sleep(3)
while True:
bFinish = True
param_list = tdSql.query(sql, row_tag=True)
for param in param_list:
if param[3] == 'offline':
tdLog.exit(
"dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1]))
if param[3] == 'leading' or param[3] == 'following':
bFinish = False
break
if bFinish:
break
self.timestamp_step = 1
self.insert_rows = 11
self.checkInsertCorrect()
self.checkAggCorrect()
def remove_directory(self, directory):
try:
shutil.rmtree(directory)
tdLog.debug("delete dir: %s " % (directory))
except OSError as e:
tdLog.exit("delete fail dir: %s " % (directory))
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

View File

@ -30,7 +30,7 @@ class AutoGen:
# _columns_sql
def gen_columns_sql(self, pre, cnt, binary_len, nchar_len):
types = [
types = [
'timestamp',
'tinyint',
'smallint',
@ -58,33 +58,33 @@ class AutoGen:
sqls += ","
sqls += sql
metas.append(sel)
return metas, sqls;
return metas, sqls;
# gen tags data
def gen_data(self, i, marr):
datas = ""
datas = ""
for c in marr:
data = ""
if c == 0 : # timestamp
if c == 0: # timestamp
data = "%d" % (self.ts + i)
elif c <= 4 : # small
data = "%d"%(i%128)
elif c <= 8 : # int
elif c <= 4: # small
data = "%d" % (i % 128)
elif c <= 8: # int
data = f"{i}"
elif c <= 10 : # float
data = "%f"%(i+i/1000)
elif c <= 11 : # bool
data = "%d"%(i%2)
elif c == 12 : # binary
elif c <= 10: # float
data = "%f" % (i + i / 1000)
elif c <= 11: # bool
data = "%d" % (i % 2)
elif c == 12: # binary
data = '"' + self.random_string(self.bin_len) + '"'
elif c == 13 : # binary
elif c == 13: # binary
data = '"' + self.random_string(self.nch_len) + '"'
if datas != "":
datas += ","
datas += data
return datas
# generate specail wide random string
@ -93,11 +93,11 @@ class AutoGen:
return ''.join(random.choice(letters) for i in range(count))
# create db
def create_db(self, dbname, vgroups = 2, replica = 1):
self.dbname = dbname
def create_db(self, dbname, vgroups=2, replica=1):
self.dbname = dbname
tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}')
tdSql.execute(f'use {dbname}')
# create table or stable
def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len):
self.bin_len = binary_len
@ -109,7 +109,7 @@ class AutoGen:
sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})"
tdSql.execute(sql)
# create child table
# create child table
def create_child(self, stbname, prename, cnt):
self.child_cnt = cnt
self.child_name = prename
@ -120,7 +120,7 @@ class AutoGen:
tdLog.info(f"create child tables {cnt} ok")
def insert_data_child(self, child_name, cnt, batch_size, step):
def insert_data_child(self, child_name, cnt, batch_size, step):
values = ""
print("insert child data")
ts = self.ts
@ -130,7 +130,7 @@ class AutoGen:
value = self.gen_data(i, self.mcols)
ts += step
values += f"({ts},{value}) "
if batch_size == 1 or (i > 0 and i % batch_size == 0) :
if batch_size == 1 or (i > 0 and i % batch_size == 0):
sql = f"insert into {child_name} values {values}"
tdSql.execute(sql)
values = ""
@ -138,18 +138,25 @@ class AutoGen:
# end batch
if values != "":
sql = f"insert into {child_name} values {values}"
tdLog.info(f" insert child data SQL{sql}")
tdSql.execute(sql)
tdLog.info(f" insert data i={i}")
values = ""
tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")
tdLog.info(f" insert child data {child_name} finished, insert rows={cnt} last_ts={ts}")
return ts
# insert data
def insert_data(self, cnt):
# insert data
def insert_data(self, cnt, bContinue=False):
if not bContinue:
self.ts = 1600000000000
currTs = 1600000000000
for i in range(self.child_cnt):
name = f"{self.child_name}{i}"
self.insert_data_child(name, cnt, self.batch_size, 1)
currTs = self.insert_data_child(name, cnt, self.batch_size, 1)
self.ts = currTs
tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
# insert same timestamp to all childs
@ -158,6 +165,4 @@ class AutoGen:
name = f"{self.child_name}{i}"
self.insert_data_child(name, cnt, self.batch_size, 0)
tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")
tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")

View File

@ -28,7 +28,7 @@ class TBase:
#
# init
def init(self, conn, logSql, replicaVar=1):
def init(self, conn, logSql, replicaVar=1, db="db", stb="stb", checkColName="ic"):
# save param
self.replicaVar = int(replicaVar)
tdSql.init(conn.cursor(), True)
@ -40,14 +40,14 @@ class TBase:
self.mLevelDisk = 0
# test case information
self.db = "db"
self.stb = "stb"
self.db = db
self.stb = stb
# sql
self.sqlSum = f"select sum(ic) from {self.stb}"
self.sqlMax = f"select max(ic) from {self.stb}"
self.sqlMin = f"select min(ic) from {self.stb}"
self.sqlAvg = f"select avg(ic) from {self.stb}"
self.sqlSum = f"select sum({checkColName}) from {self.stb}"
self.sqlMax = f"select max({checkColName}) from {self.stb}"
self.sqlMin = f"select min({checkColName}) from {self.stb}"
self.sqlAvg = f"select avg({checkColName}) from {self.stb}"
self.sqlFirst = f"select first(ts) from {self.stb}"
self.sqlLast = f"select last(ts) from {self.stb}"

View File

@ -13,23 +13,24 @@ from frame.common import *
class ClusterDnodes(TDDnodes):
"""rewrite TDDnodes and make MyDdnodes as TDDnodes child class"""
def __init__(self ,dnodes_lists):
def __init__(self):
super(ClusterDnodes,self).__init__()
self.dnodes = dnodes_lists # dnode must be TDDnode instance
self.simDeployed = False
self.testCluster = False
self.valgrind = 0
self.killValgrind = 1
def init(self, dnodes_lists, deployPath, masterIp):
self.dnodes = dnodes_lists # dnode must be TDDnode instance
super(ClusterDnodes, self).init(deployPath, masterIp)
clusterDnodes = ClusterDnodes()
class ConfigureyCluster:
"""This will create defined number of dnodes and create a cluster.
at the same time, it will return TDDnodes list: dnodes, """
hostname = socket.gethostname()
def __init__(self):
self.dnodes = []
self.dnodes = []
self.dnodeNums = 5
self.independent = True
self.startPort = 6030

View File

@ -899,6 +899,11 @@ class TDDnodes:
dnodesRootDir = "%s/sim" % (self.path)
return dnodesRootDir
def getDnodeDir(self, index):
self.check(index)
dnodesDir = "%s/sim/dnode%d" % (self.path, index)
return dnodesDir
def getSimCfgPath(self):
return self.sim.getCfgDir()

View File

@ -405,11 +405,11 @@ if __name__ == "__main__":
else :
tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
tdDnodes = ClusterDnodes(dnodeslist)
tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster)
tdDnodes.setValgrind(valgrind)
tdDnodes.stopAll()
clusterDnodes.init(dnodeslist, deployPath, masterIp)
clusterDnodes.setTestCluster(testCluster)
clusterDnodes.setValgrind(valgrind)
clusterDnodes.setAsan(asan)
clusterDnodes.stopAll()
for dnode in tdDnodes.dnodes:
tdDnodes.deploy(dnode.index, updateCfgDict)
for dnode in tdDnodes.dnodes:
@ -591,12 +591,11 @@ if __name__ == "__main__":
print(independentMnode,"independentMnode valuse")
# create dnode list
dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
tdDnodes = ClusterDnodes(dnodeslist)
tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster)
tdDnodes.setValgrind(valgrind)
tdDnodes.setAsan(asan)
tdDnodes.stopAll()
clusterDnodes.init(dnodeslist, deployPath, masterIp)
clusterDnodes.setTestCluster(testCluster)
clusterDnodes.setValgrind(valgrind)
clusterDnodes.setAsan(asan)
clusterDnodes.stopAll()
for dnode in tdDnodes.dnodes:
tdDnodes.deploy(dnode.index,updateCfgDict)
for dnode in tdDnodes.dnodes: