Merge pull request #21317 from taosdata/test/TS-3333-3.0
test: add cluster global tdDnodes
This commit is contained in:
Hui Li
GitHub
commit
f927752a5d
|
|
@ -479,6 +479,11 @@
|
|||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4
|
||||
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreQnode.py -N 5 -M 3
|
||||
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py
|
||||
|
|
|
|||
|
|
@ -93,9 +93,9 @@ class AutoGen:
|
|||
return ''.join(random.choice(letters) for i in range(count))
|
||||
|
||||
# create db
|
||||
def create_db(self, dbname):
|
||||
def create_db(self, dbname, vgroups = 2, replica = 1):
|
||||
self.dbname = dbname
|
||||
tdSql.execute(f'create database {dbname}')
|
||||
tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}')
|
||||
tdSql.execute(f'use {dbname}')
|
||||
|
||||
# create table or stable
|
||||
|
|
|
|||
|
|
@ -859,5 +859,4 @@ class TDDnodes:
|
|||
def getAsan(self):
|
||||
return self.asan
|
||||
|
||||
|
||||
tdDnodes = TDDnodes()
|
||||
tdDnodes = TDDnodes()
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
from util.log import *
|
||||
from util.cases import *
|
||||
from util.sql import *
|
||||
from util.common import *
|
||||
from util.sqlset import *
|
||||
from util.dnodes import *
|
||||
from util.autogen import *
|
||||
from util.cluster import *
|
||||
|
||||
import random
|
||||
import os
|
||||
import subprocess
|
||||
import shutil
|
||||
import time
|
||||
|
||||
|
||||
class RestoreBasic:
|
||||
# init
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdSql.init(conn.cursor())
|
||||
self.dnodes_num = 5
|
||||
|
||||
# get from global
|
||||
# test
|
||||
self.dnodes = cluster.dnodes
|
||||
num = len(self.dnodes)
|
||||
|
||||
if num < self.dnodes_num :
|
||||
tdLog.exit(f" cluster dnode is less than {self.dnodes_num}. num={num}")
|
||||
|
||||
# create data
|
||||
self.dbname = "db"
|
||||
self.stable = "st"
|
||||
self.child_count = 100
|
||||
self.insert_rows = 10000
|
||||
self.create_data()
|
||||
|
||||
# create data
|
||||
def create_data(self):
|
||||
gen = AutoGen()
|
||||
gen.create_db(self.dbname, 8, 3)
|
||||
gen.create_stable(self.stable, 5, 10, 8, 8)
|
||||
gen.create_child(self.stable, "d", self.child_count)
|
||||
gen.set_batch_size(1000)
|
||||
gen.insert_data(self.insert_rows)
|
||||
|
||||
tdSql.execute(f"flush database {self.dbname}")
|
||||
# put some duplicate ts on wal
|
||||
gen.insert_data(self.insert_rows%100)
|
||||
|
||||
for i in range(self.dnodes_num):
|
||||
sql = f"create qnode on dnode {i+1}"
|
||||
tdSql.execute(sql)
|
||||
|
||||
|
||||
# status
|
||||
def check_status_corrent(self):
|
||||
# query
|
||||
tdSql.query(f" show {self.dbname}.vgroups")
|
||||
|
||||
# check 8 vgroups
|
||||
tdSql.checkRows(8)
|
||||
|
||||
# check data corrent
|
||||
for i in range(8):
|
||||
leader = False
|
||||
for j in range(3):
|
||||
status = tdSql.getData(i, 4 + j*2)
|
||||
if status == "leader":
|
||||
leader = True
|
||||
elif status == "follower":
|
||||
pass
|
||||
else:
|
||||
tdLog.info(f" check vgroups status not leader or follower. i={i} j={j} status={status}")
|
||||
return False
|
||||
|
||||
# check leader
|
||||
if leader == False:
|
||||
tdLog.info(f" check vgroups not found leader i={i} ")
|
||||
return False
|
||||
|
||||
# info
|
||||
tdLog.info("check vgroups status successfully.")
|
||||
return True
|
||||
|
||||
# check data corrent
|
||||
def check_corrent(self):
|
||||
# check status
|
||||
status = False
|
||||
for i in range(100):
|
||||
if self.check_status_corrent():
|
||||
status = True
|
||||
break
|
||||
else:
|
||||
time.sleep(0.5)
|
||||
tdLog.info(f"sleep 500ms retry {i} to check status again...")
|
||||
|
||||
if status == False:
|
||||
tdLog.exit("check vgroups status failed, exit.")
|
||||
|
||||
# check rows count
|
||||
sql = f"select count(ts) from {self.dbname}.{self.stable}"
|
||||
tdSql.query(sql)
|
||||
tdSql.checkData(0, 0, self.child_count* self.insert_rows)
|
||||
|
||||
|
||||
# restore dnode
|
||||
def restore_dnode(self, index):
|
||||
tdLog.info(f"start restore dnode {index}")
|
||||
dnode = self.dnodes[index - 1]
|
||||
|
||||
# stop dnode
|
||||
tdLog.info(f"stop dnode {index}")
|
||||
dnode.stoptaosd()
|
||||
|
||||
# remove dnode folder
|
||||
try:
|
||||
shutil.rmtree(dnode.dataDir)
|
||||
tdLog.info(f"delete dir {dnode.dataDir} successful")
|
||||
except OSError as x:
|
||||
tdLog.exit(f"remove path {dnode.dataDir} error : {x.strerror}")
|
||||
|
||||
dnode.starttaosd()
|
||||
|
||||
# exec restore
|
||||
sql = f"restore dnode {index}"
|
||||
tdLog.info(sql)
|
||||
tdSql.execute(sql)
|
||||
self.check_corrent()
|
||||
|
||||
# restore vnode
|
||||
def restore_vnode(self, index):
|
||||
tdLog.info(f"start restore vnode on dnode {index}")
|
||||
dnode = self.dnodes[index - 1]
|
||||
del_dir = f"{dnode.dataDir}/vnode"
|
||||
|
||||
# stop dnode
|
||||
tdLog.info(f"stop dnode {index}")
|
||||
dnode.stoptaosd()
|
||||
|
||||
# remove dnode folder
|
||||
try:
|
||||
shutil.rmtree(del_dir)
|
||||
tdLog.info(f"delete dir {del_dir} successful")
|
||||
except OSError as x:
|
||||
tdLog.exit(f"remove path {del_dir} error : {x.strerror}")
|
||||
|
||||
dnode.starttaosd()
|
||||
|
||||
# exec restore
|
||||
sql = f"restore vnode on dnode {index}"
|
||||
tdLog.info(sql)
|
||||
tdSql.execute(sql)
|
||||
|
||||
# check result
|
||||
self.check_corrent()
|
||||
|
||||
|
||||
# restore mnode
|
||||
def restore_mnode(self, index):
|
||||
tdLog.info(f"start restore mnode {index}")
|
||||
dnode = self.dnodes[index - 1]
|
||||
del_dir = f"{dnode.dataDir}/mnode"
|
||||
|
||||
# stop dnode
|
||||
tdLog.info(f"stop dnode {index}")
|
||||
dnode.stoptaosd()
|
||||
|
||||
# remove dnode folder
|
||||
try:
|
||||
shutil.rmtree(del_dir)
|
||||
tdLog.info(f"delete dir {del_dir} successful")
|
||||
except OSError as x:
|
||||
tdLog.exit(f"remove path {del_dir} error : {x.strerror}")
|
||||
|
||||
dnode.starttaosd()
|
||||
|
||||
# exec restore
|
||||
sql = f"restore mnode on dnode {index}"
|
||||
tdLog.info(sql)
|
||||
tdSql.execute(sql)
|
||||
self.check_corrent()
|
||||
|
||||
|
||||
# restore qnode
|
||||
def restore_qnode(self, index):
|
||||
tdLog.info(f"start restore qnode on dnode {index}")
|
||||
dnode = self.dnodes[index - 1]
|
||||
del_dir = f"{dnode.dataDir}/qnode"
|
||||
|
||||
# stop dnode
|
||||
tdLog.info(f"stop dnode {index}")
|
||||
dnode.stoptaosd()
|
||||
|
||||
# remove dnode folder
|
||||
try:
|
||||
shutil.rmtree(del_dir)
|
||||
tdLog.info(f"delete dir {del_dir} successful")
|
||||
except OSError as x:
|
||||
tdLog.exit(f"remove path {del_dir} error : {x.strerror}")
|
||||
|
||||
# start dnode
|
||||
dnode.starttaosd()
|
||||
|
||||
# exec restore
|
||||
sql = f"restore qnode on dnode {index}"
|
||||
tdLog.info(sql)
|
||||
tdSql.execute(sql)
|
||||
self.check_corrent()
|
||||
|
||||
# path exist
|
||||
qfile = f"{del_dir}/qnode.json"
|
||||
if os.path.exists(qfile) == False:
|
||||
tdLog.exit(f"qnode restore failed. qnode.json is not exist. {qfile}")
|
||||
else:
|
||||
tdLog.info(f"check qnode.json restore ok. {qfile}")
|
||||
|
||||
# stop
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
|
||||
from util.log import *
|
||||
from util.cases import *
|
||||
from util.sql import *
|
||||
from util.common import *
|
||||
sys.path.append("./3-enterprise/restore")
|
||||
from restoreBasic import *
|
||||
|
||||
|
||||
class TDTestCase:
|
||||
# init
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
self.basic = RestoreBasic()
|
||||
self.basic.init(conn, logSql, replicaVar)
|
||||
|
||||
# run
|
||||
def run(self):
|
||||
self.basic.restore_dnode(2)
|
||||
|
||||
# stop
|
||||
def stop(self):
|
||||
self.basic.stop()
|
||||
tdLog.success("%s successfully executed" % __file__)
|
||||
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
|
||||
from util.log import *
|
||||
from util.cases import *
|
||||
from util.sql import *
|
||||
from util.common import *
|
||||
sys.path.append("./3-enterprise/restore")
|
||||
from restoreBasic import *
|
||||
|
||||
|
||||
|
||||
class TDTestCase:
|
||||
# init
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
self.basic = RestoreBasic()
|
||||
self.basic.init(conn, logSql, replicaVar)
|
||||
|
||||
# run
|
||||
def run(self):
|
||||
self.basic.restore_mnode(3)
|
||||
|
||||
# stop
|
||||
def stop(self):
|
||||
self.basic.stop()
|
||||
tdLog.success("%s successfully executed" % __file__)
|
||||
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
|
||||
from util.log import *
|
||||
from util.cases import *
|
||||
from util.sql import *
|
||||
from util.common import *
|
||||
sys.path.append("./3-enterprise/restore")
|
||||
from restoreBasic import *
|
||||
|
||||
|
||||
class TDTestCase:
|
||||
# init
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
self.basic = RestoreBasic()
|
||||
self.basic.init(conn, logSql, replicaVar)
|
||||
|
||||
# run
|
||||
def run(self):
|
||||
self.basic.restore_qnode(5)
|
||||
|
||||
# stop
|
||||
def stop(self):
|
||||
self.basic.stop()
|
||||
tdLog.success("%s successfully executed" % __file__)
|
||||
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
|
||||
from util.log import *
|
||||
from util.cases import *
|
||||
from util.sql import *
|
||||
from util.common import *
|
||||
sys.path.append("./3-enterprise/restore")
|
||||
from restoreBasic import *
|
||||
|
||||
|
||||
class TDTestCase:
|
||||
# init
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
tdLog.debug("start to execute %s" % __file__)
|
||||
self.basic = RestoreBasic()
|
||||
self.basic.init(conn, logSql, replicaVar)
|
||||
|
||||
# run
|
||||
def run(self):
|
||||
self.basic.restore_vnode(4)
|
||||
|
||||
# stop
|
||||
def stop(self):
|
||||
self.basic.stop()
|
||||
tdLog.success("%s successfully executed" % __file__)
|
||||
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
Loading…
Reference in New Issue