revote leader when stop one dnode
This commit is contained in:
wenzhouwww@live.cn
parent
74feec707b
commit
ff06586b2c
|
|
@ -0,0 +1,364 @@
|
|||
# author : wenzhouwww
|
||||
from errno import ESOCKTNOSUPPORT
|
||||
from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import TDDnodes
|
||||
from util.dnodes import TDDnode
|
||||
from util.cluster import *
|
||||
|
||||
import time
|
||||
import random
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
class TDTestCase:
|
||||
def init(self,conn ,logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
self.host = socket.gethostname()
|
||||
self.mnode_list = {}
|
||||
self.dnode_list = {}
|
||||
self.ts = 1483200000000
|
||||
self.db_name ='testdb'
|
||||
self.replica = 1
|
||||
self.vgroups = 2
|
||||
self.tb_nums = 10
|
||||
self.row_nums = 100
|
||||
self.max_vote_time_cost = 10 # seconds
|
||||
self.stop_dnode = None
|
||||
|
||||
def getBuildPath(self):
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
if ("community" in selfPath):
|
||||
projPath = selfPath[:selfPath.find("community")]
|
||||
else:
|
||||
projPath = selfPath[:selfPath.find("tests")]
|
||||
|
||||
for root, dirs, files in os.walk(projPath):
|
||||
if ("taosd" in files):
|
||||
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||
if ("packaging" not in rootRealPath):
|
||||
buildPath = root[:len(root) - len("/build/bin")]
|
||||
break
|
||||
return buildPath
|
||||
|
||||
def check_setup_cluster_status(self):
|
||||
tdSql.query("show mnodes")
|
||||
for mnode in tdSql.queryResult:
|
||||
name = mnode[1]
|
||||
info = mnode
|
||||
self.mnode_list[name] = info
|
||||
|
||||
tdSql.query("show dnodes")
|
||||
for dnode in tdSql.queryResult:
|
||||
name = dnode[1]
|
||||
info = dnode
|
||||
self.dnode_list[name] = info
|
||||
|
||||
count = 0
|
||||
is_leader = False
|
||||
mnode_name = ''
|
||||
for k,v in self.mnode_list.items():
|
||||
count +=1
|
||||
# only for 1 mnode
|
||||
mnode_name = k
|
||||
|
||||
if v[2] =='leader':
|
||||
is_leader=True
|
||||
|
||||
if count==1 and is_leader:
|
||||
tdLog.info("===== depoly cluster success with 1 mnode as leader =====")
|
||||
else:
|
||||
tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====")
|
||||
|
||||
for k ,v in self.dnode_list.items():
|
||||
if k == mnode_name:
|
||||
if v[3]==0:
|
||||
tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3]))
|
||||
else:
|
||||
tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3]))
|
||||
else:
|
||||
continue
|
||||
|
||||
def create_db_check_vgroups(self):
|
||||
|
||||
tdSql.execute("drop database if exists test")
|
||||
tdSql.execute("create database if not exists test replica 1 duration 300")
|
||||
tdSql.execute("use test")
|
||||
tdSql.execute(
|
||||
'''create table stb1
|
||||
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
|
||||
tags (t1 int)
|
||||
'''
|
||||
)
|
||||
tdSql.execute(
|
||||
'''
|
||||
create table t1
|
||||
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
|
||||
'''
|
||||
)
|
||||
|
||||
for i in range(5):
|
||||
tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i))
|
||||
tdSql.query("show stables")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.query("show tables")
|
||||
tdSql.checkRows(6)
|
||||
|
||||
tdSql.query("show test.vgroups;")
|
||||
vgroups_infos = {} # key is id: value is info list
|
||||
for vgroup_info in tdSql.queryResult:
|
||||
vgroup_id = vgroup_info[0]
|
||||
tmp_list = []
|
||||
for role in vgroup_info[3:-4]:
|
||||
if role in ['leader','follower']:
|
||||
tmp_list.append(role)
|
||||
vgroups_infos[vgroup_id]=tmp_list
|
||||
|
||||
for k , v in vgroups_infos.items():
|
||||
if len(v) ==1 and v[0]=="leader":
|
||||
tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k))
|
||||
else:
|
||||
tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k))
|
||||
|
||||
def _get_stop_dnode(self):
|
||||
only_dnode_list = self.dnode_list.keys() - self.mnode_list.keys()
|
||||
self.stop_dnode = random.sample(only_dnode_list , 1 )[0]
|
||||
return self.stop_dnode
|
||||
|
||||
|
||||
def check_vgroups_revote_leader(self,dbname):
|
||||
|
||||
status = True
|
||||
stop_dnode_id = self.dnode_list[self.stop_dnode][0]
|
||||
|
||||
tdSql.query("show {}.vgroups".format(dbname))
|
||||
for vgroup_info in tdSql.queryResult:
|
||||
vgroup_id = vgroup_info[0]
|
||||
vgroup_status = []
|
||||
vgroups_leader_follower = vgroup_info[3:-4]
|
||||
for ind , role in enumerate(vgroups_leader_follower):
|
||||
|
||||
if ind%2==0:
|
||||
if role == stop_dnode_id and vgroups_leader_follower[ind+1]=="offline":
|
||||
tdLog.info("====== dnode {} has offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode))
|
||||
elif role == stop_dnode_id :
|
||||
tdLog.exit("====== dnode {} has not offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode))
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
vgroup_status.append(role)
|
||||
if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=1 or vgroup_status.count("offline")!=1:
|
||||
status = False
|
||||
return status
|
||||
return status
|
||||
|
||||
|
||||
def wait_stop_dnode_OK(self):
|
||||
|
||||
def _get_status():
|
||||
|
||||
status = ""
|
||||
tdSql.query("show dnodes")
|
||||
dnode_infos = tdSql.queryResult
|
||||
for dnode_info in dnode_infos:
|
||||
endpoint = dnode_info[1]
|
||||
dnode_status = dnode_info[4]
|
||||
if endpoint == self.stop_dnode:
|
||||
status = dnode_status
|
||||
break
|
||||
return status
|
||||
|
||||
status = _get_status()
|
||||
while status !="offline":
|
||||
time.sleep(0.1)
|
||||
status = _get_status()
|
||||
# tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode))
|
||||
tdLog.info("==== stop_dnode has stopped , endpoint is {}".format(self.stop_dnode))
|
||||
|
||||
def wait_start_dnode_OK(self):
|
||||
|
||||
def _get_status():
|
||||
|
||||
status = ""
|
||||
tdSql.query("show dnodes")
|
||||
dnode_infos = tdSql.queryResult
|
||||
for dnode_info in dnode_infos:
|
||||
endpoint = dnode_info[1]
|
||||
dnode_status = dnode_info[4]
|
||||
if endpoint == self.stop_dnode:
|
||||
status = dnode_status
|
||||
break
|
||||
return status
|
||||
|
||||
status = _get_status()
|
||||
while status !="ready":
|
||||
time.sleep(0.1)
|
||||
status = _get_status()
|
||||
# tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode))
|
||||
tdLog.info("==== stop_dnode has restart , endpoint is {}".format(self.stop_dnode))
|
||||
|
||||
|
||||
|
||||
def random_stop_One_dnode(self):
|
||||
self.stop_dnode = self._get_stop_dnode()
|
||||
stop_dnode_id = self.dnode_list[self.stop_dnode][0]
|
||||
tdLog.info(" ==== dnode {} will offline ,endpoints is {} ====".format(stop_dnode_id , self.stop_dnode))
|
||||
tdDnodes=cluster.dnodes
|
||||
tdDnodes[stop_dnode_id-1].stoptaosd()
|
||||
self.wait_stop_dnode_OK()
|
||||
# os.system("taos -s 'show dnodes;'")
|
||||
|
||||
def Restart_stop_dnode(self):
|
||||
|
||||
tdDnodes=cluster.dnodes
|
||||
stop_dnode_id = self.dnode_list[self.stop_dnode][0]
|
||||
tdDnodes[stop_dnode_id-1].starttaosd()
|
||||
self.wait_start_dnode_OK()
|
||||
# os.system("taos -s 'show dnodes;'")
|
||||
|
||||
def check_vgroups_init_done(self,dbname):
|
||||
|
||||
status = True
|
||||
|
||||
tdSql.query("show {}.vgroups".format(dbname))
|
||||
for vgroup_info in tdSql.queryResult:
|
||||
vgroup_id = vgroup_info[0]
|
||||
vgroup_status = []
|
||||
for ind , role in enumerate(vgroup_info[3:-4]):
|
||||
|
||||
if ind%2==0:
|
||||
continue
|
||||
else:
|
||||
vgroup_status.append(role)
|
||||
if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=2:
|
||||
status = False
|
||||
return status
|
||||
return status
|
||||
|
||||
def vote_leader_time_costs(self,dbname):
|
||||
start = time.time()
|
||||
status = self.check_vgroups_init_done(dbname)
|
||||
while not status:
|
||||
time.sleep(0.1)
|
||||
status = self.check_vgroups_init_done(dbname)
|
||||
|
||||
# tdLog.info("=== database {} show vgroups vote the leader is in progress ===".format(dbname))
|
||||
end = time.time()
|
||||
cost_time = end - start
|
||||
tdLog.info(" ==== database %s vote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) )
|
||||
# os.system("taos -s 'show {}.vgroups;'".format(dbname))
|
||||
if cost_time >= self.max_vote_time_cost:
|
||||
tdLog.exit(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) )
|
||||
|
||||
return cost_time
|
||||
|
||||
|
||||
def revote_leader_time_costs(self,dbname):
|
||||
start = time.time()
|
||||
|
||||
status = self.check_vgroups_revote_leader(dbname)
|
||||
while not status:
|
||||
time.sleep(0.1)
|
||||
status = self.check_vgroups_revote_leader(dbname)
|
||||
|
||||
# tdLog.info("=== database {} show vgroups vote the leader is in progress ===".format(dbname))
|
||||
end = time.time()
|
||||
cost_time = end - start
|
||||
tdLog.info(" ==== database %s revote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) )
|
||||
# os.system("taos -s 'show {}.vgroups;'".format(dbname))
|
||||
if cost_time >= self.max_vote_time_cost:
|
||||
tdLog.exit(" ==== database %s revote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) )
|
||||
|
||||
|
||||
return cost_time
|
||||
|
||||
def exec_revote_action(self,dbname):
|
||||
|
||||
tdSql.query("show {}.vgroups".format(dbname))
|
||||
before_revote = tdSql.queryResult
|
||||
|
||||
before_vgroups = set()
|
||||
for vgroup_info in before_revote:
|
||||
before_vgroups.add(vgroup_info[3:-4])
|
||||
|
||||
self.random_stop_One_dnode()
|
||||
tdSql.query("show {}.vgroups".format(dbname))
|
||||
after_revote = tdSql.queryResult
|
||||
|
||||
after_vgroups = set()
|
||||
for vgroup_info in after_revote:
|
||||
after_vgroups.add(vgroup_info[3:-4])
|
||||
|
||||
vote_act = set(set(after_vgroups)-set(before_vgroups))
|
||||
if not vote_act:
|
||||
tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====")
|
||||
else:
|
||||
for vgroup_info in vote_act:
|
||||
for ind , role in enumerate(vgroup_info):
|
||||
if role==self.dnode_list[self.stop_dnode][0]:
|
||||
|
||||
if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info:
|
||||
tdLog.info(" === revote leader ok , leader is {} now ====".format(list(vgroup_info).index("leader")-1))
|
||||
elif vgroup_info[ind+1] !="offline":
|
||||
tdLog.exit(" === dnode {} should be offline ".format(self.stop_dnode))
|
||||
else:
|
||||
continue
|
||||
break
|
||||
|
||||
|
||||
|
||||
self.revote_leader_time_costs(dbname)
|
||||
self.Restart_stop_dnode()
|
||||
def test_init_vgroups_time_costs(self):
|
||||
|
||||
tdLog.info(" ====start check time cost about vgroups vote leaders ==== ")
|
||||
tdLog.info(" ==== current max time cost is set value : {} =======".format(self.max_vote_time_cost))
|
||||
|
||||
# create database replica 3 vgroups 1
|
||||
|
||||
db1 = 'db_1'
|
||||
create_db_replica_3_vgroups_1 = "create database {} replica 3 vgroups 1".format(db1)
|
||||
tdLog.info('=======database {} replica 3 vgroups 1 ======'.format(db1))
|
||||
tdSql.execute(create_db_replica_3_vgroups_1)
|
||||
self.vote_leader_time_costs(db1)
|
||||
self.exec_revote_action(db1)
|
||||
|
||||
# create database replica 3 vgroups 10
|
||||
db2 = 'db_2'
|
||||
create_db_replica_3_vgroups_10 = "create database {} replica 3 vgroups 10".format(db2)
|
||||
tdLog.info('=======database {} replica 3 vgroups 10 ======'.format(db2))
|
||||
tdSql.execute(create_db_replica_3_vgroups_10)
|
||||
self.vote_leader_time_costs(db2)
|
||||
self.exec_revote_action(db2)
|
||||
|
||||
# # create database replica 3 vgroups 100
|
||||
# db3 = 'db_3'
|
||||
# create_db_replica_3_vgroups_100 = "create database {} replica 3 vgroups 100".format(db3)
|
||||
# tdLog.info('=======database {} replica 3 vgroups 100 ======'.format(db3))
|
||||
# tdSql.execute(create_db_replica_3_vgroups_100)
|
||||
# self.vote_leader_time_costs(db3)
|
||||
# self.exec_revote_action(db3)
|
||||
|
||||
|
||||
|
||||
def run(self):
|
||||
self.check_setup_cluster_status()
|
||||
self.test_init_vgroups_time_costs()
|
||||
|
||||
|
||||
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
Loading…
Reference in New Issue