From ce4583e38fdf90ae4edd21016c1060274fef19f2 Mon Sep 17 00:00:00 2001 From: menshibin Date: Thu, 25 Jan 2024 13:49:56 +0800 Subject: [PATCH 1/4] add learner split vgroup case --- .../cluster/splitVgroupByLearner.json | 62 ++++++++++ .../community/cluster/splitVgroupByLearner.py | 112 ++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 tests/army/community/cluster/splitVgroupByLearner.json create mode 100644 tests/army/community/cluster/splitVgroupByLearner.py diff --git a/tests/army/community/cluster/splitVgroupByLearner.json b/tests/army/community/cluster/splitVgroupByLearner.json new file mode 100644 index 0000000000..d02cf50fda --- /dev/null +++ b/tests/army/community/cluster/splitVgroupByLearner.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 3000, + "prepared_rand": 3000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "continue_if_fail": "yes", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 3, + "duration":"1d", + "wal_retention_period": 1, + "wal_retention_size": 1, + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 100000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 10000, + "start_timestamp":"now-12d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 32} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/cluster/splitVgroupByLearner.py b/tests/army/community/cluster/splitVgroupByLearner.py new file mode 100644 index 0000000000..ce68a0c5c8 --- /dev/null +++ b/tests/army/community/cluster/splitVgroupByLearner.py @@ -0,0 +1,112 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool +import json +import threading + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame.srvCtl import * + +class TDTestCase(TBase): + def configJsonFile(self, fileName, dbName, vgroups, replica, newFileName='', insert_rows=10000000, timestamp_step=10000): + with open(fileName, 'r') as f: + data = json.load(f) + if len(newFileName) == 0: + newFileName = fileName + + data['databases']['dbinfo']['name'] = dbName + data['databases']['dbinfo']['vgroups'] = vgroups + data['databases']['dbinfo']['replica'] = replica + data['databases']['dbinfo']['replica'] = replica + data['databases']['super_tables']['insert_rows'] = insert_rows + data['databases']['super_tables']['timestamp_step'] = timestamp_step + json_data = json.dumps(data) + with open(newFileName, "w") as file: + file.write(json_data) + + def splitVgroupThread(self, configFile, event): + # self.insertData(configFile) + event.wait() + tdSql.execute('ALTER DATABASE db1 REPLICA 3') + time.sleep(5) + param_list = tdSql.query('show vgroups') + vgroupId = None + for param in param_list: + vgroupId = param[0] + tdSql.execute(f"split vgroup {vgroupId}") + # self.configJsonFile(configFile, 'db1', 1, 1, configFile, 100000000) + # self.insertData(configFile) + + def dnodeNodeStopThread(self, event): + event.wait() + time.sleep(10) + on = 2 + for i in range(5): + if i % 2 == 0: + on = 2 + else: + on = 3 + sc.dnodeStop(on) + time.sleep(5) + sc.dnodeStart(on) + time.sleep(5) + + + def dbInsertThread(self, configFile, event): + self.insertData(configFile) + event.set() + self.configJsonFile(configFile, 'db', 2, 3, configFile, 100000000) + self.insertData(configFile) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), logSql) # output sql.txt file + self.configJsonFile('splitVgroupByLearner.json', 'db', 1, 1, 'splitVgroupByLearner.json', 1000000) + + def insertData(self, configFile): + tdLog.info(f"insert data.") + # taosBenchmark run + jfile = etool.curFile(__file__, configFile) + etool.benchMark(json = jfile) + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + event = threading.Event + t1 = threading.Thread(target=self.splitVgroupThread, args=('splitVgroupByLearner1.json', event)) + t2 = threading.Thread(target=self.dbInsertThread, args=('splitVgroupByLearner.json')) + t3 = threading.Thread(target=self.dnodeNodeStopThread, args=(event)) + t1.join() + t2.join() + t3.join() + tdLog.success(f"{__file__} successfully executed") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From c03c0af0bc3363b0446bc25a836e99f2b8b89723 Mon Sep 17 00:00:00 2001 From: menshibin Date: Thu, 25 Jan 2024 17:39:43 +0800 Subject: [PATCH 2/4] add learner split vgroup case --- .../community/cluster/splitVgroupByLearner.py | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/tests/army/community/cluster/splitVgroupByLearner.py b/tests/army/community/cluster/splitVgroupByLearner.py index ce68a0c5c8..5f75db2db5 100644 --- a/tests/army/community/cluster/splitVgroupByLearner.py +++ b/tests/army/community/cluster/splitVgroupByLearner.py @@ -29,38 +29,58 @@ from frame import * from frame.autogen import * from frame.srvCtl import * + class TDTestCase(TBase): - def configJsonFile(self, fileName, dbName, vgroups, replica, newFileName='', insert_rows=10000000, timestamp_step=10000): - with open(fileName, 'r') as f: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to init {__file__}") + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + self.configJsonFile('splitVgroupByLearner.json', 'db', 1, 1, 'splitVgroupByLearner.json', 100000) + + def configJsonFile(self, fileName, dbName, vgroups, replica, newFileName='', insert_rows=100000, + timestamp_step=10000): + tdLog.debug(f"configJsonFile {fileName}") + filePath = etool.curFile(__file__, fileName) + with open(filePath, 'r') as f: data = json.load(f) + if len(newFileName) == 0: newFileName = fileName - data['databases']['dbinfo']['name'] = dbName - data['databases']['dbinfo']['vgroups'] = vgroups - data['databases']['dbinfo']['replica'] = replica - data['databases']['dbinfo']['replica'] = replica - data['databases']['super_tables']['insert_rows'] = insert_rows - data['databases']['super_tables']['timestamp_step'] = timestamp_step + data['databases'][0]['dbinfo']['name'] = dbName + data['databases'][0]['dbinfo']['vgroups'] = vgroups + data['databases'][0]['dbinfo']['replica'] = replica + data['databases'][0]['super_tables'][0]['insert_rows'] = insert_rows + data['databases'][0]['super_tables'][0]['timestamp_step'] = timestamp_step json_data = json.dumps(data) - with open(newFileName, "w") as file: + filePath = etool.curFile(__file__, newFileName) + with open(filePath, "w") as file: file.write(json_data) + tdLog.debug(f"configJsonFile {json_data}") + def splitVgroupThread(self, configFile, event): # self.insertData(configFile) event.wait() - tdSql.execute('ALTER DATABASE db1 REPLICA 3') time.sleep(5) - param_list = tdSql.query('show vgroups') - vgroupId = None - for param in param_list: - vgroupId = param[0] - tdSql.execute(f"split vgroup {vgroupId}") - # self.configJsonFile(configFile, 'db1', 1, 1, configFile, 100000000) + tdLog.debug("splitVgroupThread start") + tdSql.execute('ALTER DATABASE db REPLICA 3') + time.sleep(5) + tdSql.execute('use db') + rowLen = tdSql.query('show vgroups') + if rowLen > 0: + vgroupId = tdSql.getData(0, 0) + tdLog.debug(f"splitVgroupThread vgroupId:{vgroupId}") + tdSql.execute(f"split vgroup {vgroupId}") + else: + tdLog.exit("get vgroupId fail!") + # self.configJsonFile(configFile, 'db1', 1, 1, configFile, 100000000) # self.insertData(configFile) def dnodeNodeStopThread(self, event): event.wait() + tdLog.debug("dnodeNodeStopThread start") time.sleep(10) on = 2 for i in range(5): @@ -73,32 +93,32 @@ class TDTestCase(TBase): sc.dnodeStart(on) time.sleep(5) - def dbInsertThread(self, configFile, event): - self.insertData(configFile) - event.set() - self.configJsonFile(configFile, 'db', 2, 3, configFile, 100000000) + tdLog.debug(f"dbInsertThread start {configFile}") self.insertData(configFile) - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), logSql) # output sql.txt file - self.configJsonFile('splitVgroupByLearner.json', 'db', 1, 1, 'splitVgroupByLearner.json', 1000000) + event.set() + tdLog.debug(f"dbInsertThread first end {event}") + self.configJsonFile(configFile, 'db', 2, 3, configFile, 100000) + self.insertData(configFile) def insertData(self, configFile): tdLog.info(f"insert data.") # taosBenchmark run jfile = etool.curFile(__file__, configFile) - etool.benchMark(json = jfile) + etool.benchMark(json=jfile) # run def run(self): tdLog.debug(f"start to excute {__file__}") - event = threading.Event - t1 = threading.Thread(target=self.splitVgroupThread, args=('splitVgroupByLearner1.json', event)) - t2 = threading.Thread(target=self.dbInsertThread, args=('splitVgroupByLearner.json')) + event = threading.Event() + t1 = threading.Thread(target=self.splitVgroupThread, args=('splitVgroupByLearner.json', event)) + t2 = threading.Thread(target=self.dbInsertThread, args=('splitVgroupByLearner.json', event)) t3 = threading.Thread(target=self.dnodeNodeStopThread, args=(event)) + t1.start() + t2.start() + t3.start() + tdLog.debug("threading started!!!!!") t1.join() t2.join() t3.join() @@ -108,5 +128,6 @@ class TDTestCase(TBase): tdSql.close() tdLog.success(f"{__file__} successfully executed") + tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From ac8bb20389c87b75f24ea3dd38d10b1c978d5f83 Mon Sep 17 00:00:00 2001 From: menshibin Date: Fri, 26 Jan 2024 10:03:03 +0800 Subject: [PATCH 3/4] add learner split vgroup case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 79bec1ec76..80dde742b3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -21,7 +21,7 @@ fi ,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 - +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py From ba3f2ff0d2edf338c7211b9eb02d8db3642cf4c4 Mon Sep 17 00:00:00 2001 From: menshibin Date: Fri, 26 Jan 2024 13:02:46 +0800 Subject: [PATCH 4/4] add learner split vgroup case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 80dde742b3..d932529d0a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -21,7 +21,7 @@ fi ,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 -,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py