From fcc8185e452d37bf38e85ca3cf66cb4e5e2aa3d2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 11:35:39 +0800 Subject: [PATCH 1/5] coverage: add snapshot.json case --- tests/army/community/cluster/snapshot.json | 60 +++++++++++++++++ tests/army/community/cluster/snapshot.py | 76 ++++++++++++++++++++++ tests/army/frame/caseBase.py | 5 -- tests/parallel_test/cases.task | 2 +- 4 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 tests/army/community/cluster/snapshot.json create mode 100644 tests/army/community/cluster/snapshot.py diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json new file mode 100644 index 0000000000..56a3630bca --- /dev/null +++ b/tests/army/community/cluster/snapshot.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 3, + "duration":"1d", + "wal_retention_period": 1, + "wal_retention_size": 1, + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 10000, + "start_timestamp":"now-12d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 32} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py new file mode 100644 index 0000000000..984ea0bece --- /dev/null +++ b/tests/army/community/cluster/snapshot.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + jfile = etool.curFile(__file__, "snapshot.json") + etool.runBenchmark(json=jfile) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 10 + self.insert_rows = 100000 + self.timestamp_step = 10000 + + def doAction(self): + tdLog.info(f"do action.") + self.flushDb() + self.trimDb() + self.compactDb() + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() + + # check insert correct again + self.checkInsertCorrect() + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index ad6e744fe6..ece3d2ccc3 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -42,11 +42,6 @@ class TBase: self.db = "db" self.stb = "stb" - # variant in taosBenchmark json - self.childtable_count = 2 - self.insert_rows = 1000000 - self.timestamp_step = 1000 - # sql self.sqlSum = f"select sum(ic) from {self.stb}" self.sqlMax = f"select max(ic) from {self.stb}" diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e7395a26f1..093fec78ab 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -10,7 +10,7 @@ # ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 - +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 # From aed127ab0b92e4b99773ff6315717b573e569a20 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 16:53:57 +0800 Subject: [PATCH 2/5] coverage: snapshot add balance operator --- tests/army/community/cluster/snapshot.json | 2 +- tests/army/community/cluster/snapshot.py | 16 ++++ tests/army/frame/caseBase.py | 91 +++++++++++++++++++++- tools/shell/src/shellAuto.c | 8 +- 4 files changed, 109 insertions(+), 8 deletions(-) diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json index 56a3630bca..64bb2aaf3c 100644 --- a/tests/army/community/cluster/snapshot.json +++ b/tests/army/community/cluster/snapshot.json @@ -36,7 +36,7 @@ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, { "type": "double", "name": "dc"}, - { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, { "type": "int", "name": "ic" }, { "type": "bigint", "name": "bi" }, diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 984ea0bece..40783ad4ce 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -13,11 +13,13 @@ import sys import time +import random import taos import frame import frame.etool + from frame.log import * from frame.cases import * from frame.sql import * @@ -43,7 +45,21 @@ class TDTestCase(TBase): def doAction(self): tdLog.info(f"do action.") self.flushDb() + + # split vgroups + self.splitVGroups() self.trimDb() + + # balance vgroups + self.balanceVGroupLeader() + + # replica to 1 + self.alterReplica(1) + + vgids = self.getVGroup() + selid = random.choice(vgids) + self.balanceVGroupLeaderOn(selid) + self.compactDb() # run diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index ece3d2ccc3..05ccf975a7 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -15,6 +15,7 @@ import sys import os import time import datetime +import random from frame.log import * from frame.sql import * @@ -47,7 +48,8 @@ class TBase: self.sqlMax = f"select max(ic) from {self.stb}" self.sqlMin = f"select min(ic) from {self.stb}" self.sqlAvg = f"select avg(ic) from {self.stb}" - + self.sqlFirst = f"select first(ts) from {self.stb}" + self.sqlLast = f"select last(ts) from {self.stb}" # stop def stop(self): @@ -70,6 +72,50 @@ class TBase: def dropDb(self, show = False): tdSql.execute(f"drop database {self.db}", show = show) + def splitVGroups(self): + vgids = self.getVGroup(self.db) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + + def alterReplica(self, replica): + sql = f"alter database {self.db} replica {replica}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + def balanceVGroup(self): + sql = f"balance vgroup" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + def balanceVGroupLeader(self): + sql = f"balance vgroup leader" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + + + def balanceVGroupLeaderOn(self, vgId): + sql = f"balance vgroup leader on {vgId}" + tdSql.execute(sql, show=True) + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + return True + # # check db correct @@ -90,12 +136,13 @@ class TBase: tdSql.checkAgg(sql, 0) # save agg result - def snapshotAgg(self): - + def snapshotAgg(self): self.sum = tdSql.getFirstValue(self.sqlSum) self.avg = tdSql.getFirstValue(self.sqlAvg) self.min = tdSql.getFirstValue(self.sqlMin) self.max = tdSql.getFirstValue(self.sqlMax) + self.first = tdSql.getFirstValue(self.sqlFirst) + self.last = tdSql.getFirstValue(self.sqlLast) # check agg def checkAggCorrect(self): @@ -103,3 +150,41 @@ class TBase: tdSql.checkAgg(self.sqlAvg, self.avg) tdSql.checkAgg(self.sqlMin, self.min) tdSql.checkAgg(self.sqlMax, self.max) + tdSql.checkAgg(self.sqlFirst, self.first) + tdSql.checkAgg(self.sqlLast, self.last) + + +# +# get db information +# + + # get vgroups + def getVGroup(self, db_name): + vgidList = [] + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + res = tdSql.getResult(sql) + rows = len(res) + for i in range(rows): + vgidList.append(res[i][0]) + + return vgidList + + + +# +# util +# + + # wait transactions count to zero , return False is translation not finished + def waitTransactionZero(self, seconds = 300, interval = 1): + # wait end + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("split vgroup finished.") + return True + #tdLog.info(f"i={i} wait ...") + time.sleep(interval) + + return False diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index bd5329d810..e9b9b9e944 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -84,8 +84,8 @@ SWords shellCommands[] = { {"alter topic", 0, 0, NULL}, {"alter user ;", 0, 0, NULL}, #ifdef TD_ENTERPRISE - {"balance vgroup;", 0, 0, NULL}, - {"balance vgroup leader ", 0, 0, NULL}, + {"balance vgroup ;", 0, 0, NULL}, + {"balance vgroup leader on ", 0, 0, NULL}, #endif // 20 @@ -531,8 +531,8 @@ void showHelp() { printf( "\n\n\ ----- special commands on enterpise version ----- \n\ - balance vgroup; \n\ - balance vgroup leader \n\ + balance vgroup ;\n\ + balance vgroup leader on \n\ compact database ; \n\ redistribute vgroup dnode ;\n\ split vgroup ;"); From d318c8f9831db04533bd2e4821e49ffe60d29aa5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:04:18 +0800 Subject: [PATCH 3/5] fix: add data correct check point --- tests/army/community/cluster/snapshot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 40783ad4ce..66f9d9f598 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -49,14 +49,16 @@ class TDTestCase(TBase): # split vgroups self.splitVGroups() self.trimDb() + self.checkAggCorrect() # balance vgroups self.balanceVGroupLeader() # replica to 1 self.alterReplica(1) + self.checkAggCorrect() - vgids = self.getVGroup() + vgids = self.getVGroup(self.db) selid = random.choice(vgids) self.balanceVGroupLeaderOn(selid) From 6dba2a7aefe52a1dfd9529b17002670f7e1b4570 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:11:47 +0800 Subject: [PATCH 4/5] fix: add replica to 3 --- tests/army/community/cluster/snapshot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 66f9d9f598..b177ba3b29 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -57,12 +57,14 @@ class TDTestCase(TBase): # replica to 1 self.alterReplica(1) self.checkAggCorrect() + self.compactDb() + self.alterReplica(3) vgids = self.getVGroup(self.db) selid = random.choice(vgids) self.balanceVGroupLeaderOn(selid) - self.compactDb() + # run def run(self): From 902529352dd361b2d9625ec52e3057d1ea10c36a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 31 Dec 2023 17:16:54 +0800 Subject: [PATCH 5/5] fix: transaction tips --- tests/army/frame/caseBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 05ccf975a7..8d2c1e6d18 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -182,7 +182,7 @@ class TBase: sql ="show transactions;" rows = tdSql.query(sql) if rows == 0: - tdLog.info("split vgroup finished.") + tdLog.info("transaction count became zero.") return True #tdLog.info(f"i={i} wait ...") time.sleep(interval)