diff --git a/Jenkinsfile2 b/Jenkinsfile2 index a46736d7f3..49d5f8884b 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -151,14 +151,6 @@ def pre_test(){ cd ${WKC} git submodule update --init --recursive ''' - sh ''' - cd ${WKPY} - git reset --hard - git pull - git log -5 - echo "python connector log: `git log -5`" >>${WKDIR}/jenkins.log - echo >>${WKDIR}/jenkins.log - ''' return 1 } def pre_test_build_mac() { @@ -409,7 +401,7 @@ pipeline { } } stage('linux test') { - agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} + agent{label " slave1_50 || slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index 8ea2f249c1..d13aa327cb 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -102,7 +102,7 @@ ulimit -c unlimited TMP_DIR=$WORKDIR/tmp SOURCEDIR=$WORKDIR/src MOUNT_DIR="" -packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz" +# packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz" rm -rf ${TMP_DIR}/thread_volume/$thread_no/sim mkdir -p $SOURCEDIR mkdir -p ${TMP_DIR}/thread_volume/$thread_no/sim/tsim @@ -114,9 +114,9 @@ if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/ fi -if [ ! -f "${SOURCEDIR}/${packageName}" ]; then - wget -P ${SOURCEDIR} https://taosdata.com/assets-download/3.0/${packageName} -fi +# if [ ! -f "${SOURCEDIR}/${packageName}" ]; then +# wget -P ${SOURCEDIR} https://taosdata.com/assets-download/3.0/${packageName} +# fi MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir" echo "$thread_no -> ${exec_dir}:$cmd" diff --git a/tests/system-test/1-insert/benchmark-tbl-rsma-alter.json b/tests/system-test/1-insert/benchmark-tbl-rsma-alter.json new file mode 100644 index 0000000000..5c8186b791 --- /dev/null +++ b/tests/system-test/1-insert/benchmark-tbl-rsma-alter.json @@ -0,0 +1,105 @@ +{ + "filetype": "insert", + "cfgdir": "/home/ben/github/taosdata/3.0/TDinternal/dnodes/dnode1/cfg", + "host": "localhost", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 20, + "create_table_thread_count": 7, + "result_file": "/root/insert_rsma_10w_1vnode.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 100, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + { + "dbinfo": { + "name": "db_replica", + "drop": "yes", + "replica": 1, + "duration": 10, + "vgroups": 2, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "wal_Level": 1, + "wal_retention_period": 0, + "retentions": "-:7d, 20s:8d, 30s:9d" + }, + "super_tables": [ + { + "name": "stb1", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "ct", + "escape_character": "yes", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 5000, + "childtable_limit": 10, + "childtable_offset": 100, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "now", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "rollup": "max", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 1, + "max": 12, + "min": 8 + }, + { + "type": "INT", + "name": "voltage", + "max": 225, + "min": 215 + }, + { + "type": "FLOAT", + "name": "phase", + "max": 1, + "min": 0 + } + ], + "tags": [ + { + "type": "INT", + "name": "tag1", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": [ + "beijing", + "shanghai" + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/system-test/1-insert/benchmark-tbl-rsma.json b/tests/system-test/1-insert/benchmark-tbl-rsma.json new file mode 100644 index 0000000000..7257dc4485 --- /dev/null +++ b/tests/system-test/1-insert/benchmark-tbl-rsma.json @@ -0,0 +1,111 @@ +{ + "filetype": "insert", + "cfgdir": "", + "host": "localhost", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 20, + "create_table_thread_count": 7, + "result_file": "", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 100, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + { + "dbinfo": { + "name": "db_update", + "drop": "yes", + "replica": 1, + "duration": 10, + "vgroups": 4, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "wal_Level": 1, + "wal_retention_period": 0, + "stt_trigger": 1, + "retentions": "-:7d, 1m:30d, 1h:60d" + }, + "super_tables": [ + { + "name": "stb1", + "child_table_exists": "no", + "childtable_count": 100, + "childtable_prefix": "ct", + "escape_character": "yes", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 3600, + "childtable_limit": 10, + "childtable_offset": 100, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 5, + "disorder_range": 1000, + "update_ratio": 5, + "delete_ratio": 1, + "disorder_fill_interval": 300, + "update_fill_interval": 25, + "generate_row_rule": 2, + "timestamp_step": 1000, + "start_timestamp": "2023-11-12 00:01:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "rollup": "max", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 1, + "max": 12, + "min": 8 + }, + { + "type": "INT", + "name": "voltage", + "max": 225, + "min": 215 + }, + { + "type": "FLOAT", + "name": "phase", + "max": 1, + "min": 0 + } + ], + "tags": [ + { + "type": "INT", + "name": "tag1", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": [ + "beijing", + "shanghai" + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index a4b2cae68a..537894b406 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -57,12 +57,16 @@ class TDTestCase: "create database db1 retentions 1s:1d", "create database db1 retentions 1s:1d,2s:2d", "create database db1 retentions 1s:1d,2s:2d,3s:3d", + "create database db1 retentions 1s:1d,2s:2d,3s:3d,4s:4d", "create database db1 retentions -:1d,2s:2d,3s:3d,4s:4d", - "create database db1 retentions -:-", "create database db1 retentions --:1d", + "create database db1 retentions -:-:1d", + "create database db1 retentions 1d:-", + "create database db1 retentions -:-", "create database db1 retentions +:1d", "create database db1 retentions :1d", "create database db1 retentions -:1d,-:2d", + "create database db1 retentions -:1d,-:2d,-:3d", "create database db1 retentions -:1d,1s:-", "create database db1 retentions -:1d,15s:2d,-:3d", @@ -76,6 +80,7 @@ class TDTestCase: "create database db1 retentions -:1d,1s:86400000a", "create database db1 retentions -:1d,1s:86400000000u", "create database db1 retentions -:1d,1s:86400000000000b", + "create database db1 retentions -:1s,1s:2s", "create database db1 retentions -:1d,1s:1w", "create database db1 retentions -:1d,1s:1n", "create database db1 retentions -:1d,1s:1y", @@ -97,6 +102,8 @@ class TDTestCase: "create database db5 retentions -:1d,2s:2d,2s:3d", "create database db5 retentions -:1d,3s:2d,2s:3d", "create database db1 retentions -:1d,2s:3d,3s:2d", + "create database db1 retentions -:1d,2s:3d,1s:2d", + ] @property diff --git a/tests/system-test/1-insert/rsma.py b/tests/system-test/1-insert/rsma.py new file mode 100644 index 0000000000..bb12035177 --- /dev/null +++ b/tests/system-test/1-insert/rsma.py @@ -0,0 +1,146 @@ +from datetime import datetime +import time + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + + +PRIMARY_COL = "ts" + +INT_COL = "c_int" +BINT_COL = "c_bint" +SINT_COL = "c_sint" +TINT_COL = "c_tint" +FLOAT_COL = "c_float" +DOUBLE_COL = "c_double" +BOOL_COL = "c_bool" +TINT_UN_COL = "c_utint" +SINT_UN_COL = "c_usint" +BINT_UN_COL = "c_ubint" +INT_UN_COL = "c_uint" +BINARY_COL = "c_binary" +NCHAR_COL = "c_nchar" +TS_COL = "c_ts" + +INT_TAG = "t_int" + +TAG_COL = [INT_TAG] + +## insert data args: +TIME_STEP = 10000 +NOW = int(datetime.timestamp(datetime.now()) * 1000) + +# init db/table +DBNAME = "db" +DB1 = "db1" +DB2 = "db2" +DB3 = "db3" +DB4 = "db4" +STBNAME = "stb1" +CTBNAME = "ct1" +NTBNAME = "nt1" + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + tagValue = 'beijing' + if (i % 2 == 0): + tagValue = 'shanghai' + sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + + def run(self): + binPath = self.getPath() + + self.rows = 10 + tdLog.printNoPrefix("==========step0:all check") + dbname='d0' + tdSql.execute(f"create database {dbname} retentions -:363d,1m:364d,1h:365d STT_TRIGGER 2 vgroups 6;") + tdSql.execute(f"create stable if not exists {dbname}.st_min (ts timestamp, c1 int) tags (proid int,city binary(20)) rollup(min) watermark 0s,1s max_delay 1m,180s;;") + tdSql.execute(f"create stable if not exists {dbname}.st_avg (ts timestamp, c1 double) tags (city binary(20),district binary(20)) rollup(min) watermark 0s,1s max_delay 1m,180s;;") + self.create_ctable(tdSql, dbname, 'st_min', 'ct_min', 1000) + tdLog.printNoPrefix("==========step4:after wal, all check again ") + datetime1 = datetime.now() + print(datetime1) + tdSql.execute(f"INSERT INTO {dbname}.ct_min7 VALUES ('2023-11-07 10:01:00.001',797029643) ('2023-11-07 10:01:00.001',797029643);") + tdSql.query(f"select * from {dbname}.st_min where ts>now-363d;") + tdSql.checkData(0, 0, '2023-11-07 10:01:00.001') + sleep(6) + tdSql.query(f"select * from {dbname}.st_min where ts>now-364d;") + tdSql.checkData(0, 0, '2023-11-07 10:01:00.000') + tdSql.query(f"select * from {dbname}.st_min where ts>now-365d;") + tdSql.checkData(0, 0, '2023-11-07 10:00:00.000') + + #bug + os.system(f"{binPath} -f ./1-insert/benchmark-tbl-rsma-alter.json") + tdSql.execute(f"alter database db_replica replica 3;") + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=3,db_name="db_replica",count_number=240) + tdSql.execute(f"alter database db_replica replica 1;") + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=1,db_name="db_replica",count_number=240) + tdSql.execute(f"alter database db_replica replica 3;") + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=3,db_name="db_replica",count_number=240) + tdSql.execute(f"alter database db_replica replica 1;") + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=1,db_name="db_replica",count_number=240) + + #bug + # os.system(f"{binPath} -f ./1-insert/benchmark-tbl-rsma.json") + # tdSql.query(f" select count(*) from db_update.stb1;") + # tdSql.checkData(0, 0, 360000) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index 85d670a325..694227cea7 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -244,7 +244,7 @@ class ClusterComCheck: else: tdLog.exit(f"vgroup number of {db_name} is not correct") if self.db_replica == 1 : - if tdSql.queryResult[0][4] == 'leader' and tdSql.queryResult[1][4] == 'leader' and tdSql.queryResult[last_number][4] == 'leader': + if tdSql.queryResult[0][4] == 'leader' and tdSql.queryResult[last_number][4] == 'leader': tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';") print("db replica :",tdSql.queryResult[0][0]) if tdSql.queryResult[0][0] == db_replica: diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 95c35363a8..54a43465e7 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -15,6 +15,9 @@ from util.common import * from util.cluster import * sys.path.append("./7-tmq") from tmqCommon import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck class TDTestCase: def __init__(self): @@ -195,6 +198,9 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) + tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py index 1c2e867758..e4353d3268 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -15,6 +15,9 @@ from util.common import * from util.cluster import * sys.path.append("./7-tmq") from tmqCommon import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck class TDTestCase: def __init__(self): @@ -196,6 +199,9 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) + tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index c76188eee2..4f3f46725a 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -16,6 +16,11 @@ from util.cluster import * sys.path.append("./7-tmq") from tmqCommon import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + + class TDTestCase: def __init__(self): self.vgroups = 1 @@ -193,6 +198,9 @@ class TDTestCase: time.sleep(2) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) + + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) tdLog.printNoPrefix("======== test case 1 end ...... ") diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py index 3214c2f5c4..cda5a27919 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -16,6 +16,13 @@ from util.cluster import * sys.path.append("./7-tmq") from tmqCommon import * +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + + + class TDTestCase: def __init__(self): self.vgroups = 1 @@ -185,7 +192,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0] or expectrowcnt < resultList[0]: + if expectrowcnt / 2 >= resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) @@ -195,6 +202,8 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py index 27d296ed0e..17a427567e 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -16,6 +16,11 @@ from util.cluster import * sys.path.append("./7-tmq") from tmqCommon import * +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + class TDTestCase: def __init__(self): self.vgroups = 1 @@ -195,6 +200,8 @@ class TDTestCase: for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self):