From 5bb0a3d04b1ee594f0e4b647fa0859ed5d99a171 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Sat, 5 Jun 2021 17:17:13 +0800 Subject: [PATCH 1/8] [TD-4473] initial git --- tests/pytest/alter/alter_keep.py | 63 ++++++++++++----- .../manualTest/manual_chang_time_1_1.py | 67 +++++++++++++++++++ .../manual_change_time_1_1_A.json | 60 +++++++++++++++++ .../manual_change_time_1_1_B.json | 60 +++++++++++++++++ 4 files changed, 235 insertions(+), 15 deletions(-) create mode 100644 tests/pytest/manualTest/manual_chang_time_1_1.py create mode 100644 tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json create mode 100644 tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index cb630963a7..f9c71ed4ff 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -116,25 +116,58 @@ class TDTestCase: ##TODO: need to wait for TD-4445 to implement the following ## tests - # tdSql.prepare() - # tdSql.execute('create table tb (ts timestamp, speed int)') - # tdSql.execute('alter database db keep 10,10,10') - # tdSql.execute('insert into tb values (now, 10)') - # tdSql.execute('insert into tb values (now + 10m, 10)') - # tdSql.query('select * from tb') - # tdSql.checkRows(2) - # tdSql.execute('alter database db keep 40,40,40') - # tdSql.query('show databases') - # tdSql.checkData(0,7,'40,40,40') - # tdSql.error('insert into tb values (now-60d, 10)') - # tdSql.execute('insert into tb values (now-30d, 10)') - # tdSql.query('select * from tb') - # tdSql.checkRows(3) + + ## preset the keep + tdSql.prepare() + tdSql.execute('create table tb (ts timestamp, speed int)') + tdSql.execute('alter database db keep 10,10,10') + tdSql.execute('insert into tb values (now, 10)') + tdSql.execute('insert into tb values (now + 10m, 10)') + tdSql.query('select * from tb') + tdSql.checkRows(2) + + + #after alter from small to large, check if the alter if functioning + #test if change through test.py is consistent with change from taos client + #test case for TD-4459 and TD-4445 + tdSql.execute('alter database db keep 40,40,40') + tdSql.query('show databases') + tdSql.checkData(0,7,'40,40,40') + tdSql.error('insert into tb values (now-60d, 10)') + tdSql.execute('insert into tb values (now-30d, 10)') + tdSql.query('select * from tb') + tdSql.checkRows(3) + + rowNum = 3 + for i in range(30): + rowNum += 1 + tdSql.execute('alter database db keep 20,20,20') + tdSql.execute('alter database db keep 40,40,40') + tdSql.query('show databases') + tdSql.checkData(0,7,'40,40,40') + tdSql.error('insert into tb values (now-60d, 10)') + tdSql.execute('insert into tb values (now-30d, 10)') + tdSql.query('select * from tb') + tdSql.checkRows(rowNum) + + tdSql.execute('alter database db keep 10,10,10') + tdSql.query('show databases') + tdSql.checkData(0,7,'10,10,10') + tdSql.execute('alter database db keep 15,15,15') + tdSql.query('show databases') + tdSql.checkData(0,7,'15,15,15') + tdSql.execute('insert into tb values (now-30d, 10)') + tdSql.query('select * from tb') + tdSql.checkRows(rowNum + 1) + # tdSql.execute('alter database db keep 20,20,20') # tdSql.query('show databases') # tdSql.checkData(0,7,'20,20,20') + # tdSql.error('insert into tb values (now-30d, 10)') + # tdSql.query('show databases') + # tdSql.checkData(0,7,'20,20,20') # tdSql.query('select * from tb') - # tdSql.checkRows(2) + # tdSql.checkRows(rowNum) def stop(self): diff --git a/tests/pytest/manualTest/manual_chang_time_1_1.py b/tests/pytest/manualTest/manual_chang_time_1_1.py new file mode 100644 index 0000000000..0d82f7c871 --- /dev/null +++ b/tests/pytest/manualTest/manual_chang_time_1_1.py @@ -0,0 +1,67 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes + +##TODO: auto test version is currently unsupported, need to come up with +# an auto test version in the future +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + ## change system time to 2020/10/20 + os.system ('timedatectl set-ntp off') + os.system ('timedatectl set-time 2020-10-20') + + #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 + # + os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) + input("please check memory usage for taosd. After checking, press enter") + + def stop(self): + tdSql.close() + tdLog.debug("%s alter block manual check finish" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json new file mode 100644 index 0000000000..7b8abd6d4e --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 32766, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 1, + "cache": 4, + "blocks": 3, + "precision": "ms", + "keep": 10, + "minRows": 1000, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 864000, + "childtable_limit": 1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2020-10-11 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "BINARY", "len": 5120, "count":1}], + "tags": [{"type": "TINYINT", "count":2}] + }] + }] +} \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json new file mode 100644 index 0000000000..846730acfb --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 32766, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 1, + "cache": 4, + "blocks": 3, + "precision": "ms", + "keep": 10, + "minRows": 1000, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"yes", + "childtable_count": 1, + "childtable_prefix": "stb_", + "auto_create_table": "yes", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 86400, + "childtable_limit": 1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2020-10-17 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "BINARY", "len": 5120, "count":1}], + "tags": [{"type": "TINYINT", "count":1}] + }] + }] +} \ No newline at end of file From ed50bd47c2cdc92507dc0df68fdff880e07ef1c2 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Mon, 7 Jun 2021 09:32:23 +0800 Subject: [PATCH 2/8] [TD-4473] updating test cases --- .../manualTest/manual_chang_time_1_1.py | 28 +++++- .../manualTest/manual_chang_time_1_2.py | 86 +++++++++++++++++++ .../manual_chang_time_1_auto_test.py | 77 +++++++++++++++++ .../manual_change_time_1_1_B.json | 4 +- 4 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 tests/pytest/manualTest/manual_chang_time_1_2.py create mode 100644 tests/pytest/manualTest/manual_chang_time_1_auto_test.py diff --git a/tests/pytest/manualTest/manual_chang_time_1_1.py b/tests/pytest/manualTest/manual_chang_time_1_1.py index 0d82f7c871..69ea6f349b 100644 --- a/tests/pytest/manualTest/manual_chang_time_1_1.py +++ b/tests/pytest/manualTest/manual_chang_time_1_1.py @@ -16,6 +16,8 @@ from util.log import * from util.cases import * from util.sql import * from util.dnodes import tdDnodes +from datetime import datetime +import subprocess ##TODO: auto test version is currently unsupported, need to come up with # an auto test version in the future @@ -54,9 +56,31 @@ class TDTestCase: os.system ('timedatectl set-time 2020-10-20') #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 - # + #11 data files should be generated + #vnode at TDinternal/community/sim/dnode1/data/vnode os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) - input("please check memory usage for taosd. After checking, press enter") + input("please the data file. After checking, press enter") + + tdSql.query('select first(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) + tdSql.query('select last(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,20,23,59,59,0)) + + os.system ('timedatectl set-time 2020-10-25') + + #run taosdemo to insert data. one row per second from 2020/10/16 to 2020/10/20 + #4 oldest data file should be removed from tsdb/data + #vnode at TDinternal/community/sim/dnode1/data/vnode + #7 data file should be found + os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json" % binPath) + input("please the data file. After checking, press enter") + + tdSql.query('select first(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) + + ##test results + #2021/06/05 first check: 11 data files second check: 7 data files + # confirm with the assumption Baosheng Chang def stop(self): tdSql.close() diff --git a/tests/pytest/manualTest/manual_chang_time_1_2.py b/tests/pytest/manualTest/manual_chang_time_1_2.py new file mode 100644 index 0000000000..26661f7314 --- /dev/null +++ b/tests/pytest/manualTest/manual_chang_time_1_2.py @@ -0,0 +1,86 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from datetime import datetime + +##TODO: auto test version is currently unsupported, need to come up with +# an auto test version in the future +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + ## change system time to 2020/10/20 + os.system ('timedatectl set-ntp off') + os.system ('timedatectl set-time 2020-10-20') + + #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 + #11 data files should be generated + #vnode at TDinternal/community/sim/dnode1/data/vnode + os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) + input("please the data file. After checking, press enter") + + tdSql.query('select first(ts) from stb_0') #check the last data in the database + tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) + + os.system ('timedatectl set-time 2020-10-25') + + #restart taosd + #4 oldest data file should be removed from tsdb/data + #vnode at TDinternal/community/sim/dnode1/data/vnode + #7 data file should be found + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query('select first(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database + input("please the data file. After checking, press enter") + + + + def stop(self): + tdSql.close() + tdLog.debug("%s alter block manual check finish" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/manualTest/manual_chang_time_1_auto_test.py b/tests/pytest/manualTest/manual_chang_time_1_auto_test.py new file mode 100644 index 0000000000..86bc77404b --- /dev/null +++ b/tests/pytest/manualTest/manual_chang_time_1_auto_test.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from datetime import datetime +import subprocess + +##TODO: auto test version is currently unsupported, need to come up with +# an auto test version in the future +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + #tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + ## change system time to 2020/10/20 + os.system ('timedatectl set-ntp off') + os.system ('timedatectl set-time 2020-10-20') + + #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 + #11 data files should be generated + #vnode at TDinternal/community/sim/dnode1/data/vnode + #os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) + #input("please the data file. After checking, press enter") + commandArray = ['ls', '-l', f'{binPath}sim/dnode1/data/vnode/vnode2/tsdb/data', '|grep', '\'data\'', '|wc', '-l'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result) + # tdSql.query('select first(ts) from stb_0') + # tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) + # tdSql.query('select last(ts) from stb_0') + # tdSql.checkData(0,0,datetime(2020,10,20,23,59,59,0)) + + def stop(self): + tdSql.close() + tdLog.debug("%s alter block manual check finish" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json index 846730acfb..aeee6322e5 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json @@ -40,7 +40,7 @@ "batch_create_tbl_num": 20, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 86400, + "insert_rows": 432000, "childtable_limit": 1, "childtable_offset":0, "interlace_rows": 0, @@ -49,7 +49,7 @@ "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 1000, - "start_timestamp": "2020-10-17 00:00:00.000", + "start_timestamp": "2020-10-16 00:00:00.500", "sample_format": "csv", "sample_file": "./tools/taosdemoAllTest/sample.csv", "tags_file": "", From 5f5313b23451b594de751ec5d01d0498a41e34fc Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Mon, 7 Jun 2021 11:25:46 +0800 Subject: [PATCH 3/8] [TD-4366] modifying test case --- .../manualTest/manual_chang_time_1_2.py | 42 +++++++++++++-- .../manual_chang_time_1_auto_test.py | 54 +++++++++++++++---- .../pytest/manualTest/manual_chang_time_2.py | 46 ++++++++++++++++ 3 files changed, 129 insertions(+), 13 deletions(-) create mode 100644 tests/pytest/manualTest/manual_chang_time_2.py diff --git a/tests/pytest/manualTest/manual_chang_time_1_2.py b/tests/pytest/manualTest/manual_chang_time_1_2.py index 26661f7314..f0f11d3e72 100644 --- a/tests/pytest/manualTest/manual_chang_time_1_2.py +++ b/tests/pytest/manualTest/manual_chang_time_1_2.py @@ -17,6 +17,7 @@ from util.cases import * from util.sql import * from util.dnodes import tdDnodes from datetime import datetime +import subprocess ##TODO: auto test version is currently unsupported, need to come up with # an auto test version in the future @@ -40,7 +41,24 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + + def getTDenginePath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + for root, dirs, files in os.walk(projPath): + if ("sim" in dirs): + print(root) + rootRealPath = os.path.realpath(root) + # if ("packaging" not in rootRealPath): + # buildPath = root[:len(root)-len("/build/bin")] + # break + return rootRealPath + def run(self): tdSql.prepare() buildPath = self.getBuildPath() @@ -49,6 +67,12 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" + TDenginePath = self.getTDenginePath() + print('TD '+ TDenginePath) + if (TDenginePath == ""): + tdLog.exit("TDengine not found!") + else: + tdLog.info("TDengine found in %s" % TDenginePath) ## change system time to 2020/10/20 os.system ('timedatectl set-ntp off') @@ -58,7 +82,13 @@ class TDTestCase: #11 data files should be generated #vnode at TDinternal/community/sim/dnode1/data/vnode os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) - input("please the data file. After checking, press enter") + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 11: + tdLog.exit('wrong number of files') + else: + tdLog.info("data file number correct") tdSql.query('select first(ts) from stb_0') #check the last data in the database tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) @@ -73,7 +103,13 @@ class TDTestCase: tdDnodes.start(1) tdSql.query('select first(ts) from stb_0') tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database - input("please the data file. After checking, press enter") + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 7: + tdLog.exit('wrong number of files') + else: + tdLog.info("data file number correct") diff --git a/tests/pytest/manualTest/manual_chang_time_1_auto_test.py b/tests/pytest/manualTest/manual_chang_time_1_auto_test.py index 86bc77404b..a9e8935df4 100644 --- a/tests/pytest/manualTest/manual_chang_time_1_auto_test.py +++ b/tests/pytest/manualTest/manual_chang_time_1_auto_test.py @@ -26,6 +26,23 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) + def getTDenginePath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + for root, dirs, files in os.walk(projPath): + if ("sim" in dirs): + print(root) + rootRealPath = os.path.realpath(root) + # if ("packaging" not in rootRealPath): + # buildPath = root[:len(root)-len("/build/bin")] + # break + return rootRealPath + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -41,15 +58,21 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): - #tdSql.prepare() + tdSql.prepare() buildPath = self.getBuildPath() if (buildPath == ""): tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" + TDenginePath = self.getTDenginePath() + print('TD '+ TDenginePath) + if (TDenginePath == ""): + tdLog.exit("TDengine not found!") + else: + tdLog.info("TDengine found in %s" % TDenginePath) ## change system time to 2020/10/20 os.system ('timedatectl set-ntp off') @@ -58,15 +81,26 @@ class TDTestCase: #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 #11 data files should be generated #vnode at TDinternal/community/sim/dnode1/data/vnode - #os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) - #input("please the data file. After checking, press enter") - commandArray = ['ls', '-l', f'{binPath}sim/dnode1/data/vnode/vnode2/tsdb/data', '|grep', '\'data\'', '|wc', '-l'] + os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') - print(result) - # tdSql.query('select first(ts) from stb_0') - # tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) - # tdSql.query('select last(ts) from stb_0') - # tdSql.checkData(0,0,datetime(2020,10,20,23,59,59,0)) + print(result.count('data')) + if result.count('data') != 11: + tdLog.exit('wrong number of files') + else: + tdLog.info("data file number correct") + + os.system ('timedatectl set-time 2020-10-25') + os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json" % binPath) + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 7: + tdLog.exit('wrong number of files') + else: + tdLog.info("data file number correct") + tdSql.query('select first(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) def stop(self): tdSql.close() diff --git a/tests/pytest/manualTest/manual_chang_time_2.py b/tests/pytest/manualTest/manual_chang_time_2.py new file mode 100644 index 0000000000..5d60d557c0 --- /dev/null +++ b/tests/pytest/manualTest/manual_chang_time_2.py @@ -0,0 +1,46 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from fabric import Connection +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from datetime import datetime +import subprocess + +# class VMPnode + +# ##TODO: auto test version is currently unsupported, need to come up with +# # an auto test version in the future +# class TDTestCase: +# def init(self, conn, logSql): +# tdLog.debug("start to execute %s" % __file__) +# tdSql.init(conn.cursor(), logSql) + +# def run(self): +# return + + + +# def stop(self): +# tdSql.close() +# tdLog.debug("%s alter block manual check finish" % __file__) + + +# tdCases.addWindows(__file__, TDTestCase()) +# tdCases.addLinux(__file__, TDTestCase()) + +conn = conn = Connection("{}@{}".format('ubuntu', "lyq-q-1"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn.run('sudo timedatectl set-ntp off') +conn.run('sudo timedatectl set-time 2020-10-20') \ No newline at end of file From 60502d9b021519123b70f815ec920ad88cafeb79 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Mon, 7 Jun 2021 11:27:36 +0800 Subject: [PATCH 4/8] [TD-4773] modifying test case --- tests/pytest/alter/alter_keep.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index f9c71ed4ff..dd9fd08ba6 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -153,10 +153,10 @@ class TDTestCase: tdSql.execute('alter database db keep 10,10,10') tdSql.query('show databases') tdSql.checkData(0,7,'10,10,10') - tdSql.execute('alter database db keep 15,15,15') - tdSql.query('show databases') - tdSql.checkData(0,7,'15,15,15') - tdSql.execute('insert into tb values (now-30d, 10)') + # tdSql.execute('alter database db keep 15,15,15') + # tdSql.query('show databases') + # tdSql.checkData(0,7,'15,15,15') + tdSql.execute('insert into tb values (now-15d, 10)') tdSql.query('select * from tb') tdSql.checkRows(rowNum + 1) From ed65cfc6789ace9e0add36f2146a57eca9852101 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Sun, 25 Oct 2020 00:05:29 +0800 Subject: [PATCH 5/8] [TD-4366] updating the automatic version of test cases. Not sure if they can go through jenkins --- .../pytest/manualTest/manual_chang_time_2.py | 46 --------- ..._time_1_1.py => manual_change_time_1_1.py} | 73 +++++++++----- ..._time_1_2.py => manual_change_time_1_2.py} | 8 +- .../pytest/manualTest/manual_change_time_2.py | 97 +++++++++++++++++++ .../pytest/manualTest/manual_change_time_3.py | 97 +++++++++++++++++++ 5 files changed, 245 insertions(+), 76 deletions(-) delete mode 100644 tests/pytest/manualTest/manual_chang_time_2.py rename tests/pytest/manualTest/{manual_chang_time_1_1.py => manual_change_time_1_1.py} (54%) rename tests/pytest/manualTest/{manual_chang_time_1_2.py => manual_change_time_1_2.py} (96%) create mode 100644 tests/pytest/manualTest/manual_change_time_2.py create mode 100644 tests/pytest/manualTest/manual_change_time_3.py diff --git a/tests/pytest/manualTest/manual_chang_time_2.py b/tests/pytest/manualTest/manual_chang_time_2.py deleted file mode 100644 index 5d60d557c0..0000000000 --- a/tests/pytest/manualTest/manual_chang_time_2.py +++ /dev/null @@ -1,46 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- -from fabric import Connection -import sys -from util.log import * -from util.cases import * -from util.sql import * -from util.dnodes import tdDnodes -from datetime import datetime -import subprocess - -# class VMPnode - -# ##TODO: auto test version is currently unsupported, need to come up with -# # an auto test version in the future -# class TDTestCase: -# def init(self, conn, logSql): -# tdLog.debug("start to execute %s" % __file__) -# tdSql.init(conn.cursor(), logSql) - -# def run(self): -# return - - - -# def stop(self): -# tdSql.close() -# tdLog.debug("%s alter block manual check finish" % __file__) - - -# tdCases.addWindows(__file__, TDTestCase()) -# tdCases.addLinux(__file__, TDTestCase()) - -conn = conn = Connection("{}@{}".format('ubuntu', "lyq-q-1"), connect_kwargs={"password": "{}".format('tbase125!')}) -conn.run('sudo timedatectl set-ntp off') -conn.run('sudo timedatectl set-time 2020-10-20') \ No newline at end of file diff --git a/tests/pytest/manualTest/manual_chang_time_1_1.py b/tests/pytest/manualTest/manual_change_time_1_1.py similarity index 54% rename from tests/pytest/manualTest/manual_chang_time_1_1.py rename to tests/pytest/manualTest/manual_change_time_1_1.py index 69ea6f349b..8b444eeb6a 100644 --- a/tests/pytest/manualTest/manual_chang_time_1_1.py +++ b/tests/pytest/manualTest/manual_change_time_1_1.py @@ -19,13 +19,29 @@ from util.dnodes import tdDnodes from datetime import datetime import subprocess -##TODO: auto test version is currently unsupported, need to come up with -# an auto test version in the future +##TODO: this is now automatic, but not sure if this will run through jenkins class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) + def getTDenginePath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + for root, dirs, files in os.walk(projPath): + if ("sim" in dirs): + print(root) + rootRealPath = os.path.realpath(root) + # if ("packaging" not in rootRealPath): + # buildPath = root[:len(root)-len("/build/bin")] + # break + return rootRealPath + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -41,7 +57,7 @@ class TDTestCase: buildPath = root[:len(root)-len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() @@ -50,37 +66,42 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" + TDenginePath = self.getTDenginePath() + print('TD '+ TDenginePath) + if (TDenginePath == ""): + tdLog.exit("TDengine not found!") + else: + tdLog.info("TDengine found in %s" % TDenginePath) ## change system time to 2020/10/20 - os.system ('timedatectl set-ntp off') - os.system ('timedatectl set-time 2020-10-20') + os.system('sudo timedatectl set-ntp off') + os.system('sudo timedatectl set-time 2020-10-20') #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 #11 data files should be generated #vnode at TDinternal/community/sim/dnode1/data/vnode os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) - input("please the data file. After checking, press enter") - - tdSql.query('select first(ts) from stb_0') - tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) - tdSql.query('select last(ts) from stb_0') - tdSql.checkData(0,0,datetime(2020,10,20,23,59,59,0)) - - os.system ('timedatectl set-time 2020-10-25') - - #run taosdemo to insert data. one row per second from 2020/10/16 to 2020/10/20 - #4 oldest data file should be removed from tsdb/data - #vnode at TDinternal/community/sim/dnode1/data/vnode - #7 data file should be found - os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json" % binPath) - input("please the data file. After checking, press enter") - - tdSql.query('select first(ts) from stb_0') - tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 11: + tdLog.exit('wrong number of files') + else: + tdLog.info("data file number correct") - ##test results - #2021/06/05 first check: 11 data files second check: 7 data files - # confirm with the assumption Baosheng Chang + #move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write + #leaving 7 data files. + os.system ('timedatectl set-time 2020-10-25') + os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json" % binPath) + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 7: + tdLog.exit('wrong number of files') + else: + tdLog.info("data file number correct") + tdSql.query('select first(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database def stop(self): tdSql.close() diff --git a/tests/pytest/manualTest/manual_chang_time_1_2.py b/tests/pytest/manualTest/manual_change_time_1_2.py similarity index 96% rename from tests/pytest/manualTest/manual_chang_time_1_2.py rename to tests/pytest/manualTest/manual_change_time_1_2.py index f0f11d3e72..076099edae 100644 --- a/tests/pytest/manualTest/manual_chang_time_1_2.py +++ b/tests/pytest/manualTest/manual_change_time_1_2.py @@ -19,8 +19,7 @@ from util.dnodes import tdDnodes from datetime import datetime import subprocess -##TODO: auto test version is currently unsupported, need to come up with -# an auto test version in the future +##TODO: this is now automatic, but not sure if this will run through jenkins class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) @@ -95,10 +94,11 @@ class TDTestCase: os.system ('timedatectl set-time 2020-10-25') - #restart taosd + #moves 5 days ahead to 2020/10/25 and restart taosd #4 oldest data file should be removed from tsdb/data - #vnode at TDinternal/community/sim/dnode1/data/vnode #7 data file should be found + #vnode at TDinternal/community/sim/dnode1/data/vnode + os.system ('timedatectl set-time 2020-10-25') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query('select first(ts) from stb_0') diff --git a/tests/pytest/manualTest/manual_change_time_2.py b/tests/pytest/manualTest/manual_change_time_2.py new file mode 100644 index 0000000000..4aeacb9f63 --- /dev/null +++ b/tests/pytest/manualTest/manual_change_time_2.py @@ -0,0 +1,97 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from fabric import Connection +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from datetime import datetime +import subprocess +import time +import taos +##TODO: this is now automatic, but not sure if this will run through jenkins + +#the initial time used for this test is 2020/10/20 + +#setting local machine's time for later connecting to the server +os.system('sudo timedatectl set-ntp off') +os.system('sudo timedatectl set-time 2020-10-25') + +#connect to VM lyq-1, and initalize the environment at lyq-1 +conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn1.run("sudo systemctl stop taosd") +conn1.run('ls -l') +conn1.run('sudo timedatectl set-ntp off') +conn1.run('sudo timedatectl set-time 2020-10-20') + +with conn1.cd('/data/taos/log'): + conn1.run('sudo rm -rf *') + +with conn1.cd('/data/taos/data'): + conn1.run('sudo rm -rf *') + +#lanuch taosd and start taosdemo +conn1.run("sudo systemctl start taosd") +time.sleep(5) +with conn1.cd('~/bschang_test'): + conn1.run('taosdemo -f manual_change_time_1_1_A.json') + +#force everything onto disk +conn1.run("sudo systemctl restart taosd") +time.sleep(10) + +#change lyq-1 to 2020/10/25 for testing if the server +#will send data that is out of time range +conn1.run('sudo timedatectl set-time 2020-10-25') + +#connect to VM lyq-2, initalize the environment at lyq-2, and run taosd +#on that +conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn2.run('sudo timedatectl set-ntp off') +conn2.run('sudo timedatectl set-time 2020-10-20') +conn2.run("sudo systemctl stop taosd") +with conn2.cd('/data/taos/log'): + conn2.run('sudo rm -rf *') +with conn2.cd('/data/taos/data'): + conn2.run('sudo rm -rf *') +conn2.run("sudo systemctl start taosd") + +#set replica to 2 +connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos') +c1 = connTaos.cursor() +c1.execute('create dnode \'lyq-2:6030\'') +c1.execute('alter database db replica 2') +c1.close() +connTaos.close() +time.sleep(5) + +#force everything onto the disk for lyq-2 +#stopping taosd on lyq-1 for future testing +conn2.run("sudo systemctl stop taosd") +conn1.run("sudo systemctl stop taosd") + +#reset the time +conn1.run('sudo timedatectl set-ntp on') +conn2.run('sudo timedatectl set-ntp on') +os.system('sudo timedatectl set-ntp on') + +#check if the number of file received is 7 +#the 4 oldest data files should be dropped +#4 files because of moving 5 days ahead +with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'): + result = conn2.run('ls -l |grep \'data\' |wc -l') + if result.stdout.strip() != '7': + tdLog.exit('the file number is wrong') + else: + tdLog.success('the file number is the same. test pass') diff --git a/tests/pytest/manualTest/manual_change_time_3.py b/tests/pytest/manualTest/manual_change_time_3.py new file mode 100644 index 0000000000..a288147b56 --- /dev/null +++ b/tests/pytest/manualTest/manual_change_time_3.py @@ -0,0 +1,97 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from fabric import Connection +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from datetime import datetime +import subprocess +import time +import taos + +##TODO: this is now automatic, but not sure if this will run through jenkins + +#the initial time used for this test is 2020/10/20 + +#setting local machine's time for later connecting to the server +os.system('sudo timedatectl set-ntp off') +os.system('sudo timedatectl set-time 2020-10-20') + +#connect to VM lyq-1, and initalize the environment at lyq-1 +conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn1.run("sudo systemctl stop taosd") +conn1.run('sudo timedatectl set-ntp off') +conn1.run('sudo timedatectl set-time 2020-10-20') +with conn1.cd('/data/taos/log'): + conn1.run('sudo rm -rf *') + +with conn1.cd('/data/taos/data'): + conn1.run('sudo rm -rf *') + +#lanuch taosd and start taosdemo +conn1.run("sudo systemctl start taosd") +time.sleep(5) +with conn1.cd('~/bschang_test'): + conn1.run('taosdemo -f manual_change_time_1_1_A.json') #the json file is placed in lyq-1 already + +#force everything onto disk +conn1.run("sudo systemctl restart taosd") +time.sleep(10) + +#connect to VM lyq-2, and initalize the environment at lyq-2 +conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn2.run('sudo timedatectl set-ntp off') +conn2.run('sudo timedatectl set-time 2020-10-20') +conn2.run("sudo systemctl stop taosd") +with conn2.cd('/data/taos/log'): + conn2.run('sudo rm -rf *') +with conn2.cd('/data/taos/data'): + conn2.run('sudo rm -rf *') + +#the date of lyq-2 is going to be set to 2020/10/25 +#for testing if other pnode will accpet file out of local time range +conn2.run("sudo systemctl start taosd") +conn2.run('sudo timedatectl set-time 2020-10-25') + +#set the replica to 2 +connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos') +c1 = connTaos.cursor() +c1.execute('create dnode \'lyq-2:6030\'') +c1.execute('alter database db replica 2') +time.sleep(5) + +#force everything onto the disk for lyq-2 +#stopping taosd on lyq-1 for future testing +conn2.run("sudo systemctl stop taosd") +conn1.run("sudo systemctl stop taosd") + +#reset the time +conn1.run('sudo timedatectl set-ntp on') +conn2.run('sudo timedatectl set-ntp on') +os.system('sudo timedatectl set-ntp on') + +#check if the number of file received is 7 +#the 4 oldest data files should be dropped +#4 files because of moving 5 days ahead +with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'): + result = conn2.run('ls -l |grep \'data\' |wc -l') + if result.stdout.strip() != '7': + tdLog.exit('the file number is wrong') + else: + tdLog.success('the file number is the same. test pass') + +c1.close() +connTaos.close() + From 881b267d2fb3b008d5b582f353471bcf8b33420d Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 8 Jun 2021 09:13:25 +0800 Subject: [PATCH 6/8] [TD-4473] revertalterkeep.py to older version --- tests/pytest/alter/alter_keep.py | 61 ++++++++------------------------ 1 file changed, 14 insertions(+), 47 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index dd9fd08ba6..cb630963a7 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -116,58 +116,25 @@ class TDTestCase: ##TODO: need to wait for TD-4445 to implement the following ## tests - - ## preset the keep - tdSql.prepare() - tdSql.execute('create table tb (ts timestamp, speed int)') - tdSql.execute('alter database db keep 10,10,10') - tdSql.execute('insert into tb values (now, 10)') - tdSql.execute('insert into tb values (now + 10m, 10)') - tdSql.query('select * from tb') - tdSql.checkRows(2) - - - #after alter from small to large, check if the alter if functioning - #test if change through test.py is consistent with change from taos client - #test case for TD-4459 and TD-4445 - tdSql.execute('alter database db keep 40,40,40') - tdSql.query('show databases') - tdSql.checkData(0,7,'40,40,40') - tdSql.error('insert into tb values (now-60d, 10)') - tdSql.execute('insert into tb values (now-30d, 10)') - tdSql.query('select * from tb') - tdSql.checkRows(3) - - rowNum = 3 - for i in range(30): - rowNum += 1 - tdSql.execute('alter database db keep 20,20,20') - tdSql.execute('alter database db keep 40,40,40') - tdSql.query('show databases') - tdSql.checkData(0,7,'40,40,40') - tdSql.error('insert into tb values (now-60d, 10)') - tdSql.execute('insert into tb values (now-30d, 10)') - tdSql.query('select * from tb') - tdSql.checkRows(rowNum) - - tdSql.execute('alter database db keep 10,10,10') - tdSql.query('show databases') - tdSql.checkData(0,7,'10,10,10') - # tdSql.execute('alter database db keep 15,15,15') + # tdSql.prepare() + # tdSql.execute('create table tb (ts timestamp, speed int)') + # tdSql.execute('alter database db keep 10,10,10') + # tdSql.execute('insert into tb values (now, 10)') + # tdSql.execute('insert into tb values (now + 10m, 10)') + # tdSql.query('select * from tb') + # tdSql.checkRows(2) + # tdSql.execute('alter database db keep 40,40,40') # tdSql.query('show databases') - # tdSql.checkData(0,7,'15,15,15') - tdSql.execute('insert into tb values (now-15d, 10)') - tdSql.query('select * from tb') - tdSql.checkRows(rowNum + 1) - + # tdSql.checkData(0,7,'40,40,40') + # tdSql.error('insert into tb values (now-60d, 10)') + # tdSql.execute('insert into tb values (now-30d, 10)') + # tdSql.query('select * from tb') + # tdSql.checkRows(3) # tdSql.execute('alter database db keep 20,20,20') # tdSql.query('show databases') # tdSql.checkData(0,7,'20,20,20') - # tdSql.error('insert into tb values (now-30d, 10)') - # tdSql.query('show databases') - # tdSql.checkData(0,7,'20,20,20') # tdSql.query('select * from tb') - # tdSql.checkRows(rowNum) + # tdSql.checkRows(2) def stop(self): From b7c0995f8744d5c5a8badd1551276cb82cfeae45 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 8 Jun 2021 09:15:10 +0800 Subject: [PATCH 7/8] [TD-4473] remove old file --- .../manual_chang_time_1_auto_test.py | 111 ------------------ 1 file changed, 111 deletions(-) delete mode 100644 tests/pytest/manualTest/manual_chang_time_1_auto_test.py diff --git a/tests/pytest/manualTest/manual_chang_time_1_auto_test.py b/tests/pytest/manualTest/manual_chang_time_1_auto_test.py deleted file mode 100644 index a9e8935df4..0000000000 --- a/tests/pytest/manualTest/manual_chang_time_1_auto_test.py +++ /dev/null @@ -1,111 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -from util.log import * -from util.cases import * -from util.sql import * -from util.dnodes import tdDnodes -from datetime import datetime -import subprocess - -##TODO: auto test version is currently unsupported, need to come up with -# an auto test version in the future -class TDTestCase: - def init(self, conn, logSql): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) - - def getTDenginePath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - print(projPath) - for root, dirs, files in os.walk(projPath): - if ("sim" in dirs): - print(root) - rootRealPath = os.path.realpath(root) - # if ("packaging" not in rootRealPath): - # buildPath = root[:len(root)-len("/build/bin")] - # break - return rootRealPath - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def run(self): - tdSql.prepare() - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - TDenginePath = self.getTDenginePath() - print('TD '+ TDenginePath) - if (TDenginePath == ""): - tdLog.exit("TDengine not found!") - else: - tdLog.info("TDengine found in %s" % TDenginePath) - - ## change system time to 2020/10/20 - os.system ('timedatectl set-ntp off') - os.system ('timedatectl set-time 2020-10-20') - - #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 - #11 data files should be generated - #vnode at TDinternal/community/sim/dnode1/data/vnode - os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) - commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] - result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') - print(result.count('data')) - if result.count('data') != 11: - tdLog.exit('wrong number of files') - else: - tdLog.info("data file number correct") - - os.system ('timedatectl set-time 2020-10-25') - os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json" % binPath) - commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] - result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') - print(result.count('data')) - if result.count('data') != 7: - tdLog.exit('wrong number of files') - else: - tdLog.info("data file number correct") - tdSql.query('select first(ts) from stb_0') - tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) - - def stop(self): - tdSql.close() - tdLog.debug("%s alter block manual check finish" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) From 9c1457a3aa27afe86359111c28918e78708cca7b Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 8 Jun 2021 11:16:28 +0800 Subject: [PATCH 8/8] [TD-4473] modifying two cases to auto test with minor modification --- .../change_time_1_1.py} | 60 +++----------- .../change_time_1_2.py} | 58 ++----------- tests/pytest/fulltest.sh | 2 + tests/pytest/util/pathFinding.py | 83 +++++++++++++++++++ 4 files changed, 104 insertions(+), 99 deletions(-) rename tests/pytest/{manualTest/manual_change_time_1_1.py => client/change_time_1_1.py} (53%) rename tests/pytest/{manualTest/manual_change_time_1_2.py => client/change_time_1_2.py} (58%) create mode 100644 tests/pytest/util/pathFinding.py diff --git a/tests/pytest/manualTest/manual_change_time_1_1.py b/tests/pytest/client/change_time_1_1.py similarity index 53% rename from tests/pytest/manualTest/manual_change_time_1_1.py rename to tests/pytest/client/change_time_1_1.py index 8b444eeb6a..24e0195a9c 100644 --- a/tests/pytest/manualTest/manual_change_time_1_1.py +++ b/tests/pytest/client/change_time_1_1.py @@ -15,6 +15,7 @@ import sys from util.log import * from util.cases import * from util.sql import * +from util.pathFinding import * from util.dnodes import tdDnodes from datetime import datetime import subprocess @@ -24,54 +25,12 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - - def getTDenginePath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - print(projPath) - for root, dirs, files in os.walk(projPath): - if ("sim" in dirs): - print(root) - rootRealPath = os.path.realpath(root) - # if ("packaging" not in rootRealPath): - # buildPath = root[:len(root)-len("/build/bin")] - # break - return rootRealPath - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath + tdFindPath.init(__file__) def run(self): tdSql.prepare() - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - TDenginePath = self.getTDenginePath() - print('TD '+ TDenginePath) - if (TDenginePath == ""): - tdLog.exit("TDengine not found!") - else: - tdLog.info("TDengine found in %s" % TDenginePath) + binPath = tdFindPath.getTaosdemoPath() + TDenginePath = tdFindPath.getTDenginePath() ## change system time to 2020/10/20 os.system('sudo timedatectl set-ntp off') @@ -80,32 +39,33 @@ class TDTestCase: #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 #11 data files should be generated #vnode at TDinternal/community/sim/dnode1/data/vnode - os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') print(result.count('data')) if result.count('data') != 11: tdLog.exit('wrong number of files') else: - tdLog.info("data file number correct") + tdLog.debug("data file number correct") #move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write #leaving 7 data files. os.system ('timedatectl set-time 2020-10-25') - os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json" % binPath) + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json") commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') print(result.count('data')) if result.count('data') != 7: tdLog.exit('wrong number of files') else: - tdLog.info("data file number correct") + tdLog.debug("data file number correct") tdSql.query('select first(ts) from stb_0') tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database def stop(self): + os.system('sudo timedatectl set-ntp on') tdSql.close() - tdLog.debug("%s alter block manual check finish" % __file__) + tdLog.success("alter block manual check finish") tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/pytest/manualTest/manual_change_time_1_2.py b/tests/pytest/client/change_time_1_2.py similarity index 58% rename from tests/pytest/manualTest/manual_change_time_1_2.py rename to tests/pytest/client/change_time_1_2.py index 076099edae..cd1a17926d 100644 --- a/tests/pytest/manualTest/manual_change_time_1_2.py +++ b/tests/pytest/client/change_time_1_2.py @@ -16,6 +16,7 @@ from util.log import * from util.cases import * from util.sql import * from util.dnodes import tdDnodes +from util.pathFinding import * from datetime import datetime import subprocess @@ -24,54 +25,12 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def getTDenginePath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - print(projPath) - for root, dirs, files in os.walk(projPath): - if ("sim" in dirs): - print(root) - rootRealPath = os.path.realpath(root) - # if ("packaging" not in rootRealPath): - # buildPath = root[:len(root)-len("/build/bin")] - # break - return rootRealPath + tdFindPath.init(__file__) def run(self): tdSql.prepare() - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - TDenginePath = self.getTDenginePath() - print('TD '+ TDenginePath) - if (TDenginePath == ""): - tdLog.exit("TDengine not found!") - else: - tdLog.info("TDengine found in %s" % TDenginePath) + binPath = tdFindPath.getTaosdemoPath() + TDenginePath = tdFindPath.getTDenginePath() ## change system time to 2020/10/20 os.system ('timedatectl set-ntp off') @@ -80,14 +39,14 @@ class TDTestCase: #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 #11 data files should be generated #vnode at TDinternal/community/sim/dnode1/data/vnode - os.system("%staosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json" % binPath) + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') print(result.count('data')) if result.count('data') != 11: tdLog.exit('wrong number of files') else: - tdLog.info("data file number correct") + tdLog.debug("data file number correct") tdSql.query('select first(ts) from stb_0') #check the last data in the database tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) @@ -109,13 +68,14 @@ class TDTestCase: if result.count('data') != 7: tdLog.exit('wrong number of files') else: - tdLog.info("data file number correct") + tdLog.debug("data file number correct") def stop(self): + os.system('sudo timedatectl set-ntp on') tdSql.close() - tdLog.debug("%s alter block manual check finish" % __file__) + tdLog.success("alter block manual check finish") tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 8fed9aa81c..6a09a41b93 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -256,6 +256,8 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py +python3 test.py -f client/change_time_1_1.py +python3 test.py -f client/change_time_1_2.py # Misc python3 testCompress.py diff --git a/tests/pytest/util/pathFinding.py b/tests/pytest/util/pathFinding.py new file mode 100644 index 0000000000..9dee5142ce --- /dev/null +++ b/tests/pytest/util/pathFinding.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +import os +from util.log import * + + + +class TDFindPath: + """This class is for finding path within TDengine + """ + def __init__(self): + self.file = "" + + + def init(self, file): + """[summary] + + Args: + file (str): the file location you want to start the query. Generally using __file__ + """ + self.file = file + + def getTaosdemoPath(self): + """for finding the path of directory containing taosdemo + + Returns: + str: the path to directory containing taosdemo + """ + selfPath = os.path.dirname(os.path.realpath(self.file)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + return buildPath + "/build/bin/" + + def getTDenginePath(self): + """for finding the root path of TDengine + + Returns: + str: the root path of TDengine + """ + selfPath = os.path.dirname(os.path.realpath(self.file)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + for root, dirs, files in os.walk(projPath): + if ("sim" in dirs): + print(root) + rootRealPath = os.path.realpath(root) + if (rootRealPath == ""): + tdLog.exit("TDengine not found!") + else: + tdLog.info(f"TDengine found in {rootRealPath}") + return rootRealPath + +tdFindPath = TDFindPath() \ No newline at end of file