diff --git a/tests/pytest/client/change_time_1_1.py b/tests/pytest/client/change_time_1_1.py new file mode 100644 index 0000000000..24e0195a9c --- /dev/null +++ b/tests/pytest/client/change_time_1_1.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.pathFinding import * +from util.dnodes import tdDnodes +from datetime import datetime +import subprocess + +##TODO: this is now automatic, but not sure if this will run through jenkins +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + tdFindPath.init(__file__) + + def run(self): + tdSql.prepare() + binPath = tdFindPath.getTaosdemoPath() + TDenginePath = tdFindPath.getTDenginePath() + + ## change system time to 2020/10/20 + os.system('sudo timedatectl set-ntp off') + os.system('sudo timedatectl set-time 2020-10-20') + + #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 + #11 data files should be generated + #vnode at TDinternal/community/sim/dnode1/data/vnode + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 11: + tdLog.exit('wrong number of files') + else: + tdLog.debug("data file number correct") + + #move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write + #leaving 7 data files. + os.system ('timedatectl set-time 2020-10-25') + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json") + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 7: + tdLog.exit('wrong number of files') + else: + tdLog.debug("data file number correct") + tdSql.query('select first(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database + + def stop(self): + os.system('sudo timedatectl set-ntp on') + tdSql.close() + tdLog.success("alter block manual check finish") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/client/change_time_1_2.py b/tests/pytest/client/change_time_1_2.py new file mode 100644 index 0000000000..cd1a17926d --- /dev/null +++ b/tests/pytest/client/change_time_1_2.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from util.pathFinding import * +from datetime import datetime +import subprocess + +##TODO: this is now automatic, but not sure if this will run through jenkins +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + tdFindPath.init(__file__) + + def run(self): + tdSql.prepare() + binPath = tdFindPath.getTaosdemoPath() + TDenginePath = tdFindPath.getTDenginePath() + + ## change system time to 2020/10/20 + os.system ('timedatectl set-ntp off') + os.system ('timedatectl set-time 2020-10-20') + + #run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20 + #11 data files should be generated + #vnode at TDinternal/community/sim/dnode1/data/vnode + os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json") + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 11: + tdLog.exit('wrong number of files') + else: + tdLog.debug("data file number correct") + + tdSql.query('select first(ts) from stb_0') #check the last data in the database + tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0)) + + os.system ('timedatectl set-time 2020-10-25') + + #moves 5 days ahead to 2020/10/25 and restart taosd + #4 oldest data file should be removed from tsdb/data + #7 data file should be found + #vnode at TDinternal/community/sim/dnode1/data/vnode + os.system ('timedatectl set-time 2020-10-25') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query('select first(ts) from stb_0') + tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database + commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data'] + result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8') + print(result.count('data')) + if result.count('data') != 7: + tdLog.exit('wrong number of files') + else: + tdLog.debug("data file number correct") + + + + def stop(self): + os.system('sudo timedatectl set-ntp on') + tdSql.close() + tdLog.success("alter block manual check finish") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 68d7bbefee..a93a98a145 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -256,6 +256,8 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py +python3 test.py -f client/change_time_1_1.py +python3 test.py -f client/change_time_1_2.py # Misc python3 testCompress.py diff --git a/tests/pytest/manualTest/manual_change_time_2.py b/tests/pytest/manualTest/manual_change_time_2.py new file mode 100644 index 0000000000..4aeacb9f63 --- /dev/null +++ b/tests/pytest/manualTest/manual_change_time_2.py @@ -0,0 +1,97 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from fabric import Connection +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from datetime import datetime +import subprocess +import time +import taos +##TODO: this is now automatic, but not sure if this will run through jenkins + +#the initial time used for this test is 2020/10/20 + +#setting local machine's time for later connecting to the server +os.system('sudo timedatectl set-ntp off') +os.system('sudo timedatectl set-time 2020-10-25') + +#connect to VM lyq-1, and initalize the environment at lyq-1 +conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn1.run("sudo systemctl stop taosd") +conn1.run('ls -l') +conn1.run('sudo timedatectl set-ntp off') +conn1.run('sudo timedatectl set-time 2020-10-20') + +with conn1.cd('/data/taos/log'): + conn1.run('sudo rm -rf *') + +with conn1.cd('/data/taos/data'): + conn1.run('sudo rm -rf *') + +#lanuch taosd and start taosdemo +conn1.run("sudo systemctl start taosd") +time.sleep(5) +with conn1.cd('~/bschang_test'): + conn1.run('taosdemo -f manual_change_time_1_1_A.json') + +#force everything onto disk +conn1.run("sudo systemctl restart taosd") +time.sleep(10) + +#change lyq-1 to 2020/10/25 for testing if the server +#will send data that is out of time range +conn1.run('sudo timedatectl set-time 2020-10-25') + +#connect to VM lyq-2, initalize the environment at lyq-2, and run taosd +#on that +conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn2.run('sudo timedatectl set-ntp off') +conn2.run('sudo timedatectl set-time 2020-10-20') +conn2.run("sudo systemctl stop taosd") +with conn2.cd('/data/taos/log'): + conn2.run('sudo rm -rf *') +with conn2.cd('/data/taos/data'): + conn2.run('sudo rm -rf *') +conn2.run("sudo systemctl start taosd") + +#set replica to 2 +connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos') +c1 = connTaos.cursor() +c1.execute('create dnode \'lyq-2:6030\'') +c1.execute('alter database db replica 2') +c1.close() +connTaos.close() +time.sleep(5) + +#force everything onto the disk for lyq-2 +#stopping taosd on lyq-1 for future testing +conn2.run("sudo systemctl stop taosd") +conn1.run("sudo systemctl stop taosd") + +#reset the time +conn1.run('sudo timedatectl set-ntp on') +conn2.run('sudo timedatectl set-ntp on') +os.system('sudo timedatectl set-ntp on') + +#check if the number of file received is 7 +#the 4 oldest data files should be dropped +#4 files because of moving 5 days ahead +with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'): + result = conn2.run('ls -l |grep \'data\' |wc -l') + if result.stdout.strip() != '7': + tdLog.exit('the file number is wrong') + else: + tdLog.success('the file number is the same. test pass') diff --git a/tests/pytest/manualTest/manual_change_time_3.py b/tests/pytest/manualTest/manual_change_time_3.py new file mode 100644 index 0000000000..a288147b56 --- /dev/null +++ b/tests/pytest/manualTest/manual_change_time_3.py @@ -0,0 +1,97 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from fabric import Connection +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from datetime import datetime +import subprocess +import time +import taos + +##TODO: this is now automatic, but not sure if this will run through jenkins + +#the initial time used for this test is 2020/10/20 + +#setting local machine's time for later connecting to the server +os.system('sudo timedatectl set-ntp off') +os.system('sudo timedatectl set-time 2020-10-20') + +#connect to VM lyq-1, and initalize the environment at lyq-1 +conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn1.run("sudo systemctl stop taosd") +conn1.run('sudo timedatectl set-ntp off') +conn1.run('sudo timedatectl set-time 2020-10-20') +with conn1.cd('/data/taos/log'): + conn1.run('sudo rm -rf *') + +with conn1.cd('/data/taos/data'): + conn1.run('sudo rm -rf *') + +#lanuch taosd and start taosdemo +conn1.run("sudo systemctl start taosd") +time.sleep(5) +with conn1.cd('~/bschang_test'): + conn1.run('taosdemo -f manual_change_time_1_1_A.json') #the json file is placed in lyq-1 already + +#force everything onto disk +conn1.run("sudo systemctl restart taosd") +time.sleep(10) + +#connect to VM lyq-2, and initalize the environment at lyq-2 +conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')}) +conn2.run('sudo timedatectl set-ntp off') +conn2.run('sudo timedatectl set-time 2020-10-20') +conn2.run("sudo systemctl stop taosd") +with conn2.cd('/data/taos/log'): + conn2.run('sudo rm -rf *') +with conn2.cd('/data/taos/data'): + conn2.run('sudo rm -rf *') + +#the date of lyq-2 is going to be set to 2020/10/25 +#for testing if other pnode will accpet file out of local time range +conn2.run("sudo systemctl start taosd") +conn2.run('sudo timedatectl set-time 2020-10-25') + +#set the replica to 2 +connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos') +c1 = connTaos.cursor() +c1.execute('create dnode \'lyq-2:6030\'') +c1.execute('alter database db replica 2') +time.sleep(5) + +#force everything onto the disk for lyq-2 +#stopping taosd on lyq-1 for future testing +conn2.run("sudo systemctl stop taosd") +conn1.run("sudo systemctl stop taosd") + +#reset the time +conn1.run('sudo timedatectl set-ntp on') +conn2.run('sudo timedatectl set-ntp on') +os.system('sudo timedatectl set-ntp on') + +#check if the number of file received is 7 +#the 4 oldest data files should be dropped +#4 files because of moving 5 days ahead +with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'): + result = conn2.run('ls -l |grep \'data\' |wc -l') + if result.stdout.strip() != '7': + tdLog.exit('the file number is wrong') + else: + tdLog.success('the file number is the same. test pass') + +c1.close() +connTaos.close() + diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json new file mode 100644 index 0000000000..7b8abd6d4e --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 32766, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 1, + "cache": 4, + "blocks": 3, + "precision": "ms", + "keep": 10, + "minRows": 1000, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 864000, + "childtable_limit": 1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2020-10-11 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "BINARY", "len": 5120, "count":1}], + "tags": [{"type": "TINYINT", "count":2}] + }] + }] +} \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json new file mode 100644 index 0000000000..aeee6322e5 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json @@ -0,0 +1,60 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 32766, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 1, + "cache": 4, + "blocks": 3, + "precision": "ms", + "keep": 10, + "minRows": 1000, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"yes", + "childtable_count": 1, + "childtable_prefix": "stb_", + "auto_create_table": "yes", + "batch_create_tbl_num": 20, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 432000, + "childtable_limit": 1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2020-10-16 00:00:00.500", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "BINARY", "len": 5120, "count":1}], + "tags": [{"type": "TINYINT", "count":1}] + }] + }] +} \ No newline at end of file diff --git a/tests/pytest/util/pathFinding.py b/tests/pytest/util/pathFinding.py new file mode 100644 index 0000000000..9dee5142ce --- /dev/null +++ b/tests/pytest/util/pathFinding.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +import os +from util.log import * + + + +class TDFindPath: + """This class is for finding path within TDengine + """ + def __init__(self): + self.file = "" + + + def init(self, file): + """[summary] + + Args: + file (str): the file location you want to start the query. Generally using __file__ + """ + self.file = file + + def getTaosdemoPath(self): + """for finding the path of directory containing taosdemo + + Returns: + str: the path to directory containing taosdemo + """ + selfPath = os.path.dirname(os.path.realpath(self.file)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + return buildPath + "/build/bin/" + + def getTDenginePath(self): + """for finding the root path of TDengine + + Returns: + str: the root path of TDengine + """ + selfPath = os.path.dirname(os.path.realpath(self.file)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + for root, dirs, files in os.walk(projPath): + if ("sim" in dirs): + print(root) + rootRealPath = os.path.realpath(root) + if (rootRealPath == ""): + tdLog.exit("TDengine not found!") + else: + tdLog.info(f"TDengine found in {rootRealPath}") + return rootRealPath + +tdFindPath = TDFindPath() \ No newline at end of file