xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #6405 from taosdata/test/TD-4366 

Test/td 4366
This commit is contained in:
Shengliang Guan 2021-06-09 20:03:47 +08:00 committed by GitHub
parent 99de15f157 9c1457a3aa
commit 47ec533542
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 553 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
tests/pytest/client/change_time_1_1.py Normal file
View File

@ -0,0 +1,72 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.pathFinding import *
from util.dnodes import tdDnodes
from datetime import datetime
import subprocess
##TODO: this is now automatic, but not sure if this will run through jenkins
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
tdFindPath.init(__file__)
def run(self):
tdSql.prepare()
binPath = tdFindPath.getTaosdemoPath()
TDenginePath = tdFindPath.getTDenginePath()
## change system time to 2020/10/20
os.system('sudo timedatectl set-ntp off')
os.system('sudo timedatectl set-time 2020-10-20')
#run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20
#11 data files should be generated
#vnode at TDinternal/community/sim/dnode1/data/vnode
os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json")
commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
print(result.count('data'))
if result.count('data') != 11:
tdLog.exit('wrong number of files')
else:
tdLog.debug("data file number correct")
#move 5 days ahead to 2020/10/25. 4 oldest files should be removed during the new write
#leaving 7 data files.
os.system ('timedatectl set-time 2020-10-25')
os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_B.json")
commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
print(result.count('data'))
if result.count('data') != 7:
tdLog.exit('wrong number of files')
else:
tdLog.debug("data file number correct")
tdSql.query('select first(ts) from stb_0')
tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database
def stop(self):
os.system('sudo timedatectl set-ntp on')
tdSql.close()
tdLog.success("alter block manual check finish")
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

82
tests/pytest/client/change_time_1_2.py Normal file
View File

@ -0,0 +1,82 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import tdDnodes
from util.pathFinding import *
from datetime import datetime
import subprocess
##TODO: this is now automatic, but not sure if this will run through jenkins
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
tdFindPath.init(__file__)
def run(self):
tdSql.prepare()
binPath = tdFindPath.getTaosdemoPath()
TDenginePath = tdFindPath.getTDenginePath()
## change system time to 2020/10/20
os.system ('timedatectl set-ntp off')
os.system ('timedatectl set-time 2020-10-20')
#run taosdemo to insert data. one row per second from 2020/10/11 to 2020/10/20
#11 data files should be generated
#vnode at TDinternal/community/sim/dnode1/data/vnode
os.system(f"{binPath}taosdemo -f tools/taosdemoAllTest/manual_change_time_1_1_A.json")
commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
print(result.count('data'))
if result.count('data') != 11:
tdLog.exit('wrong number of files')
else:
tdLog.debug("data file number correct")
tdSql.query('select first(ts) from stb_0') #check the last data in the database
tdSql.checkData(0,0,datetime(2020,10,11,0,0,0,0))
os.system ('timedatectl set-time 2020-10-25')
#moves 5 days ahead to 2020/10/25 and restart taosd
#4 oldest data file should be removed from tsdb/data
#7 data file should be found
#vnode at TDinternal/community/sim/dnode1/data/vnode
os.system ('timedatectl set-time 2020-10-25')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query('select first(ts) from stb_0')
tdSql.checkData(0,0,datetime(2020,10,14,8,0,0,0)) #check the last data in the database
commandArray = ['ls', '-l', f'{TDenginePath}/sim/dnode1/data/vnode/vnode2/tsdb/data']
result = subprocess.run(commandArray, stdout=subprocess.PIPE).stdout.decode('utf-8')
print(result.count('data'))
if result.count('data') != 7:
tdLog.exit('wrong number of files')
else:
tdLog.debug("data file number correct")
def stop(self):
os.system('sudo timedatectl set-ntp on')
tdSql.close()
tdLog.success("alter block manual check finish")
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

2
tests/pytest/fulltest.sh
View File

@ -256,6 +256,8 @@ python3 ./test.py -f client/client.py
python3 ./test.py -f client/version.py
python3 ./test.py -f client/alterDatabase.py
python3 ./test.py -f client/noConnectionErrorTest.py
python3 test.py -f client/change_time_1_1.py
python3 test.py -f client/change_time_1_2.py
# Misc
python3 testCompress.py

97
tests/pytest/manualTest/manual_change_time_2.py Normal file
View File

@ -0,0 +1,97 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from fabric import Connection
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import tdDnodes
from datetime import datetime
import subprocess
import time
import taos
##TODO: this is now automatic, but not sure if this will run through jenkins
#the initial time used for this test is 2020/10/20
#setting local machine's time for later connecting to the server
os.system('sudo timedatectl set-ntp off')
os.system('sudo timedatectl set-time 2020-10-25')
#connect to VM lyq-1, and initalize the environment at lyq-1
conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')})
conn1.run("sudo systemctl stop taosd")
conn1.run('ls -l')
conn1.run('sudo timedatectl set-ntp off')
conn1.run('sudo timedatectl set-time 2020-10-20')
with conn1.cd('/data/taos/log'):
conn1.run('sudo rm -rf *')
with conn1.cd('/data/taos/data'):
conn1.run('sudo rm -rf *')
#lanuch taosd and start taosdemo
conn1.run("sudo systemctl start taosd")
time.sleep(5)
with conn1.cd('~/bschang_test'):
conn1.run('taosdemo -f manual_change_time_1_1_A.json')
#force everything onto disk
conn1.run("sudo systemctl restart taosd")
time.sleep(10)
#change lyq-1 to 2020/10/25 for testing if the server
#will send data that is out of time range
conn1.run('sudo timedatectl set-time 2020-10-25')
#connect to VM lyq-2, initalize the environment at lyq-2, and run taosd
#on that
conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')})
conn2.run('sudo timedatectl set-ntp off')
conn2.run('sudo timedatectl set-time 2020-10-20')
conn2.run("sudo systemctl stop taosd")
with conn2.cd('/data/taos/log'):
conn2.run('sudo rm -rf *')
with conn2.cd('/data/taos/data'):
conn2.run('sudo rm -rf *')
conn2.run("sudo systemctl start taosd")
#set replica to 2
connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos')
c1 = connTaos.cursor()
c1.execute('create dnode \'lyq-2:6030\'')
c1.execute('alter database db replica 2')
c1.close()
connTaos.close()
time.sleep(5)
#force everything onto the disk for lyq-2
#stopping taosd on lyq-1 for future testing
conn2.run("sudo systemctl stop taosd")
conn1.run("sudo systemctl stop taosd")
#reset the time
conn1.run('sudo timedatectl set-ntp on')
conn2.run('sudo timedatectl set-ntp on')
os.system('sudo timedatectl set-ntp on')
#check if the number of file received is 7
#the 4 oldest data files should be dropped
#4 files because of moving 5 days ahead
with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'):
result = conn2.run('ls -l |grep \'data\' |wc -l')
if result.stdout.strip() != '7':
tdLog.exit('the file number is wrong')
else:
tdLog.success('the file number is the same. test pass')

97
tests/pytest/manualTest/manual_change_time_3.py Normal file
View File

@ -0,0 +1,97 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from fabric import Connection
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import tdDnodes
from datetime import datetime
import subprocess
import time
import taos
##TODO: this is now automatic, but not sure if this will run through jenkins
#the initial time used for this test is 2020/10/20
#setting local machine's time for later connecting to the server
os.system('sudo timedatectl set-ntp off')
os.system('sudo timedatectl set-time 2020-10-20')
#connect to VM lyq-1, and initalize the environment at lyq-1
conn1 = Connection("{}@{}".format('ubuntu', "192.168.1.125"), connect_kwargs={"password": "{}".format('tbase125!')})
conn1.run("sudo systemctl stop taosd")
conn1.run('sudo timedatectl set-ntp off')
conn1.run('sudo timedatectl set-time 2020-10-20')
with conn1.cd('/data/taos/log'):
conn1.run('sudo rm -rf *')
with conn1.cd('/data/taos/data'):
conn1.run('sudo rm -rf *')
#lanuch taosd and start taosdemo
conn1.run("sudo systemctl start taosd")
time.sleep(5)
with conn1.cd('~/bschang_test'):
conn1.run('taosdemo -f manual_change_time_1_1_A.json') #the json file is placed in lyq-1 already
#force everything onto disk
conn1.run("sudo systemctl restart taosd")
time.sleep(10)
#connect to VM lyq-2, and initalize the environment at lyq-2
conn2 = Connection("{}@{}".format('ubuntu', "192.168.1.126"), connect_kwargs={"password": "{}".format('tbase125!')})
conn2.run('sudo timedatectl set-ntp off')
conn2.run('sudo timedatectl set-time 2020-10-20')
conn2.run("sudo systemctl stop taosd")
with conn2.cd('/data/taos/log'):
conn2.run('sudo rm -rf *')
with conn2.cd('/data/taos/data'):
conn2.run('sudo rm -rf *')
#the date of lyq-2 is going to be set to 2020/10/25
#for testing if other pnode will accpet file out of local time range
conn2.run("sudo systemctl start taosd")
conn2.run('sudo timedatectl set-time 2020-10-25')
#set the replica to 2
connTaos = taos.connect(host = '192.168.1.125', user = 'root', password = 'taosdata', cnfig = '/etc/taos')
c1 = connTaos.cursor()
c1.execute('create dnode \'lyq-2:6030\'')
c1.execute('alter database db replica 2')
time.sleep(5)
#force everything onto the disk for lyq-2
#stopping taosd on lyq-1 for future testing
conn2.run("sudo systemctl stop taosd")
conn1.run("sudo systemctl stop taosd")
#reset the time
conn1.run('sudo timedatectl set-ntp on')
conn2.run('sudo timedatectl set-ntp on')
os.system('sudo timedatectl set-ntp on')
#check if the number of file received is 7
#the 4 oldest data files should be dropped
#4 files because of moving 5 days ahead
with conn2.cd('/data/taos/data/vnode/vnode3/tsdb/data'):
result = conn2.run('ls -l |grep \'data\' |wc -l')
if result.stdout.strip() != '7':
tdLog.exit('the file number is wrong')
else:
tdLog.success('the file number is the same. test pass')
c1.close()
connTaos.close()

60
tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json Normal file
View File

@ -0,0 +1,60 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 100,
"num_of_records_per_req": 32766,
"databases": [{
"dbinfo": {
"name": "db",
"drop": "yes",
"replica": 1,
"days": 1,
"cache": 4,
"blocks": 3,
"precision": "ms",
"keep": 10,
"minRows": 1000,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb",
"child_table_exists":"no",
"childtable_count": 1,
"childtable_prefix": "stb_",
"auto_create_table": "no",
"batch_create_tbl_num": 20,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 864000,
"childtable_limit": 1,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1000,
"start_timestamp": "2020-10-11 00:00:00.000",
"sample_format": "csv",
"sample_file": "./tools/taosdemoAllTest/sample.csv",
"tags_file": "",
"columns": [{"type": "BINARY", "len": 5120, "count":1}],
"tags": [{"type": "TINYINT", "count":2}]
}]
}]
}

60
tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json Normal file
View File

@ -0,0 +1,60 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 100,
"num_of_records_per_req": 32766,
"databases": [{
"dbinfo": {
"name": "db",
"drop": "no",
"replica": 1,
"days": 1,
"cache": 4,
"blocks": 3,
"precision": "ms",
"keep": 10,
"minRows": 1000,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb",
"child_table_exists":"yes",
"childtable_count": 1,
"childtable_prefix": "stb_",
"auto_create_table": "yes",
"batch_create_tbl_num": 20,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 432000,
"childtable_limit": 1,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1000,
"start_timestamp": "2020-10-16 00:00:00.500",
"sample_format": "csv",
"sample_file": "./tools/taosdemoAllTest/sample.csv",
"tags_file": "",
"columns": [{"type": "BINARY", "len": 5120, "count":1}],
"tags": [{"type": "TINYINT", "count":1}]
}]
}]
}

83
tests/pytest/util/pathFinding.py Normal file
View File

@ -0,0 +1,83 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
from util.log import *
class TDFindPath:
"""This class is for finding path within TDengine
"""
def __init__(self):
self.file = ""
def init(self, file):
"""[summary]
Args:
file (str): the file location you want to start the query. Generally using __file__
"""
self.file = file
def getTaosdemoPath(self):
"""for finding the path of directory containing taosdemo
Returns:
str: the path to directory containing taosdemo
"""
selfPath = os.path.dirname(os.path.realpath(self.file))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info(f"taosd found in {buildPath}")
return buildPath + "/build/bin/"
def getTDenginePath(self):
"""for finding the root path of TDengine
Returns:
str: the root path of TDengine
"""
selfPath = os.path.dirname(os.path.realpath(self.file))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
print(projPath)
for root, dirs, files in os.walk(projPath):
if ("sim" in dirs):
print(root)
rootRealPath = os.path.realpath(root)
if (rootRealPath == ""):
tdLog.exit("TDengine not found!")
else:
tdLog.info(f"TDengine found in {rootRealPath}")
return rootRealPath
tdFindPath = TDFindPath()