From b66428c33bce88c1fe3ef4a50ed2b8514df594da Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 11 Jan 2021 16:48:50 +0800 Subject: [PATCH 1/7] multi-level --- tests/pytest/multilevel/basic.py | 50 ++++++++++++++++++++++++++++++++ tests/pytest/util/sql.py | 43 +++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 tests/pytest/multilevel/basic.py diff --git a/tests/pytest/multilevel/basic.py b/tests/pytest/multilevel/basic.py new file mode 100644 index 0000000000..8ae8d9868a --- /dev/null +++ b/tests/pytest/multilevel/basic.py @@ -0,0 +1,50 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + cfg={ + 'dataDir': '/mnt/data1', + 'dataDir': '/mnt/data2 0 0' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b2ed6212fd..6deeae2d0b 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -16,9 +16,12 @@ import os import time import datetime import inspect +import psutil +import shutil from util.log import * + class TDSql: def __init__(self): self.queryRows = 0 @@ -183,6 +186,46 @@ class TDSql: tdLog.exit("%s(%d) failed: sql:%s, affectedRows:%d != expect:%d" % args) tdLog.info("sql:%s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows)) + + def taosdStatus(self, state): + pstate = 0 + loop = 0 + for i in range(30): + pl = psutil.pids() + for pid in pl: + if psutil.Process(pid).name == 'taosd': + pstate = 1 + loop = 1 + break + if loop:break + time.sleep(5) + if pstate == state: + tdLog.info("taosd state is %d == expect:%d",pstate,state) + else: + tdLog.exit("taosd state is %d != expect:%d" ,pstate,state) + pass + def haveFile(self, dir, state): + if os.path.exists(dir) and os.path.isdir(dir): + if not os.listdir(dir): + if state : + tdLog.exit("dir: %s is empty, expect: not empty" ,dir) + else: + tdLog.info("dir: %s is empty, expect: empty" ,dir) + else: + if state : + tdLog.info("dir: %s is empty, expect: not empty" ,dir) + else: + tdLog.exit("dir: %s is empty, expect: empty" ,dir) + else: + tdLog.exit("dir: %s doesn't exist" ,dir) + def createDir(self, dir): + if os.path.exists(dir): + shutil.rmtree(dir) + tdLog.info("dir: %s is removed" ,dir) + os.makedirs( dir, 755 ) + tdLog.info("dir: %s is created" ,dir) + pass + tdSql = TDSql() From 5746eb5c655b6600284b70245b0c4deb46435576 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 11 Jan 2021 18:42:31 +0800 Subject: [PATCH 2/7] fix dict error --- tests/pytest/multilevel/basic.py | 7 +- tests/pytest/multilevel/dirNotExist.py | 49 ++++++++++++ tests/pytest/multilevel/moreThan16disks.py | 80 +++++++++++++++++++ .../pytest/multilevel/multiMainMountPoint.py | 50 ++++++++++++ tests/pytest/multilevel/noneMainMountPoint.py | 50 ++++++++++++ tests/pytest/multilevel/tt.py | 20 +++++ tests/pytest/util/dnodes.py | 18 ++++- tests/pytest/util/sql.py | 20 ++--- 8 files changed, 278 insertions(+), 16 deletions(-) create mode 100644 tests/pytest/multilevel/dirNotExist.py create mode 100644 tests/pytest/multilevel/moreThan16disks.py create mode 100644 tests/pytest/multilevel/multiMainMountPoint.py create mode 100644 tests/pytest/multilevel/noneMainMountPoint.py create mode 100644 tests/pytest/multilevel/tt.py diff --git a/tests/pytest/multilevel/basic.py b/tests/pytest/multilevel/basic.py index 8ae8d9868a..b3e38cb6ce 100644 --- a/tests/pytest/multilevel/basic.py +++ b/tests/pytest/multilevel/basic.py @@ -30,12 +30,15 @@ class TDTestCase: self.startTime = 1520000010000 tdDnodes.stop(1) + print('-'*40) cfg={ - 'dataDir': '/mnt/data1', - 'dataDir': '/mnt/data2 0 0' + '/mnt/data1' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' } + print('*'*40) tdSql.createDir('/mnt/data1') tdSql.createDir('/mnt/data2') + print('+'*40) tdDnodes.deploy(1,cfg) tdDnodes.startWithoutSleep(1) diff --git a/tests/pytest/multilevel/dirNotExist.py b/tests/pytest/multilevel/dirNotExist.py new file mode 100644 index 0000000000..a0d9d4c5bf --- /dev/null +++ b/tests/pytest/multilevel/dirNotExist.py @@ -0,0 +1,49 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + cfg={ + 'dataDir': '/mnt/data1 0 0 ', + 'dataDir': '/mnt/data2 0 0' + } + tdSql.createDir('/mnt/data1') + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/moreThan16disks.py b/tests/pytest/multilevel/moreThan16disks.py new file mode 100644 index 0000000000..09af90f160 --- /dev/null +++ b/tests/pytest/multilevel/moreThan16disks.py @@ -0,0 +1,80 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + cfg={ + 'dataDir': '/mnt/data1 0 0', + 'dataDir': '/mnt/data2 0 0', + 'dataDir': '/mnt/data3 0 0', + 'dataDir': '/mnt/data4 0 0', + 'dataDir': '/mnt/data5 0 0', + 'dataDir': '/mnt/data6 0 0', + 'dataDir': '/mnt/data7 0 0', + 'dataDir': '/mnt/data8 0 0', + 'dataDir': '/mnt/data9 0 0', + 'dataDir': '/mnt/data10 0 0', + 'dataDir': '/mnt/data11 0 0', + 'dataDir': '/mnt/data12 0 0', + 'dataDir': '/mnt/data13 0 0', + 'dataDir': '/mnt/data14 0 0', + 'dataDir': '/mnt/data15 0 0', + 'dataDir': '/mnt/data16 0 0', + 'dataDir': '/mnt/data17 0 0' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + tdSql.createDir('/mnt/data3') + tdSql.createDir('/mnt/data4') + tdSql.createDir('/mnt/data5') + tdSql.createDir('/mnt/data6') + tdSql.createDir('/mnt/data7') + tdSql.createDir('/mnt/data8') + tdSql.createDir('/mnt/data9') + tdSql.createDir('/mnt/data10') + tdSql.createDir('/mnt/data11') + tdSql.createDir('/mnt/data12') + tdSql.createDir('/mnt/data13') + tdSql.createDir('/mnt/data14') + tdSql.createDir('/mnt/data15') + tdSql.createDir('/mnt/data16') + tdSql.createDir('/mnt/data17') + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/multiMainMountPoint.py b/tests/pytest/multilevel/multiMainMountPoint.py new file mode 100644 index 0000000000..4a0a913b23 --- /dev/null +++ b/tests/pytest/multilevel/multiMainMountPoint.py @@ -0,0 +1,50 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + cfg={ + 'dataDir': '/mnt/data1', + 'dataDir': '/mnt/data2' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/noneMainMountPoint.py b/tests/pytest/multilevel/noneMainMountPoint.py new file mode 100644 index 0000000000..2a6e092bf7 --- /dev/null +++ b/tests/pytest/multilevel/noneMainMountPoint.py @@ -0,0 +1,50 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + cfg={ + 'dataDir': '/mnt/data1 0 0', + 'dataDir': '/mnt/data2 0 0' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/tt.py b/tests/pytest/multilevel/tt.py new file mode 100644 index 0000000000..a84e3f80c9 --- /dev/null +++ b/tests/pytest/multilevel/tt.py @@ -0,0 +1,20 @@ +cfg={ + '/mnt/data1 0 0': 'dataDir' , + 'dataDir': '/mnt/data2 0 0', + 'dataDir': '/mnt/data3 0 0', + 'dataDir': '/mnt/data4 0 0', + 'dataDir': '/mnt/data5 0 0', + 'dataDir': '/mnt/data6 0 0', + 'dataDir': '/mnt/data7 0 0', + 'dataDir': '/mnt/data8 0 0', + 'dataDir': '/mnt/data9 0 0', + 'dataDir': '/mnt/data10 0 0', + 'dataDir': '/mnt/data11 0 0', + 'dataDir': '/mnt/data12 0 0', + 'dataDir': '/mnt/data13 0 0', + 'dataDir': '/mnt/data14 0 0', + 'dataDir': '/mnt/data15 0 0', + 'dataDir': '/mnt/data16 0 0', + 'dataDir': '/mnt/data17 0 0' + } +print(cfg) \ No newline at end of file diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index afea23372f..117dfe5f07 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -208,14 +208,24 @@ class TDDnode: self.cfg("publicIp", "192.168.0.%d" % (self.index)) self.cfg("internalIp", "192.168.0.%d" % (self.index)) self.cfg("privateIp", "192.168.0.%d" % (self.index)) - - self.cfg("dataDir",self.dataDir) - self.cfg("logDir",self.logDir) + self.cfgDict["dataDir"] = self.dataDir + self.cfgDict["logDir"] = self.logDir + # self.cfg("dataDir",self.dataDir) + # self.cfg("logDir",self.logDir) print(updatecfgDict) + isFirstDir = 1 if updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) for key,value in updatecfgDict[0][0].items(): - self.addExtraCfg(key,value) + if value == 'dataDir' : + if isFirstDir: + self.cfgDict.pop('dataDir') + self.cfg(value,key) + isFirstDir = 0 + else: + self.cfg(value,key) + else: + self.addExtraCfg(key,value) for key, value in self.cfgDict.items(): self.cfg(key, value) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 6deeae2d0b..904a647b7f 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -199,33 +199,33 @@ class TDSql: break if loop:break time.sleep(5) - + args=(pstate,state) if pstate == state: - tdLog.info("taosd state is %d == expect:%d",pstate,state) + tdLog.info("taosd state is %d == expect:%d" %args) else: - tdLog.exit("taosd state is %d != expect:%d" ,pstate,state) + tdLog.exit("taosd state is %d != expect:%d" %args) pass def haveFile(self, dir, state): if os.path.exists(dir) and os.path.isdir(dir): if not os.listdir(dir): if state : - tdLog.exit("dir: %s is empty, expect: not empty" ,dir) + tdLog.exit("dir: %s is empty, expect: not empty" %dir) else: - tdLog.info("dir: %s is empty, expect: empty" ,dir) + tdLog.info("dir: %s is empty, expect: empty" %dir) else: if state : - tdLog.info("dir: %s is empty, expect: not empty" ,dir) + tdLog.info("dir: %s is empty, expect: not empty" %dir) else: - tdLog.exit("dir: %s is empty, expect: empty" ,dir) + tdLog.exit("dir: %s is empty, expect: empty" %dir) else: - tdLog.exit("dir: %s doesn't exist" ,dir) + tdLog.exit("dir: %s doesn't exist" %dir) def createDir(self, dir): if os.path.exists(dir): shutil.rmtree(dir) - tdLog.info("dir: %s is removed" ,dir) + tdLog.info("dir: %s is removed" %dir) os.makedirs( dir, 755 ) - tdLog.info("dir: %s is created" ,dir) + tdLog.info("dir: %s is created" %dir) pass tdSql = TDSql() From f9b2c27c0a597aa0a50c4571cda2e9130f1da9ef Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 12 Jan 2021 10:42:46 +0800 Subject: [PATCH 3/7] [TD-2723]add case for multi-level storage --- tests/pytest/multilevel/basic.py | 16 +++--- tests/pytest/multilevel/dirNotExist.py | 15 +++--- .../pytest/multilevel/dirPermissionDenied.py | 51 +++++++++++++++++++ tests/pytest/multilevel/moreThan16disks.py | 51 +++++-------------- .../pytest/multilevel/multiMainMountPoint.py | 13 +++-- tests/pytest/multilevel/noneMainMountPoint.py | 13 +++-- 6 files changed, 90 insertions(+), 69 deletions(-) create mode 100644 tests/pytest/multilevel/dirPermissionDenied.py diff --git a/tests/pytest/multilevel/basic.py b/tests/pytest/multilevel/basic.py index b3e38cb6ce..95549409c9 100644 --- a/tests/pytest/multilevel/basic.py +++ b/tests/pytest/multilevel/basic.py @@ -25,24 +25,22 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def run(self): - self.ntables = 10 - self.rowsPerTable = 10 - self.startTime = 1520000010000 - - tdDnodes.stop(1) - print('-'*40) cfg={ '/mnt/data1' : 'dataDir', '/mnt/data2 0 0' : 'dataDir' } - print('*'*40) tdSql.createDir('/mnt/data1') tdSql.createDir('/mnt/data2') - print('+'*40) + + tdLog.info("================= step1") + tdDnodes.stop(1) tdDnodes.deploy(1,cfg) tdDnodes.startWithoutSleep(1) - tdSql.taosdStatus(0) + tdLog.info("================= step2") + tdSql.haveFile('/mnt/data1',1) + tdSql.haveFile('/mnt/data2',1) + def stop(self): tdSql.close() diff --git a/tests/pytest/multilevel/dirNotExist.py b/tests/pytest/multilevel/dirNotExist.py index a0d9d4c5bf..e895ac31ad 100644 --- a/tests/pytest/multilevel/dirNotExist.py +++ b/tests/pytest/multilevel/dirNotExist.py @@ -25,19 +25,20 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def run(self): - self.ntables = 10 - self.rowsPerTable = 10 - self.startTime = 1520000010000 - - tdDnodes.stop(1) cfg={ - 'dataDir': '/mnt/data1 0 0 ', - 'dataDir': '/mnt/data2 0 0' + '/mnt/data1 0 0' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' } tdSql.createDir('/mnt/data1') + os.system('rm -rf /mnt/data2') + + + tdLog.info("================= step1") + tdDnodes.stop(1) tdDnodes.deploy(1,cfg) tdDnodes.startWithoutSleep(1) + tdLog.info("================= step2") tdSql.taosdStatus(0) def stop(self): diff --git a/tests/pytest/multilevel/dirPermissionDenied.py b/tests/pytest/multilevel/dirPermissionDenied.py new file mode 100644 index 0000000000..db2cdcf757 --- /dev/null +++ b/tests/pytest/multilevel/dirPermissionDenied.py @@ -0,0 +1,51 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + cfg={ + '/mnt/data1 0 0' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + os.system('chmod 111 /mnt/data2') + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.taosdStatus(0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/moreThan16disks.py b/tests/pytest/multilevel/moreThan16disks.py index 09af90f160..407e5aa697 100644 --- a/tests/pytest/multilevel/moreThan16disks.py +++ b/tests/pytest/multilevel/moreThan16disks.py @@ -25,50 +25,23 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def run(self): - self.ntables = 10 - self.rowsPerTable = 10 - self.startTime = 1520000010000 + cfg={} + for i in range(17): + if i == 0 : + datadir = '/mnt/data%d 0 1' % (i+1) + else: + datadir = '/mnt/data%d 0 0' % (i+1) + cfg.update({ datadir : 'dataDir' }) + tdSql.createDir('/mnt/data%d' % (i+1)) + + tdLog.info("================= step1") tdDnodes.stop(1) - cfg={ - 'dataDir': '/mnt/data1 0 0', - 'dataDir': '/mnt/data2 0 0', - 'dataDir': '/mnt/data3 0 0', - 'dataDir': '/mnt/data4 0 0', - 'dataDir': '/mnt/data5 0 0', - 'dataDir': '/mnt/data6 0 0', - 'dataDir': '/mnt/data7 0 0', - 'dataDir': '/mnt/data8 0 0', - 'dataDir': '/mnt/data9 0 0', - 'dataDir': '/mnt/data10 0 0', - 'dataDir': '/mnt/data11 0 0', - 'dataDir': '/mnt/data12 0 0', - 'dataDir': '/mnt/data13 0 0', - 'dataDir': '/mnt/data14 0 0', - 'dataDir': '/mnt/data15 0 0', - 'dataDir': '/mnt/data16 0 0', - 'dataDir': '/mnt/data17 0 0' - } - tdSql.createDir('/mnt/data1') - tdSql.createDir('/mnt/data2') - tdSql.createDir('/mnt/data3') - tdSql.createDir('/mnt/data4') - tdSql.createDir('/mnt/data5') - tdSql.createDir('/mnt/data6') - tdSql.createDir('/mnt/data7') - tdSql.createDir('/mnt/data8') - tdSql.createDir('/mnt/data9') - tdSql.createDir('/mnt/data10') - tdSql.createDir('/mnt/data11') - tdSql.createDir('/mnt/data12') - tdSql.createDir('/mnt/data13') - tdSql.createDir('/mnt/data14') - tdSql.createDir('/mnt/data15') - tdSql.createDir('/mnt/data16') - tdSql.createDir('/mnt/data17') tdDnodes.deploy(1,cfg) tdDnodes.startWithoutSleep(1) + tdLog.info("================= step2") + tdSql.taosdStatus(0) def stop(self): diff --git a/tests/pytest/multilevel/multiMainMountPoint.py b/tests/pytest/multilevel/multiMainMountPoint.py index 4a0a913b23..c5af816205 100644 --- a/tests/pytest/multilevel/multiMainMountPoint.py +++ b/tests/pytest/multilevel/multiMainMountPoint.py @@ -25,20 +25,19 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def run(self): - self.ntables = 10 - self.rowsPerTable = 10 - self.startTime = 1520000010000 - - tdDnodes.stop(1) cfg={ - 'dataDir': '/mnt/data1', - 'dataDir': '/mnt/data2' + '/mnt/data1' : 'dataDir', + '/mnt/data2' : 'dataDir' } tdSql.createDir('/mnt/data1') tdSql.createDir('/mnt/data2') + + tdLog.info("================= step1") + tdDnodes.stop(1) tdDnodes.deploy(1,cfg) tdDnodes.startWithoutSleep(1) + tdLog.info("================= step2") tdSql.taosdStatus(0) def stop(self): diff --git a/tests/pytest/multilevel/noneMainMountPoint.py b/tests/pytest/multilevel/noneMainMountPoint.py index 2a6e092bf7..67fab7c7a2 100644 --- a/tests/pytest/multilevel/noneMainMountPoint.py +++ b/tests/pytest/multilevel/noneMainMountPoint.py @@ -25,20 +25,19 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def run(self): - self.ntables = 10 - self.rowsPerTable = 10 - self.startTime = 1520000010000 - - tdDnodes.stop(1) cfg={ - 'dataDir': '/mnt/data1 0 0', - 'dataDir': '/mnt/data2 0 0' + '/mnt/data1 0 0' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' } tdSql.createDir('/mnt/data1') tdSql.createDir('/mnt/data2') + + tdLog.info("================= step1") + tdDnodes.stop(1) tdDnodes.deploy(1,cfg) tdDnodes.startWithoutSleep(1) + tdLog.info("================= step2") tdSql.taosdStatus(0) def stop(self): From 02c96a2fa318ca422400470b75400e5b6966deec Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 12 Jan 2021 10:46:41 +0800 Subject: [PATCH 4/7] remove temp file --- tests/pytest/multilevel/tt.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 tests/pytest/multilevel/tt.py diff --git a/tests/pytest/multilevel/tt.py b/tests/pytest/multilevel/tt.py deleted file mode 100644 index a84e3f80c9..0000000000 --- a/tests/pytest/multilevel/tt.py +++ /dev/null @@ -1,20 +0,0 @@ -cfg={ - '/mnt/data1 0 0': 'dataDir' , - 'dataDir': '/mnt/data2 0 0', - 'dataDir': '/mnt/data3 0 0', - 'dataDir': '/mnt/data4 0 0', - 'dataDir': '/mnt/data5 0 0', - 'dataDir': '/mnt/data6 0 0', - 'dataDir': '/mnt/data7 0 0', - 'dataDir': '/mnt/data8 0 0', - 'dataDir': '/mnt/data9 0 0', - 'dataDir': '/mnt/data10 0 0', - 'dataDir': '/mnt/data11 0 0', - 'dataDir': '/mnt/data12 0 0', - 'dataDir': '/mnt/data13 0 0', - 'dataDir': '/mnt/data14 0 0', - 'dataDir': '/mnt/data15 0 0', - 'dataDir': '/mnt/data16 0 0', - 'dataDir': '/mnt/data17 0 0' - } -print(cfg) \ No newline at end of file From 037816547b8e29ae77da32e6dd9c43a1b93e841c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 12 Jan 2021 11:54:06 +0800 Subject: [PATCH 5/7] improve taosdStatus --- tests/pytest/util/sql.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 904a647b7f..4f7607fc6c 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -188,17 +188,21 @@ class TDSql: tdLog.info("sql:%s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows)) def taosdStatus(self, state): + tdLog.sleep(5) pstate = 0 - loop = 0 for i in range(30): + pstate = 0 pl = psutil.pids() for pid in pl: if psutil.Process(pid).name == 'taosd': pstate = 1 - loop = 1 break - if loop:break - time.sleep(5) + if pstate: + tdLog.sleep(5) + continue + pstate = 0 + break + args=(pstate,state) if pstate == state: tdLog.info("taosd state is %d == expect:%d" %args) From b89d65c99c067600bbae0865a491e2028f6fc001 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 12 Jan 2021 13:46:51 +0800 Subject: [PATCH 6/7] [TD-2722]: add test cases for muliple level storage --- .../multilevel/fileDistributionSameLevel.py | 71 ++++++++++++++++ .../multilevel/illegelLevelMountPoint.py | 59 +++++++++++++ tests/pytest/multilevel/middleLevelMissing.py | 64 +++++++++++++++ tests/pytest/multilevel/retentionTest.py | 82 +++++++++++++++++++ .../pytest/multilevel/threeLevelMountPoint.py | 65 +++++++++++++++ .../pytest/multilevel/wrongLevelMountPoint.py | 49 +++++++++++ 6 files changed, 390 insertions(+) create mode 100644 tests/pytest/multilevel/fileDistributionSameLevel.py create mode 100644 tests/pytest/multilevel/illegelLevelMountPoint.py create mode 100644 tests/pytest/multilevel/middleLevelMissing.py create mode 100644 tests/pytest/multilevel/retentionTest.py create mode 100644 tests/pytest/multilevel/threeLevelMountPoint.py create mode 100644 tests/pytest/multilevel/wrongLevelMountPoint.py diff --git a/tests/pytest/multilevel/fileDistributionSameLevel.py b/tests/pytest/multilevel/fileDistributionSameLevel.py new file mode 100644 index 0000000000..e2880283b5 --- /dev/null +++ b/tests/pytest/multilevel/fileDistributionSameLevel.py @@ -0,0 +1,71 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.ts = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '10' : 'maxVgroupsPerDb', + '/mnt/data00 0 1' : 'dataDir', + '/mnt/data01 0 0' : 'dataDir', + '/mnt/data02 0 0' : 'dataDir', + '/mnt/data03 1 0' : 'dataDir', + '/mnt/data04 1 0' : 'dataDir' + } + tdSql.createDir('/mnt/data00') + tdSql.createDir('/mnt/data01') + tdSql.createDir('/mnt/data02') + tdSql.createDir('/mnt/data03') + tdSql.createDir('/mnt/data04') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.execute("create database test days 1") + tdSql.execute("use test") + + tdSql.execute("create table tb(ts timestamp, c int)") + + for i in range(self.rowsPerTable): + tdSql.execute("insert into tb values(%d, 1)" % (self.ts + i * 86400000)) + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select * from test.tb") + tdSql.checkRows(10) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/illegelLevelMountPoint.py b/tests/pytest/multilevel/illegelLevelMountPoint.py new file mode 100644 index 0000000000..cbf7fc1fe9 --- /dev/null +++ b/tests/pytest/multilevel/illegelLevelMountPoint.py @@ -0,0 +1,59 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 3 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 -1 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/middleLevelMissing.py b/tests/pytest/multilevel/middleLevelMissing.py new file mode 100644 index 0000000000..727c2380ae --- /dev/null +++ b/tests/pytest/multilevel/middleLevelMissing.py @@ -0,0 +1,64 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 1 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + + tdDnodes.stop(1) + # Test2 2 dataDir + cfg={ + '/mnt/data1 0 1' : 'dataDir', + '/mnt/data1 2 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/retentionTest.py b/tests/pytest/multilevel/retentionTest.py new file mode 100644 index 0000000000..9ea59d9d59 --- /dev/null +++ b/tests/pytest/multilevel/retentionTest.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data00 0 1' : 'dataDir', + '/mnt/data01 0 0' : 'dataDir', + '/mnt/data02 0 0' : 'dataDir', + '/mnt/data10 1 0' : 'dataDir', + '/mnt/data11 1 0' : 'dataDir', + '/mnt/data12 1 0' : 'dataDir', + '/mnt/data20 2 0' : 'dataDir', + '/mnt/data21 2 0' : 'dataDir', + '/mnt/data22 2 0' : 'dataDir' + } + tdSql.createDir('/mnt/data00') + tdSql.createDir('/mnt/data01') + tdSql.createDir('/mnt/data02') + tdSql.createDir('/mnt/data10') + tdSql.createDir('/mnt/data11') + tdSql.createDir('/mnt/data12') + tdSql.createDir('/mnt/data20') + tdSql.createDir('/mnt/data21') + tdSql.createDir('/mnt/data22') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.execute("create database test days 1 keep 15,5,10") + tdSql.execute("use test") + + tdSql.execute("create table tb(ts timestamp, c int)") + + count = 0 + os.system("sudo timedatectl set-ntp false") + + for i in range(self.rowsPerTable): + tdSql.execute("insert into tb values(now, 1)") + count += 1 + tdSql.query("select * from tb") + tdSql.checkRows(count) + tdDnodes.stop(1) + os.system("sudo date -s $(date -d \"${DATE} 1 day\" \"+%Y%m%d\")") + tdDnodes.start(1) + + def stop(self): + os.system("sudo timedatectl set-ntp true") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/threeLevelMountPoint.py b/tests/pytest/multilevel/threeLevelMountPoint.py new file mode 100644 index 0000000000..ccaf95f47d --- /dev/null +++ b/tests/pytest/multilevel/threeLevelMountPoint.py @@ -0,0 +1,65 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data00 0 1' : 'dataDir', + '/mnt/data01 0 0' : 'dataDir', + '/mnt/data02 0 0' : 'dataDir', + '/mnt/data10 1 0' : 'dataDir', + '/mnt/data11 1 0' : 'dataDir', + '/mnt/data12 1 0' : 'dataDir', + '/mnt/data20 2 0' : 'dataDir', + '/mnt/data21 2 0' : 'dataDir', + '/mnt/data22 2 0' : 'dataDir' + } + tdSql.createDir('/mnt/data00') + tdSql.createDir('/mnt/data01') + tdSql.createDir('/mnt/data02') + tdSql.createDir('/mnt/data10') + tdSql.createDir('/mnt/data11') + tdSql.createDir('/mnt/data12') + tdSql.createDir('/mnt/data20') + tdSql.createDir('/mnt/data21') + tdSql.createDir('/mnt/data22') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/wrongLevelMountPoint.py b/tests/pytest/multilevel/wrongLevelMountPoint.py new file mode 100644 index 0000000000..00b9eb89c6 --- /dev/null +++ b/tests/pytest/multilevel/wrongLevelMountPoint.py @@ -0,0 +1,49 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 1 1' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 0616475e505be9f395fa26edf113e59203345a4a Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 12 Jan 2021 16:56:01 +0800 Subject: [PATCH 7/7] [TD-2734]fix timeout error --- tests/pytest/util/dnodes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 117dfe5f07..91c75efe97 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -212,7 +212,7 @@ class TDDnode: self.cfgDict["logDir"] = self.logDir # self.cfg("dataDir",self.dataDir) # self.cfg("logDir",self.logDir) - print(updatecfgDict) + # print(updatecfgDict) isFirstDir = 1 if updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) @@ -292,13 +292,15 @@ class TDDnode: break popen = subprocess.Popen('tail -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) pid = popen.pid - print('Popen.pid:' + str(pid)) + # print('Popen.pid:' + str(pid)) + timeout = time.time() + 60*2 while True: line = popen.stdout.readline().strip() if bkey in line: - print(line) popen.kill() break + if time.time() > timeout: + tdLog.exit('wait too long for taosd start') tdLog.debug("the dnode:%d has been started." % (self.index)) else: tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index))