From dc451984b5ae4acc77cf4c21479df75d3c054023 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 12:59:46 +0800 Subject: [PATCH 1/7] feat: add level and disk param on test.py --- tests/army/frame/caseBase.py | 25 ++ tests/army/frame/common.py | 2 +- tests/army/frame/dnodes-default.py | 502 ---------------------- tests/army/frame/dnodes-no-random-fail.py | 500 --------------------- tests/army/frame/dnodes-random-fail.py | 497 --------------------- tests/army/frame/{ => server}/cluster.py | 10 +- tests/army/frame/{ => server}/dnodes.py | 23 +- tests/army/frame/sql.py | 3 + tests/army/frame/srvCtl.py | 27 ++ tests/army/test.py | 21 +- 10 files changed, 96 insertions(+), 1514 deletions(-) create mode 100644 tests/army/frame/caseBase.py delete mode 100644 tests/army/frame/dnodes-default.py delete mode 100644 tests/army/frame/dnodes-no-random-fail.py delete mode 100644 tests/army/frame/dnodes-random-fail.py rename tests/army/frame/{ => server}/cluster.py (91%) rename tests/army/frame/{ => server}/dnodes.py (98%) create mode 100644 tests/army/frame/srvCtl.py diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py new file mode 100644 index 0000000000..abaa92a51c --- /dev/null +++ b/tests/army/frame/caseBase.py @@ -0,0 +1,25 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime + +class caseBase: + def __init__(self): + # record server information + self.dnodeNum = 0 + self.mnodeNum = 0 + self.mLevel = 0 + self.mLevelDisk = 0 diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py index 93059ff078..b8704080c0 100644 --- a/tests/army/frame/common.py +++ b/tests/army/frame/common.py @@ -23,7 +23,7 @@ import taos from frame.log import * from frame.sql import * from frame.cases import * -from frame.dnodes import * +from TDengine.tests.army.frame.server.dnodes import * from frame.common import * from frame.constant import * from dataclasses import dataclass,field diff --git a/tests/army/frame/dnodes-default.py b/tests/army/frame/dnodes-default.py deleted file mode 100644 index a1d1698b00..0000000000 --- a/tests/army/frame/dnodes-default.py +++ /dev/null @@ -1,502 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "minTablesPerVnode": "4", - "maxTablesPerVnode": "1000", - "tableIncStepPerVnode": "10000", - "maxVgroupsPerDb": "1000", - "sdbDebugFlag": "143", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/dnodes-no-random-fail.py b/tests/army/frame/dnodes-no-random-fail.py deleted file mode 100644 index 3b3083396b..0000000000 --- a/tests/army/frame/dnodes-no-random-fail.py +++ /dev/null @@ -1,500 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "anyIp": "0", - "sdbDebugFlag": "135", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s --random-file-fail-factor 0 > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/dnodes-random-fail.py b/tests/army/frame/dnodes-random-fail.py deleted file mode 100644 index 794adfd7ed..0000000000 --- a/tests/army/frame/dnodes-random-fail.py +++ /dev/null @@ -1,497 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s --alloc-random-fail --random-file-fail-factor 5 > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/cluster.py b/tests/army/frame/server/cluster.py similarity index 91% rename from tests/army/frame/cluster.py rename to tests/army/frame/server/cluster.py index 4da53840c0..ade8ac39a2 100644 --- a/tests/army/frame/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -8,7 +8,7 @@ import socket from frame.log import * from frame.sql import * from frame.cases import * -from frame.dnodes import * +from frame.server.dnodes import * from frame.common import * class ClusterDnodes(TDDnodes): @@ -35,17 +35,21 @@ class ConfigureyCluster: self.startPort = 6030 self.portStep = 100 self.mnodeNums = 0 + self.level = 0 + self.disk = 0 - def configure_cluster(self ,dnodeNums=5,mnodeNums=0,independentMnode=True,startPort=6030,portStep=100,hostname="%s"%hostname): + def configure_cluster(self ,dnodeNums=5, mnodeNums=0, independentMnode=True, startPort=6030, portStep=100, hostname="%s"%hostname, level=1, disk=1): self.startPort=int(startPort) self.portStep=int(portStep) self.hostname=hostname self.dnodeNums = int(dnodeNums) self.mnodeNums = int(mnodeNums) + self.level = int(level) + self.disk = int(disk) self.dnodes = [] startPort_sec = int(startPort+portStep) for num in range(1, (self.dnodeNums+1)): - dnode = TDDnode(num) + dnode = TDDnode(num, self.level, self.disk) dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}") dnode.addExtraCfg("fqdn", f"{hostname}") dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}") diff --git a/tests/army/frame/dnodes.py b/tests/army/frame/server/dnodes.py similarity index 98% rename from tests/army/frame/dnodes.py rename to tests/army/frame/server/dnodes.py index 5581f29a57..7eee6f4bb6 100644 --- a/tests/army/frame/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -115,8 +115,11 @@ class TDSimClient: class TDDnode: - def __init__(self, index): + def __init__(self, index, level, disk): self.index = index + self.level = level + self.disk = disk + self.dataDir = [] self.running = 0 self.deployed = 0 self.testCluster = False @@ -210,13 +213,18 @@ class TDDnode: def deploy(self, *updatecfgDict): self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") + simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) + for i in range(1, self.level): + for j in range(1, self.disk): + eDir = os.path.join(simPath, f"data{i}{j}") + self.dataDir.append(eDir) self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + + for eDir in self.dataDir: + cmd = "rm -rf " + eDir + if os.system(cmd) != 0: + tdLog.exit(cmd) cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -229,7 +237,8 @@ class TDDnode: # cmd = "mkdir -p " + self.dataDir # if os.system(cmd) != 0: # tdLog.exit(cmd) - os.makedirs(self.dataDir) + for eDir in self.dataDir: + os.makedirs(self.dataDir) # cmd = "mkdir -p " + self.logDir # if os.system(cmd) != 0: diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index d7dce2bc3e..409dc6c96c 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -630,6 +630,7 @@ class TDSql: tdLog.exit("taosd state is %d != expect:%d" %args) pass + ''' def haveFile(self, dir, state): if os.path.exists(dir) and os.path.isdir(dir): if not os.listdir(dir): @@ -644,6 +645,7 @@ class TDSql: tdLog.exit("dir: %s is not empty, expect: empty" %dir) else: tdLog.exit("dir: %s doesn't exist" %dir) + def createDir(self, dir): if os.path.exists(dir): shrmtree(dir) @@ -651,5 +653,6 @@ class TDSql: os.makedirs( dir, 755 ) tdLog.info("dir: %s is created" %dir) pass +''' tdSql = TDSql() diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py new file mode 100644 index 0000000000..e7ef08cde9 --- /dev/null +++ b/tests/army/frame/srvCtl.py @@ -0,0 +1,27 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime + +class srvCtl: + def __init__(self): + # record server information + self.dnodeNum = 0 + self.mnodeNum = 0 + self.mLevel = 0 + self.mLevelDisk = 0 + +sc = srvCtl() \ No newline at end of file diff --git a/tests/army/test.py b/tests/army/test.py index cc114e0d16..ae873fb9d6 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -28,9 +28,9 @@ import importlib import toml from frame.log import * -from frame.dnodes import * +from frame.server.dnodes import * +from frame.server.cluster import * from frame.cases import * -from frame.cluster import * from frame.taosadapter import * import taos @@ -111,6 +111,9 @@ if __name__ == "__main__": asan = False independentMnode = False previousCluster = False + level = 1 + disk = 1 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) for key, value in opts: @@ -139,6 +142,8 @@ if __name__ == "__main__": tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.') + tdLog.printNoPrefix('-L set multiple level number. range 1 ~ 3') + tdLog.printNoPrefix('-D set disk number on each level. range 1 ~ 10') sys.exit(0) @@ -226,6 +231,12 @@ if __name__ == "__main__": if key in ['-P', '--previous']: previousCluster = True + if key in ['-L', '--level']: + level = value + + if key in ['-D', '--disk']: + disk = value + # # do exeCmd command # @@ -391,7 +402,7 @@ if __name__ == "__main__": tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -498,6 +509,7 @@ if __name__ == "__main__": else: tdLog.info("not need to query") else: + # except windows tdDnodes.setKillValgrind(killValgrind) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -574,7 +586,8 @@ if __name__ == "__main__": # dnode > 1 cluster tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) print(independentMnode,"independentMnode valuse") - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) + # create dnode list + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) From 9e7a4b8a868df69614f3f7d474cf3a09f39b7799 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 13:09:02 +0800 Subject: [PATCH 2/7] feat: test.py add -L --level -D --disk options --- tests/army/test.py | 2 +- tests/parallel_test/cases.task | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/army/test.py b/tests/army/test.py index ae873fb9d6..25cb9b383d 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -218,7 +218,7 @@ if __name__ == "__main__": if key in ['-a', '--asan']: asan = True - if key in ['-D', '--adaptercfgupdate']: + if key in ['-U', '--adaptercfgupdate']: try: adaptercfgupdate = eval(base64.b64decode(value.encode()).decode()) except: diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..19a44767a7 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,7 +6,7 @@ ,,y,unit-test,bash test.sh #army-test -,,y,army,./pytest.sh python3 ./test.py -f empty.py +,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 #system test ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py From 266ac30d96336204f4ebe040faf624d01dcebb37 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 16:28:39 +0800 Subject: [PATCH 3/7] feat: support -L -D for test.py submit --- tests/army/frame/common.py | 2 +- tests/army/frame/server/dnodes.py | 23 +++++++++++++++++------ tests/army/test.py | 7 ++++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py index b8704080c0..5cdb3f9f46 100644 --- a/tests/army/frame/common.py +++ b/tests/army/frame/common.py @@ -23,7 +23,7 @@ import taos from frame.log import * from frame.sql import * from frame.cases import * -from TDengine.tests.army.frame.server.dnodes import * +from frame.server.dnodes import * from frame.common import * from frame.constant import * from dataclasses import dataclass,field diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 7eee6f4bb6..8368a1573e 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -115,7 +115,7 @@ class TDSimClient: class TDDnode: - def __init__(self, index, level, disk): + def __init__(self, index=1, level=1, disk=1): self.index = index self.level = level self.disk = disk @@ -212,12 +212,19 @@ class TDDnode: self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) def deploy(self, *updatecfgDict): + # logDir self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") + # dataDir simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) - for i in range(1, self.level): - for j in range(1, self.disk): + primary = 1 + for i in range(self.level): + for j in range(self.disk): eDir = os.path.join(simPath, f"data{i}{j}") - self.dataDir.append(eDir) + self.dataDir.append(f"{eDir} {i} {primary}") + if primary == 1: + primary = 0 + + # taos.cfg self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") @@ -238,7 +245,7 @@ class TDDnode: # if os.system(cmd) != 0: # tdLog.exit(cmd) for eDir in self.dataDir: - os.makedirs(self.dataDir) + os.makedirs(eDir.split(' ')[0]) # cmd = "mkdir -p " + self.logDir # if os.system(cmd) != 0: @@ -284,7 +291,11 @@ class TDDnode: self.addExtraCfg(key, value) if (self.remoteIP == ""): for key, value in self.cfgDict.items(): - self.cfg(key, value) + if type(value) == list: + for v in value: + self.cfg(key, v) + else: + self.cfg(key, value) else: self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)"%self.index) diff --git a/tests/army/test.py b/tests/army/test.py index 25cb9b383d..83d065b24b 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -114,8 +114,9 @@ if __name__ == "__main__": level = 1 disk = 1 - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWU:n:i:aP:L:D:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums', + 'queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode',"asan",'previous','level','disk']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -137,7 +138,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') tdLog.printNoPrefix('-R restful realization form') tdLog.printNoPrefix('-W websocket connection') - tdLog.printNoPrefix('-D taosadapter update cfg dict ') + tdLog.printNoPrefix('-U taosadapter update cfg dict ') tdLog.printNoPrefix('-n the number of replicas') tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') From 4b6bd14295954abf828efd2a78bcdd04512b1d17 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 16:39:13 +0800 Subject: [PATCH 4/7] fix: if level=1 disk=1 save old data folder --- tests/army/frame/server/dnodes.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 8368a1573e..1e51fbc3be 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -217,12 +217,16 @@ class TDDnode: # dataDir simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) primary = 1 - for i in range(self.level): - for j in range(self.disk): - eDir = os.path.join(simPath, f"data{i}{j}") - self.dataDir.append(f"{eDir} {i} {primary}") - if primary == 1: - primary = 0 + if self.level == 1 and self.disk == 1: + eDir = os.path.join(simPath, "data") + self.dataDir.append(eDir) + else: + for i in range(self.level): + for j in range(self.disk): + eDir = os.path.join(simPath, f"data{i}{j}") + self.dataDir.append(f"{eDir} {i} {primary}") + if primary == 1: + primary = 0 # taos.cfg self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") From f567c19d05acd296799aa0d34551f5745f5177bf Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 25 Dec 2023 15:59:29 +0800 Subject: [PATCH 5/7] fix: add new-frame public file --- .../enterprise/multi-level/mlevel_basic.json | 58 +++++++++++++ .../enterprise/multi-level/mlevel_basic.py | 61 +++++++++++--- tests/army/frame/caseBase.py | 83 ++++++++++++++++++- tests/army/frame/eos.py | 35 ++++++++ tests/army/frame/epath.py | 53 ++++++++++++ tests/army/frame/etime.py | 23 +++++ tests/army/frame/etool.py | 40 +++++++++ tests/army/frame/pathFinding.py | 83 ------------------- tests/army/frame/sql.py | 13 ++- 9 files changed, 352 insertions(+), 97 deletions(-) create mode 100644 tests/army/enterprise/multi-level/mlevel_basic.json create mode 100644 tests/army/frame/eos.py create mode 100644 tests/army/frame/epath.py create mode 100644 tests/army/frame/etime.py create mode 100644 tests/army/frame/etool.py delete mode 100644 tests/army/frame/pathFinding.py diff --git a/tests/army/enterprise/multi-level/mlevel_basic.json b/tests/army/enterprise/multi-level/mlevel_basic.json new file mode 100644 index 0000000000..7df5b1caa8 --- /dev/null +++ b/tests/army/enterprise/multi-level/mlevel_basic.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "duration":"1d", + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"now-10d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index e3a3f57c74..a12b1b59eb 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -15,26 +15,65 @@ import sys import time import taos +import frame + from frame.log import * from frame.cases import * from frame.sql import * +from frame.caseBase import * + + + +class TDTestCase(TBase): + + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + json = frame.etool.curFile(__file__, "mlevel_basic.json") + frame.etool.runBenchmark(json=json) + + # set insert data information + self.childtable_count = 2 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + def doAction(self): + tdLog.info(f"trim database.") + self.trimDb() + + def saveData(self): + tdLog.info(f"check correct.") + + def checkSaveCorrent(self): + tdLog.info(f"check correct.") + + def checkCorrect(self): + tdLog.info(f"check correct.") -class TDTestCase: - # init - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) # run def run(self): - # check two db query result same - tdLog.info(f"hello world.") + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() - # stop - def stop(self): - tdSql.close() tdLog.success(f"{__file__} successfully executed") + + tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index abaa92a51c..7b371cd822 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -16,10 +16,89 @@ import os import time import datetime -class caseBase: - def __init__(self): +from log import * +from sql import * + +# test case base +class DbBase: + +# +# frame call +# + + # init + def init(self, conn, logSql, replicaVar=1): + # save param + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor(), True) + # record server information self.dnodeNum = 0 self.mnodeNum = 0 self.mLevel = 0 self.mLevelDisk = 0 + + # test case information + self.db = "db" + self.stb = "stb" + + # variant in taosBenchmark json + self.childtable_count = 2 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + # sql + self.sqlSum = f"select sum(ic) from {self.stb}" + self.sqlMax = f"select max(ic) from {self.stb}" + self.sqlMin = f"select min(ic) from {self.stb}" + self.sqlAvg = f"select avg(ic) from {self.stb}" + + + # stop + def stop(self): + tdSql.close() + + +# +# db action +# + + def trimDb(self): + tdSql.execute(f"trim database {self.db}") + + def compactDb(self): + tdSql.execute(f"compact database {self.db}") + + +# +# check db correct +# + + # basic + def checkInsertCorrect(self): + # check count + sql = f"select count(*) from {self.stb}" + tdSql.checkAgg(sql, self.childtable_count * self.insert_rows) + + # check child table count + sql = f" select count(*) from (select count(*) as cnt , tbname from {self.stb} group by tbname) where cnt = {self.insert_rows} " + tdSql.checkAgg(sql, self.childtable_count) + + # check step + sql = f"select count(*) from (select diff(ts) as dif from {self.stb} group by tbname) where dif != {self.timestamp_step}" + tdSql.checkAgg(sql, 0) + + # save agg result + def snapshotAgg(self): + + self.sum = tdSql.getFirstValue(self.sqlSum) + self.avg = tdSql.getFirstValue(self.sqlAvg) + self.min = tdSql.getFirstValue(self.sqlMin) + self.max = tdSql.getFirstValue(self.sqlMax) + + # check agg + def checkAggCorrect(self): + tdSql.checkAgg(self.sqlSum, self.sum) + tdSql.checkAgg(self.sqlAvg, self.avg) + tdSql.checkAgg(self.sqlMin, self.min) + tdSql.checkAgg(self.sqlMax, self.max) diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py new file mode 100644 index 0000000000..2d8559f3eb --- /dev/null +++ b/tests/army/frame/eos.py @@ -0,0 +1,35 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about system funciton extension +# + +import sys +import os +import time +import datetime + +# if windows platform return True +def isWin(): + return sys.platform.system().lower() == 'windows' + +# wait util execute file finished +def exe(file): + return os.system(file) + +# execute file and return immediately +def exeNoWait(file): + print("exe no wait") + + diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py new file mode 100644 index 0000000000..644e448fb7 --- /dev/null +++ b/tests/army/frame/epath.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about path function extension +# + +import os +from frame.log import * + +# build/bin path +binPath = "" + +def binPath(self): + global binPath + + if binPath != "": + return binPath + + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community/tests" in selfPath): + projPath = selfPath[:selfPath.find("community/tests")] + else: + projPath = selfPath[:selfPath.find("TDengine/tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + # check + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + # return + binPath = buildPath + "/build/bin/" + return binPath + +def binFile(self, file): + return binPath() + file \ No newline at end of file diff --git a/tests/army/frame/etime.py b/tests/army/frame/etime.py new file mode 100644 index 0000000000..2ee223e122 --- /dev/null +++ b/tests/army/frame/etime.py @@ -0,0 +1,23 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about tools funciton extension +# + +import sys +import os +import time +import datetime + + diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py new file mode 100644 index 0000000000..5ba29ee1c3 --- /dev/null +++ b/tests/army/frame/etool.py @@ -0,0 +1,40 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about system funciton extension +# + +import sys +import os +import time +import datetime +import epath +import eos + +# run taosBenchmark with command or json file mode +def runBenchmark(command = "", json = "") : + # get taosBenchmark path + bmFile = epath.binFile("taosBenchmark") + if eos.isWin(): + bmFile += ".exe" + + # run + if command != "": + eos.exe(bmFile + " " + command) + if json != "": + eos.exe(bmFile + " -f " + json) + +# get current directory file name +def curFile(fullPath, file): + return os.path.dirname(fullPath) + "/" + file diff --git a/tests/army/frame/pathFinding.py b/tests/army/frame/pathFinding.py deleted file mode 100644 index df03f0ed68..0000000000 --- a/tests/army/frame/pathFinding.py +++ /dev/null @@ -1,83 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - - -import os -from frame.log import * - - - -class TDFindPath: - """This class is for finding path within TDengine - """ - def __init__(self): - self.file = "" - - - def init(self, file): - """[summary] - - Args: - file (str): the file location you want to start the query. Generally using __file__ - """ - self.file = file - - def getTaosdemoPath(self): - """for finding the path of directory containing taosdemo - - Returns: - str: the path to directory containing taosdemo - """ - selfPath = os.path.dirname(os.path.realpath(self.file)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info(f"taosd found in {buildPath}") - return buildPath + "/build/bin/" - - def getTDenginePath(self): - """for finding the root path of TDengine - - Returns: - str: the root path of TDengine - """ - selfPath = os.path.dirname(os.path.realpath(self.file)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - print(projPath) - for root, dirs, files in os.walk(projPath): - if ("sim" in dirs): - print(root) - rootRealPath = os.path.realpath(root) - if (rootRealPath == ""): - tdLog.exit("TDengine not found!") - else: - tdLog.info(f"TDengine found in {rootRealPath}") - return rootRealPath - -tdFindPath = TDFindPath() \ No newline at end of file diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index 409dc6c96c..b3ee1ed5f8 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -535,6 +535,16 @@ class TDSql: tdLog.info("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args) raise Exception + # check like select count(*) ... sql + def checkAgg(self, sql, expectCnt): + self.query(sql) + self.checkRowCol(0, 0, expectCnt) + + # get first value + def getFirstValue(self, sql) : + self.query(sql) + return self.getData(0, 0) + def get_times(self, time_str, precision="ms"): caller = inspect.getframeinfo(inspect.stack()[1][0]) if time_str[-1] not in TAOS_TIME_INIT: @@ -602,6 +612,7 @@ class TDSql: if self.cursor.istype(col, "BIGINT UNSIGNED"): return "BIGINT UNSIGNED" + ''' def taosdStatus(self, state): tdLog.sleep(5) pstate = 0 @@ -630,7 +641,7 @@ class TDSql: tdLog.exit("taosd state is %d != expect:%d" %args) pass - ''' + def haveFile(self, dir, state): if os.path.exists(dir) and os.path.isdir(dir): if not os.listdir(dir): From 5f0b0b4338e69437e34bbe82ab6d524e8e3392fd Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Dec 2023 19:56:56 +0800 Subject: [PATCH 6/7] case: mlevel_basic.py passed --- .../enterprise/multi-level/mlevel_basic.py | 10 ++++++---- tests/army/frame/caseBase.py | 8 ++++---- tests/army/frame/eos.py | 2 +- tests/army/frame/epath.py | 18 +++++++++--------- tests/army/frame/etool.py | 18 ++++++++++-------- tests/army/frame/sql.py | 2 +- 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index a12b1b59eb..70bfb9b66e 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -16,12 +16,13 @@ import time import taos import frame +import frame.etool from frame.log import * from frame.cases import * from frame.sql import * from frame.caseBase import * - +from frame import * class TDTestCase(TBase): @@ -30,9 +31,10 @@ class TDTestCase(TBase): def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run - json = frame.etool.curFile(__file__, "mlevel_basic.json") - frame.etool.runBenchmark(json=json) + json = etool.curFile(__file__, "mlevel_basic.json") + etool.runBenchmark(json=json) + tdSql.execute(f"use {self.db}") # set insert data information self.childtable_count = 2 self.insert_rows = 1000000 @@ -76,4 +78,4 @@ class TDTestCase(TBase): tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 7b371cd822..11c44b28e9 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -16,11 +16,11 @@ import os import time import datetime -from log import * -from sql import * +from frame.log import * +from frame.sql import * # test case base -class DbBase: +class TBase: # # frame call @@ -85,7 +85,7 @@ class DbBase: tdSql.checkAgg(sql, self.childtable_count) # check step - sql = f"select count(*) from (select diff(ts) as dif from {self.stb} group by tbname) where dif != {self.timestamp_step}" + sql = f"select count(*) from (select diff(ts) as dif from {self.stb} partition by tbname) where dif != {self.timestamp_step}" tdSql.checkAgg(sql, 0) # save agg result diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index 2d8559f3eb..ab4d2c1bd0 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -22,7 +22,7 @@ import datetime # if windows platform return True def isWin(): - return sys.platform.system().lower() == 'windows' + return sys.platform.lower() == 'windows' # wait util execute file finished def exe(file): diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py index 644e448fb7..edff9c78a4 100644 --- a/tests/army/frame/epath.py +++ b/tests/army/frame/epath.py @@ -19,13 +19,13 @@ import os from frame.log import * # build/bin path -binPath = "" +binDir = "" -def binPath(self): - global binPath +def binPath(): + global binDir - if binPath != "": - return binPath + if binDir != "": + return binDir selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -46,8 +46,8 @@ def binPath(self): else: tdLog.info(f"taosd found in {buildPath}") # return - binPath = buildPath + "/build/bin/" - return binPath + binDir = buildPath + "/build/bin/" + return binDir -def binFile(self, file): - return binPath() + file \ No newline at end of file +def binFile(filename): + return binPath() + filename diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index 5ba29ee1c3..f24871b3dd 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -19,22 +19,24 @@ import sys import os import time import datetime -import epath -import eos +import frame.epath +import frame.eos # run taosBenchmark with command or json file mode def runBenchmark(command = "", json = "") : # get taosBenchmark path - bmFile = epath.binFile("taosBenchmark") - if eos.isWin(): + bmFile = frame.epath.binFile("taosBenchmark") + if frame.eos.isWin(): bmFile += ".exe" # run if command != "": - eos.exe(bmFile + " " + command) + frame.eos.exe(bmFile + " " + command) if json != "": - eos.exe(bmFile + " -f " + json) + cmd = f"{bmFile} -f {json}" + print(cmd) + frame.eos.exe(cmd) # get current directory file name -def curFile(fullPath, file): - return os.path.dirname(fullPath) + "/" + file +def curFile(fullPath, filename): + return os.path.dirname(fullPath) + "/" + filename diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index b3ee1ed5f8..2e14f0c2f0 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -538,7 +538,7 @@ class TDSql: # check like select count(*) ... sql def checkAgg(self, sql, expectCnt): self.query(sql) - self.checkRowCol(0, 0, expectCnt) + self.checkData(0, 0, expectCnt) # get first value def getFirstValue(self, sql) : From 9071ff39736daa9ec3d61c76cddbffa189955246 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Dec 2023 21:20:59 +0800 Subject: [PATCH 7/7] fix: add compactDb flushDb --- .../enterprise/multi-level/mlevel_basic.json | 4 ++-- .../enterprise/multi-level/mlevel_basic.py | 19 +++++++------------ tests/army/frame/caseBase.py | 2 ++ tests/army/frame/eos.py | 3 ++- tests/army/frame/etool.py | 6 +++++- tests/army/frame/server/dnodes.py | 8 +++++++- tests/army/test.py | 2 ++ 7 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/army/enterprise/multi-level/mlevel_basic.json b/tests/army/enterprise/multi-level/mlevel_basic.json index 7df5b1caa8..1c2b9274d2 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.json +++ b/tests/army/enterprise/multi-level/mlevel_basic.json @@ -24,12 +24,12 @@ { "name": "stb", "child_table_exists": "no", - "childtable_count": 2, + "childtable_count": 4, "insert_rows": 1000000, "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-10d", + "start_timestamp":"now-12d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index 70bfb9b66e..3bec2bfb72 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -36,23 +36,15 @@ class TDTestCase(TBase): tdSql.execute(f"use {self.db}") # set insert data information - self.childtable_count = 2 + self.childtable_count = 4 self.insert_rows = 1000000 self.timestamp_step = 1000 def doAction(self): - tdLog.info(f"trim database.") + tdLog.info(f"do action.") + self.flushDb() self.trimDb() - - def saveData(self): - tdLog.info(f"check correct.") - - def checkSaveCorrent(self): - tdLog.info(f"check correct.") - - def checkCorrect(self): - tdLog.info(f"check correct.") - + self.compactDb() # run def run(self): @@ -73,6 +65,9 @@ class TDTestCase(TBase): # check save agg result correct self.checkAggCorrect() + # check insert correct again + self.checkInsertCorrect() + tdLog.success(f"{__file__} successfully executed") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 11c44b28e9..441196f050 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -69,6 +69,8 @@ class TBase: def compactDb(self): tdSql.execute(f"compact database {self.db}") + def flushDb(self): + tdSql.execute(f"flush database {self.db}") # # check db correct diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index ab4d2c1bd0..dcb63ff3aa 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -19,10 +19,11 @@ import sys import os import time import datetime +import platform # if windows platform return True def isWin(): - return sys.platform.lower() == 'windows' + return platform.system().lower() == 'windows' # wait util execute file finished def exe(file): diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index f24871b3dd..35c390dc1a 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -21,6 +21,7 @@ import time import datetime import frame.epath import frame.eos +from frame.log import * # run taosBenchmark with command or json file mode def runBenchmark(command = "", json = "") : @@ -35,7 +36,10 @@ def runBenchmark(command = "", json = "") : if json != "": cmd = f"{bmFile} -f {json}" print(cmd) - frame.eos.exe(cmd) + status = frame.eos.exe(cmd) + if status !=0: + tdLog.exit(f"run failed {cmd} status={status}") + # get current directory file name def curFile(fullPath, filename): diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 1e51fbc3be..0d40b665dd 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -911,4 +911,10 @@ class TDDnodes: def getAsan(self): return self.asan -tdDnodes = TDDnodes() \ No newline at end of file + def setLevelDisk(self, level, disk): + for i in range(len(self.dnodes)): + self.dnodes[i].level = int(level) + self.dnodes[i].disk = int(disk) + + +tdDnodes = TDDnodes() diff --git a/tests/army/test.py b/tests/army/test.py index 83d065b24b..a3e28b772d 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -373,6 +373,7 @@ if __name__ == "__main__": tAdapter.stop(force_kill=True) if dnodeNums == 1 : + tdDnodes.setLevelDisk(level, disk) tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) @@ -542,6 +543,7 @@ if __name__ == "__main__": if dnodeNums == 1 : # dnode is one + tdDnodes.setLevelDisk(level, disk) tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql)