xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #13157 from taosdata/fix/ZhiqiangWang/TD-15849-case-run-remote-taosd 

fix(os): case run remote taosd
This commit is contained in:
Zhiqiang Wang 2022-05-28 17:35:12 +08:00 committed by GitHub
parent 9c8ad8a148 e164e194b1
commit dc719f3c2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 124 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
tests/pytest/util/dnodes.py
View File

@ -17,6 +17,10 @@ import os.path
import platform
import subprocess
from time import sleep
import base64
import json
import copy
from fabric2 import Connection
from util.log import *
@ -111,6 +115,7 @@ class TDDnode:
self.deployed = 0
self.testCluster = False
self.valgrind = 0
self.remoteIP = ""
self.cfgDict = {
"walLevel": "2",
"fsync": "1000",
@ -137,8 +142,9 @@ class TDDnode:
"telemetryReporting": "0"
}
def init(self, path):
def init(self, path, remoteIP = ""):
self.path = path
self.remoteIP = remoteIP
def setTestCluster(self, value):
self.testCluster = value
@ -162,6 +168,24 @@ class TDDnode:
def addExtraCfg(self, option, value):
self.cfgDict.update({option: value})
def remoteExec(self, updateCfgDict, execCmd):
remote_conn = Connection(self.remoteIP, port=22, user='root', connect_kwargs={'password':'123456'})
remote_top_dir = '~/test'
valgrindStr = ''
if (self.valgrind==1):
valgrindStr = '-g'
remoteCfgDict = copy.deepcopy(updateCfgDict)
if ("logDir" in remoteCfgDict):
del remoteCfgDict["logDir"]
if ("dataDir" in remoteCfgDict):
del remoteCfgDict["dataDir"]
if ("cfgDir" in remoteCfgDict):
del remoteCfgDict["cfgDir"]
remoteCfgDictStr = base64.b64encode(json.dumps(remoteCfgDict).encode()).decode()
execCmdStr = base64.b64encode(execCmd.encode()).decode()
with remote_conn.cd((remote_top_dir+sys.path[0].replace(self.path, '')).replace('\\','/')):
remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr))
def deploy(self, *updatecfgDict):
self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index)
self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index)
@ -229,8 +253,11 @@ class TDDnode:
self.cfg(value, key)
else:
self.addExtraCfg(key, value)
if (self.remoteIP == ""):
for key, value in self.cfgDict.items():
self.cfg(key, value)
else:
self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)"%self.index)
self.deployed = 1
tdLog.debug(
@ -268,19 +295,32 @@ class TDDnode:
tdLog.exit("dnode:%d is not deployed" % (self.index))
if self.valgrind == 0:
if platform.system().lower() == 'windows':
cmd = "mintty -h never -w hide %s -c %s" % (
binPath, self.cfgDir)
else:
cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
binPath, self.cfgDir)
else:
valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir
if platform.system().lower() == 'windows':
cmd = "mintty -h never -w hide %s %s -c %s" % (
valgrindCmdline, binPath, self.cfgDir)
else:
cmd = "nohup %s %s -c %s 2>&1 & " % (
valgrindCmdline, binPath, self.cfgDir)
print(cmd)
if (not self.remoteIP == ""):
self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.start(%d)"%(self.index, self.index))
self.running = 1
else:
if os.system(cmd) != 0:
tdLog.exit(cmd)
self.running = 1
print("dnode:%d is running with %s " % (self.index, cmd))
tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
if self.valgrind == 0:
time.sleep(0.1)
@ -293,73 +333,11 @@ class TDDnode:
i += 1
if i > 50:
break
tailCmdStr = 'tail -f '
if platform.system().lower() == 'windows':
tailCmdStr = 'tail -n +0 -f '
popen = subprocess.Popen(
'tail -f ' + logFile,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
pid = popen.pid
# print('Popen.pid:' + str(pid))
timeout = time.time() + 60 * 2
while True:
line = popen.stdout.readline().strip()
if bkey in line:
popen.kill()
break
if time.time() > timeout:
tdLog.exit('wait too long for taosd start')
tdLog.debug("the dnode:%d has been started." % (self.index))
else:
tdLog.debug(
"wait 10 seconds for the dnode:%d to start." %
(self.index))
time.sleep(10)
# time.sleep(5)
def startWin(self):
binPath = self.getPath("taosd.exe")
if (binPath == ""):
tdLog.exit("taosd.exe not found!")
else:
tdLog.info("taosd.exe found: %s" % binPath)
taosadapterBinPath = self.getPath("taosadapter.exe")
if (taosadapterBinPath == ""):
tdLog.info("taosAdapter.exe not found!")
else:
tdLog.info("taosAdapter.exe found in %s" % taosadapterBuildPath)
if self.deployed == 0:
tdLog.exit("dnode:%d is not deployed" % (self.index))
cmd = "mintty -h never -w hide %s -c %s" % (
binPath, self.cfgDir)
if (taosadapterBinPath != ""):
taosadapterCmd = "mintty -h never -w hide %s --monitor.writeToTD=false " % (
taosadapterBinPath)
if os.system(taosadapterCmd) != 0:
tdLog.exit(taosadapterCmd)
if os.system(cmd) != 0:
tdLog.exit(cmd)
self.running = 1
tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
if self.valgrind == 0:
time.sleep(0.1)
key = 'from offline to online'
bkey = bytes(key, encoding="utf8")
logFile = self.logDir + "/taosdlog.0"
i = 0
while not os.path.exists(logFile):
sleep(0.1)
i += 1
if i > 50:
break
popen = subprocess.Popen(
'tail -n +0 -f ' + logFile,
tailCmdStr + logFile,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
@ -402,12 +380,19 @@ class TDDnode:
print(cmd)
if (self.remoteIP == ""):
if os.system(cmd) != 0:
tdLog.exit(cmd)
else:
self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.startWithoutSleep(%d)"%(self.index, self.index))
self.running = 1
tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
def stop(self):
if (not self.remoteIP == ""):
self.remoteExec(self.cfgDict, "tdDnodes.stop(%d)"%self.index)
return
if self.valgrind == 0:
toBeKilled = "taosd"
else:
@ -435,6 +420,9 @@ class TDDnode:
tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
def forcestop(self):
if (not self.remoteIP == ""):
self.remoteExec(self.cfgDict, "tdDnodes.forcestop(%d)"%self.index)
return
if self.valgrind == 0:
toBeKilled = "taosd"
else:
@ -499,8 +487,10 @@ class TDDnodes:
self.dnodes.append(TDDnode(9))
self.dnodes.append(TDDnode(10))
self.simDeployed = False
self.testCluster = False
self.valgrind = 0
def init(self, path):
def init(self, path, remoteIP = ""):
psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
@ -520,9 +510,9 @@ class TDDnodes:
psCmd, shell=True).decode("utf-8")
binPath = self.dnodes[0].getPath() + "/../../../"
tdLog.debug("binPath %s" % (binPath))
# tdLog.debug("binPath %s" % (binPath))
binPath = os.path.realpath(binPath)
tdLog.debug("binPath real path %s" % (binPath))
# tdLog.debug("binPath real path %s" % (binPath))
# cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
# tdLog.debug(cmd)
@ -545,7 +535,7 @@ class TDDnodes:
self.path = os.path.realpath(path)
for i in range(len(self.dnodes)):
self.dnodes[i].init(self.path)
self.dnodes[i].init(self.path, remoteIP)
self.sim = TDSimClient(self.path)
def setTestCluster(self, value):
@ -572,9 +562,6 @@ class TDDnodes:
def start(self, index):
self.check(index)
if platform.system().lower() == 'windows':
self.dnodes[index - 1].startWin()
else:
self.dnodes[index - 1].start()
def startWithoutSleep(self, index):

31
tests/system-test/test.py
View File

@ -44,8 +44,9 @@ if __name__ == "__main__":
if platform.system().lower() == 'windows':
windows = 1
updateCfgDict = {}
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict'])
execCmd = ""
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:e:', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'execCmd'])
for key, value in opts:
if key in ['-h', '--help']:
tdLog.printNoPrefix(
@ -59,6 +60,7 @@ if __name__ == "__main__":
tdLog.printNoPrefix('-g valgrind Test Flag')
tdLog.printNoPrefix('-r taosd restart test')
tdLog.printNoPrefix('-d update cfg dict, base64 json str')
tdLog.printNoPrefix('-e eval str to run')
sys.exit(0)
if key in ['-r', '--restart']:
@ -97,6 +99,19 @@ if __name__ == "__main__":
except:
print('updateCfgDict convert fail.')
sys.exit(0)
if key in ['-e', '--execCmd']:
try:
execCmd = base64.b64decode(value.encode()).decode()
except:
print('updateCfgDict convert fail.')
sys.exit(0)
if not execCmd == "":
tdDnodes.init(deployPath)
exec(execCmd)
quit()
if (stop != 0):
if (valgrind == 0):
toBeKilled = "taosd"
@ -136,7 +151,7 @@ if __name__ == "__main__":
if windows:
tdCases.logSql(logSql)
tdLog.info("Procedures for testing self-deployment")
tdDnodes.init(deployPath)
tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster)
tdDnodes.setValgrind(valgrind)
tdDnodes.stopAll()
@ -161,15 +176,7 @@ if __name__ == "__main__":
else:
pass
tdDnodes.deploy(1,updateCfgDict)
if masterIp == "" or masterIp == "localhost":
tdDnodes.start(1)
else:
remote_conn = Connection("root@%s"%host)
with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
remote_conn.run("python3 ./test.py %s"%updateCfgDictStr)
# print("docker exec -d cross_platform bash -c \"cd ~/test/community/tests/system-test && python3 ./test.py %s\""%updateCfgDictStr)
# os.system("docker exec -d cross_platform bash -c \"cd ~/test/community/tests/system-test && (ps -aux | grep taosd | head -n 1 | awk '{print $2}' | xargs kill -9) && rm -rf /root/test/sim/dnode1/data/ && python3 ./test.py %s\""%updateCfgDictStr)
# time.sleep(2)
conn = taos.connect(
host="%s"%(host),
config=tdDnodes.sim.getCfgDir())
@ -178,7 +185,7 @@ if __name__ == "__main__":
else:
tdCases.runAllWindows(conn)
else:
tdDnodes.init(deployPath)
tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster)
tdDnodes.setValgrind(valgrind)
tdDnodes.stopAll()