xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #28471 from taosdata/test/fix_memleak_deadlock_conflick 

test: format code and add dead-lock return code to crash_gen
This commit is contained in:
Shengliang Guan 2024-11-05 11:08:19 +08:00 committed by GitHub
parent 9588581b5d 1ba0b39582
commit 382aff37f7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 102 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
tests/pytest/auto_crash_gen.py
View File

@ -6,15 +6,12 @@ import requests
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os ,sys import os ,sys
import random import random
import argparse
import subprocess import subprocess
import time
import platform
# valgrind mode ? # valgrind mode ?
valgrind_mode = False valgrind_mode = False
msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" } msg_dict = {0: "success", 1: "failed", 2: "other errors", 3: "crash occured", 4: "Invalid read/write", 5: "memory leak", 6: "dead locked"}
# formal # formal
hostname = socket.gethostname() hostname = socket.gethostname()
@ -112,9 +109,9 @@ def random_args(args_list):
# args_list["--connector-type"]=connect_types[random.randint(0,2)] # args_list["--connector-type"]=connect_types[random.randint(0,2)]
args_list["--connector-type"]= connect_types[0] args_list["--connector-type"]= connect_types[0]
args_list["--max-dbs"]= random.randint(1,10) args_list["--max-dbs"]= random.randint(1,10)
# dnodes = [1,3] # set single dnodes; # dnodes = [1,3] # set single dnodes;
# args_list["--num-dnodes"]= random.sample(dnodes,1)[0] # args_list["--num-dnodes"]= random.sample(dnodes,1)[0]
# args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"]) # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"])
args_list["--debug"]=False args_list["--debug"]=False
@ -125,7 +122,7 @@ def random_args(args_list):
# args_list["--ignore-errors"]=[] ## can add error codes for detail # args_list["--ignore-errors"]=[] ## can add error codes for detail
args_list["--run-tdengine"]= False args_list["--run-tdengine"]= False
args_list["--use-shadow-db"]= False args_list["--use-shadow-db"]= False
args_list["--dynamic-db-table-names"]= True args_list["--dynamic-db-table-names"]= True
@ -162,7 +159,7 @@ def random_args(args_list):
if args_list["--larger-data"]: if args_list["--larger-data"]:
threads = [16,32] threads = [16,32]
else: else:
threads = [32,64,128,256] threads = [32,64,128,256]
args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug
return args_list return args_list
@ -176,7 +173,7 @@ def limits(args_list):
pass pass
# env is start by test frame , not crash_gen instance # env is start by test frame , not crash_gen instance
# elif args_list["--num-replicas"]==0: # elif args_list["--num-replicas"]==0:
# print(" make sure num-replicas is at least 1 ") # print(" make sure num-replicas is at least 1 ")
# args_list["--num-replicas"]=1 # args_list["--num-replicas"]=1
@ -186,10 +183,10 @@ def limits(args_list):
# elif args_list["--num-replicas"]>1: # elif args_list["--num-replicas"]>1:
# if not args_list["--auto-start-service"]: # if not args_list["--auto-start-service"]:
# print("it should be deployed by crash_gen auto-start-service for multi replicas") # print("it should be deployed by crash_gen auto-start-service for multi replicas")
# else: # else:
# pass # pass
return args_list return args_list
def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
@ -216,9 +213,9 @@ def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
arguments+="" arguments+=""
else: else:
arguments+=(k+"="+str(v)+" ") arguments+=(k+"="+str(v)+" ")
if valgrind : if valgrind :
crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203,0x4012 '%(crash_gen_path ,arguments) crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203,0x4012 '%(crash_gen_path ,arguments)
else: else:
@ -239,7 +236,7 @@ def start_taosd():
start_cmd = 'cd %s && python3 test.py >>/dev/null '%(start_path) start_cmd = 'cd %s && python3 test.py >>/dev/null '%(start_path)
os.system(start_cmd) os.system(start_cmd)
def get_cmds(args_list): def get_cmds(args_list):
crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode) crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode)
return crash_gen_cmd return crash_gen_cmd
@ -276,11 +273,15 @@ def check_status():
os.system("tail -n 50 %s>>%s"%(result_file,exit_status_logs)) os.system("tail -n 50 %s>>%s"%(result_file,exit_status_logs))
core_check = subprocess.Popen('ls -l %s | grep "^-" | wc -l'%core_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") core_check = subprocess.Popen('ls -l %s | grep "^-" | wc -l'%core_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
dead_lock_check = subprocess.Popen("grep -i 'dead locked' %s "%result_file, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if int(core_check.strip().rstrip()) > 0: if int(core_check.strip().rstrip()) > 0:
# it means core files has occured # it means core files has occured
return 3 return 3
if dead_lock_check:
return 6
if "Crash_Gen is now exiting with status code: 1" in run_code: if "Crash_Gen is now exiting with status code: 1" in run_code:
return 1 return 1
elif "Crash_Gen is now exiting with status code: 0" in run_code: elif "Crash_Gen is now exiting with status code: 0" in run_code:
@ -293,7 +294,7 @@ def main():
args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[], args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[],
"--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False, "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False,
"--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False ,
"--continue-on-exception":False } "--continue-on-exception":False }
args = random_args(args_list) args = random_args(args_list)
@ -301,24 +302,24 @@ def main():
build_path = get_path() build_path = get_path()
if repo =="community": if repo =="community":
crash_gen_path = build_path[:-5]+"community/tests/pytest/" crash_gen_path = build_path[:-5]+"community/tests/pytest/"
elif repo =="TDengine": elif repo =="TDengine":
crash_gen_path = build_path[:-5]+"/tests/pytest/" crash_gen_path = build_path[:-5]+"/tests/pytest/"
else: else:
pass pass
if os.path.exists(crash_gen_path+"crash_gen.sh"): if os.path.exists(crash_gen_path+"crash_gen.sh"):
print(" make sure crash_gen.sh is ready") print(" make sure crash_gen.sh is ready")
else: else:
print( " crash_gen.sh is not exists ") print( " crash_gen.sh is not exists ")
sys.exit(1) sys.exit(1)
git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16] git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16]
# crash_cmds = get_cmds() # crash_cmds = get_cmds()
crash_cmds = get_cmds(args) crash_cmds = get_cmds(args)
# clean run_dir # clean run_dir
os.system('rm -rf %s'%run_dir ) os.system('rm -rf %s'%run_dir )
@ -329,9 +330,9 @@ def main():
run_crash_gen(crash_cmds) run_crash_gen(crash_cmds)
endtime = datetime.datetime.now() endtime = datetime.datetime.now()
status = check_status() status = check_status()
print("exit status : ", status) print("exit status : ", status)
if status ==4: if status ==4:
print('======== crash_gen found memory bugs ========') print('======== crash_gen found memory bugs ========')
if status ==5: if status ==5:
@ -344,15 +345,15 @@ def main():
try: try:
cmd = crash_cmds.split('&')[2] cmd = crash_cmds.split('&')[2]
if status == 0: if status == 0:
log_dir = "none" log_dir = "none"
else: else:
log_dir= "/root/pxiao/crash_gen_logs" log_dir= "/root/pxiao/crash_gen_logs"
if status == 3: if status == 3:
core_dir = "/root/pxiao/crash_gen_logs" core_dir = "/root/pxiao/crash_gen_logs"
else: else:
core_dir = "none" core_dir = "none"
text = f''' text = f'''
exit status: {msg_dict[status]} exit status: {msg_dict[status]}
test scope: crash_gen test scope: crash_gen
@ -364,12 +365,12 @@ def main():
log dir: {log_dir} log dir: {log_dir}
core dir: {core_dir} core dir: {core_dir}
cmd: {cmd}''' cmd: {cmd}'''
send_msg(get_msg(text)) send_msg(get_msg(text))
except Exception as e: except Exception as e:
print("exception:", e) print("exception:", e)
exit(status) exit(status)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

74
tests/pytest/auto_crash_gen_valgrind.py
View File

@ -9,15 +9,12 @@ import requests
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os ,sys import os ,sys
import random import random
import argparse
import subprocess import subprocess
import time
import platform
# valgrind mode ? # valgrind mode ?
valgrind_mode = True valgrind_mode = True
msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" } msg_dict = {0: "success", 1: "failed", 2: "other errors", 3: "crash occured", 4: "Invalid read/write", 5: "memory leak", 6: "dead locked"}
# formal # formal
hostname = socket.gethostname() hostname = socket.gethostname()
@ -48,6 +45,7 @@ def send_msg(json):
'Content-Type': 'application/json' 'Content-Type': 'application/json'
} }
req = requests.post(url=group_url, headers=headers, json=json) req = requests.post(url=group_url, headers=headers, json=json)
inf = req.json() inf = req.json()
if "StatusCode" in inf and inf["StatusCode"] == 0: if "StatusCode" in inf and inf["StatusCode"] == 0:
@ -115,9 +113,9 @@ def random_args(args_list):
# args_list["--connector-type"]=connect_types[random.randint(0,2)] # args_list["--connector-type"]=connect_types[random.randint(0,2)]
args_list["--connector-type"]= connect_types[0] args_list["--connector-type"]= connect_types[0]
args_list["--max-dbs"]= random.randint(1,10) args_list["--max-dbs"]= random.randint(1,10)
# dnodes = [1,3] # set single dnodes; # dnodes = [1,3] # set single dnodes;
# args_list["--num-dnodes"]= random.sample(dnodes,1)[0] # args_list["--num-dnodes"]= random.sample(dnodes,1)[0]
# args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"]) # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"])
args_list["--debug"]=False args_list["--debug"]=False
@ -125,13 +123,13 @@ def random_args(args_list):
args_list["--track-memory-leaks"]=False args_list["--track-memory-leaks"]=False
args_list["--max-steps"]=random.randint(200,500) args_list["--max-steps"]=random.randint(200,500)
threads = [16,32] threads = [16,32]
args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug
# args_list["--ignore-errors"]=[] ## can add error codes for detail # args_list["--ignore-errors"]=[] ## can add error codes for detail
args_list["--run-tdengine"]= False args_list["--run-tdengine"]= False
args_list["--use-shadow-db"]= False args_list["--use-shadow-db"]= False
args_list["--dynamic-db-table-names"]= True args_list["--dynamic-db-table-names"]= True
@ -177,7 +175,7 @@ def limits(args_list):
pass pass
# env is start by test frame , not crash_gen instance # env is start by test frame , not crash_gen instance
# elif args_list["--num-replicas"]==0: # elif args_list["--num-replicas"]==0:
# print(" make sure num-replicas is at least 1 ") # print(" make sure num-replicas is at least 1 ")
# args_list["--num-replicas"]=1 # args_list["--num-replicas"]=1
@ -187,10 +185,10 @@ def limits(args_list):
# elif args_list["--num-replicas"]>1: # elif args_list["--num-replicas"]>1:
# if not args_list["--auto-start-service"]: # if not args_list["--auto-start-service"]:
# print("it should be deployed by crash_gen auto-start-service for multi replicas") # print("it should be deployed by crash_gen auto-start-service for multi replicas")
# else: # else:
# pass # pass
return args_list return args_list
def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
@ -217,9 +215,9 @@ def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
arguments+="" arguments+=""
else: else:
arguments+=(k+"="+str(v)+" ") arguments+=(k+"="+str(v)+" ")
if valgrind : if valgrind :
crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203,0x4012 '%(crash_gen_path ,arguments) crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0203,0x4012 '%(crash_gen_path ,arguments)
else: else:
@ -228,7 +226,6 @@ def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
return crash_gen_cmd return crash_gen_cmd
def start_taosd(): def start_taosd():
build_path = get_path() build_path = get_path()
if repo == "community": if repo == "community":
@ -272,7 +269,7 @@ def check_status():
if int(core_check.strip().rstrip()) > 0: if int(core_check.strip().rstrip()) > 0:
# it means core files has occured # it means core files has occured
return 3 return 3
mem_status = check_memory() mem_status = check_memory()
if mem_status >0: if mem_status >0:
return mem_status return mem_status
@ -281,8 +278,8 @@ def check_status():
elif "Crash_Gen is now exiting with status code: 0" in run_code: elif "Crash_Gen is now exiting with status code: 0" in run_code:
return 0 return 0
else: else:
return 2 return 2
def check_memory(): def check_memory():
@ -301,34 +298,37 @@ def check_memory():
os.mkdir(back_path) os.mkdir(back_path)
stderr_file = os.path.join(crash_gen_path , "valgrind.err") stderr_file = os.path.join(crash_gen_path , "valgrind.err")
stdout_file = os.path.join(crash_gen_path, 'valgrind.out')
status = 0 status = 0
grep_res = subprocess.Popen("grep -i 'Invalid read' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") grep_res = subprocess.Popen("grep -i 'Invalid read' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if grep_res: if grep_res:
# os.system("cp %s %s"%(stderr_file , back_path)) # os.system("cp %s %s"%(stderr_file , back_path))
status = 4 status = 4
grep_res = subprocess.Popen("grep -i 'Invalid write' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") grep_res = subprocess.Popen("grep -i 'Invalid write' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if grep_res: if grep_res:
# os.system("cp %s %s"%(stderr_file , back_path)) # os.system("cp %s %s"%(stderr_file , back_path))
status = 4 status = 4
grep_res = subprocess.Popen("grep -i 'taosMemoryMalloc' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") grep_res = subprocess.Popen("grep -i 'taosMemoryMalloc' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if grep_res: if grep_res:
# mem-leak can be also occure when exit normally when dead lock
# os.system("cp %s %s"%(stderr_file , back_path)) # os.system("cp %s %s"%(stderr_file , back_path))
status = 5 dead_lock_res = subprocess.Popen("grep -i 'dead locked' %s "%stdout_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
status = 6 if dead_lock_res else 5
return status return status
def main(): def main():
args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[], args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[],
"--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False, "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False,
"--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False ,
"--continue-on-exception":False } "--continue-on-exception":False }
args = random_args(args_list) args = random_args(args_list)
@ -341,17 +341,17 @@ def main():
crash_gen_path = build_path[:-5]+"/tests/pytest/" crash_gen_path = build_path[:-5]+"/tests/pytest/"
else: else:
pass pass
if os.path.exists(crash_gen_path+"crash_gen.sh"): if os.path.exists(crash_gen_path+"crash_gen.sh"):
print(" make sure crash_gen.sh is ready") print(" make sure crash_gen.sh is ready")
else: else:
print( " crash_gen.sh is not exists ") print( " crash_gen.sh is not exists ")
sys.exit(1) sys.exit(1)
git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16] git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16]
# crash_cmds = get_cmds() # crash_cmds = get_cmds()
crash_cmds = get_cmds(args) crash_cmds = get_cmds(args)
# clean run_dir # clean run_dir
@ -364,9 +364,9 @@ def main():
endtime = datetime.datetime.now() endtime = datetime.datetime.now()
status = check_status() status = check_status()
# back_path = os.path.join(core_path,"valgrind_report") # back_path = os.path.join(core_path,"valgrind_report")
print("exit status : ", status) print("exit status : ", status)
if status ==4: if status ==4:
print('======== crash_gen found memory bugs ========') print('======== crash_gen found memory bugs ========')
if status ==5: if status ==5:
@ -379,15 +379,15 @@ def main():
try: try:
cmd = crash_cmds.split('&')[2] cmd = crash_cmds.split('&')[2]
if status == 0: if status == 0:
log_dir = "none" log_dir = "none"
else: else:
log_dir= "/root/pxiao/crash_gen_logs" log_dir= "/root/pxiao/crash_gen_logs"
if status == 3: if status == 3:
core_dir = "/root/pxiao/crash_gen_logs" core_dir = "/root/pxiao/crash_gen_logs"
else: else:
core_dir = "none" core_dir = "none"
text = f''' text = f'''
exit status: {msg_dict[status]} exit status: {msg_dict[status]}
test scope: crash_gen test scope: crash_gen
@ -399,12 +399,12 @@ def main():
log dir: {log_dir} log dir: {log_dir}
core dir: {core_dir} core dir: {core_dir}
cmd: {cmd}''' cmd: {cmd}'''
send_msg(get_msg(text)) send_msg(get_msg(text))
except Exception as e: except Exception as e:
print("exception:", e) print("exception:", e)
exit(status) exit(status)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

82
tests/pytest/auto_crash_gen_valgrind_cluster.py
View File

@ -1,23 +1,17 @@
#!/usr/bin/python3 #!/usr/bin/python3
# -*- coding: utf-8 -*-
import datetime import datetime
import os import os
import socket import socket
import requests import requests
# -*- coding: utf-8 -*-
import os ,sys import os ,sys
import random import random
import argparse
import subprocess import subprocess
import time
import platform
# valgrind mode ? # valgrind mode ?
valgrind_mode = True valgrind_mode = True
msg_dict = {0:"success" , 1:"failed" , 2:"other errors" , 3:"crash occured" , 4:"Invalid read/write" , 5:"memory leak" } msg_dict = {0: "success", 1: "failed", 2: "other errors", 3: "crash occured", 4: "Invalid read/write", 5: "memory leak", 6: "dead locked"}
# formal # formal
hostname = socket.gethostname() hostname = socket.gethostname()
@ -115,9 +109,9 @@ def random_args(args_list):
# args_list["--connector-type"]=connect_types[random.randint(0,2)] # args_list["--connector-type"]=connect_types[random.randint(0,2)]
args_list["--connector-type"]= connect_types[0] args_list["--connector-type"]= connect_types[0]
args_list["--max-dbs"]= random.randint(1,10) args_list["--max-dbs"]= random.randint(1,10)
# dnodes = [1,3] # set single dnodes; # dnodes = [1,3] # set single dnodes;
# args_list["--num-dnodes"]= random.sample(dnodes,1)[0] # args_list["--num-dnodes"]= random.sample(dnodes,1)[0]
# args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"]) # args_list["--num-replicas"]= random.randint(1,args_list["--num-dnodes"])
args_list["--debug"]=False args_list["--debug"]=False
@ -125,13 +119,12 @@ def random_args(args_list):
args_list["--track-memory-leaks"]=False args_list["--track-memory-leaks"]=False
args_list["--max-steps"]=random.randint(200,500) args_list["--max-steps"]=random.randint(200,500)
threads = [16,32] threads = [16,32]
args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug args_list["--num-threads"]=random.sample(threads,1)[0] #$ debug
# args_list["--ignore-errors"]=[] ## can add error codes for detail # args_list["--ignore-errors"]=[] ## can add error codes for detail
args_list["--run-tdengine"]= False args_list["--run-tdengine"]= False
args_list["--use-shadow-db"]= False args_list["--use-shadow-db"]= False
args_list["--dynamic-db-table-names"]= True args_list["--dynamic-db-table-names"]= True
@ -177,7 +170,7 @@ def limits(args_list):
pass pass
# env is start by test frame , not crash_gen instance # env is start by test frame , not crash_gen instance
# elif args_list["--num-replicas"]==0: # elif args_list["--num-replicas"]==0:
# print(" make sure num-replicas is at least 1 ") # print(" make sure num-replicas is at least 1 ")
# args_list["--num-replicas"]=1 # args_list["--num-replicas"]=1
@ -187,10 +180,9 @@ def limits(args_list):
# elif args_list["--num-replicas"]>1: # elif args_list["--num-replicas"]>1:
# if not args_list["--auto-start-service"]: # if not args_list["--auto-start-service"]:
# print("it should be deployed by crash_gen auto-start-service for multi replicas") # print("it should be deployed by crash_gen auto-start-service for multi replicas")
# else: # else:
# pass # pass
return args_list return args_list
def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode): def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
@ -217,18 +209,13 @@ def get_auto_mix_cmds(args_list ,valgrind=valgrind_mode):
arguments+="" arguments+=""
else: else:
arguments+=(k+"="+str(v)+" ") arguments+=(k+"="+str(v)+" ")
if valgrind : if valgrind :
crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0707,0x0203,0x4012 '%(crash_gen_path ,arguments) crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0707,0x0203,0x4012 '%(crash_gen_path ,arguments)
else: else:
crash_gen_cmd = 'cd %s && ./crash_gen.sh -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0014,0x0707,0x0203,0x4012'%(crash_gen_path ,arguments) crash_gen_cmd = 'cd %s && ./crash_gen.sh -i 3 %s -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550,0x0014,0x0707,0x0203,0x4012'%(crash_gen_path ,arguments)
return crash_gen_cmd return crash_gen_cmd
def start_taosd(): def start_taosd():
build_path = get_path() build_path = get_path()
if repo == "community": if repo == "community":
@ -242,7 +229,7 @@ def start_taosd():
os.system(start_cmd +">>/dev/null") os.system(start_cmd +">>/dev/null")
def get_cmds(args_list): def get_cmds(args_list):
crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode) crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode)
return crash_gen_cmd return crash_gen_cmd
@ -272,7 +259,7 @@ def check_status():
if int(core_check.strip().rstrip()) > 0: if int(core_check.strip().rstrip()) > 0:
# it means core files has occured # it means core files has occured
return 3 return 3
mem_status = check_memory() mem_status = check_memory()
if mem_status >0: if mem_status >0:
return mem_status return mem_status
@ -281,8 +268,7 @@ def check_status():
elif "Crash_Gen is now exiting with status code: 0" in run_code: elif "Crash_Gen is now exiting with status code: 0" in run_code:
return 0 return 0
else: else:
return 2 return 2
def check_memory(): def check_memory():
@ -301,57 +287,58 @@ def check_memory():
os.mkdir(back_path) os.mkdir(back_path)
stderr_file = os.path.join(crash_gen_path , "valgrind.err") stderr_file = os.path.join(crash_gen_path , "valgrind.err")
stdout_file = os.path.join(crash_gen_path, 'valgrind.out')
status = 0 status = 0
grep_res = subprocess.Popen("grep -i 'Invalid read' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") grep_res = subprocess.Popen("grep -i 'Invalid read' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if grep_res: if grep_res:
# os.system("cp %s %s"%(stderr_file , back_path)) # os.system("cp %s %s"%(stderr_file , back_path))
status = 4 status = 4
grep_res = subprocess.Popen("grep -i 'Invalid write' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") grep_res = subprocess.Popen("grep -i 'Invalid write' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if grep_res: if grep_res:
# os.system("cp %s %s"%(stderr_file , back_path)) # os.system("cp %s %s"%(stderr_file , back_path))
status = 4 status = 4
grep_res = subprocess.Popen("grep -i 'taosMemoryMalloc' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") grep_res = subprocess.Popen("grep -i 'taosMemoryMalloc' %s "%stderr_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
if grep_res: if grep_res:
# mem-leak can be also occure when exit normally when dead lock
# os.system("cp %s %s"%(stderr_file , back_path)) # os.system("cp %s %s"%(stderr_file , back_path))
status = 5 dead_lock_res = subprocess.Popen("grep -i 'dead locked' %s "%stdout_file , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
status = 6 if dead_lock_res else 5
return status return status
def main(): def main():
args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[], args_list = {"--auto-start-service":False ,"--max-dbs":0,"--connector-type":"native","--debug":False,"--run-tdengine":False,"--ignore-errors":[],
"--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False, "--track-memory-leaks":False , "--larger-data":False, "--mix-oos-data":False, "--dynamic-db-table-names":False,
"--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False , "--per-thread-db-connection":False , "--record-ops":False , "--max-steps":100, "--num-threads":10, "--verify-data":False,"--use-shadow-db":False ,
"--continue-on-exception":False } "--continue-on-exception":False }
args = random_args(args_list) args = random_args(args_list)
args = limits(args) args = limits(args)
build_path = get_path() build_path = get_path()
if repo =="community": if repo =="community":
crash_gen_path = build_path[:-5]+"community/tests/pytest/" crash_gen_path = build_path[:-5]+"community/tests/pytest/"
elif repo =="TDengine": elif repo =="TDengine":
crash_gen_path = build_path[:-5]+"/tests/pytest/" crash_gen_path = build_path[:-5]+"/tests/pytest/"
else: else:
pass pass
if os.path.exists(crash_gen_path+"crash_gen.sh"): if os.path.exists(crash_gen_path+"crash_gen.sh"):
print(" make sure crash_gen.sh is ready") print(" make sure crash_gen.sh is ready")
else: else:
print( " crash_gen.sh is not exists ") print( " crash_gen.sh is not exists ")
sys.exit(1) sys.exit(1)
git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16] git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16]
# crash_cmds = get_cmds() # crash_cmds = get_cmds()
crash_cmds = get_cmds(args) crash_cmds = get_cmds(args)
# clean run_dir # clean run_dir
@ -364,9 +351,9 @@ def main():
endtime = datetime.datetime.now() endtime = datetime.datetime.now()
status = check_status() status = check_status()
# back_path = os.path.join(core_path,"valgrind_report") # back_path = os.path.join(core_path,"valgrind_report")
print("exit status : ", status) print("exit status : ", status)
if status ==4: if status ==4:
print('======== crash_gen found memory bugs ========') print('======== crash_gen found memory bugs ========')
if status ==5: if status ==5:
@ -379,15 +366,15 @@ def main():
try: try:
cmd = crash_cmds.split('&')[2] cmd = crash_cmds.split('&')[2]
if status == 0: if status == 0:
log_dir = "none" log_dir = "none"
else: else:
log_dir= "/root/pxiao/crash_gen_logs" log_dir= "/root/pxiao/crash_gen_logs"
if status == 3: if status == 3:
core_dir = "/root/pxiao/crash_gen_logs" core_dir = "/root/pxiao/crash_gen_logs"
else: else:
core_dir = "none" core_dir = "none"
text = f''' text = f'''
exit status: {msg_dict[status]} exit status: {msg_dict[status]}
test scope: crash_gen test scope: crash_gen
@ -399,12 +386,11 @@ def main():
log dir: {log_dir} log dir: {log_dir}
core dir: {core_dir} core dir: {core_dir}
cmd: {cmd}''' cmd: {cmd}'''
send_msg(get_msg(text)) send_msg(get_msg(text))
except Exception as e: except Exception as e:
print("exception:", e) print("exception:", e)
exit(status) exit(status)
if __name__ == '__main__': if __name__ == '__main__':
main() main()