From 4504f5a6fdc7277fef277e32987fe7f41d4ce9e9 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 23 Jan 2025 13:45:56 +0800 Subject: [PATCH] enh: rename --- .../telemetry/{ => crash-report}/.env.example | 0 .../telemetry/crash-report/CrashCounter.py | 302 +++++++++++++---- .../crash-report/CrashCounter.py.old | 128 ++++++++ .../crash-report/CrashCounter_new.py | 308 ------------------ 4 files changed, 369 insertions(+), 369 deletions(-) rename tests/script/telemetry/{ => crash-report}/.env.example (100%) create mode 100644 tests/script/telemetry/crash-report/CrashCounter.py.old delete mode 100644 tests/script/telemetry/crash-report/CrashCounter_new.py diff --git a/tests/script/telemetry/.env.example b/tests/script/telemetry/crash-report/.env.example similarity index 100% rename from tests/script/telemetry/.env.example rename to tests/script/telemetry/crash-report/.env.example diff --git a/tests/script/telemetry/crash-report/CrashCounter.py b/tests/script/telemetry/crash-report/CrashCounter.py index 66edc8d63e..a89567da3d 100644 --- a/tests/script/telemetry/crash-report/CrashCounter.py +++ b/tests/script/telemetry/crash-report/CrashCounter.py @@ -1,26 +1,45 @@ from datetime import date from datetime import timedelta import os +import json import re import requests +import subprocess from dotenv import load_dotenv # load .env +# You should have a .env file in the same directory as this script +# You can exec: cp .env.example .env load_dotenv() # define version -version = "3.3.*" +version = "3.3.2.*" +version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+') +version_pattern = re.compile(rf'^{version_pattern_str}$') +version_stack_list = list() + +# define ip ip = os.getenv("EXCLUDE_IP") server_ip = os.getenv("SERVER_IP") +http_serv_ip = os.getenv("HTTP_SERV_IP") +http_serv_port = os.getenv("HTTP_SERV_PORT") owner = os.getenv("OWNER") # feishu-msg url feishu_msg_url = os.getenv("FEISHU_MSG_URL") +# get today today = date.today() -#today = date(2023,8,7) + +# Define the file and parameters path="/data/telemetry/crash-report/" +trace_report_path = path + "trace_report" +os.makedirs(path, exist_ok=True) +os.makedirs(trace_report_path, exist_ok=True) + +assert_script_path = path + "filter_assert.sh" +nassert_script_path = path + "filter_nassert.sh" # get files for the past 7 days def get_files(): @@ -28,30 +47,161 @@ def get_files(): for i in range(1,8): #print ((today - timedelta(days=i)).strftime("%Y%m%d")) files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt " + return files.strip().split(" ") - return files +# Define the AWK script as a string with proper escaping +def get_res(file_path): + # Execute the script + command = ['bash', file_path, version, ip] + get_files() + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) -# for none-taosAssertDebug -filter1_cmd = '''grep '"version":"%s"' %s \ -| grep "taosd(" \ -| awk -F "stackInfo" '{print $2}' \ -| grep -v "taosAssertDebug" \ -| grep -v %s \ -| awk -F "taosd" '{print $3}' \ -| cut -d")" -f 1 \ -| cut -d"(" -f 2 \ -| sort | uniq -c ''' % (version, get_files(), ip) + # Capture the output and errors + output, errors = process.communicate() -# for taosAssertDebug -filter2_cmd = '''grep '"version":"%s"' %s \ -| grep "taosd(" \ -| awk -F "stackInfo" '{print $2}' \ -| grep "taosAssertDebug" \ -| grep -v %s \ -| awk -F "taosd" '{print $3}' \ -| cut -d")" -f 1 \ -| cut -d"(" -f 2 \ -| sort | uniq -c ''' % (version, get_files(), ip) + # Check for errors + if process.returncode != 0: + return errors + else: + return output.rstrip() + +def get_sum(output): + # Split the output into lines + lines = output.strip().split('\n') + + # Initialize the sum + total_sum = 0 + + # Iterate over each line + for line in lines: + # Split each line by space to separate the columns + parts = line.split() + + # The first part of the line is the number, convert it to integer + if parts: # Check if there are any elements in the parts list + number = int(parts[0]) + total_sum += number + + return total_sum + +def convert_html(data): + # convert data to json + start_time = get_files()[6].split("/")[-1].split(".")[0] + end_time = get_files()[0].split("/")[-1].split(".")[0] + html_report_file = f'{start_time}_{end_time}.html' + json_data = json.dumps(data) + + # Create HTML content + html_content = f''' + + + + + + Stack Trace Report + + + +

Stack Trace Report From {start_time} To {end_time}

+ + + + + + + + + + + + +
Key Stack InfoVersionsNum Of CrashesFull Stack Info
+ + + + +''' + # Write the HTML content to a file + + with open(f'{trace_report_path}/{html_report_file}', 'w') as f: + f.write(html_content) + return html_report_file + +def get_version_stack_list(res): + for line in res.strip().split('\n'): + version_list = list() + version_stack_dict = dict() + count = line.split()[0] + key_stack_info = line.split()[1] + for file in get_files(): + with open(file, 'r') as infile: + for line in infile: + line = line.strip() + data = json.loads(line) + # print(line) + if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line: + if data["version"] not in version_list: + version_list.append(data["version"]) + full_stack_info = data["stackInfo"] + version_stack_dict["key_stack_info"] = key_stack_info + version_stack_dict["full_stack_info"] = full_stack_info + version_stack_dict["version_list"] = version_list + version_stack_dict["count"] = count + # print(version_stack_dict) + version_stack_list.append(version_stack_dict) + return version_stack_list # get msg info def get_msg(text): @@ -78,51 +228,81 @@ def send_msg(json): 'Content-Type': 'application/json' } - req = requests.post(url=group_url, headers=headers, json=json) + req = requests.post(url=feishu_msg_url, headers=headers, json=json) inf = req.json() if "StatusCode" in inf and inf["StatusCode"] == 0: pass else: print(inf) -# exec cmd and return res -def get_output(cmd): - text = os.popen(cmd) - lines = text.read() - text.close() - return lines + +def format_results(results): + # Split the results into lines + lines = results.strip().split('\n') + + # Parse lines into a list of tuples (number, rest_of_line) + parsed_lines = [] + for line in lines: + parts = line.split(maxsplit=1) + if len(parts) == 2: + number = int(parts[0]) # Convert the number part to an integer + parsed_lines.append((number, parts[1])) + + # Sort the parsed lines by the first element (number) in descending order + parsed_lines.sort(reverse=True, key=lambda x: x[0]) + + # Determine the maximum width of the first column for alignment + # max_width = max(len(str(item[0])) for item in parsed_lines) + if parsed_lines: + max_width = max(len(str(item[0])) for item in parsed_lines) + else: + max_width = 0 + + # Format each line to align the numbers and function names with indentation + formatted_lines = [] + for number, text in parsed_lines: + formatted_line = f" {str(number).rjust(max_width)} {text}" + formatted_lines.append(formatted_line) + + # Join the formatted lines into a single string + return '\n'.join(formatted_lines) + +# # send report to feishu +def send_report(res, sum, html_report_file): + content = f''' + version: v{version} + from: {get_files()[6].split("/")[-1].split(".")[0]} + to: {get_files()[0].split("/")[-1].split(".")[0]} + ip: {server_ip} + owner: {owner} + result: \n{format_results(res)}\n + total crashes: {sum}\n + details: http://{http_serv_ip}:{http_serv_port}/{html_report_file} + ''' + print(get_msg(content)) + send_msg(get_msg(content)) + # print(content) + +# for none-taosAssertDebug +nassert_res = get_res(nassert_script_path) +# print(nassert_res) + +# for taosAssertDebug +assert_res = get_res(assert_script_path) +# print(assert_res) + +# combine the results +res = nassert_res + assert_res + +# get version stack list +version_stack_list = get_version_stack_list(res) if len(res) > 0 else list() + +# convert to html +html_report_file = convert_html(version_stack_list) # get sum -def get_count(output): - res = re.findall(" \d+ ", output) - sum1 = 0 - for r in res: - sum1 = sum1 + int(r.strip()) - return sum1 +sum = get_sum(res) -# print total crash count -def print_result(): - #print(f"Files for statistics: {get_files()}\n") - sum1 = get_count(get_output(filter1_cmd)) - sum2 = get_count(get_output(filter2_cmd)) - total = sum1 + sum2 - #print(f"total crashes: {total}") - return total +# send report +send_report(res, sum, html_report_file) -# send report to feishu -def send_report(): - content = f''' - test scope: Telemetry Statistics - owner: {owner} - ip: {server_ip} - from: {get_files().split(" ")[6].split("/")[4].split(".")[0]} - to: {get_files().split(" ")[0].split("/")[4].split(".")[0]} - filter1 result: {get_output(filter1_cmd)} - filter2 result: {get_output(filter2_cmd)} - total crashes: {print_result()} - ''' - #send_msg(get_msg(content)) - print(content) - -print_result() -send_report() diff --git a/tests/script/telemetry/crash-report/CrashCounter.py.old b/tests/script/telemetry/crash-report/CrashCounter.py.old new file mode 100644 index 0000000000..66edc8d63e --- /dev/null +++ b/tests/script/telemetry/crash-report/CrashCounter.py.old @@ -0,0 +1,128 @@ +from datetime import date +from datetime import timedelta +import os +import re +import requests +from dotenv import load_dotenv + +# load .env +load_dotenv() + +# define version +version = "3.3.*" + +ip = os.getenv("EXCLUDE_IP") +server_ip = os.getenv("SERVER_IP") +owner = os.getenv("OWNER") + +# feishu-msg url +feishu_msg_url = os.getenv("FEISHU_MSG_URL") + +today = date.today() +#today = date(2023,8,7) +path="/data/telemetry/crash-report/" + +# get files for the past 7 days +def get_files(): + files = "" + for i in range(1,8): + #print ((today - timedelta(days=i)).strftime("%Y%m%d")) + files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt " + + return files + +# for none-taosAssertDebug +filter1_cmd = '''grep '"version":"%s"' %s \ +| grep "taosd(" \ +| awk -F "stackInfo" '{print $2}' \ +| grep -v "taosAssertDebug" \ +| grep -v %s \ +| awk -F "taosd" '{print $3}' \ +| cut -d")" -f 1 \ +| cut -d"(" -f 2 \ +| sort | uniq -c ''' % (version, get_files(), ip) + +# for taosAssertDebug +filter2_cmd = '''grep '"version":"%s"' %s \ +| grep "taosd(" \ +| awk -F "stackInfo" '{print $2}' \ +| grep "taosAssertDebug" \ +| grep -v %s \ +| awk -F "taosd" '{print $3}' \ +| cut -d")" -f 1 \ +| cut -d"(" -f 2 \ +| sort | uniq -c ''' % (version, get_files(), ip) + +# get msg info +def get_msg(text): + return { + "msg_type": "post", + "content": { + "post": { + "zh_cn": { + "title": "Telemetry Statistics", + "content": [ + [{ + "tag": "text", + "text": text + } + ]] + } + } + } + } + +# post msg +def send_msg(json): + headers = { + 'Content-Type': 'application/json' + } + + req = requests.post(url=group_url, headers=headers, json=json) + inf = req.json() + if "StatusCode" in inf and inf["StatusCode"] == 0: + pass + else: + print(inf) + +# exec cmd and return res +def get_output(cmd): + text = os.popen(cmd) + lines = text.read() + text.close() + return lines + +# get sum +def get_count(output): + res = re.findall(" \d+ ", output) + sum1 = 0 + for r in res: + sum1 = sum1 + int(r.strip()) + return sum1 + +# print total crash count +def print_result(): + #print(f"Files for statistics: {get_files()}\n") + sum1 = get_count(get_output(filter1_cmd)) + sum2 = get_count(get_output(filter2_cmd)) + total = sum1 + sum2 + #print(f"total crashes: {total}") + return total + +# send report to feishu +def send_report(): + content = f''' + test scope: Telemetry Statistics + owner: {owner} + ip: {server_ip} + from: {get_files().split(" ")[6].split("/")[4].split(".")[0]} + to: {get_files().split(" ")[0].split("/")[4].split(".")[0]} + filter1 result: {get_output(filter1_cmd)} + filter2 result: {get_output(filter2_cmd)} + total crashes: {print_result()} + ''' + #send_msg(get_msg(content)) + print(content) + +print_result() +send_report() diff --git a/tests/script/telemetry/crash-report/CrashCounter_new.py b/tests/script/telemetry/crash-report/CrashCounter_new.py deleted file mode 100644 index a89567da3d..0000000000 --- a/tests/script/telemetry/crash-report/CrashCounter_new.py +++ /dev/null @@ -1,308 +0,0 @@ -from datetime import date -from datetime import timedelta -import os -import json -import re -import requests -import subprocess -from dotenv import load_dotenv - -# load .env -# You should have a .env file in the same directory as this script -# You can exec: cp .env.example .env -load_dotenv() - -# define version -version = "3.3.2.*" -version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+') -version_pattern = re.compile(rf'^{version_pattern_str}$') -version_stack_list = list() - -# define ip - -ip = os.getenv("EXCLUDE_IP") -server_ip = os.getenv("SERVER_IP") -http_serv_ip = os.getenv("HTTP_SERV_IP") -http_serv_port = os.getenv("HTTP_SERV_PORT") -owner = os.getenv("OWNER") - -# feishu-msg url -feishu_msg_url = os.getenv("FEISHU_MSG_URL") - -# get today -today = date.today() - -# Define the file and parameters -path="/data/telemetry/crash-report/" -trace_report_path = path + "trace_report" -os.makedirs(path, exist_ok=True) -os.makedirs(trace_report_path, exist_ok=True) - -assert_script_path = path + "filter_assert.sh" -nassert_script_path = path + "filter_nassert.sh" - -# get files for the past 7 days -def get_files(): - files = "" - for i in range(1,8): - #print ((today - timedelta(days=i)).strftime("%Y%m%d")) - files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt " - return files.strip().split(" ") - -# Define the AWK script as a string with proper escaping -def get_res(file_path): - # Execute the script - command = ['bash', file_path, version, ip] + get_files() - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - - # Capture the output and errors - output, errors = process.communicate() - - # Check for errors - if process.returncode != 0: - return errors - else: - return output.rstrip() - -def get_sum(output): - # Split the output into lines - lines = output.strip().split('\n') - - # Initialize the sum - total_sum = 0 - - # Iterate over each line - for line in lines: - # Split each line by space to separate the columns - parts = line.split() - - # The first part of the line is the number, convert it to integer - if parts: # Check if there are any elements in the parts list - number = int(parts[0]) - total_sum += number - - return total_sum - -def convert_html(data): - # convert data to json - start_time = get_files()[6].split("/")[-1].split(".")[0] - end_time = get_files()[0].split("/")[-1].split(".")[0] - html_report_file = f'{start_time}_{end_time}.html' - json_data = json.dumps(data) - - # Create HTML content - html_content = f''' - - - - - - Stack Trace Report - - - -

Stack Trace Report From {start_time} To {end_time}

- - - - - - - - - - - - -
Key Stack InfoVersionsNum Of CrashesFull Stack Info
- - - - -''' - # Write the HTML content to a file - - with open(f'{trace_report_path}/{html_report_file}', 'w') as f: - f.write(html_content) - return html_report_file - -def get_version_stack_list(res): - for line in res.strip().split('\n'): - version_list = list() - version_stack_dict = dict() - count = line.split()[0] - key_stack_info = line.split()[1] - for file in get_files(): - with open(file, 'r') as infile: - for line in infile: - line = line.strip() - data = json.loads(line) - # print(line) - if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line: - if data["version"] not in version_list: - version_list.append(data["version"]) - full_stack_info = data["stackInfo"] - version_stack_dict["key_stack_info"] = key_stack_info - version_stack_dict["full_stack_info"] = full_stack_info - version_stack_dict["version_list"] = version_list - version_stack_dict["count"] = count - # print(version_stack_dict) - version_stack_list.append(version_stack_dict) - return version_stack_list - -# get msg info -def get_msg(text): - return { - "msg_type": "post", - "content": { - "post": { - "zh_cn": { - "title": "Telemetry Statistics", - "content": [ - [{ - "tag": "text", - "text": text - } - ]] - } - } - } - } - -# post msg -def send_msg(json): - headers = { - 'Content-Type': 'application/json' - } - - req = requests.post(url=feishu_msg_url, headers=headers, json=json) - inf = req.json() - if "StatusCode" in inf and inf["StatusCode"] == 0: - pass - else: - print(inf) - - -def format_results(results): - # Split the results into lines - lines = results.strip().split('\n') - - # Parse lines into a list of tuples (number, rest_of_line) - parsed_lines = [] - for line in lines: - parts = line.split(maxsplit=1) - if len(parts) == 2: - number = int(parts[0]) # Convert the number part to an integer - parsed_lines.append((number, parts[1])) - - # Sort the parsed lines by the first element (number) in descending order - parsed_lines.sort(reverse=True, key=lambda x: x[0]) - - # Determine the maximum width of the first column for alignment - # max_width = max(len(str(item[0])) for item in parsed_lines) - if parsed_lines: - max_width = max(len(str(item[0])) for item in parsed_lines) - else: - max_width = 0 - - # Format each line to align the numbers and function names with indentation - formatted_lines = [] - for number, text in parsed_lines: - formatted_line = f" {str(number).rjust(max_width)} {text}" - formatted_lines.append(formatted_line) - - # Join the formatted lines into a single string - return '\n'.join(formatted_lines) - -# # send report to feishu -def send_report(res, sum, html_report_file): - content = f''' - version: v{version} - from: {get_files()[6].split("/")[-1].split(".")[0]} - to: {get_files()[0].split("/")[-1].split(".")[0]} - ip: {server_ip} - owner: {owner} - result: \n{format_results(res)}\n - total crashes: {sum}\n - details: http://{http_serv_ip}:{http_serv_port}/{html_report_file} - ''' - print(get_msg(content)) - send_msg(get_msg(content)) - # print(content) - -# for none-taosAssertDebug -nassert_res = get_res(nassert_script_path) -# print(nassert_res) - -# for taosAssertDebug -assert_res = get_res(assert_script_path) -# print(assert_res) - -# combine the results -res = nassert_res + assert_res - -# get version stack list -version_stack_list = get_version_stack_list(res) if len(res) > 0 else list() - -# convert to html -html_report_file = convert_html(version_stack_list) - -# get sum -sum = get_sum(res) - -# send report -send_report(res, sum, html_report_file) -