diff --git a/tests/script/telemetry/.env.example b/tests/script/telemetry/crash-report/.env.example
similarity index 100%
rename from tests/script/telemetry/.env.example
rename to tests/script/telemetry/crash-report/.env.example
diff --git a/tests/script/telemetry/crash-report/CrashCounter.py b/tests/script/telemetry/crash-report/CrashCounter.py
index 66edc8d63e..a89567da3d 100644
--- a/tests/script/telemetry/crash-report/CrashCounter.py
+++ b/tests/script/telemetry/crash-report/CrashCounter.py
@@ -1,26 +1,45 @@
from datetime import date
from datetime import timedelta
import os
+import json
import re
import requests
+import subprocess
from dotenv import load_dotenv
# load .env
+# You should have a .env file in the same directory as this script
+# You can exec: cp .env.example .env
load_dotenv()
# define version
-version = "3.3.*"
+version = "3.3.2.*"
+version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+')
+version_pattern = re.compile(rf'^{version_pattern_str}$')
+version_stack_list = list()
+
+# define ip
ip = os.getenv("EXCLUDE_IP")
server_ip = os.getenv("SERVER_IP")
+http_serv_ip = os.getenv("HTTP_SERV_IP")
+http_serv_port = os.getenv("HTTP_SERV_PORT")
owner = os.getenv("OWNER")
# feishu-msg url
feishu_msg_url = os.getenv("FEISHU_MSG_URL")
+# get today
today = date.today()
-#today = date(2023,8,7)
+
+# Define the file and parameters
path="/data/telemetry/crash-report/"
+trace_report_path = path + "trace_report"
+os.makedirs(path, exist_ok=True)
+os.makedirs(trace_report_path, exist_ok=True)
+
+assert_script_path = path + "filter_assert.sh"
+nassert_script_path = path + "filter_nassert.sh"
# get files for the past 7 days
def get_files():
@@ -28,30 +47,161 @@ def get_files():
for i in range(1,8):
#print ((today - timedelta(days=i)).strftime("%Y%m%d"))
files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
+ return files.strip().split(" ")
- return files
+# Define the AWK script as a string with proper escaping
+def get_res(file_path):
+ # Execute the script
+ command = ['bash', file_path, version, ip] + get_files()
+ process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-# for none-taosAssertDebug
-filter1_cmd = '''grep '"version":"%s"' %s \
-| grep "taosd(" \
-| awk -F "stackInfo" '{print $2}' \
-| grep -v "taosAssertDebug" \
-| grep -v %s \
-| awk -F "taosd" '{print $3}' \
-| cut -d")" -f 1 \
-| cut -d"(" -f 2 \
-| sort | uniq -c ''' % (version, get_files(), ip)
+ # Capture the output and errors
+ output, errors = process.communicate()
-# for taosAssertDebug
-filter2_cmd = '''grep '"version":"%s"' %s \
-| grep "taosd(" \
-| awk -F "stackInfo" '{print $2}' \
-| grep "taosAssertDebug" \
-| grep -v %s \
-| awk -F "taosd" '{print $3}' \
-| cut -d")" -f 1 \
-| cut -d"(" -f 2 \
-| sort | uniq -c ''' % (version, get_files(), ip)
+ # Check for errors
+ if process.returncode != 0:
+ return errors
+ else:
+ return output.rstrip()
+
+def get_sum(output):
+ # Split the output into lines
+ lines = output.strip().split('\n')
+
+ # Initialize the sum
+ total_sum = 0
+
+ # Iterate over each line
+ for line in lines:
+ # Split each line by space to separate the columns
+ parts = line.split()
+
+ # The first part of the line is the number, convert it to integer
+ if parts: # Check if there are any elements in the parts list
+ number = int(parts[0])
+ total_sum += number
+
+ return total_sum
+
+def convert_html(data):
+ # convert data to json
+ start_time = get_files()[6].split("/")[-1].split(".")[0]
+ end_time = get_files()[0].split("/")[-1].split(".")[0]
+ html_report_file = f'{start_time}_{end_time}.html'
+ json_data = json.dumps(data)
+
+ # Create HTML content
+ html_content = f'''
+
+
+
+
+
+ Stack Trace Report
+
+
+
+ Stack Trace Report From {start_time} To {end_time}
+
+
+
+
+ | Key Stack Info |
+ Versions |
+ Num Of Crashes |
+ Full Stack Info |
+
+
+
+
+
+
+
+
+
+'''
+ # Write the HTML content to a file
+
+ with open(f'{trace_report_path}/{html_report_file}', 'w') as f:
+ f.write(html_content)
+ return html_report_file
+
+def get_version_stack_list(res):
+ for line in res.strip().split('\n'):
+ version_list = list()
+ version_stack_dict = dict()
+ count = line.split()[0]
+ key_stack_info = line.split()[1]
+ for file in get_files():
+ with open(file, 'r') as infile:
+ for line in infile:
+ line = line.strip()
+ data = json.loads(line)
+ # print(line)
+ if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line:
+ if data["version"] not in version_list:
+ version_list.append(data["version"])
+ full_stack_info = data["stackInfo"]
+ version_stack_dict["key_stack_info"] = key_stack_info
+ version_stack_dict["full_stack_info"] = full_stack_info
+ version_stack_dict["version_list"] = version_list
+ version_stack_dict["count"] = count
+ # print(version_stack_dict)
+ version_stack_list.append(version_stack_dict)
+ return version_stack_list
# get msg info
def get_msg(text):
@@ -78,51 +228,81 @@ def send_msg(json):
'Content-Type': 'application/json'
}
- req = requests.post(url=group_url, headers=headers, json=json)
+ req = requests.post(url=feishu_msg_url, headers=headers, json=json)
inf = req.json()
if "StatusCode" in inf and inf["StatusCode"] == 0:
pass
else:
print(inf)
-# exec cmd and return res
-def get_output(cmd):
- text = os.popen(cmd)
- lines = text.read()
- text.close()
- return lines
+
+def format_results(results):
+ # Split the results into lines
+ lines = results.strip().split('\n')
+
+ # Parse lines into a list of tuples (number, rest_of_line)
+ parsed_lines = []
+ for line in lines:
+ parts = line.split(maxsplit=1)
+ if len(parts) == 2:
+ number = int(parts[0]) # Convert the number part to an integer
+ parsed_lines.append((number, parts[1]))
+
+ # Sort the parsed lines by the first element (number) in descending order
+ parsed_lines.sort(reverse=True, key=lambda x: x[0])
+
+ # Determine the maximum width of the first column for alignment
+ # max_width = max(len(str(item[0])) for item in parsed_lines)
+ if parsed_lines:
+ max_width = max(len(str(item[0])) for item in parsed_lines)
+ else:
+ max_width = 0
+
+ # Format each line to align the numbers and function names with indentation
+ formatted_lines = []
+ for number, text in parsed_lines:
+ formatted_line = f" {str(number).rjust(max_width)} {text}"
+ formatted_lines.append(formatted_line)
+
+ # Join the formatted lines into a single string
+ return '\n'.join(formatted_lines)
+
+# # send report to feishu
+def send_report(res, sum, html_report_file):
+ content = f'''
+ version: v{version}
+ from: {get_files()[6].split("/")[-1].split(".")[0]}
+ to: {get_files()[0].split("/")[-1].split(".")[0]}
+ ip: {server_ip}
+ owner: {owner}
+ result: \n{format_results(res)}\n
+ total crashes: {sum}\n
+ details: http://{http_serv_ip}:{http_serv_port}/{html_report_file}
+ '''
+ print(get_msg(content))
+ send_msg(get_msg(content))
+ # print(content)
+
+# for none-taosAssertDebug
+nassert_res = get_res(nassert_script_path)
+# print(nassert_res)
+
+# for taosAssertDebug
+assert_res = get_res(assert_script_path)
+# print(assert_res)
+
+# combine the results
+res = nassert_res + assert_res
+
+# get version stack list
+version_stack_list = get_version_stack_list(res) if len(res) > 0 else list()
+
+# convert to html
+html_report_file = convert_html(version_stack_list)
# get sum
-def get_count(output):
- res = re.findall(" \d+ ", output)
- sum1 = 0
- for r in res:
- sum1 = sum1 + int(r.strip())
- return sum1
+sum = get_sum(res)
-# print total crash count
-def print_result():
- #print(f"Files for statistics: {get_files()}\n")
- sum1 = get_count(get_output(filter1_cmd))
- sum2 = get_count(get_output(filter2_cmd))
- total = sum1 + sum2
- #print(f"total crashes: {total}")
- return total
+# send report
+send_report(res, sum, html_report_file)
-# send report to feishu
-def send_report():
- content = f'''
- test scope: Telemetry Statistics
- owner: {owner}
- ip: {server_ip}
- from: {get_files().split(" ")[6].split("/")[4].split(".")[0]}
- to: {get_files().split(" ")[0].split("/")[4].split(".")[0]}
- filter1 result: {get_output(filter1_cmd)}
- filter2 result: {get_output(filter2_cmd)}
- total crashes: {print_result()}
- '''
- #send_msg(get_msg(content))
- print(content)
-
-print_result()
-send_report()
diff --git a/tests/script/telemetry/crash-report/CrashCounter.py.old b/tests/script/telemetry/crash-report/CrashCounter.py.old
new file mode 100644
index 0000000000..66edc8d63e
--- /dev/null
+++ b/tests/script/telemetry/crash-report/CrashCounter.py.old
@@ -0,0 +1,128 @@
+from datetime import date
+from datetime import timedelta
+import os
+import re
+import requests
+from dotenv import load_dotenv
+
+# load .env
+load_dotenv()
+
+# define version
+version = "3.3.*"
+
+ip = os.getenv("EXCLUDE_IP")
+server_ip = os.getenv("SERVER_IP")
+owner = os.getenv("OWNER")
+
+# feishu-msg url
+feishu_msg_url = os.getenv("FEISHU_MSG_URL")
+
+today = date.today()
+#today = date(2023,8,7)
+path="/data/telemetry/crash-report/"
+
+# get files for the past 7 days
+def get_files():
+ files = ""
+ for i in range(1,8):
+ #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
+ files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
+
+ return files
+
+# for none-taosAssertDebug
+filter1_cmd = '''grep '"version":"%s"' %s \
+| grep "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep -v "taosAssertDebug" \
+| grep -v %s \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c ''' % (version, get_files(), ip)
+
+# for taosAssertDebug
+filter2_cmd = '''grep '"version":"%s"' %s \
+| grep "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep "taosAssertDebug" \
+| grep -v %s \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c ''' % (version, get_files(), ip)
+
+# get msg info
+def get_msg(text):
+ return {
+ "msg_type": "post",
+ "content": {
+ "post": {
+ "zh_cn": {
+ "title": "Telemetry Statistics",
+ "content": [
+ [{
+ "tag": "text",
+ "text": text
+ }
+ ]]
+ }
+ }
+ }
+ }
+
+# post msg
+def send_msg(json):
+ headers = {
+ 'Content-Type': 'application/json'
+ }
+
+ req = requests.post(url=group_url, headers=headers, json=json)
+ inf = req.json()
+ if "StatusCode" in inf and inf["StatusCode"] == 0:
+ pass
+ else:
+ print(inf)
+
+# exec cmd and return res
+def get_output(cmd):
+ text = os.popen(cmd)
+ lines = text.read()
+ text.close()
+ return lines
+
+# get sum
+def get_count(output):
+ res = re.findall(" \d+ ", output)
+ sum1 = 0
+ for r in res:
+ sum1 = sum1 + int(r.strip())
+ return sum1
+
+# print total crash count
+def print_result():
+ #print(f"Files for statistics: {get_files()}\n")
+ sum1 = get_count(get_output(filter1_cmd))
+ sum2 = get_count(get_output(filter2_cmd))
+ total = sum1 + sum2
+ #print(f"total crashes: {total}")
+ return total
+
+# send report to feishu
+def send_report():
+ content = f'''
+ test scope: Telemetry Statistics
+ owner: {owner}
+ ip: {server_ip}
+ from: {get_files().split(" ")[6].split("/")[4].split(".")[0]}
+ to: {get_files().split(" ")[0].split("/")[4].split(".")[0]}
+ filter1 result: {get_output(filter1_cmd)}
+ filter2 result: {get_output(filter2_cmd)}
+ total crashes: {print_result()}
+ '''
+ #send_msg(get_msg(content))
+ print(content)
+
+print_result()
+send_report()
diff --git a/tests/script/telemetry/crash-report/CrashCounter_new.py b/tests/script/telemetry/crash-report/CrashCounter_new.py
deleted file mode 100644
index a89567da3d..0000000000
--- a/tests/script/telemetry/crash-report/CrashCounter_new.py
+++ /dev/null
@@ -1,308 +0,0 @@
-from datetime import date
-from datetime import timedelta
-import os
-import json
-import re
-import requests
-import subprocess
-from dotenv import load_dotenv
-
-# load .env
-# You should have a .env file in the same directory as this script
-# You can exec: cp .env.example .env
-load_dotenv()
-
-# define version
-version = "3.3.2.*"
-version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+')
-version_pattern = re.compile(rf'^{version_pattern_str}$')
-version_stack_list = list()
-
-# define ip
-
-ip = os.getenv("EXCLUDE_IP")
-server_ip = os.getenv("SERVER_IP")
-http_serv_ip = os.getenv("HTTP_SERV_IP")
-http_serv_port = os.getenv("HTTP_SERV_PORT")
-owner = os.getenv("OWNER")
-
-# feishu-msg url
-feishu_msg_url = os.getenv("FEISHU_MSG_URL")
-
-# get today
-today = date.today()
-
-# Define the file and parameters
-path="/data/telemetry/crash-report/"
-trace_report_path = path + "trace_report"
-os.makedirs(path, exist_ok=True)
-os.makedirs(trace_report_path, exist_ok=True)
-
-assert_script_path = path + "filter_assert.sh"
-nassert_script_path = path + "filter_nassert.sh"
-
-# get files for the past 7 days
-def get_files():
- files = ""
- for i in range(1,8):
- #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
- files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
- return files.strip().split(" ")
-
-# Define the AWK script as a string with proper escaping
-def get_res(file_path):
- # Execute the script
- command = ['bash', file_path, version, ip] + get_files()
- process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-
- # Capture the output and errors
- output, errors = process.communicate()
-
- # Check for errors
- if process.returncode != 0:
- return errors
- else:
- return output.rstrip()
-
-def get_sum(output):
- # Split the output into lines
- lines = output.strip().split('\n')
-
- # Initialize the sum
- total_sum = 0
-
- # Iterate over each line
- for line in lines:
- # Split each line by space to separate the columns
- parts = line.split()
-
- # The first part of the line is the number, convert it to integer
- if parts: # Check if there are any elements in the parts list
- number = int(parts[0])
- total_sum += number
-
- return total_sum
-
-def convert_html(data):
- # convert data to json
- start_time = get_files()[6].split("/")[-1].split(".")[0]
- end_time = get_files()[0].split("/")[-1].split(".")[0]
- html_report_file = f'{start_time}_{end_time}.html'
- json_data = json.dumps(data)
-
- # Create HTML content
- html_content = f'''
-
-
-
-
-
- Stack Trace Report
-
-
-
- Stack Trace Report From {start_time} To {end_time}
-
-
-
-
- | Key Stack Info |
- Versions |
- Num Of Crashes |
- Full Stack Info |
-
-
-
-
-
-
-
-
-
-'''
- # Write the HTML content to a file
-
- with open(f'{trace_report_path}/{html_report_file}', 'w') as f:
- f.write(html_content)
- return html_report_file
-
-def get_version_stack_list(res):
- for line in res.strip().split('\n'):
- version_list = list()
- version_stack_dict = dict()
- count = line.split()[0]
- key_stack_info = line.split()[1]
- for file in get_files():
- with open(file, 'r') as infile:
- for line in infile:
- line = line.strip()
- data = json.loads(line)
- # print(line)
- if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line:
- if data["version"] not in version_list:
- version_list.append(data["version"])
- full_stack_info = data["stackInfo"]
- version_stack_dict["key_stack_info"] = key_stack_info
- version_stack_dict["full_stack_info"] = full_stack_info
- version_stack_dict["version_list"] = version_list
- version_stack_dict["count"] = count
- # print(version_stack_dict)
- version_stack_list.append(version_stack_dict)
- return version_stack_list
-
-# get msg info
-def get_msg(text):
- return {
- "msg_type": "post",
- "content": {
- "post": {
- "zh_cn": {
- "title": "Telemetry Statistics",
- "content": [
- [{
- "tag": "text",
- "text": text
- }
- ]]
- }
- }
- }
- }
-
-# post msg
-def send_msg(json):
- headers = {
- 'Content-Type': 'application/json'
- }
-
- req = requests.post(url=feishu_msg_url, headers=headers, json=json)
- inf = req.json()
- if "StatusCode" in inf and inf["StatusCode"] == 0:
- pass
- else:
- print(inf)
-
-
-def format_results(results):
- # Split the results into lines
- lines = results.strip().split('\n')
-
- # Parse lines into a list of tuples (number, rest_of_line)
- parsed_lines = []
- for line in lines:
- parts = line.split(maxsplit=1)
- if len(parts) == 2:
- number = int(parts[0]) # Convert the number part to an integer
- parsed_lines.append((number, parts[1]))
-
- # Sort the parsed lines by the first element (number) in descending order
- parsed_lines.sort(reverse=True, key=lambda x: x[0])
-
- # Determine the maximum width of the first column for alignment
- # max_width = max(len(str(item[0])) for item in parsed_lines)
- if parsed_lines:
- max_width = max(len(str(item[0])) for item in parsed_lines)
- else:
- max_width = 0
-
- # Format each line to align the numbers and function names with indentation
- formatted_lines = []
- for number, text in parsed_lines:
- formatted_line = f" {str(number).rjust(max_width)} {text}"
- formatted_lines.append(formatted_line)
-
- # Join the formatted lines into a single string
- return '\n'.join(formatted_lines)
-
-# # send report to feishu
-def send_report(res, sum, html_report_file):
- content = f'''
- version: v{version}
- from: {get_files()[6].split("/")[-1].split(".")[0]}
- to: {get_files()[0].split("/")[-1].split(".")[0]}
- ip: {server_ip}
- owner: {owner}
- result: \n{format_results(res)}\n
- total crashes: {sum}\n
- details: http://{http_serv_ip}:{http_serv_port}/{html_report_file}
- '''
- print(get_msg(content))
- send_msg(get_msg(content))
- # print(content)
-
-# for none-taosAssertDebug
-nassert_res = get_res(nassert_script_path)
-# print(nassert_res)
-
-# for taosAssertDebug
-assert_res = get_res(assert_script_path)
-# print(assert_res)
-
-# combine the results
-res = nassert_res + assert_res
-
-# get version stack list
-version_stack_list = get_version_stack_list(res) if len(res) > 0 else list()
-
-# convert to html
-html_report_file = convert_html(version_stack_list)
-
-# get sum
-sum = get_sum(res)
-
-# send report
-send_report(res, sum, html_report_file)
-