From 7c083234f76671cde36d3f6a36af789e5cf7e00b Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 23 Jan 2025 11:25:40 +0800 Subject: [PATCH] enh: change path --- tests/script/telemetry/CrashCounter.py | 128 --------- tests/script/telemetry/CrashCounter_new.py | 308 --------------------- tests/script/telemetry/filter1.sh | 15 - tests/script/telemetry/filter2.sh | 14 - tests/script/telemetry/filter_assert.sh | 67 ----- tests/script/telemetry/filter_nassert.sh | 74 ----- 6 files changed, 606 deletions(-) delete mode 100644 tests/script/telemetry/CrashCounter.py delete mode 100644 tests/script/telemetry/CrashCounter_new.py delete mode 100755 tests/script/telemetry/filter1.sh delete mode 100755 tests/script/telemetry/filter2.sh delete mode 100755 tests/script/telemetry/filter_assert.sh delete mode 100755 tests/script/telemetry/filter_nassert.sh diff --git a/tests/script/telemetry/CrashCounter.py b/tests/script/telemetry/CrashCounter.py deleted file mode 100644 index 66edc8d63e..0000000000 --- a/tests/script/telemetry/CrashCounter.py +++ /dev/null @@ -1,128 +0,0 @@ -from datetime import date -from datetime import timedelta -import os -import re -import requests -from dotenv import load_dotenv - -# load .env -load_dotenv() - -# define version -version = "3.3.*" - -ip = os.getenv("EXCLUDE_IP") -server_ip = os.getenv("SERVER_IP") -owner = os.getenv("OWNER") - -# feishu-msg url -feishu_msg_url = os.getenv("FEISHU_MSG_URL") - -today = date.today() -#today = date(2023,8,7) -path="/data/telemetry/crash-report/" - -# get files for the past 7 days -def get_files(): - files = "" - for i in range(1,8): - #print ((today - timedelta(days=i)).strftime("%Y%m%d")) - files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt " - - return files - -# for none-taosAssertDebug -filter1_cmd = '''grep '"version":"%s"' %s \ -| grep "taosd(" \ -| awk -F "stackInfo" '{print $2}' \ -| grep -v "taosAssertDebug" \ -| grep -v %s \ -| awk -F "taosd" '{print $3}' \ -| cut -d")" -f 1 \ -| cut -d"(" -f 2 \ -| sort | uniq -c ''' % (version, get_files(), ip) - -# for taosAssertDebug -filter2_cmd = '''grep '"version":"%s"' %s \ -| grep "taosd(" \ -| awk -F "stackInfo" '{print $2}' \ -| grep "taosAssertDebug" \ -| grep -v %s \ -| awk -F "taosd" '{print $3}' \ -| cut -d")" -f 1 \ -| cut -d"(" -f 2 \ -| sort | uniq -c ''' % (version, get_files(), ip) - -# get msg info -def get_msg(text): - return { - "msg_type": "post", - "content": { - "post": { - "zh_cn": { - "title": "Telemetry Statistics", - "content": [ - [{ - "tag": "text", - "text": text - } - ]] - } - } - } - } - -# post msg -def send_msg(json): - headers = { - 'Content-Type': 'application/json' - } - - req = requests.post(url=group_url, headers=headers, json=json) - inf = req.json() - if "StatusCode" in inf and inf["StatusCode"] == 0: - pass - else: - print(inf) - -# exec cmd and return res -def get_output(cmd): - text = os.popen(cmd) - lines = text.read() - text.close() - return lines - -# get sum -def get_count(output): - res = re.findall(" \d+ ", output) - sum1 = 0 - for r in res: - sum1 = sum1 + int(r.strip()) - return sum1 - -# print total crash count -def print_result(): - #print(f"Files for statistics: {get_files()}\n") - sum1 = get_count(get_output(filter1_cmd)) - sum2 = get_count(get_output(filter2_cmd)) - total = sum1 + sum2 - #print(f"total crashes: {total}") - return total - -# send report to feishu -def send_report(): - content = f''' - test scope: Telemetry Statistics - owner: {owner} - ip: {server_ip} - from: {get_files().split(" ")[6].split("/")[4].split(".")[0]} - to: {get_files().split(" ")[0].split("/")[4].split(".")[0]} - filter1 result: {get_output(filter1_cmd)} - filter2 result: {get_output(filter2_cmd)} - total crashes: {print_result()} - ''' - #send_msg(get_msg(content)) - print(content) - -print_result() -send_report() diff --git a/tests/script/telemetry/CrashCounter_new.py b/tests/script/telemetry/CrashCounter_new.py deleted file mode 100644 index a89567da3d..0000000000 --- a/tests/script/telemetry/CrashCounter_new.py +++ /dev/null @@ -1,308 +0,0 @@ -from datetime import date -from datetime import timedelta -import os -import json -import re -import requests -import subprocess -from dotenv import load_dotenv - -# load .env -# You should have a .env file in the same directory as this script -# You can exec: cp .env.example .env -load_dotenv() - -# define version -version = "3.3.2.*" -version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+') -version_pattern = re.compile(rf'^{version_pattern_str}$') -version_stack_list = list() - -# define ip - -ip = os.getenv("EXCLUDE_IP") -server_ip = os.getenv("SERVER_IP") -http_serv_ip = os.getenv("HTTP_SERV_IP") -http_serv_port = os.getenv("HTTP_SERV_PORT") -owner = os.getenv("OWNER") - -# feishu-msg url -feishu_msg_url = os.getenv("FEISHU_MSG_URL") - -# get today -today = date.today() - -# Define the file and parameters -path="/data/telemetry/crash-report/" -trace_report_path = path + "trace_report" -os.makedirs(path, exist_ok=True) -os.makedirs(trace_report_path, exist_ok=True) - -assert_script_path = path + "filter_assert.sh" -nassert_script_path = path + "filter_nassert.sh" - -# get files for the past 7 days -def get_files(): - files = "" - for i in range(1,8): - #print ((today - timedelta(days=i)).strftime("%Y%m%d")) - files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt " - return files.strip().split(" ") - -# Define the AWK script as a string with proper escaping -def get_res(file_path): - # Execute the script - command = ['bash', file_path, version, ip] + get_files() - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - - # Capture the output and errors - output, errors = process.communicate() - - # Check for errors - if process.returncode != 0: - return errors - else: - return output.rstrip() - -def get_sum(output): - # Split the output into lines - lines = output.strip().split('\n') - - # Initialize the sum - total_sum = 0 - - # Iterate over each line - for line in lines: - # Split each line by space to separate the columns - parts = line.split() - - # The first part of the line is the number, convert it to integer - if parts: # Check if there are any elements in the parts list - number = int(parts[0]) - total_sum += number - - return total_sum - -def convert_html(data): - # convert data to json - start_time = get_files()[6].split("/")[-1].split(".")[0] - end_time = get_files()[0].split("/")[-1].split(".")[0] - html_report_file = f'{start_time}_{end_time}.html' - json_data = json.dumps(data) - - # Create HTML content - html_content = f''' - - - - - - Stack Trace Report - - - -

Stack Trace Report From {start_time} To {end_time}

- - - - - - - - - - - - -
Key Stack InfoVersionsNum Of CrashesFull Stack Info
- - - - -''' - # Write the HTML content to a file - - with open(f'{trace_report_path}/{html_report_file}', 'w') as f: - f.write(html_content) - return html_report_file - -def get_version_stack_list(res): - for line in res.strip().split('\n'): - version_list = list() - version_stack_dict = dict() - count = line.split()[0] - key_stack_info = line.split()[1] - for file in get_files(): - with open(file, 'r') as infile: - for line in infile: - line = line.strip() - data = json.loads(line) - # print(line) - if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line: - if data["version"] not in version_list: - version_list.append(data["version"]) - full_stack_info = data["stackInfo"] - version_stack_dict["key_stack_info"] = key_stack_info - version_stack_dict["full_stack_info"] = full_stack_info - version_stack_dict["version_list"] = version_list - version_stack_dict["count"] = count - # print(version_stack_dict) - version_stack_list.append(version_stack_dict) - return version_stack_list - -# get msg info -def get_msg(text): - return { - "msg_type": "post", - "content": { - "post": { - "zh_cn": { - "title": "Telemetry Statistics", - "content": [ - [{ - "tag": "text", - "text": text - } - ]] - } - } - } - } - -# post msg -def send_msg(json): - headers = { - 'Content-Type': 'application/json' - } - - req = requests.post(url=feishu_msg_url, headers=headers, json=json) - inf = req.json() - if "StatusCode" in inf and inf["StatusCode"] == 0: - pass - else: - print(inf) - - -def format_results(results): - # Split the results into lines - lines = results.strip().split('\n') - - # Parse lines into a list of tuples (number, rest_of_line) - parsed_lines = [] - for line in lines: - parts = line.split(maxsplit=1) - if len(parts) == 2: - number = int(parts[0]) # Convert the number part to an integer - parsed_lines.append((number, parts[1])) - - # Sort the parsed lines by the first element (number) in descending order - parsed_lines.sort(reverse=True, key=lambda x: x[0]) - - # Determine the maximum width of the first column for alignment - # max_width = max(len(str(item[0])) for item in parsed_lines) - if parsed_lines: - max_width = max(len(str(item[0])) for item in parsed_lines) - else: - max_width = 0 - - # Format each line to align the numbers and function names with indentation - formatted_lines = [] - for number, text in parsed_lines: - formatted_line = f" {str(number).rjust(max_width)} {text}" - formatted_lines.append(formatted_line) - - # Join the formatted lines into a single string - return '\n'.join(formatted_lines) - -# # send report to feishu -def send_report(res, sum, html_report_file): - content = f''' - version: v{version} - from: {get_files()[6].split("/")[-1].split(".")[0]} - to: {get_files()[0].split("/")[-1].split(".")[0]} - ip: {server_ip} - owner: {owner} - result: \n{format_results(res)}\n - total crashes: {sum}\n - details: http://{http_serv_ip}:{http_serv_port}/{html_report_file} - ''' - print(get_msg(content)) - send_msg(get_msg(content)) - # print(content) - -# for none-taosAssertDebug -nassert_res = get_res(nassert_script_path) -# print(nassert_res) - -# for taosAssertDebug -assert_res = get_res(assert_script_path) -# print(assert_res) - -# combine the results -res = nassert_res + assert_res - -# get version stack list -version_stack_list = get_version_stack_list(res) if len(res) > 0 else list() - -# convert to html -html_report_file = convert_html(version_stack_list) - -# get sum -sum = get_sum(res) - -# send report -send_report(res, sum, html_report_file) - diff --git a/tests/script/telemetry/filter1.sh b/tests/script/telemetry/filter1.sh deleted file mode 100755 index 3cb36a18ad..0000000000 --- a/tests/script/telemetry/filter1.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -source .env -filesPath="/data/telemetry/crash-report" -version="3.0.4.1" -taosdataIp=$EXCLUDE_IP -grep "\"version\":\"${version}\"" ${filesPath}/*.txt \ -| grep "taosd(" \ -| awk -F "stackInfo" '{print $2}' \ -| grep -v "taosAssertDebug" \ -| grep -v ${taosdataIp} \ -| awk -F "taosd" '{print $2}' \ -| cut -d")" -f 1 \ -| cut -d"(" -f 2 \ -| sort | uniq -c - diff --git a/tests/script/telemetry/filter2.sh b/tests/script/telemetry/filter2.sh deleted file mode 100755 index 4ad545345e..0000000000 --- a/tests/script/telemetry/filter2.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -source .env -filesPath="/data/telemetry/crash-report" -version="3.0.4.1" -taosdataIp=$EXCLUDE_IP -grep "\"version\":\"${version}\"" ${filesPath}/*.txt \ -| grep "taosd(" \ -| awk -F "stackInfo" '{print $2}' \ -| grep "taosAssertDebug" \ -| grep -v ${taosdataIp} \ -| awk -F "taosd" '{print $3}' \ -| cut -d")" -f 1 \ -| cut -d"(" -f 2 \ -| sort | uniq -c diff --git a/tests/script/telemetry/filter_assert.sh b/tests/script/telemetry/filter_assert.sh deleted file mode 100755 index 2d56287fc9..0000000000 --- a/tests/script/telemetry/filter_assert.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# Extract version and IP from the first two arguments -version="$1" -ip="$2" -shift 2 # Remove the first two arguments, leaving only file paths - -# All remaining arguments are considered as file paths -file_paths="$@" - -# Execute the awk script and capture the output -readarray -t output < <(awk -v version="$version" -v ip="$ip" ' -BEGIN { - RS = "\\n"; # Set the record separator to newline - FS = ","; # Set the field separator to comma - total = 0; # Initialize total count - version_regex = version; # Use the passed version pattern - ip_regex = ip; # Use the passed IP pattern -} -{ - start_collecting = 0; - version_matched = 0; - ip_excluded = 0; - - # Check each field within a record - for (i = 1; i <= NF; i++) { - if ($i ~ /"ip":"[^"]*"/ && $i ~ ip_regex) { - ip_excluded = 1; - } - if ($i ~ /"version":"[^"]*"/ && $i ~ version_regex) { - version_matched = 1; - } - } - - if (!ip_excluded && version_matched) { - for (i = 1; i <= NF; i++) { - if ($i ~ /taosAssertDebug/ && start_collecting == 0) { - start_collecting = 1; - continue; - } - if (start_collecting == 1 && $i ~ /taosd\(([^)]+)\)/) { - match($i, /taosd\(([^)]+)\)/, arr); - if (arr[1] != "") { - count[arr[1]]++; - total++; - break; - } - } - } - } -} -END { - for (c in count) { - printf "%d %s\n", count[c], c; - } - print "Total count:", total; -}' $file_paths) - -# Capture the function details and total count into separate variables -function_details=$(printf "%s\n" "${output[@]::${#output[@]}-1}") -total_count="${output[-1]}" - -# Output or use the variables as needed -#echo "Function Details:" -echo "$function_details" -#echo "Total Count:" -#echo "$total_count" diff --git a/tests/script/telemetry/filter_nassert.sh b/tests/script/telemetry/filter_nassert.sh deleted file mode 100755 index 2a5acdfbf1..0000000000 --- a/tests/script/telemetry/filter_nassert.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -# Pass version, ip, and file paths as arguments -version="$1" -ip="$2" -shift 2 # Shift the first two arguments to get file paths -file_paths="$@" - -# Execute awk and capture the output -readarray -t output < <(awk -v version="$version" -v ip="$ip" ' -BEGIN { - RS = "\\n"; # Set the record separator to newline - total = 0; # Initialize total count - version_regex = "\"version\":\"" version; # Construct the regex for version - ip_regex = "\"ip\":\"" ip "\""; # Construct the regex for IP -} -{ - found = 0; # Initialize the found flag to false - start_collecting = 1; # Start collecting by default, unless taosAssertDebug is encountered - split($0, parts, "\\n"); # Split each record by newline - - # Check for version and IP in each part - version_matched = 0; - ip_excluded = 0; - for (i in parts) { - if (parts[i] ~ version_regex) { - version_matched = 1; # Set flag if version is matched - } - if (parts[i] ~ ip_regex) { - ip_excluded = 1; # Set flag if IP is matched - break; # No need to continue if IP is excluded - } - } - - # Process only if version is matched and IP is not excluded - if (version_matched && !ip_excluded) { - for (i in parts) { - if (parts[i] ~ /taosAssertDebug/) { - start_collecting = 0; # Skip this record if taosAssertDebug is encountered - break; # Exit the loop - } - } - if (start_collecting == 1) { # Continue processing if taosAssertDebug is not found - for (i in parts) { - if (found == 0 && parts[i] ~ /frame:.*taosd\([^)]+\)/) { - # Match the first frame that meets the condition - match(parts[i], /taosd\(([^)]+)\)/, a); # Extract the function name - if (a[1] != "") { - count[a[1]]++; # Increment the count for this function name - total++; # Increment the total count - found = 1; # Set found flag to true - break; # Exit the loop once the function is found - } - } - } - } - } -} -END { - for (c in count) { - printf "%d %s\n", count[c], c; # Print the count and function name formatted - } - print total; # Print the total count alone -}' $file_paths) # Note the removal of quotes around "$file_paths" to handle multiple paths - -# Capture the function details and total count into separate variables -function_details=$(printf "%s\n" "${output[@]::${#output[@]}-1}") # Join array elements with newlines -total_count="${output[-1]}" # The last element - -# Output or use the variables as needed -#echo "Function Details:" -echo "$function_details" -#echo "Total Count:" -#echo "$total_count"