From e587bddf9a52c24249c4032df39a636bdf8a6a43 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Tue, 21 Jan 2025 10:22:49 +0800
Subject: [PATCH 01/11] enh: backup old scripts

---
 tests/script/telemetry/CrashCounter.py | 121 +++++++++++++++++++++++++
 tests/script/telemetry/filter1.sh      |  15 +++
 tests/script/telemetry/filter2.sh      |  14 +++
 3 files changed, 150 insertions(+)
 create mode 100644 tests/script/telemetry/CrashCounter.py
 create mode 100755 tests/script/telemetry/filter1.sh
 create mode 100755 tests/script/telemetry/filter2.sh

diff --git a/tests/script/telemetry/CrashCounter.py b/tests/script/telemetry/CrashCounter.py
new file mode 100644
index 0000000000..4a6700cf5b
--- /dev/null
+++ b/tests/script/telemetry/CrashCounter.py
@@ -0,0 +1,121 @@
+from datetime import date
+from datetime import timedelta
+import os
+import re
+import requests
+
+# define version
+version = "3.3.*"
+ip = "103.229.218.146"
+
+# feishu-msg url
+#group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9'
+group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/11e9e452-34a0-4c88-b014-10e21cb521dd'
+today = date.today()
+#today = date(2023,8,7)
+path="/data/telemetry/crash-report/"
+
+# get files for the past 7 days
+def get_files():
+    files = ""
+    for i in range(1,8):
+        #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
+        files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
+
+    return files
+
+# for none-taosAssertDebug
+filter1_cmd = '''grep '"version":"%s"'  %s \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep -v "taosAssertDebug" \
+| grep -v %s \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c ''' % (version, get_files(), ip)
+
+# for taosAssertDebug
+filter2_cmd = '''grep '"version":"%s"'  %s \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep "taosAssertDebug" \
+| grep -v %s \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c ''' % (version, get_files(), ip)
+
+# get msg info
+def get_msg(text):
+    return {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": "Telemetry Statistics",
+                    "content": [
+                        [{
+                            "tag": "text",
+                            "text": text
+                        }
+                        ]]
+                }
+            }
+        }
+    }
+
+# post msg
+def send_msg(json):
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    req = requests.post(url=group_url, headers=headers, json=json)
+    inf = req.json()
+    if "StatusCode" in inf and inf["StatusCode"] == 0:
+        pass
+    else:
+        print(inf)
+
+# exec cmd and return res
+def get_output(cmd):
+    text = os.popen(cmd)
+    lines = text.read()
+    text.close()
+    return lines
+
+# get sum
+def get_count(output):
+    res = re.findall(" \d+ ", output)
+    sum1 = 0
+    for r in res:
+        sum1 = sum1 + int(r.strip())
+    return sum1
+
+# print total crash count
+def print_result():
+    #print(f"Files for statistics: {get_files()}\n")
+    sum1 = get_count(get_output(filter1_cmd))
+    sum2 = get_count(get_output(filter2_cmd))
+    total = sum1 + sum2
+    #print(f"total crashes: {total}")
+    return total
+
+# send report to feishu
+def send_report():
+    content = f'''
+    test scope: Telemetry Statistics
+    owner: Jayden Jia
+    ip: 20.124.239.6
+    from: {get_files().split(" ")[6].split("/")[4].split(".")[0]}
+    to: {get_files().split(" ")[0].split("/")[4].split(".")[0]}
+    filter1 result: {get_output(filter1_cmd)}
+    filter2 result: {get_output(filter2_cmd)}
+    total crashes: {print_result()}
+    '''    
+    #send_msg(get_msg(content))
+    print(content)
+
+print_result()
+send_report()
diff --git a/tests/script/telemetry/filter1.sh b/tests/script/telemetry/filter1.sh
new file mode 100755
index 0000000000..65d34808cb
--- /dev/null
+++ b/tests/script/telemetry/filter1.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+filesPath="/data/telemetry/crash-report"
+version="3.0.4.1"
+taosdataIp="103.229.218.146"
+grep "\"version\":\"${version}\"" ${filesPath}/*.txt \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep -v "taosAssertDebug" \
+| grep -v ${taosdataIp} \
+| awk -F "taosd" '{print $2}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c
+
diff --git a/tests/script/telemetry/filter2.sh b/tests/script/telemetry/filter2.sh
new file mode 100755
index 0000000000..33f0fd08b5
--- /dev/null
+++ b/tests/script/telemetry/filter2.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+filesPath="/data/telemetry/crash-report"
+version="3.0.4.1"
+taosdataIp="103.229.218.146"
+grep "\"version\":\"${version}\"" ${filesPath}/*.txt \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep "taosAssertDebug" \
+| grep -v ${taosdataIp} \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c

From 98144cee081bde5a9c04786fc794eacb4c5d692a Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Tue, 21 Jan 2025 10:23:15 +0800
Subject: [PATCH 02/11] enh: add new scripts

---
 tests/script/telemetry/CrashCounter_new.py | 301 +++++++++++++++++++++
 tests/script/telemetry/filter_assert.sh    |  67 +++++
 tests/script/telemetry/filter_nassert.sh   |  74 +++++
 3 files changed, 442 insertions(+)
 create mode 100644 tests/script/telemetry/CrashCounter_new.py
 create mode 100755 tests/script/telemetry/filter_assert.sh
 create mode 100755 tests/script/telemetry/filter_nassert.sh

diff --git a/tests/script/telemetry/CrashCounter_new.py b/tests/script/telemetry/CrashCounter_new.py
new file mode 100644
index 0000000000..af88558c6a
--- /dev/null
+++ b/tests/script/telemetry/CrashCounter_new.py
@@ -0,0 +1,301 @@
+from datetime import date
+from datetime import timedelta
+import os
+import json
+import re
+import requests
+import subprocess
+
+# define version
+version = "3.3.2.*"
+version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+')
+version_pattern = re.compile(rf'^{version_pattern_str}$')
+version_stack_list = list()
+
+# define ip
+ip = "103.229.218.146"
+server_ip = "20.124.239.6"
+http_ip = "192.168.2.92"
+owner = "Jayden Jia"
+
+# feishu-msg url
+group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9'
+# group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/11e9e452-34a0-4c88-b014-10e21cb521dd'
+
+# get today
+today = date.today()
+
+# Define the file and parameters
+path="/data/telemetry/crash-report/"
+trace_report_path = path + "trace_report"
+os.makedirs(path, exist_ok=True)
+os.makedirs(trace_report_path, exist_ok=True)
+
+assert_script_path = path + "filter_assert.sh"
+nassert_script_path = path + "filter_nassert.sh"
+
+# get files for the past 7 days
+def get_files():
+    files = ""
+    for i in range(1,8):
+        #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
+        files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
+    return files.strip().split(" ")
+
+# Define the AWK script as a string with proper escaping
+def get_res(file_path):
+    # Execute the script
+    command = ['bash', file_path, version, ip] + get_files()
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+
+    # Capture the output and errors
+    output, errors = process.communicate()
+
+    # Check for errors
+    if process.returncode != 0:
+        return errors
+    else:
+        return output.rstrip()
+
+def get_sum(output):
+    # Split the output into lines
+    lines = output.strip().split('\n')
+
+    # Initialize the sum
+    total_sum = 0
+
+    # Iterate over each line
+    for line in lines:
+        # Split each line by space to separate the columns
+        parts = line.split()
+
+        # The first part of the line is the number, convert it to integer
+        if parts:  # Check if there are any elements in the parts list
+            number = int(parts[0])
+            total_sum += number
+
+    return total_sum
+
+def convert_html(data):
+    # convert data to json
+    start_time = get_files()[6].split("/")[-1].split(".")[0]
+    end_time = get_files()[0].split("/")[-1].split(".")[0]
+    html_report_file = f'{start_time}_{end_time}.html'
+    json_data = json.dumps(data)
+
+    # Create HTML content
+    html_content = f'''
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Stack Trace Report</title>
+    <style>
+        body {{
+            font-family: Arial, sans-serif;
+            margin: 20px;
+            background-color: #f0f0f5;
+        }}
+        h1 {{
+            color: #2c3e50;
+            text-align: center;
+        }}
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            margin-bottom: 20px;
+            box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
+        }}
+        th, td {{
+            border: 1px solid #ddd;
+            padding: 10px;
+            text-align: left;
+        }}
+        th {{
+            background-color: #3498db;
+            color: white;
+        }}
+        tr:nth-child(even) {{
+            background-color: #ecf0f1;
+        }}
+        tr:hover {{
+            background-color: #d1e7fd;
+        }}
+        pre {{
+            background-color: #f7f7f7;
+            padding: 10px;
+            border: 1px solid #ddd;
+            overflow-x: auto;
+            white-space: pre-wrap;
+            border-radius: 5px;
+        }}
+    </style>
+</head>
+<body>
+    <h1>Stack Trace Report From {start_time} To {end_time} </h1>
+
+    <table>
+        <thead>
+            <tr>
+                <th>Key Stack Info</th>
+                <th>Versions</th>
+                <th>Num Of Crashes</th>
+                <th>Full Stack Info</th>
+            </tr>
+        </thead>
+        <tbody id="report">
+        </tbody>
+    </table>
+
+    <script>
+        const data = {json_data};
+
+        const reportBody = document.getElementById('report');
+        data.forEach(entry => {{
+            const row = document.createElement('tr');
+            row.innerHTML = `
+                <td>${{entry.key_stack_info}}</td>
+                <td>${{entry.version_list.join('<br>')}}</td>
+                <td>${{entry.count}}</td>
+                <td><pre>${{entry.full_stack_info}}</pre></td>
+            `;
+            reportBody.appendChild(row);
+        }});
+    </script>
+</body>
+</html>
+'''
+    # Write the HTML content to a file
+
+    with open(f'{trace_report_path}/{html_report_file}', 'w') as f:
+        f.write(html_content)
+    return html_report_file
+
+def get_version_stack_list(res):
+    for line in res.strip().split('\n'):
+        version_list = list()
+        version_stack_dict = dict()
+        count = line.split()[0]
+        key_stack_info = line.split()[1]
+        for file in get_files():
+            with open(file, 'r') as infile:
+                for line in infile:
+                    line = line.strip()
+                    data = json.loads(line)
+                    # print(line)
+                    if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line:
+                        if data["version"] not in version_list:
+                            version_list.append(data["version"])
+                            full_stack_info = data["stackInfo"]
+        version_stack_dict["key_stack_info"] = key_stack_info
+        version_stack_dict["full_stack_info"] = full_stack_info
+        version_stack_dict["version_list"] = version_list
+        version_stack_dict["count"] = count
+        # print(version_stack_dict)
+        version_stack_list.append(version_stack_dict)
+    return version_stack_list
+
+# get msg info
+def get_msg(text):
+    return {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": "Telemetry Statistics",
+                    "content": [
+                        [{
+                            "tag": "text",
+                            "text": text
+                        }
+                        ]]
+                }
+            }
+        }
+    }
+
+# post msg
+def send_msg(json):
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    req = requests.post(url=group_url, headers=headers, json=json)
+    inf = req.json()
+    if "StatusCode" in inf and inf["StatusCode"] == 0:
+        pass
+    else:
+        print(inf)
+
+
+def format_results(results):
+    # Split the results into lines
+    lines = results.strip().split('\n')
+
+    # Parse lines into a list of tuples (number, rest_of_line)
+    parsed_lines = []
+    for line in lines:
+        parts = line.split(maxsplit=1)
+        if len(parts) == 2:
+            number = int(parts[0])  # Convert the number part to an integer
+            parsed_lines.append((number, parts[1]))
+
+    # Sort the parsed lines by the first element (number) in descending order
+    parsed_lines.sort(reverse=True, key=lambda x: x[0])
+
+    # Determine the maximum width of the first column for alignment
+    # max_width = max(len(str(item[0])) for item in parsed_lines)
+    if parsed_lines:
+        max_width = max(len(str(item[0])) for item in parsed_lines)
+    else:
+        max_width = 0
+
+    # Format each line to align the numbers and function names with indentation
+    formatted_lines = []
+    for number, text in parsed_lines:
+        formatted_line = f"       {str(number).rjust(max_width)} {text}"
+        formatted_lines.append(formatted_line)
+
+    # Join the formatted lines into a single string
+    return '\n'.join(formatted_lines)
+
+# # send report to feishu
+def send_report(res, sum, html_report_file):
+    content = f'''
+    version: v{version}
+    from: {get_files()[6].split("/")[-1].split(".")[0]}
+    to: {get_files()[0].split("/")[-1].split(".")[0]}
+    ip: {server_ip}
+    owner: {owner}
+    result: \n{format_results(res)}\n
+    total crashes: {sum}\n
+    details: http://{http_ip}:8000/{html_report_file}
+    '''
+    print(get_msg(content))
+    send_msg(get_msg(content))
+    # print(content)
+
+# for none-taosAssertDebug
+nassert_res = get_res(nassert_script_path)
+# print(nassert_res)
+
+# for taosAssertDebug
+assert_res = get_res(assert_script_path)
+# print(assert_res)
+
+# combine the results
+res = nassert_res + assert_res
+
+# get version stack list
+version_stack_list = get_version_stack_list(res) if len(res) > 0 else list()
+
+# convert to html
+html_report_file = convert_html(version_stack_list)
+
+# get sum
+sum = get_sum(res)
+
+# send report
+send_report(res, sum, html_report_file)
+
diff --git a/tests/script/telemetry/filter_assert.sh b/tests/script/telemetry/filter_assert.sh
new file mode 100755
index 0000000000..2d56287fc9
--- /dev/null
+++ b/tests/script/telemetry/filter_assert.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Extract version and IP from the first two arguments
+version="$1"
+ip="$2"
+shift 2  # Remove the first two arguments, leaving only file paths
+
+# All remaining arguments are considered as file paths
+file_paths="$@"
+
+# Execute the awk script and capture the output
+readarray -t output < <(awk -v version="$version" -v ip="$ip" '
+BEGIN {
+    RS = "\\n";  # Set the record separator to newline
+    FS = ",";    # Set the field separator to comma
+    total = 0;   # Initialize total count
+    version_regex = version;  # Use the passed version pattern
+    ip_regex = ip;  # Use the passed IP pattern
+}
+{
+    start_collecting = 0;
+    version_matched = 0;
+    ip_excluded = 0;
+
+    # Check each field within a record
+    for (i = 1; i <= NF; i++) {
+        if ($i ~ /"ip":"[^"]*"/ && $i ~ ip_regex) {
+            ip_excluded = 1;
+        }
+        if ($i ~ /"version":"[^"]*"/ && $i ~ version_regex) {
+            version_matched = 1;
+        }
+    }
+
+    if (!ip_excluded && version_matched) {
+        for (i = 1; i <= NF; i++) {
+            if ($i ~ /taosAssertDebug/ && start_collecting == 0) {
+                start_collecting = 1;
+                continue;
+            }
+            if (start_collecting == 1 && $i ~ /taosd\(([^)]+)\)/) {
+                match($i, /taosd\(([^)]+)\)/, arr);
+                if (arr[1] != "") {
+                    count[arr[1]]++;
+                    total++;
+                    break;
+                }
+            }
+        }
+    }
+}
+END {
+    for (c in count) {
+        printf "%d %s\n", count[c], c;
+    }
+    print "Total count:", total;
+}' $file_paths)
+
+# Capture the function details and total count into separate variables
+function_details=$(printf "%s\n" "${output[@]::${#output[@]}-1}")
+total_count="${output[-1]}"
+
+# Output or use the variables as needed
+#echo "Function Details:"
+echo "$function_details"
+#echo "Total Count:"
+#echo "$total_count"
diff --git a/tests/script/telemetry/filter_nassert.sh b/tests/script/telemetry/filter_nassert.sh
new file mode 100755
index 0000000000..2a5acdfbf1
--- /dev/null
+++ b/tests/script/telemetry/filter_nassert.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Pass version, ip, and file paths as arguments
+version="$1"
+ip="$2"
+shift 2  # Shift the first two arguments to get file paths
+file_paths="$@"
+
+# Execute awk and capture the output
+readarray -t output < <(awk -v version="$version" -v ip="$ip" '
+BEGIN {
+    RS = "\\n";  # Set the record separator to newline
+    total = 0;   # Initialize total count
+    version_regex = "\"version\":\"" version;  # Construct the regex for version
+    ip_regex = "\"ip\":\"" ip "\"";  # Construct the regex for IP
+}
+{
+    found = 0;  # Initialize the found flag to false
+    start_collecting = 1;  # Start collecting by default, unless taosAssertDebug is encountered
+    split($0, parts, "\\n");  # Split each record by newline
+
+    # Check for version and IP in each part
+    version_matched = 0;
+    ip_excluded = 0;
+    for (i in parts) {
+        if (parts[i] ~ version_regex) {
+            version_matched = 1;  # Set flag if version is matched
+        }
+        if (parts[i] ~ ip_regex) {
+            ip_excluded = 1;  # Set flag if IP is matched
+            break;  # No need to continue if IP is excluded
+        }
+    }
+
+    # Process only if version is matched and IP is not excluded
+    if (version_matched && !ip_excluded) {
+        for (i in parts) {
+            if (parts[i] ~ /taosAssertDebug/) {
+                start_collecting = 0;  # Skip this record if taosAssertDebug is encountered
+                break;  # Exit the loop
+            }
+        }
+        if (start_collecting == 1) {  # Continue processing if taosAssertDebug is not found
+            for (i in parts) {
+                if (found == 0 && parts[i] ~ /frame:.*taosd\([^)]+\)/) {
+                    # Match the first frame that meets the condition
+                    match(parts[i], /taosd\(([^)]+)\)/, a);  # Extract the function name
+                    if (a[1] != "") {
+                        count[a[1]]++;  # Increment the count for this function name
+                        total++;  # Increment the total count
+                        found = 1;  # Set found flag to true
+                        break;  # Exit the loop once the function is found
+                    }
+                }
+            }
+        }
+    }
+}
+END {
+    for (c in count) {
+        printf "%d %s\n", count[c], c;  # Print the count and function name formatted
+    }
+    print total;  # Print the total count alone
+}' $file_paths)  # Note the removal of quotes around "$file_paths" to handle multiple paths
+
+# Capture the function details and total count into separate variables
+function_details=$(printf "%s\n" "${output[@]::${#output[@]}-1}")  # Join array elements with newlines
+total_count="${output[-1]}"  # The last element
+
+# Output or use the variables as needed
+#echo "Function Details:"
+echo "$function_details"
+#echo "Total Count:"
+#echo "$total_count"

From 79099ffd0e2ddd97534280eea3b87eb59000840d Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Wed, 22 Jan 2025 19:18:39 +0800
Subject: [PATCH 03/11] enh: add dotenv

---
 tests/script/telemetry/.env.example        |  6 ++++++
 tests/script/telemetry/CrashCounter.py     | 19 ++++++++++++------
 tests/script/telemetry/CrashCounter_new.py | 23 ++++++++++++++--------
 tests/script/telemetry/filter1.sh          |  4 ++--
 tests/script/telemetry/filter2.sh          |  4 ++--
 5 files changed, 38 insertions(+), 18 deletions(-)
 create mode 100644 tests/script/telemetry/.env.example

diff --git a/tests/script/telemetry/.env.example b/tests/script/telemetry/.env.example
new file mode 100644
index 0000000000..f7d50f40c9
--- /dev/null
+++ b/tests/script/telemetry/.env.example
@@ -0,0 +1,6 @@
+EXCLUDE_IP="192.168.1.10"
+SERVER_IP="192.168.1.11"
+HTTP_SERV_IP="192.168.1.12"
+HTTP_SERV_PORT=8080
+FEISHU_MSG_URL="https://open.feishu.cn/open-apis/bot/v2/hook/*******"
+OWNER="Jayden Jia"
diff --git a/tests/script/telemetry/CrashCounter.py b/tests/script/telemetry/CrashCounter.py
index 4a6700cf5b..66edc8d63e 100644
--- a/tests/script/telemetry/CrashCounter.py
+++ b/tests/script/telemetry/CrashCounter.py
@@ -3,14 +3,21 @@ from datetime import timedelta
 import os
 import re
 import requests
+from dotenv import load_dotenv
+
+# load .env
+load_dotenv()
 
 # define version
 version = "3.3.*"
-ip = "103.229.218.146"
+
+ip = os.getenv("EXCLUDE_IP")
+server_ip = os.getenv("SERVER_IP")
+owner = os.getenv("OWNER")
 
 # feishu-msg url
-#group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9'
-group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/11e9e452-34a0-4c88-b014-10e21cb521dd'
+feishu_msg_url = os.getenv("FEISHU_MSG_URL")
+
 today = date.today()
 #today = date(2023,8,7)
 path="/data/telemetry/crash-report/"
@@ -106,14 +113,14 @@ def print_result():
 def send_report():
     content = f'''
     test scope: Telemetry Statistics
-    owner: Jayden Jia
-    ip: 20.124.239.6
+    owner: {owner}
+    ip: {server_ip}
     from: {get_files().split(" ")[6].split("/")[4].split(".")[0]}
     to: {get_files().split(" ")[0].split("/")[4].split(".")[0]}
     filter1 result: {get_output(filter1_cmd)}
     filter2 result: {get_output(filter2_cmd)}
     total crashes: {print_result()}
-    '''    
+    '''
     #send_msg(get_msg(content))
     print(content)
 
diff --git a/tests/script/telemetry/CrashCounter_new.py b/tests/script/telemetry/CrashCounter_new.py
index af88558c6a..a89567da3d 100644
--- a/tests/script/telemetry/CrashCounter_new.py
+++ b/tests/script/telemetry/CrashCounter_new.py
@@ -5,6 +5,12 @@ import json
 import re
 import requests
 import subprocess
+from dotenv import load_dotenv
+
+# load .env
+# You should have a .env file in the same directory as this script
+# You can exec: cp .env.example .env
+load_dotenv()
 
 # define version
 version = "3.3.2.*"
@@ -13,14 +19,15 @@ version_pattern = re.compile(rf'^{version_pattern_str}$')
 version_stack_list = list()
 
 # define ip
-ip = "103.229.218.146"
-server_ip = "20.124.239.6"
-http_ip = "192.168.2.92"
-owner = "Jayden Jia"
+
+ip = os.getenv("EXCLUDE_IP")
+server_ip = os.getenv("SERVER_IP")
+http_serv_ip = os.getenv("HTTP_SERV_IP")
+http_serv_port = os.getenv("HTTP_SERV_PORT")
+owner = os.getenv("OWNER")
 
 # feishu-msg url
-group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9'
-# group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/11e9e452-34a0-4c88-b014-10e21cb521dd'
+feishu_msg_url = os.getenv("FEISHU_MSG_URL")
 
 # get today
 today = date.today()
@@ -221,7 +228,7 @@ def send_msg(json):
         'Content-Type': 'application/json'
     }
 
-    req = requests.post(url=group_url, headers=headers, json=json)
+    req = requests.post(url=feishu_msg_url, headers=headers, json=json)
     inf = req.json()
     if "StatusCode" in inf and inf["StatusCode"] == 0:
         pass
@@ -270,7 +277,7 @@ def send_report(res, sum, html_report_file):
     owner: {owner}
     result: \n{format_results(res)}\n
     total crashes: {sum}\n
-    details: http://{http_ip}:8000/{html_report_file}
+    details: http://{http_serv_ip}:{http_serv_port}/{html_report_file}
     '''
     print(get_msg(content))
     send_msg(get_msg(content))
diff --git a/tests/script/telemetry/filter1.sh b/tests/script/telemetry/filter1.sh
index 65d34808cb..3cb36a18ad 100755
--- a/tests/script/telemetry/filter1.sh
+++ b/tests/script/telemetry/filter1.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
-
+source .env
 filesPath="/data/telemetry/crash-report"
 version="3.0.4.1"
-taosdataIp="103.229.218.146"
+taosdataIp=$EXCLUDE_IP
 grep "\"version\":\"${version}\"" ${filesPath}/*.txt \
 | grep  "taosd(" \
 | awk -F "stackInfo" '{print $2}' \
diff --git a/tests/script/telemetry/filter2.sh b/tests/script/telemetry/filter2.sh
index 33f0fd08b5..4ad545345e 100755
--- a/tests/script/telemetry/filter2.sh
+++ b/tests/script/telemetry/filter2.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
-
+source .env
 filesPath="/data/telemetry/crash-report"
 version="3.0.4.1"
-taosdataIp="103.229.218.146"
+taosdataIp=$EXCLUDE_IP
 grep "\"version\":\"${version}\"" ${filesPath}/*.txt \
 | grep  "taosd(" \
 | awk -F "stackInfo" '{print $2}' \

From b6f5152e9db1059c894e6ea5587b6dc20c72477a Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Jan 2025 11:24:38 +0800
Subject: [PATCH 04/11] enh: change path

---
 .../telemetry/crash-report/CrashCounter.py    | 128 ++++++++
 .../crash-report/CrashCounter_new.py          | 308 ++++++++++++++++++
 tests/script/telemetry/crash-report/README.md | 210 ++++++++++++
 .../script/telemetry/crash-report/filter1.sh  |  15 +
 .../script/telemetry/crash-report/filter2.sh  |  14 +
 .../telemetry/crash-report/filter_assert.sh   |  67 ++++
 .../telemetry/crash-report/filter_nassert.sh  |  74 +++++
 7 files changed, 816 insertions(+)
 create mode 100644 tests/script/telemetry/crash-report/CrashCounter.py
 create mode 100644 tests/script/telemetry/crash-report/CrashCounter_new.py
 create mode 100644 tests/script/telemetry/crash-report/README.md
 create mode 100755 tests/script/telemetry/crash-report/filter1.sh
 create mode 100755 tests/script/telemetry/crash-report/filter2.sh
 create mode 100755 tests/script/telemetry/crash-report/filter_assert.sh
 create mode 100755 tests/script/telemetry/crash-report/filter_nassert.sh

diff --git a/tests/script/telemetry/crash-report/CrashCounter.py b/tests/script/telemetry/crash-report/CrashCounter.py
new file mode 100644
index 0000000000..66edc8d63e
--- /dev/null
+++ b/tests/script/telemetry/crash-report/CrashCounter.py
@@ -0,0 +1,128 @@
+from datetime import date
+from datetime import timedelta
+import os
+import re
+import requests
+from dotenv import load_dotenv
+
+# load .env
+load_dotenv()
+
+# define version
+version = "3.3.*"
+
+ip = os.getenv("EXCLUDE_IP")
+server_ip = os.getenv("SERVER_IP")
+owner = os.getenv("OWNER")
+
+# feishu-msg url
+feishu_msg_url = os.getenv("FEISHU_MSG_URL")
+
+today = date.today()
+#today = date(2023,8,7)
+path="/data/telemetry/crash-report/"
+
+# get files for the past 7 days
+def get_files():
+    files = ""
+    for i in range(1,8):
+        #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
+        files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
+
+    return files
+
+# for none-taosAssertDebug
+filter1_cmd = '''grep '"version":"%s"'  %s \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep -v "taosAssertDebug" \
+| grep -v %s \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c ''' % (version, get_files(), ip)
+
+# for taosAssertDebug
+filter2_cmd = '''grep '"version":"%s"'  %s \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep "taosAssertDebug" \
+| grep -v %s \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c ''' % (version, get_files(), ip)
+
+# get msg info
+def get_msg(text):
+    return {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": "Telemetry Statistics",
+                    "content": [
+                        [{
+                            "tag": "text",
+                            "text": text
+                        }
+                        ]]
+                }
+            }
+        }
+    }
+
+# post msg
+def send_msg(json):
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    req = requests.post(url=group_url, headers=headers, json=json)
+    inf = req.json()
+    if "StatusCode" in inf and inf["StatusCode"] == 0:
+        pass
+    else:
+        print(inf)
+
+# exec cmd and return res
+def get_output(cmd):
+    text = os.popen(cmd)
+    lines = text.read()
+    text.close()
+    return lines
+
+# get sum
+def get_count(output):
+    res = re.findall(" \d+ ", output)
+    sum1 = 0
+    for r in res:
+        sum1 = sum1 + int(r.strip())
+    return sum1
+
+# print total crash count
+def print_result():
+    #print(f"Files for statistics: {get_files()}\n")
+    sum1 = get_count(get_output(filter1_cmd))
+    sum2 = get_count(get_output(filter2_cmd))
+    total = sum1 + sum2
+    #print(f"total crashes: {total}")
+    return total
+
+# send report to feishu
+def send_report():
+    content = f'''
+    test scope: Telemetry Statistics
+    owner: {owner}
+    ip: {server_ip}
+    from: {get_files().split(" ")[6].split("/")[4].split(".")[0]}
+    to: {get_files().split(" ")[0].split("/")[4].split(".")[0]}
+    filter1 result: {get_output(filter1_cmd)}
+    filter2 result: {get_output(filter2_cmd)}
+    total crashes: {print_result()}
+    '''
+    #send_msg(get_msg(content))
+    print(content)
+
+print_result()
+send_report()
diff --git a/tests/script/telemetry/crash-report/CrashCounter_new.py b/tests/script/telemetry/crash-report/CrashCounter_new.py
new file mode 100644
index 0000000000..a89567da3d
--- /dev/null
+++ b/tests/script/telemetry/crash-report/CrashCounter_new.py
@@ -0,0 +1,308 @@
+from datetime import date
+from datetime import timedelta
+import os
+import json
+import re
+import requests
+import subprocess
+from dotenv import load_dotenv
+
+# load .env
+# You should have a .env file in the same directory as this script
+# You can exec: cp .env.example .env
+load_dotenv()
+
+# define version
+version = "3.3.2.*"
+version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+')
+version_pattern = re.compile(rf'^{version_pattern_str}$')
+version_stack_list = list()
+
+# define ip
+
+ip = os.getenv("EXCLUDE_IP")
+server_ip = os.getenv("SERVER_IP")
+http_serv_ip = os.getenv("HTTP_SERV_IP")
+http_serv_port = os.getenv("HTTP_SERV_PORT")
+owner = os.getenv("OWNER")
+
+# feishu-msg url
+feishu_msg_url = os.getenv("FEISHU_MSG_URL")
+
+# get today
+today = date.today()
+
+# Define the file and parameters
+path="/data/telemetry/crash-report/"
+trace_report_path = path + "trace_report"
+os.makedirs(path, exist_ok=True)
+os.makedirs(trace_report_path, exist_ok=True)
+
+assert_script_path = path + "filter_assert.sh"
+nassert_script_path = path + "filter_nassert.sh"
+
+# get files for the past 7 days
+def get_files():
+    files = ""
+    for i in range(1,8):
+        #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
+        files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
+    return files.strip().split(" ")
+
+# Define the AWK script as a string with proper escaping
+def get_res(file_path):
+    # Execute the script
+    command = ['bash', file_path, version, ip] + get_files()
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+
+    # Capture the output and errors
+    output, errors = process.communicate()
+
+    # Check for errors
+    if process.returncode != 0:
+        return errors
+    else:
+        return output.rstrip()
+
+def get_sum(output):
+    # Split the output into lines
+    lines = output.strip().split('\n')
+
+    # Initialize the sum
+    total_sum = 0
+
+    # Iterate over each line
+    for line in lines:
+        # Split each line by space to separate the columns
+        parts = line.split()
+
+        # The first part of the line is the number, convert it to integer
+        if parts:  # Check if there are any elements in the parts list
+            number = int(parts[0])
+            total_sum += number
+
+    return total_sum
+
+def convert_html(data):
+    # convert data to json
+    start_time = get_files()[6].split("/")[-1].split(".")[0]
+    end_time = get_files()[0].split("/")[-1].split(".")[0]
+    html_report_file = f'{start_time}_{end_time}.html'
+    json_data = json.dumps(data)
+
+    # Create HTML content
+    html_content = f'''
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Stack Trace Report</title>
+    <style>
+        body {{
+            font-family: Arial, sans-serif;
+            margin: 20px;
+            background-color: #f0f0f5;
+        }}
+        h1 {{
+            color: #2c3e50;
+            text-align: center;
+        }}
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            margin-bottom: 20px;
+            box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
+        }}
+        th, td {{
+            border: 1px solid #ddd;
+            padding: 10px;
+            text-align: left;
+        }}
+        th {{
+            background-color: #3498db;
+            color: white;
+        }}
+        tr:nth-child(even) {{
+            background-color: #ecf0f1;
+        }}
+        tr:hover {{
+            background-color: #d1e7fd;
+        }}
+        pre {{
+            background-color: #f7f7f7;
+            padding: 10px;
+            border: 1px solid #ddd;
+            overflow-x: auto;
+            white-space: pre-wrap;
+            border-radius: 5px;
+        }}
+    </style>
+</head>
+<body>
+    <h1>Stack Trace Report From {start_time} To {end_time} </h1>
+
+    <table>
+        <thead>
+            <tr>
+                <th>Key Stack Info</th>
+                <th>Versions</th>
+                <th>Num Of Crashes</th>
+                <th>Full Stack Info</th>
+            </tr>
+        </thead>
+        <tbody id="report">
+        </tbody>
+    </table>
+
+    <script>
+        const data = {json_data};
+
+        const reportBody = document.getElementById('report');
+        data.forEach(entry => {{
+            const row = document.createElement('tr');
+            row.innerHTML = `
+                <td>${{entry.key_stack_info}}</td>
+                <td>${{entry.version_list.join('<br>')}}</td>
+                <td>${{entry.count}}</td>
+                <td><pre>${{entry.full_stack_info}}</pre></td>
+            `;
+            reportBody.appendChild(row);
+        }});
+    </script>
+</body>
+</html>
+'''
+    # Write the HTML content to a file
+
+    with open(f'{trace_report_path}/{html_report_file}', 'w') as f:
+        f.write(html_content)
+    return html_report_file
+
+def get_version_stack_list(res):
+    for line in res.strip().split('\n'):
+        version_list = list()
+        version_stack_dict = dict()
+        count = line.split()[0]
+        key_stack_info = line.split()[1]
+        for file in get_files():
+            with open(file, 'r') as infile:
+                for line in infile:
+                    line = line.strip()
+                    data = json.loads(line)
+                    # print(line)
+                    if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line:
+                        if data["version"] not in version_list:
+                            version_list.append(data["version"])
+                            full_stack_info = data["stackInfo"]
+        version_stack_dict["key_stack_info"] = key_stack_info
+        version_stack_dict["full_stack_info"] = full_stack_info
+        version_stack_dict["version_list"] = version_list
+        version_stack_dict["count"] = count
+        # print(version_stack_dict)
+        version_stack_list.append(version_stack_dict)
+    return version_stack_list
+
+# get msg info
+def get_msg(text):
+    return {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": "Telemetry Statistics",
+                    "content": [
+                        [{
+                            "tag": "text",
+                            "text": text
+                        }
+                        ]]
+                }
+            }
+        }
+    }
+
+# post msg
+def send_msg(json):
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    req = requests.post(url=feishu_msg_url, headers=headers, json=json)
+    inf = req.json()
+    if "StatusCode" in inf and inf["StatusCode"] == 0:
+        pass
+    else:
+        print(inf)
+
+
+def format_results(results):
+    # Split the results into lines
+    lines = results.strip().split('\n')
+
+    # Parse lines into a list of tuples (number, rest_of_line)
+    parsed_lines = []
+    for line in lines:
+        parts = line.split(maxsplit=1)
+        if len(parts) == 2:
+            number = int(parts[0])  # Convert the number part to an integer
+            parsed_lines.append((number, parts[1]))
+
+    # Sort the parsed lines by the first element (number) in descending order
+    parsed_lines.sort(reverse=True, key=lambda x: x[0])
+
+    # Determine the maximum width of the first column for alignment
+    # max_width = max(len(str(item[0])) for item in parsed_lines)
+    if parsed_lines:
+        max_width = max(len(str(item[0])) for item in parsed_lines)
+    else:
+        max_width = 0
+
+    # Format each line to align the numbers and function names with indentation
+    formatted_lines = []
+    for number, text in parsed_lines:
+        formatted_line = f"       {str(number).rjust(max_width)} {text}"
+        formatted_lines.append(formatted_line)
+
+    # Join the formatted lines into a single string
+    return '\n'.join(formatted_lines)
+
+# # send report to feishu
+def send_report(res, sum, html_report_file):
+    content = f'''
+    version: v{version}
+    from: {get_files()[6].split("/")[-1].split(".")[0]}
+    to: {get_files()[0].split("/")[-1].split(".")[0]}
+    ip: {server_ip}
+    owner: {owner}
+    result: \n{format_results(res)}\n
+    total crashes: {sum}\n
+    details: http://{http_serv_ip}:{http_serv_port}/{html_report_file}
+    '''
+    print(get_msg(content))
+    send_msg(get_msg(content))
+    # print(content)
+
+# for none-taosAssertDebug
+nassert_res = get_res(nassert_script_path)
+# print(nassert_res)
+
+# for taosAssertDebug
+assert_res = get_res(assert_script_path)
+# print(assert_res)
+
+# combine the results
+res = nassert_res + assert_res
+
+# get version stack list
+version_stack_list = get_version_stack_list(res) if len(res) > 0 else list()
+
+# convert to html
+html_report_file = convert_html(version_stack_list)
+
+# get sum
+sum = get_sum(res)
+
+# send report
+send_report(res, sum, html_report_file)
+
diff --git a/tests/script/telemetry/crash-report/README.md b/tests/script/telemetry/crash-report/README.md
new file mode 100644
index 0000000000..ffee4998c7
--- /dev/null
+++ b/tests/script/telemetry/crash-report/README.md
@@ -0,0 +1,210 @@
+# Table of Contents
+
+1. [Introduction](#1-introduction)
+1. [Prerequisites](#2-prerequisites)
+1. [Running](#3-running)
+
+# 1. Introduction
+
+This manual is intended to give developers comprehensive guidance to collect crash information from the past 7 days and report it to the FeiShu notification group.
+
+> [!NOTE]
+> - The commands and scripts below are verified on Linux (CentOs 7.9.2009).
+> - The commands and steps described below are to run the tests on a single host.
+
+# 2. Prerequisites
+
+- Install Python3
+
+```bash
+yum install python3
+yum install python3-pip
+```
+
+- Install Python dependencies
+
+```bash
+pip3 install requests python-dotenv
+```
+
+# 3. Running
+
+In `crash-report` directory, there are different types of tests for TDengine. Below is a brief introduction about how to run them and how to add new cases.
+
+### 3.1 Unit Test
+
+Unit tests are the smallest testable units, which are used to test functions, methods or classes in TDengine code.
+
+### 3.1.1 How to run single test case?
+
+```bash
+cd debug/build/bin
+./osTimeTests
+```
+
+### 3.1.2 How to run all unit test cases?
+
+```bash
+cd tests/unit-test/
+bash test.sh -e 0
+```
+
+### 3.1.3 How to add new cases?
+
+<details>
+
+<summary>Detailed steps to add new unit test case</summary>
+
+The Google test framwork is used for unit testing to specific function module, please refer to steps below to add a new test case:
+
+##### a. Create test case file and develop the test scripts
+
+In the test directory corresponding to the target function module, create test files in CPP format and write corresponding test cases.
+
+##### b. Update build configuration
+
+Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the `source/os/test/CMakeLists.txt` file for configuration examples.
+
+##### c. Compile test code
+
+In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., `cmake .. -DBUILD_TEST=1`) to generate a compilation file,
+
+and then run a compilation command (e.g. make) to complete the compilation of the test code.
+
+##### d. Execute the test program
+
+Find the executable file in the compiled directory(e.g. `TDengine/debug/build/bin/`) and run it.
+
+##### e. Integrate into CI tests
+
+Use the add_test command to add new compiled test cases into CI test collection, ensure that the new added test cases can be run for every build.
+
+</details>
+
+## 3.2 System Test
+
+System tests are end-to-end test cases written in Python from a system point of view. Some of them are designed to test features only in enterprise ediiton, so when running on community edition, they may fail. We'll fix this issue by separating the cases into different gruops in the future.
+
+### 3.2.1 How to run a single test case?
+
+Take test file `system-test/2-query/avg.py` for example:
+
+```bash
+cd tests/system-test
+python3 ./test.py -f 2-query/avg.py
+```
+
+### 3.2.2 How to run all system test cases?
+
+```bash
+cd tests
+./run_all_ci_cases.sh -t python # all python cases
+```
+
+### 3.2.3 How to add new case?
+
+<details>
+
+<summary>Detailed steps to add new system test case</summary>
+
+The Python test framework is developed by TDengine team, and test.py is the test case execution and monitoring of the entry program, Use `python3 ./test.py -h` to view more features.
+
+Please refer to steps below for how to add a new test case:
+
+##### a. Create a test case file and develop the test cases
+
+Create a file in `tests/system-test` containing each functional directory and refer to the use case template `tests/system-test/0-others/test_case_template.py` to add a new test case.
+
+##### b. Execute the test case
+
+Ensure the test case execution is successful.
+
+``` bash
+cd tests/system-test && python3 ./test.py -f 0-others/test_case_template.py
+```
+
+##### c. Integrate into CI tests
+
+Edit `tests/parallel_test/cases.task` and add the testcase path and executions in the specified format. The third column indicates whether to use Address Sanitizer mode for testing.
+
+```bash
+#caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand
+,,n,system-test, python3 ./test.py  -f 0-others/test_case_template.py
+```
+
+</details>
+
+## 3.3 Legacy Test
+
+In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++.
+
+### 3.3.1 How to run single test case?
+
+To run the legacy test cases, please execute the following commands:
+
+```bash
+cd tests/script
+./test.sh -f tsim/db/basic1.sim
+```
+
+### 3.3.2 How to run all legacy test cases?
+
+```bash
+cd tests
+./run_all_ci_cases.sh -t legacy # all legacy cases
+```
+
+### 3.3.3 How to add new cases?
+
+> [!NOTE]
+> TSIM test framwork is deprecated by system test now, it is encouraged to add new test cases in system test, please refer to [System Test](#32-system-test) for details.
+
+## 3.4 Smoke Test
+
+Smoke test is a group of test cases selected from system test, which is also known as sanity test to ensure the critical functionalities of TDengine.
+
+### 3.4.1 How to run test?
+
+```bash
+cd /root/TDengine/packaging/smokeTest
+./test_smoking_selfhost.sh
+```
+
+### 3.4.2 How to add new cases?
+
+New cases can be added by updating the value of `commands` variable in `test_smoking_selfhost.sh`.
+
+## 3.5 Chaos Test
+
+A simple tool to execute various functions of the system in a randomized way, hoping to expose potential problems without a pre-defined test scenario.
+
+### 3.5.1 How to run test?
+
+```bash
+cd tests/pytest
+python3 auto_crash_gen.py
+```
+
+### 3.5.2 How to add new cases?
+
+1. Add a function, such as `TaskCreateNewFunction` in `pytest/crash_gen/crash_gen_main.py`.
+2. Integrate `TaskCreateNewFunction` into the `balance_pickTaskType` function in `crash_gen_main.py`.
+
+## 3.6 CI Test
+
+CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability.
+
+TDengine CI testing will run all the test cases from the following three types of tests: unit test, system test and legacy test.
+
+### 3.6.1 How to run all CI test cases?
+
+If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands:
+
+```bash
+cd tests
+./run_all_ci_cases.sh -b main # on main branch
+```
+
+### 3.6.2 How to add new cases?
+
+Please refer to the [Unit Test](#31-unit-test)、[System Test](#32-system-test) and [Legacy Test](#33-legacy-test) sections for detailed steps to add new test cases, when new cases are added in aboved tests, they will be run automatically by CI test.
diff --git a/tests/script/telemetry/crash-report/filter1.sh b/tests/script/telemetry/crash-report/filter1.sh
new file mode 100755
index 0000000000..3cb36a18ad
--- /dev/null
+++ b/tests/script/telemetry/crash-report/filter1.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+source .env
+filesPath="/data/telemetry/crash-report"
+version="3.0.4.1"
+taosdataIp=$EXCLUDE_IP
+grep "\"version\":\"${version}\"" ${filesPath}/*.txt \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep -v "taosAssertDebug" \
+| grep -v ${taosdataIp} \
+| awk -F "taosd" '{print $2}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c
+
diff --git a/tests/script/telemetry/crash-report/filter2.sh b/tests/script/telemetry/crash-report/filter2.sh
new file mode 100755
index 0000000000..4ad545345e
--- /dev/null
+++ b/tests/script/telemetry/crash-report/filter2.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+source .env
+filesPath="/data/telemetry/crash-report"
+version="3.0.4.1"
+taosdataIp=$EXCLUDE_IP
+grep "\"version\":\"${version}\"" ${filesPath}/*.txt \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep "taosAssertDebug" \
+| grep -v ${taosdataIp} \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c
diff --git a/tests/script/telemetry/crash-report/filter_assert.sh b/tests/script/telemetry/crash-report/filter_assert.sh
new file mode 100755
index 0000000000..2d56287fc9
--- /dev/null
+++ b/tests/script/telemetry/crash-report/filter_assert.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Extract version and IP from the first two arguments
+version="$1"
+ip="$2"
+shift 2  # Remove the first two arguments, leaving only file paths
+
+# All remaining arguments are considered as file paths
+file_paths="$@"
+
+# Execute the awk script and capture the output
+readarray -t output < <(awk -v version="$version" -v ip="$ip" '
+BEGIN {
+    RS = "\\n";  # Set the record separator to newline
+    FS = ",";    # Set the field separator to comma
+    total = 0;   # Initialize total count
+    version_regex = version;  # Use the passed version pattern
+    ip_regex = ip;  # Use the passed IP pattern
+}
+{
+    start_collecting = 0;
+    version_matched = 0;
+    ip_excluded = 0;
+
+    # Check each field within a record
+    for (i = 1; i <= NF; i++) {
+        if ($i ~ /"ip":"[^"]*"/ && $i ~ ip_regex) {
+            ip_excluded = 1;
+        }
+        if ($i ~ /"version":"[^"]*"/ && $i ~ version_regex) {
+            version_matched = 1;
+        }
+    }
+
+    if (!ip_excluded && version_matched) {
+        for (i = 1; i <= NF; i++) {
+            if ($i ~ /taosAssertDebug/ && start_collecting == 0) {
+                start_collecting = 1;
+                continue;
+            }
+            if (start_collecting == 1 && $i ~ /taosd\(([^)]+)\)/) {
+                match($i, /taosd\(([^)]+)\)/, arr);
+                if (arr[1] != "") {
+                    count[arr[1]]++;
+                    total++;
+                    break;
+                }
+            }
+        }
+    }
+}
+END {
+    for (c in count) {
+        printf "%d %s\n", count[c], c;
+    }
+    print "Total count:", total;
+}' $file_paths)
+
+# Capture the function details and total count into separate variables
+function_details=$(printf "%s\n" "${output[@]::${#output[@]}-1}")
+total_count="${output[-1]}"
+
+# Output or use the variables as needed
+#echo "Function Details:"
+echo "$function_details"
+#echo "Total Count:"
+#echo "$total_count"
diff --git a/tests/script/telemetry/crash-report/filter_nassert.sh b/tests/script/telemetry/crash-report/filter_nassert.sh
new file mode 100755
index 0000000000..2a5acdfbf1
--- /dev/null
+++ b/tests/script/telemetry/crash-report/filter_nassert.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Pass version, ip, and file paths as arguments
+version="$1"
+ip="$2"
+shift 2  # Shift the first two arguments to get file paths
+file_paths="$@"
+
+# Execute awk and capture the output
+readarray -t output < <(awk -v version="$version" -v ip="$ip" '
+BEGIN {
+    RS = "\\n";  # Set the record separator to newline
+    total = 0;   # Initialize total count
+    version_regex = "\"version\":\"" version;  # Construct the regex for version
+    ip_regex = "\"ip\":\"" ip "\"";  # Construct the regex for IP
+}
+{
+    found = 0;  # Initialize the found flag to false
+    start_collecting = 1;  # Start collecting by default, unless taosAssertDebug is encountered
+    split($0, parts, "\\n");  # Split each record by newline
+
+    # Check for version and IP in each part
+    version_matched = 0;
+    ip_excluded = 0;
+    for (i in parts) {
+        if (parts[i] ~ version_regex) {
+            version_matched = 1;  # Set flag if version is matched
+        }
+        if (parts[i] ~ ip_regex) {
+            ip_excluded = 1;  # Set flag if IP is matched
+            break;  # No need to continue if IP is excluded
+        }
+    }
+
+    # Process only if version is matched and IP is not excluded
+    if (version_matched && !ip_excluded) {
+        for (i in parts) {
+            if (parts[i] ~ /taosAssertDebug/) {
+                start_collecting = 0;  # Skip this record if taosAssertDebug is encountered
+                break;  # Exit the loop
+            }
+        }
+        if (start_collecting == 1) {  # Continue processing if taosAssertDebug is not found
+            for (i in parts) {
+                if (found == 0 && parts[i] ~ /frame:.*taosd\([^)]+\)/) {
+                    # Match the first frame that meets the condition
+                    match(parts[i], /taosd\(([^)]+)\)/, a);  # Extract the function name
+                    if (a[1] != "") {
+                        count[a[1]]++;  # Increment the count for this function name
+                        total++;  # Increment the total count
+                        found = 1;  # Set found flag to true
+                        break;  # Exit the loop once the function is found
+                    }
+                }
+            }
+        }
+    }
+}
+END {
+    for (c in count) {
+        printf "%d %s\n", count[c], c;  # Print the count and function name formatted
+    }
+    print total;  # Print the total count alone
+}' $file_paths)  # Note the removal of quotes around "$file_paths" to handle multiple paths
+
+# Capture the function details and total count into separate variables
+function_details=$(printf "%s\n" "${output[@]::${#output[@]}-1}")  # Join array elements with newlines
+total_count="${output[-1]}"  # The last element
+
+# Output or use the variables as needed
+#echo "Function Details:"
+echo "$function_details"
+#echo "Total Count:"
+#echo "$total_count"

From 7c083234f76671cde36d3f6a36af789e5cf7e00b Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Jan 2025 11:25:40 +0800
Subject: [PATCH 05/11] enh: change path

---
 tests/script/telemetry/CrashCounter.py     | 128 ---------
 tests/script/telemetry/CrashCounter_new.py | 308 ---------------------
 tests/script/telemetry/filter1.sh          |  15 -
 tests/script/telemetry/filter2.sh          |  14 -
 tests/script/telemetry/filter_assert.sh    |  67 -----
 tests/script/telemetry/filter_nassert.sh   |  74 -----
 6 files changed, 606 deletions(-)
 delete mode 100644 tests/script/telemetry/CrashCounter.py
 delete mode 100644 tests/script/telemetry/CrashCounter_new.py
 delete mode 100755 tests/script/telemetry/filter1.sh
 delete mode 100755 tests/script/telemetry/filter2.sh
 delete mode 100755 tests/script/telemetry/filter_assert.sh
 delete mode 100755 tests/script/telemetry/filter_nassert.sh

diff --git a/tests/script/telemetry/CrashCounter.py b/tests/script/telemetry/CrashCounter.py
deleted file mode 100644
index 66edc8d63e..0000000000
--- a/tests/script/telemetry/CrashCounter.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from datetime import date
-from datetime import timedelta
-import os
-import re
-import requests
-from dotenv import load_dotenv
-
-# load .env
-load_dotenv()
-
-# define version
-version = "3.3.*"
-
-ip = os.getenv("EXCLUDE_IP")
-server_ip = os.getenv("SERVER_IP")
-owner = os.getenv("OWNER")
-
-# feishu-msg url
-feishu_msg_url = os.getenv("FEISHU_MSG_URL")
-
-today = date.today()
-#today = date(2023,8,7)
-path="/data/telemetry/crash-report/"
-
-# get files for the past 7 days
-def get_files():
-    files = ""
-    for i in range(1,8):
-        #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
-        files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
-
-    return files
-
-# for none-taosAssertDebug
-filter1_cmd = '''grep '"version":"%s"'  %s \
-| grep  "taosd(" \
-| awk -F "stackInfo" '{print $2}' \
-| grep -v "taosAssertDebug" \
-| grep -v %s \
-| awk -F "taosd" '{print $3}' \
-| cut -d")" -f 1 \
-| cut -d"(" -f 2 \
-| sort | uniq -c ''' % (version, get_files(), ip)
-
-# for taosAssertDebug
-filter2_cmd = '''grep '"version":"%s"'  %s \
-| grep  "taosd(" \
-| awk -F "stackInfo" '{print $2}' \
-| grep "taosAssertDebug" \
-| grep -v %s \
-| awk -F "taosd" '{print $3}' \
-| cut -d")" -f 1 \
-| cut -d"(" -f 2 \
-| sort | uniq -c ''' % (version, get_files(), ip)
-
-# get msg info
-def get_msg(text):
-    return {
-        "msg_type": "post",
-        "content": {
-            "post": {
-                "zh_cn": {
-                    "title": "Telemetry Statistics",
-                    "content": [
-                        [{
-                            "tag": "text",
-                            "text": text
-                        }
-                        ]]
-                }
-            }
-        }
-    }
-
-# post msg
-def send_msg(json):
-    headers = {
-        'Content-Type': 'application/json'
-    }
-
-    req = requests.post(url=group_url, headers=headers, json=json)
-    inf = req.json()
-    if "StatusCode" in inf and inf["StatusCode"] == 0:
-        pass
-    else:
-        print(inf)
-
-# exec cmd and return res
-def get_output(cmd):
-    text = os.popen(cmd)
-    lines = text.read()
-    text.close()
-    return lines
-
-# get sum
-def get_count(output):
-    res = re.findall(" \d+ ", output)
-    sum1 = 0
-    for r in res:
-        sum1 = sum1 + int(r.strip())
-    return sum1
-
-# print total crash count
-def print_result():
-    #print(f"Files for statistics: {get_files()}\n")
-    sum1 = get_count(get_output(filter1_cmd))
-    sum2 = get_count(get_output(filter2_cmd))
-    total = sum1 + sum2
-    #print(f"total crashes: {total}")
-    return total
-
-# send report to feishu
-def send_report():
-    content = f'''
-    test scope: Telemetry Statistics
-    owner: {owner}
-    ip: {server_ip}
-    from: {get_files().split(" ")[6].split("/")[4].split(".")[0]}
-    to: {get_files().split(" ")[0].split("/")[4].split(".")[0]}
-    filter1 result: {get_output(filter1_cmd)}
-    filter2 result: {get_output(filter2_cmd)}
-    total crashes: {print_result()}
-    '''
-    #send_msg(get_msg(content))
-    print(content)
-
-print_result()
-send_report()
diff --git a/tests/script/telemetry/CrashCounter_new.py b/tests/script/telemetry/CrashCounter_new.py
deleted file mode 100644
index a89567da3d..0000000000
--- a/tests/script/telemetry/CrashCounter_new.py
+++ /dev/null
@@ -1,308 +0,0 @@
-from datetime import date
-from datetime import timedelta
-import os
-import json
-import re
-import requests
-import subprocess
-from dotenv import load_dotenv
-
-# load .env
-# You should have a .env file in the same directory as this script
-# You can exec: cp .env.example .env
-load_dotenv()
-
-# define version
-version = "3.3.2.*"
-version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+')
-version_pattern = re.compile(rf'^{version_pattern_str}$')
-version_stack_list = list()
-
-# define ip
-
-ip = os.getenv("EXCLUDE_IP")
-server_ip = os.getenv("SERVER_IP")
-http_serv_ip = os.getenv("HTTP_SERV_IP")
-http_serv_port = os.getenv("HTTP_SERV_PORT")
-owner = os.getenv("OWNER")
-
-# feishu-msg url
-feishu_msg_url = os.getenv("FEISHU_MSG_URL")
-
-# get today
-today = date.today()
-
-# Define the file and parameters
-path="/data/telemetry/crash-report/"
-trace_report_path = path + "trace_report"
-os.makedirs(path, exist_ok=True)
-os.makedirs(trace_report_path, exist_ok=True)
-
-assert_script_path = path + "filter_assert.sh"
-nassert_script_path = path + "filter_nassert.sh"
-
-# get files for the past 7 days
-def get_files():
-    files = ""
-    for i in range(1,8):
-        #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
-        files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
-    return files.strip().split(" ")
-
-# Define the AWK script as a string with proper escaping
-def get_res(file_path):
-    # Execute the script
-    command = ['bash', file_path, version, ip] + get_files()
-    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-
-    # Capture the output and errors
-    output, errors = process.communicate()
-
-    # Check for errors
-    if process.returncode != 0:
-        return errors
-    else:
-        return output.rstrip()
-
-def get_sum(output):
-    # Split the output into lines
-    lines = output.strip().split('\n')
-
-    # Initialize the sum
-    total_sum = 0
-
-    # Iterate over each line
-    for line in lines:
-        # Split each line by space to separate the columns
-        parts = line.split()
-
-        # The first part of the line is the number, convert it to integer
-        if parts:  # Check if there are any elements in the parts list
-            number = int(parts[0])
-            total_sum += number
-
-    return total_sum
-
-def convert_html(data):
-    # convert data to json
-    start_time = get_files()[6].split("/")[-1].split(".")[0]
-    end_time = get_files()[0].split("/")[-1].split(".")[0]
-    html_report_file = f'{start_time}_{end_time}.html'
-    json_data = json.dumps(data)
-
-    # Create HTML content
-    html_content = f'''
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Stack Trace Report</title>
-    <style>
-        body {{
-            font-family: Arial, sans-serif;
-            margin: 20px;
-            background-color: #f0f0f5;
-        }}
-        h1 {{
-            color: #2c3e50;
-            text-align: center;
-        }}
-        table {{
-            width: 100%;
-            border-collapse: collapse;
-            margin-bottom: 20px;
-            box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
-        }}
-        th, td {{
-            border: 1px solid #ddd;
-            padding: 10px;
-            text-align: left;
-        }}
-        th {{
-            background-color: #3498db;
-            color: white;
-        }}
-        tr:nth-child(even) {{
-            background-color: #ecf0f1;
-        }}
-        tr:hover {{
-            background-color: #d1e7fd;
-        }}
-        pre {{
-            background-color: #f7f7f7;
-            padding: 10px;
-            border: 1px solid #ddd;
-            overflow-x: auto;
-            white-space: pre-wrap;
-            border-radius: 5px;
-        }}
-    </style>
-</head>
-<body>
-    <h1>Stack Trace Report From {start_time} To {end_time} </h1>
-
-    <table>
-        <thead>
-            <tr>
-                <th>Key Stack Info</th>
-                <th>Versions</th>
-                <th>Num Of Crashes</th>
-                <th>Full Stack Info</th>
-            </tr>
-        </thead>
-        <tbody id="report">
-        </tbody>
-    </table>
-
-    <script>
-        const data = {json_data};
-
-        const reportBody = document.getElementById('report');
-        data.forEach(entry => {{
-            const row = document.createElement('tr');
-            row.innerHTML = `
-                <td>${{entry.key_stack_info}}</td>
-                <td>${{entry.version_list.join('<br>')}}</td>
-                <td>${{entry.count}}</td>
-                <td><pre>${{entry.full_stack_info}}</pre></td>
-            `;
-            reportBody.appendChild(row);
-        }});
-    </script>
-</body>
-</html>
-'''
-    # Write the HTML content to a file
-
-    with open(f'{trace_report_path}/{html_report_file}', 'w') as f:
-        f.write(html_content)
-    return html_report_file
-
-def get_version_stack_list(res):
-    for line in res.strip().split('\n'):
-        version_list = list()
-        version_stack_dict = dict()
-        count = line.split()[0]
-        key_stack_info = line.split()[1]
-        for file in get_files():
-            with open(file, 'r') as infile:
-                for line in infile:
-                    line = line.strip()
-                    data = json.loads(line)
-                    # print(line)
-                    if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line:
-                        if data["version"] not in version_list:
-                            version_list.append(data["version"])
-                            full_stack_info = data["stackInfo"]
-        version_stack_dict["key_stack_info"] = key_stack_info
-        version_stack_dict["full_stack_info"] = full_stack_info
-        version_stack_dict["version_list"] = version_list
-        version_stack_dict["count"] = count
-        # print(version_stack_dict)
-        version_stack_list.append(version_stack_dict)
-    return version_stack_list
-
-# get msg info
-def get_msg(text):
-    return {
-        "msg_type": "post",
-        "content": {
-            "post": {
-                "zh_cn": {
-                    "title": "Telemetry Statistics",
-                    "content": [
-                        [{
-                            "tag": "text",
-                            "text": text
-                        }
-                        ]]
-                }
-            }
-        }
-    }
-
-# post msg
-def send_msg(json):
-    headers = {
-        'Content-Type': 'application/json'
-    }
-
-    req = requests.post(url=feishu_msg_url, headers=headers, json=json)
-    inf = req.json()
-    if "StatusCode" in inf and inf["StatusCode"] == 0:
-        pass
-    else:
-        print(inf)
-
-
-def format_results(results):
-    # Split the results into lines
-    lines = results.strip().split('\n')
-
-    # Parse lines into a list of tuples (number, rest_of_line)
-    parsed_lines = []
-    for line in lines:
-        parts = line.split(maxsplit=1)
-        if len(parts) == 2:
-            number = int(parts[0])  # Convert the number part to an integer
-            parsed_lines.append((number, parts[1]))
-
-    # Sort the parsed lines by the first element (number) in descending order
-    parsed_lines.sort(reverse=True, key=lambda x: x[0])
-
-    # Determine the maximum width of the first column for alignment
-    # max_width = max(len(str(item[0])) for item in parsed_lines)
-    if parsed_lines:
-        max_width = max(len(str(item[0])) for item in parsed_lines)
-    else:
-        max_width = 0
-
-    # Format each line to align the numbers and function names with indentation
-    formatted_lines = []
-    for number, text in parsed_lines:
-        formatted_line = f"       {str(number).rjust(max_width)} {text}"
-        formatted_lines.append(formatted_line)
-
-    # Join the formatted lines into a single string
-    return '\n'.join(formatted_lines)
-
-# # send report to feishu
-def send_report(res, sum, html_report_file):
-    content = f'''
-    version: v{version}
-    from: {get_files()[6].split("/")[-1].split(".")[0]}
-    to: {get_files()[0].split("/")[-1].split(".")[0]}
-    ip: {server_ip}
-    owner: {owner}
-    result: \n{format_results(res)}\n
-    total crashes: {sum}\n
-    details: http://{http_serv_ip}:{http_serv_port}/{html_report_file}
-    '''
-    print(get_msg(content))
-    send_msg(get_msg(content))
-    # print(content)
-
-# for none-taosAssertDebug
-nassert_res = get_res(nassert_script_path)
-# print(nassert_res)
-
-# for taosAssertDebug
-assert_res = get_res(assert_script_path)
-# print(assert_res)
-
-# combine the results
-res = nassert_res + assert_res
-
-# get version stack list
-version_stack_list = get_version_stack_list(res) if len(res) > 0 else list()
-
-# convert to html
-html_report_file = convert_html(version_stack_list)
-
-# get sum
-sum = get_sum(res)
-
-# send report
-send_report(res, sum, html_report_file)
-
diff --git a/tests/script/telemetry/filter1.sh b/tests/script/telemetry/filter1.sh
deleted file mode 100755
index 3cb36a18ad..0000000000
--- a/tests/script/telemetry/filter1.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-source .env
-filesPath="/data/telemetry/crash-report"
-version="3.0.4.1"
-taosdataIp=$EXCLUDE_IP
-grep "\"version\":\"${version}\"" ${filesPath}/*.txt \
-| grep  "taosd(" \
-| awk -F "stackInfo" '{print $2}' \
-| grep -v "taosAssertDebug" \
-| grep -v ${taosdataIp} \
-| awk -F "taosd" '{print $2}' \
-| cut -d")" -f 1 \
-| cut -d"(" -f 2 \
-| sort | uniq -c
-
diff --git a/tests/script/telemetry/filter2.sh b/tests/script/telemetry/filter2.sh
deleted file mode 100755
index 4ad545345e..0000000000
--- a/tests/script/telemetry/filter2.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-source .env
-filesPath="/data/telemetry/crash-report"
-version="3.0.4.1"
-taosdataIp=$EXCLUDE_IP
-grep "\"version\":\"${version}\"" ${filesPath}/*.txt \
-| grep  "taosd(" \
-| awk -F "stackInfo" '{print $2}' \
-| grep "taosAssertDebug" \
-| grep -v ${taosdataIp} \
-| awk -F "taosd" '{print $3}' \
-| cut -d")" -f 1 \
-| cut -d"(" -f 2 \
-| sort | uniq -c
diff --git a/tests/script/telemetry/filter_assert.sh b/tests/script/telemetry/filter_assert.sh
deleted file mode 100755
index 2d56287fc9..0000000000
--- a/tests/script/telemetry/filter_assert.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash
-
-# Extract version and IP from the first two arguments
-version="$1"
-ip="$2"
-shift 2  # Remove the first two arguments, leaving only file paths
-
-# All remaining arguments are considered as file paths
-file_paths="$@"
-
-# Execute the awk script and capture the output
-readarray -t output < <(awk -v version="$version" -v ip="$ip" '
-BEGIN {
-    RS = "\\n";  # Set the record separator to newline
-    FS = ",";    # Set the field separator to comma
-    total = 0;   # Initialize total count
-    version_regex = version;  # Use the passed version pattern
-    ip_regex = ip;  # Use the passed IP pattern
-}
-{
-    start_collecting = 0;
-    version_matched = 0;
-    ip_excluded = 0;
-
-    # Check each field within a record
-    for (i = 1; i <= NF; i++) {
-        if ($i ~ /"ip":"[^"]*"/ && $i ~ ip_regex) {
-            ip_excluded = 1;
-        }
-        if ($i ~ /"version":"[^"]*"/ && $i ~ version_regex) {
-            version_matched = 1;
-        }
-    }
-
-    if (!ip_excluded && version_matched) {
-        for (i = 1; i <= NF; i++) {
-            if ($i ~ /taosAssertDebug/ && start_collecting == 0) {
-                start_collecting = 1;
-                continue;
-            }
-            if (start_collecting == 1 && $i ~ /taosd\(([^)]+)\)/) {
-                match($i, /taosd\(([^)]+)\)/, arr);
-                if (arr[1] != "") {
-                    count[arr[1]]++;
-                    total++;
-                    break;
-                }
-            }
-        }
-    }
-}
-END {
-    for (c in count) {
-        printf "%d %s\n", count[c], c;
-    }
-    print "Total count:", total;
-}' $file_paths)
-
-# Capture the function details and total count into separate variables
-function_details=$(printf "%s\n" "${output[@]::${#output[@]}-1}")
-total_count="${output[-1]}"
-
-# Output or use the variables as needed
-#echo "Function Details:"
-echo "$function_details"
-#echo "Total Count:"
-#echo "$total_count"
diff --git a/tests/script/telemetry/filter_nassert.sh b/tests/script/telemetry/filter_nassert.sh
deleted file mode 100755
index 2a5acdfbf1..0000000000
--- a/tests/script/telemetry/filter_nassert.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/bin/bash
-
-# Pass version, ip, and file paths as arguments
-version="$1"
-ip="$2"
-shift 2  # Shift the first two arguments to get file paths
-file_paths="$@"
-
-# Execute awk and capture the output
-readarray -t output < <(awk -v version="$version" -v ip="$ip" '
-BEGIN {
-    RS = "\\n";  # Set the record separator to newline
-    total = 0;   # Initialize total count
-    version_regex = "\"version\":\"" version;  # Construct the regex for version
-    ip_regex = "\"ip\":\"" ip "\"";  # Construct the regex for IP
-}
-{
-    found = 0;  # Initialize the found flag to false
-    start_collecting = 1;  # Start collecting by default, unless taosAssertDebug is encountered
-    split($0, parts, "\\n");  # Split each record by newline
-
-    # Check for version and IP in each part
-    version_matched = 0;
-    ip_excluded = 0;
-    for (i in parts) {
-        if (parts[i] ~ version_regex) {
-            version_matched = 1;  # Set flag if version is matched
-        }
-        if (parts[i] ~ ip_regex) {
-            ip_excluded = 1;  # Set flag if IP is matched
-            break;  # No need to continue if IP is excluded
-        }
-    }
-
-    # Process only if version is matched and IP is not excluded
-    if (version_matched && !ip_excluded) {
-        for (i in parts) {
-            if (parts[i] ~ /taosAssertDebug/) {
-                start_collecting = 0;  # Skip this record if taosAssertDebug is encountered
-                break;  # Exit the loop
-            }
-        }
-        if (start_collecting == 1) {  # Continue processing if taosAssertDebug is not found
-            for (i in parts) {
-                if (found == 0 && parts[i] ~ /frame:.*taosd\([^)]+\)/) {
-                    # Match the first frame that meets the condition
-                    match(parts[i], /taosd\(([^)]+)\)/, a);  # Extract the function name
-                    if (a[1] != "") {
-                        count[a[1]]++;  # Increment the count for this function name
-                        total++;  # Increment the total count
-                        found = 1;  # Set found flag to true
-                        break;  # Exit the loop once the function is found
-                    }
-                }
-            }
-        }
-    }
-}
-END {
-    for (c in count) {
-        printf "%d %s\n", count[c], c;  # Print the count and function name formatted
-    }
-    print total;  # Print the total count alone
-}' $file_paths)  # Note the removal of quotes around "$file_paths" to handle multiple paths
-
-# Capture the function details and total count into separate variables
-function_details=$(printf "%s\n" "${output[@]::${#output[@]}-1}")  # Join array elements with newlines
-total_count="${output[-1]}"  # The last element
-
-# Output or use the variables as needed
-#echo "Function Details:"
-echo "$function_details"
-#echo "Total Count:"
-#echo "$total_count"

From 4504f5a6fdc7277fef277e32987fe7f41d4ce9e9 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Jan 2025 13:45:56 +0800
Subject: [PATCH 06/11] enh: rename

---
 .../telemetry/{ => crash-report}/.env.example |   0
 .../telemetry/crash-report/CrashCounter.py    | 302 +++++++++++++----
 .../crash-report/CrashCounter.py.old          | 128 ++++++++
 .../crash-report/CrashCounter_new.py          | 308 ------------------
 4 files changed, 369 insertions(+), 369 deletions(-)
 rename tests/script/telemetry/{ => crash-report}/.env.example (100%)
 create mode 100644 tests/script/telemetry/crash-report/CrashCounter.py.old
 delete mode 100644 tests/script/telemetry/crash-report/CrashCounter_new.py

diff --git a/tests/script/telemetry/.env.example b/tests/script/telemetry/crash-report/.env.example
similarity index 100%
rename from tests/script/telemetry/.env.example
rename to tests/script/telemetry/crash-report/.env.example
diff --git a/tests/script/telemetry/crash-report/CrashCounter.py b/tests/script/telemetry/crash-report/CrashCounter.py
index 66edc8d63e..a89567da3d 100644
--- a/tests/script/telemetry/crash-report/CrashCounter.py
+++ b/tests/script/telemetry/crash-report/CrashCounter.py
@@ -1,26 +1,45 @@
 from datetime import date
 from datetime import timedelta
 import os
+import json
 import re
 import requests
+import subprocess
 from dotenv import load_dotenv
 
 # load .env
+# You should have a .env file in the same directory as this script
+# You can exec: cp .env.example .env
 load_dotenv()
 
 # define version
-version = "3.3.*"
+version = "3.3.2.*"
+version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+')
+version_pattern = re.compile(rf'^{version_pattern_str}$')
+version_stack_list = list()
+
+# define ip
 
 ip = os.getenv("EXCLUDE_IP")
 server_ip = os.getenv("SERVER_IP")
+http_serv_ip = os.getenv("HTTP_SERV_IP")
+http_serv_port = os.getenv("HTTP_SERV_PORT")
 owner = os.getenv("OWNER")
 
 # feishu-msg url
 feishu_msg_url = os.getenv("FEISHU_MSG_URL")
 
+# get today
 today = date.today()
-#today = date(2023,8,7)
+
+# Define the file and parameters
 path="/data/telemetry/crash-report/"
+trace_report_path = path + "trace_report"
+os.makedirs(path, exist_ok=True)
+os.makedirs(trace_report_path, exist_ok=True)
+
+assert_script_path = path + "filter_assert.sh"
+nassert_script_path = path + "filter_nassert.sh"
 
 # get files for the past 7 days
 def get_files():
@@ -28,30 +47,161 @@ def get_files():
     for i in range(1,8):
         #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
         files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
+    return files.strip().split(" ")
 
-    return files
+# Define the AWK script as a string with proper escaping
+def get_res(file_path):
+    # Execute the script
+    command = ['bash', file_path, version, ip] + get_files()
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
 
-# for none-taosAssertDebug
-filter1_cmd = '''grep '"version":"%s"'  %s \
-| grep  "taosd(" \
-| awk -F "stackInfo" '{print $2}' \
-| grep -v "taosAssertDebug" \
-| grep -v %s \
-| awk -F "taosd" '{print $3}' \
-| cut -d")" -f 1 \
-| cut -d"(" -f 2 \
-| sort | uniq -c ''' % (version, get_files(), ip)
+    # Capture the output and errors
+    output, errors = process.communicate()
 
-# for taosAssertDebug
-filter2_cmd = '''grep '"version":"%s"'  %s \
-| grep  "taosd(" \
-| awk -F "stackInfo" '{print $2}' \
-| grep "taosAssertDebug" \
-| grep -v %s \
-| awk -F "taosd" '{print $3}' \
-| cut -d")" -f 1 \
-| cut -d"(" -f 2 \
-| sort | uniq -c ''' % (version, get_files(), ip)
+    # Check for errors
+    if process.returncode != 0:
+        return errors
+    else:
+        return output.rstrip()
+
+def get_sum(output):
+    # Split the output into lines
+    lines = output.strip().split('\n')
+
+    # Initialize the sum
+    total_sum = 0
+
+    # Iterate over each line
+    for line in lines:
+        # Split each line by space to separate the columns
+        parts = line.split()
+
+        # The first part of the line is the number, convert it to integer
+        if parts:  # Check if there are any elements in the parts list
+            number = int(parts[0])
+            total_sum += number
+
+    return total_sum
+
+def convert_html(data):
+    # convert data to json
+    start_time = get_files()[6].split("/")[-1].split(".")[0]
+    end_time = get_files()[0].split("/")[-1].split(".")[0]
+    html_report_file = f'{start_time}_{end_time}.html'
+    json_data = json.dumps(data)
+
+    # Create HTML content
+    html_content = f'''
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Stack Trace Report</title>
+    <style>
+        body {{
+            font-family: Arial, sans-serif;
+            margin: 20px;
+            background-color: #f0f0f5;
+        }}
+        h1 {{
+            color: #2c3e50;
+            text-align: center;
+        }}
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            margin-bottom: 20px;
+            box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
+        }}
+        th, td {{
+            border: 1px solid #ddd;
+            padding: 10px;
+            text-align: left;
+        }}
+        th {{
+            background-color: #3498db;
+            color: white;
+        }}
+        tr:nth-child(even) {{
+            background-color: #ecf0f1;
+        }}
+        tr:hover {{
+            background-color: #d1e7fd;
+        }}
+        pre {{
+            background-color: #f7f7f7;
+            padding: 10px;
+            border: 1px solid #ddd;
+            overflow-x: auto;
+            white-space: pre-wrap;
+            border-radius: 5px;
+        }}
+    </style>
+</head>
+<body>
+    <h1>Stack Trace Report From {start_time} To {end_time} </h1>
+
+    <table>
+        <thead>
+            <tr>
+                <th>Key Stack Info</th>
+                <th>Versions</th>
+                <th>Num Of Crashes</th>
+                <th>Full Stack Info</th>
+            </tr>
+        </thead>
+        <tbody id="report">
+        </tbody>
+    </table>
+
+    <script>
+        const data = {json_data};
+
+        const reportBody = document.getElementById('report');
+        data.forEach(entry => {{
+            const row = document.createElement('tr');
+            row.innerHTML = `
+                <td>${{entry.key_stack_info}}</td>
+                <td>${{entry.version_list.join('<br>')}}</td>
+                <td>${{entry.count}}</td>
+                <td><pre>${{entry.full_stack_info}}</pre></td>
+            `;
+            reportBody.appendChild(row);
+        }});
+    </script>
+</body>
+</html>
+'''
+    # Write the HTML content to a file
+
+    with open(f'{trace_report_path}/{html_report_file}', 'w') as f:
+        f.write(html_content)
+    return html_report_file
+
+def get_version_stack_list(res):
+    for line in res.strip().split('\n'):
+        version_list = list()
+        version_stack_dict = dict()
+        count = line.split()[0]
+        key_stack_info = line.split()[1]
+        for file in get_files():
+            with open(file, 'r') as infile:
+                for line in infile:
+                    line = line.strip()
+                    data = json.loads(line)
+                    # print(line)
+                    if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line:
+                        if data["version"] not in version_list:
+                            version_list.append(data["version"])
+                            full_stack_info = data["stackInfo"]
+        version_stack_dict["key_stack_info"] = key_stack_info
+        version_stack_dict["full_stack_info"] = full_stack_info
+        version_stack_dict["version_list"] = version_list
+        version_stack_dict["count"] = count
+        # print(version_stack_dict)
+        version_stack_list.append(version_stack_dict)
+    return version_stack_list
 
 # get msg info
 def get_msg(text):
@@ -78,51 +228,81 @@ def send_msg(json):
         'Content-Type': 'application/json'
     }
 
-    req = requests.post(url=group_url, headers=headers, json=json)
+    req = requests.post(url=feishu_msg_url, headers=headers, json=json)
     inf = req.json()
     if "StatusCode" in inf and inf["StatusCode"] == 0:
         pass
     else:
         print(inf)
 
-# exec cmd and return res
-def get_output(cmd):
-    text = os.popen(cmd)
-    lines = text.read()
-    text.close()
-    return lines
+
+def format_results(results):
+    # Split the results into lines
+    lines = results.strip().split('\n')
+
+    # Parse lines into a list of tuples (number, rest_of_line)
+    parsed_lines = []
+    for line in lines:
+        parts = line.split(maxsplit=1)
+        if len(parts) == 2:
+            number = int(parts[0])  # Convert the number part to an integer
+            parsed_lines.append((number, parts[1]))
+
+    # Sort the parsed lines by the first element (number) in descending order
+    parsed_lines.sort(reverse=True, key=lambda x: x[0])
+
+    # Determine the maximum width of the first column for alignment
+    # max_width = max(len(str(item[0])) for item in parsed_lines)
+    if parsed_lines:
+        max_width = max(len(str(item[0])) for item in parsed_lines)
+    else:
+        max_width = 0
+
+    # Format each line to align the numbers and function names with indentation
+    formatted_lines = []
+    for number, text in parsed_lines:
+        formatted_line = f"       {str(number).rjust(max_width)} {text}"
+        formatted_lines.append(formatted_line)
+
+    # Join the formatted lines into a single string
+    return '\n'.join(formatted_lines)
+
+# # send report to feishu
+def send_report(res, sum, html_report_file):
+    content = f'''
+    version: v{version}
+    from: {get_files()[6].split("/")[-1].split(".")[0]}
+    to: {get_files()[0].split("/")[-1].split(".")[0]}
+    ip: {server_ip}
+    owner: {owner}
+    result: \n{format_results(res)}\n
+    total crashes: {sum}\n
+    details: http://{http_serv_ip}:{http_serv_port}/{html_report_file}
+    '''
+    print(get_msg(content))
+    send_msg(get_msg(content))
+    # print(content)
+
+# for none-taosAssertDebug
+nassert_res = get_res(nassert_script_path)
+# print(nassert_res)
+
+# for taosAssertDebug
+assert_res = get_res(assert_script_path)
+# print(assert_res)
+
+# combine the results
+res = nassert_res + assert_res
+
+# get version stack list
+version_stack_list = get_version_stack_list(res) if len(res) > 0 else list()
+
+# convert to html
+html_report_file = convert_html(version_stack_list)
 
 # get sum
-def get_count(output):
-    res = re.findall(" \d+ ", output)
-    sum1 = 0
-    for r in res:
-        sum1 = sum1 + int(r.strip())
-    return sum1
+sum = get_sum(res)
 
-# print total crash count
-def print_result():
-    #print(f"Files for statistics: {get_files()}\n")
-    sum1 = get_count(get_output(filter1_cmd))
-    sum2 = get_count(get_output(filter2_cmd))
-    total = sum1 + sum2
-    #print(f"total crashes: {total}")
-    return total
+# send report
+send_report(res, sum, html_report_file)
 
-# send report to feishu
-def send_report():
-    content = f'''
-    test scope: Telemetry Statistics
-    owner: {owner}
-    ip: {server_ip}
-    from: {get_files().split(" ")[6].split("/")[4].split(".")[0]}
-    to: {get_files().split(" ")[0].split("/")[4].split(".")[0]}
-    filter1 result: {get_output(filter1_cmd)}
-    filter2 result: {get_output(filter2_cmd)}
-    total crashes: {print_result()}
-    '''
-    #send_msg(get_msg(content))
-    print(content)
-
-print_result()
-send_report()
diff --git a/tests/script/telemetry/crash-report/CrashCounter.py.old b/tests/script/telemetry/crash-report/CrashCounter.py.old
new file mode 100644
index 0000000000..66edc8d63e
--- /dev/null
+++ b/tests/script/telemetry/crash-report/CrashCounter.py.old
@@ -0,0 +1,128 @@
+from datetime import date
+from datetime import timedelta
+import os
+import re
+import requests
+from dotenv import load_dotenv
+
+# load .env
+load_dotenv()
+
+# define version
+version = "3.3.*"
+
+ip = os.getenv("EXCLUDE_IP")
+server_ip = os.getenv("SERVER_IP")
+owner = os.getenv("OWNER")
+
+# feishu-msg url
+feishu_msg_url = os.getenv("FEISHU_MSG_URL")
+
+today = date.today()
+#today = date(2023,8,7)
+path="/data/telemetry/crash-report/"
+
+# get files for the past 7 days
+def get_files():
+    files = ""
+    for i in range(1,8):
+        #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
+        files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
+
+    return files
+
+# for none-taosAssertDebug
+filter1_cmd = '''grep '"version":"%s"'  %s \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep -v "taosAssertDebug" \
+| grep -v %s \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c ''' % (version, get_files(), ip)
+
+# for taosAssertDebug
+filter2_cmd = '''grep '"version":"%s"'  %s \
+| grep  "taosd(" \
+| awk -F "stackInfo" '{print $2}' \
+| grep "taosAssertDebug" \
+| grep -v %s \
+| awk -F "taosd" '{print $3}' \
+| cut -d")" -f 1 \
+| cut -d"(" -f 2 \
+| sort | uniq -c ''' % (version, get_files(), ip)
+
+# get msg info
+def get_msg(text):
+    return {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": "Telemetry Statistics",
+                    "content": [
+                        [{
+                            "tag": "text",
+                            "text": text
+                        }
+                        ]]
+                }
+            }
+        }
+    }
+
+# post msg
+def send_msg(json):
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    req = requests.post(url=group_url, headers=headers, json=json)
+    inf = req.json()
+    if "StatusCode" in inf and inf["StatusCode"] == 0:
+        pass
+    else:
+        print(inf)
+
+# exec cmd and return res
+def get_output(cmd):
+    text = os.popen(cmd)
+    lines = text.read()
+    text.close()
+    return lines
+
+# get sum
+def get_count(output):
+    res = re.findall(" \d+ ", output)
+    sum1 = 0
+    for r in res:
+        sum1 = sum1 + int(r.strip())
+    return sum1
+
+# print total crash count
+def print_result():
+    #print(f"Files for statistics: {get_files()}\n")
+    sum1 = get_count(get_output(filter1_cmd))
+    sum2 = get_count(get_output(filter2_cmd))
+    total = sum1 + sum2
+    #print(f"total crashes: {total}")
+    return total
+
+# send report to feishu
+def send_report():
+    content = f'''
+    test scope: Telemetry Statistics
+    owner: {owner}
+    ip: {server_ip}
+    from: {get_files().split(" ")[6].split("/")[4].split(".")[0]}
+    to: {get_files().split(" ")[0].split("/")[4].split(".")[0]}
+    filter1 result: {get_output(filter1_cmd)}
+    filter2 result: {get_output(filter2_cmd)}
+    total crashes: {print_result()}
+    '''
+    #send_msg(get_msg(content))
+    print(content)
+
+print_result()
+send_report()
diff --git a/tests/script/telemetry/crash-report/CrashCounter_new.py b/tests/script/telemetry/crash-report/CrashCounter_new.py
deleted file mode 100644
index a89567da3d..0000000000
--- a/tests/script/telemetry/crash-report/CrashCounter_new.py
+++ /dev/null
@@ -1,308 +0,0 @@
-from datetime import date
-from datetime import timedelta
-import os
-import json
-import re
-import requests
-import subprocess
-from dotenv import load_dotenv
-
-# load .env
-# You should have a .env file in the same directory as this script
-# You can exec: cp .env.example .env
-load_dotenv()
-
-# define version
-version = "3.3.2.*"
-version_pattern_str = version.replace('.', r'\.').replace('*', r'\d+')
-version_pattern = re.compile(rf'^{version_pattern_str}$')
-version_stack_list = list()
-
-# define ip
-
-ip = os.getenv("EXCLUDE_IP")
-server_ip = os.getenv("SERVER_IP")
-http_serv_ip = os.getenv("HTTP_SERV_IP")
-http_serv_port = os.getenv("HTTP_SERV_PORT")
-owner = os.getenv("OWNER")
-
-# feishu-msg url
-feishu_msg_url = os.getenv("FEISHU_MSG_URL")
-
-# get today
-today = date.today()
-
-# Define the file and parameters
-path="/data/telemetry/crash-report/"
-trace_report_path = path + "trace_report"
-os.makedirs(path, exist_ok=True)
-os.makedirs(trace_report_path, exist_ok=True)
-
-assert_script_path = path + "filter_assert.sh"
-nassert_script_path = path + "filter_nassert.sh"
-
-# get files for the past 7 days
-def get_files():
-    files = ""
-    for i in range(1,8):
-        #print ((today - timedelta(days=i)).strftime("%Y%m%d"))
-        files = files + path + (today - timedelta(days=i)).strftime("%Y%m%d") + ".txt "
-    return files.strip().split(" ")
-
-# Define the AWK script as a string with proper escaping
-def get_res(file_path):
-    # Execute the script
-    command = ['bash', file_path, version, ip] + get_files()
-    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-
-    # Capture the output and errors
-    output, errors = process.communicate()
-
-    # Check for errors
-    if process.returncode != 0:
-        return errors
-    else:
-        return output.rstrip()
-
-def get_sum(output):
-    # Split the output into lines
-    lines = output.strip().split('\n')
-
-    # Initialize the sum
-    total_sum = 0
-
-    # Iterate over each line
-    for line in lines:
-        # Split each line by space to separate the columns
-        parts = line.split()
-
-        # The first part of the line is the number, convert it to integer
-        if parts:  # Check if there are any elements in the parts list
-            number = int(parts[0])
-            total_sum += number
-
-    return total_sum
-
-def convert_html(data):
-    # convert data to json
-    start_time = get_files()[6].split("/")[-1].split(".")[0]
-    end_time = get_files()[0].split("/")[-1].split(".")[0]
-    html_report_file = f'{start_time}_{end_time}.html'
-    json_data = json.dumps(data)
-
-    # Create HTML content
-    html_content = f'''
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Stack Trace Report</title>
-    <style>
-        body {{
-            font-family: Arial, sans-serif;
-            margin: 20px;
-            background-color: #f0f0f5;
-        }}
-        h1 {{
-            color: #2c3e50;
-            text-align: center;
-        }}
-        table {{
-            width: 100%;
-            border-collapse: collapse;
-            margin-bottom: 20px;
-            box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
-        }}
-        th, td {{
-            border: 1px solid #ddd;
-            padding: 10px;
-            text-align: left;
-        }}
-        th {{
-            background-color: #3498db;
-            color: white;
-        }}
-        tr:nth-child(even) {{
-            background-color: #ecf0f1;
-        }}
-        tr:hover {{
-            background-color: #d1e7fd;
-        }}
-        pre {{
-            background-color: #f7f7f7;
-            padding: 10px;
-            border: 1px solid #ddd;
-            overflow-x: auto;
-            white-space: pre-wrap;
-            border-radius: 5px;
-        }}
-    </style>
-</head>
-<body>
-    <h1>Stack Trace Report From {start_time} To {end_time} </h1>
-
-    <table>
-        <thead>
-            <tr>
-                <th>Key Stack Info</th>
-                <th>Versions</th>
-                <th>Num Of Crashes</th>
-                <th>Full Stack Info</th>
-            </tr>
-        </thead>
-        <tbody id="report">
-        </tbody>
-    </table>
-
-    <script>
-        const data = {json_data};
-
-        const reportBody = document.getElementById('report');
-        data.forEach(entry => {{
-            const row = document.createElement('tr');
-            row.innerHTML = `
-                <td>${{entry.key_stack_info}}</td>
-                <td>${{entry.version_list.join('<br>')}}</td>
-                <td>${{entry.count}}</td>
-                <td><pre>${{entry.full_stack_info}}</pre></td>
-            `;
-            reportBody.appendChild(row);
-        }});
-    </script>
-</body>
-</html>
-'''
-    # Write the HTML content to a file
-
-    with open(f'{trace_report_path}/{html_report_file}', 'w') as f:
-        f.write(html_content)
-    return html_report_file
-
-def get_version_stack_list(res):
-    for line in res.strip().split('\n'):
-        version_list = list()
-        version_stack_dict = dict()
-        count = line.split()[0]
-        key_stack_info = line.split()[1]
-        for file in get_files():
-            with open(file, 'r') as infile:
-                for line in infile:
-                    line = line.strip()
-                    data = json.loads(line)
-                    # print(line)
-                    if ip not in line and version_pattern.search(data["version"]) and key_stack_info in line:
-                        if data["version"] not in version_list:
-                            version_list.append(data["version"])
-                            full_stack_info = data["stackInfo"]
-        version_stack_dict["key_stack_info"] = key_stack_info
-        version_stack_dict["full_stack_info"] = full_stack_info
-        version_stack_dict["version_list"] = version_list
-        version_stack_dict["count"] = count
-        # print(version_stack_dict)
-        version_stack_list.append(version_stack_dict)
-    return version_stack_list
-
-# get msg info
-def get_msg(text):
-    return {
-        "msg_type": "post",
-        "content": {
-            "post": {
-                "zh_cn": {
-                    "title": "Telemetry Statistics",
-                    "content": [
-                        [{
-                            "tag": "text",
-                            "text": text
-                        }
-                        ]]
-                }
-            }
-        }
-    }
-
-# post msg
-def send_msg(json):
-    headers = {
-        'Content-Type': 'application/json'
-    }
-
-    req = requests.post(url=feishu_msg_url, headers=headers, json=json)
-    inf = req.json()
-    if "StatusCode" in inf and inf["StatusCode"] == 0:
-        pass
-    else:
-        print(inf)
-
-
-def format_results(results):
-    # Split the results into lines
-    lines = results.strip().split('\n')
-
-    # Parse lines into a list of tuples (number, rest_of_line)
-    parsed_lines = []
-    for line in lines:
-        parts = line.split(maxsplit=1)
-        if len(parts) == 2:
-            number = int(parts[0])  # Convert the number part to an integer
-            parsed_lines.append((number, parts[1]))
-
-    # Sort the parsed lines by the first element (number) in descending order
-    parsed_lines.sort(reverse=True, key=lambda x: x[0])
-
-    # Determine the maximum width of the first column for alignment
-    # max_width = max(len(str(item[0])) for item in parsed_lines)
-    if parsed_lines:
-        max_width = max(len(str(item[0])) for item in parsed_lines)
-    else:
-        max_width = 0
-
-    # Format each line to align the numbers and function names with indentation
-    formatted_lines = []
-    for number, text in parsed_lines:
-        formatted_line = f"       {str(number).rjust(max_width)} {text}"
-        formatted_lines.append(formatted_line)
-
-    # Join the formatted lines into a single string
-    return '\n'.join(formatted_lines)
-
-# # send report to feishu
-def send_report(res, sum, html_report_file):
-    content = f'''
-    version: v{version}
-    from: {get_files()[6].split("/")[-1].split(".")[0]}
-    to: {get_files()[0].split("/")[-1].split(".")[0]}
-    ip: {server_ip}
-    owner: {owner}
-    result: \n{format_results(res)}\n
-    total crashes: {sum}\n
-    details: http://{http_serv_ip}:{http_serv_port}/{html_report_file}
-    '''
-    print(get_msg(content))
-    send_msg(get_msg(content))
-    # print(content)
-
-# for none-taosAssertDebug
-nassert_res = get_res(nassert_script_path)
-# print(nassert_res)
-
-# for taosAssertDebug
-assert_res = get_res(assert_script_path)
-# print(assert_res)
-
-# combine the results
-res = nassert_res + assert_res
-
-# get version stack list
-version_stack_list = get_version_stack_list(res) if len(res) > 0 else list()
-
-# convert to html
-html_report_file = convert_html(version_stack_list)
-
-# get sum
-sum = get_sum(res)
-
-# send report
-send_report(res, sum, html_report_file)
-

From bcdb50e49979a089bc1628b48420b15d6a9be013 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Jan 2025 14:21:42 +0800
Subject: [PATCH 07/11] doc: add README

---
 .../telemetry/crash-report/README-CN.md       |  43 ++++
 tests/script/telemetry/crash-report/README.md | 187 +-----------------
 2 files changed, 53 insertions(+), 177 deletions(-)
 create mode 100644 tests/script/telemetry/crash-report/README-CN.md

diff --git a/tests/script/telemetry/crash-report/README-CN.md b/tests/script/telemetry/crash-report/README-CN.md
new file mode 100644
index 0000000000..7a0b974f0c
--- /dev/null
+++ b/tests/script/telemetry/crash-report/README-CN.md
@@ -0,0 +1,43 @@
+# 目录
+
+1. [介绍](#1-介绍)
+1. [前置条件](#2-前置条件)
+1. [运行](#3-运行)
+
+# 1. 介绍
+
+本手册旨在为开发人员提供全面的指导，以收集过去7天的崩溃信息并将其报告到飞书通知群。
+
+> [!注意]
+> - 下面的命令和脚本已在 Linux（CentOS 7.9.2009）上验证.
+
+# 2. 前置条件
+
+- 安装 Python3
+
+```bash
+yum install python3
+yum install python3-pip
+```
+
+- 安装 Python 依赖
+
+```bash
+pip3 install requests python-dotenv
+```
+
+- 调整 .env 文件
+
+```bash
+cd $DIR/telemetry/crash-report
+cp .env.example .env
+```
+
+# 3. 运行
+
+在 $DIR/telemetry/crash-report 目录中，有类似文件名为 202501**.txt 的一些文件。Python 脚本会将从这些文本文件中收集崩溃信息，并将报告发送到您的飞书机器人群组中。
+
+```bash
+cd $DIR/telemetry/crash-report
+python3 CrashCounter.py
+```
diff --git a/tests/script/telemetry/crash-report/README.md b/tests/script/telemetry/crash-report/README.md
index ffee4998c7..537f3538f3 100644
--- a/tests/script/telemetry/crash-report/README.md
+++ b/tests/script/telemetry/crash-report/README.md
@@ -10,7 +10,6 @@ This manual is intended to give developers comprehensive guidance to collect cra
 
 > [!NOTE]
 > - The commands and scripts below are verified on Linux (CentOs 7.9.2009).
-> - The commands and steps described below are to run the tests on a single host.
 
 # 2. Prerequisites
 
@@ -27,184 +26,18 @@ yum install python3-pip
 pip3 install requests python-dotenv
 ```
 
+- Adjust .env file
+
+```bash
+cd $DIR/telemetry/crash-report
+cp .env.example .env
+```
+
 # 3. Running
 
-In `crash-report` directory, there are different types of tests for TDengine. Below is a brief introduction about how to run them and how to add new cases.
-
-### 3.1 Unit Test
-
-Unit tests are the smallest testable units, which are used to test functions, methods or classes in TDengine code.
-
-### 3.1.1 How to run single test case?
+In `$DIR/telemetry/crash-report` directory, there are several files with names like 202501**.txt. The python script will collect crash information from these text files and send report to your Feishu bot group.
 
 ```bash
-cd debug/build/bin
-./osTimeTests
+cd $DIR/telemetry/crash-report
+python3 CrashCounter.py
 ```
-
-### 3.1.2 How to run all unit test cases?
-
-```bash
-cd tests/unit-test/
-bash test.sh -e 0
-```
-
-### 3.1.3 How to add new cases?
-
-<details>
-
-<summary>Detailed steps to add new unit test case</summary>
-
-The Google test framwork is used for unit testing to specific function module, please refer to steps below to add a new test case:
-
-##### a. Create test case file and develop the test scripts
-
-In the test directory corresponding to the target function module, create test files in CPP format and write corresponding test cases.
-
-##### b. Update build configuration
-
-Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the `source/os/test/CMakeLists.txt` file for configuration examples.
-
-##### c. Compile test code
-
-In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., `cmake .. -DBUILD_TEST=1`) to generate a compilation file,
-
-and then run a compilation command (e.g. make) to complete the compilation of the test code.
-
-##### d. Execute the test program
-
-Find the executable file in the compiled directory(e.g. `TDengine/debug/build/bin/`) and run it.
-
-##### e. Integrate into CI tests
-
-Use the add_test command to add new compiled test cases into CI test collection, ensure that the new added test cases can be run for every build.
-
-</details>
-
-## 3.2 System Test
-
-System tests are end-to-end test cases written in Python from a system point of view. Some of them are designed to test features only in enterprise ediiton, so when running on community edition, they may fail. We'll fix this issue by separating the cases into different gruops in the future.
-
-### 3.2.1 How to run a single test case?
-
-Take test file `system-test/2-query/avg.py` for example:
-
-```bash
-cd tests/system-test
-python3 ./test.py -f 2-query/avg.py
-```
-
-### 3.2.2 How to run all system test cases?
-
-```bash
-cd tests
-./run_all_ci_cases.sh -t python # all python cases
-```
-
-### 3.2.3 How to add new case?
-
-<details>
-
-<summary>Detailed steps to add new system test case</summary>
-
-The Python test framework is developed by TDengine team, and test.py is the test case execution and monitoring of the entry program, Use `python3 ./test.py -h` to view more features.
-
-Please refer to steps below for how to add a new test case:
-
-##### a. Create a test case file and develop the test cases
-
-Create a file in `tests/system-test` containing each functional directory and refer to the use case template `tests/system-test/0-others/test_case_template.py` to add a new test case.
-
-##### b. Execute the test case
-
-Ensure the test case execution is successful.
-
-``` bash
-cd tests/system-test && python3 ./test.py -f 0-others/test_case_template.py
-```
-
-##### c. Integrate into CI tests
-
-Edit `tests/parallel_test/cases.task` and add the testcase path and executions in the specified format. The third column indicates whether to use Address Sanitizer mode for testing.
-
-```bash
-#caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand
-,,n,system-test, python3 ./test.py  -f 0-others/test_case_template.py
-```
-
-</details>
-
-## 3.3 Legacy Test
-
-In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++.
-
-### 3.3.1 How to run single test case?
-
-To run the legacy test cases, please execute the following commands:
-
-```bash
-cd tests/script
-./test.sh -f tsim/db/basic1.sim
-```
-
-### 3.3.2 How to run all legacy test cases?
-
-```bash
-cd tests
-./run_all_ci_cases.sh -t legacy # all legacy cases
-```
-
-### 3.3.3 How to add new cases?
-
-> [!NOTE]
-> TSIM test framwork is deprecated by system test now, it is encouraged to add new test cases in system test, please refer to [System Test](#32-system-test) for details.
-
-## 3.4 Smoke Test
-
-Smoke test is a group of test cases selected from system test, which is also known as sanity test to ensure the critical functionalities of TDengine.
-
-### 3.4.1 How to run test?
-
-```bash
-cd /root/TDengine/packaging/smokeTest
-./test_smoking_selfhost.sh
-```
-
-### 3.4.2 How to add new cases?
-
-New cases can be added by updating the value of `commands` variable in `test_smoking_selfhost.sh`.
-
-## 3.5 Chaos Test
-
-A simple tool to execute various functions of the system in a randomized way, hoping to expose potential problems without a pre-defined test scenario.
-
-### 3.5.1 How to run test?
-
-```bash
-cd tests/pytest
-python3 auto_crash_gen.py
-```
-
-### 3.5.2 How to add new cases?
-
-1. Add a function, such as `TaskCreateNewFunction` in `pytest/crash_gen/crash_gen_main.py`.
-2. Integrate `TaskCreateNewFunction` into the `balance_pickTaskType` function in `crash_gen_main.py`.
-
-## 3.6 CI Test
-
-CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability.
-
-TDengine CI testing will run all the test cases from the following three types of tests: unit test, system test and legacy test.
-
-### 3.6.1 How to run all CI test cases?
-
-If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands:
-
-```bash
-cd tests
-./run_all_ci_cases.sh -b main # on main branch
-```
-
-### 3.6.2 How to add new cases?
-
-Please refer to the [Unit Test](#31-unit-test)、[System Test](#32-system-test) and [Legacy Test](#33-legacy-test) sections for detailed steps to add new test cases, when new cases are added in aboved tests, they will be run automatically by CI test.

From f165817a31c4acb4544c20f92286da58549a06e2 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Jan 2025 14:38:11 +0800
Subject: [PATCH 08/11] doc: modify readme

---
 tests/script/telemetry/crash-report/README-CN.md | 2 ++
 tests/script/telemetry/crash-report/README.md    | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tests/script/telemetry/crash-report/README-CN.md b/tests/script/telemetry/crash-report/README-CN.md
index 7a0b974f0c..f755a23b18 100644
--- a/tests/script/telemetry/crash-report/README-CN.md
+++ b/tests/script/telemetry/crash-report/README-CN.md
@@ -31,6 +31,8 @@ pip3 install requests python-dotenv
 ```bash
 cd $DIR/telemetry/crash-report
 cp .env.example .env
+vim .env
+...
 ```
 
 # 3. 运行
diff --git a/tests/script/telemetry/crash-report/README.md b/tests/script/telemetry/crash-report/README.md
index 537f3538f3..cbe70dcda4 100644
--- a/tests/script/telemetry/crash-report/README.md
+++ b/tests/script/telemetry/crash-report/README.md
@@ -31,6 +31,8 @@ pip3 install requests python-dotenv
 ```bash
 cd $DIR/telemetry/crash-report
 cp .env.example .env
+vim .env
+...
 ```
 
 # 3. Running

From 850586ca4e3b9d9453273269d8bc9992da202d85 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Jan 2025 14:42:14 +0800
Subject: [PATCH 09/11] doc: modify readme

---
 tests/script/telemetry/crash-report/README-CN.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/script/telemetry/crash-report/README-CN.md b/tests/script/telemetry/crash-report/README-CN.md
index f755a23b18..0e4507c844 100644
--- a/tests/script/telemetry/crash-report/README-CN.md
+++ b/tests/script/telemetry/crash-report/README-CN.md
@@ -8,7 +8,7 @@
 
 本手册旨在为开发人员提供全面的指导，以收集过去7天的崩溃信息并将其报告到飞书通知群。
 
-> [!注意]
+> [!]注意
 > - 下面的命令和脚本已在 Linux（CentOS 7.9.2009）上验证.
 
 # 2. 前置条件

From 3b51d7bafba447864b96441da897031d4edec639 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Jan 2025 14:42:54 +0800
Subject: [PATCH 10/11] doc: modify readme

---
 tests/script/telemetry/crash-report/README-CN.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/script/telemetry/crash-report/README-CN.md b/tests/script/telemetry/crash-report/README-CN.md
index 0e4507c844..6570ad913d 100644
--- a/tests/script/telemetry/crash-report/README-CN.md
+++ b/tests/script/telemetry/crash-report/README-CN.md
@@ -8,7 +8,7 @@
 
 本手册旨在为开发人员提供全面的指导，以收集过去7天的崩溃信息并将其报告到飞书通知群。
 
-> [!]注意
+> [!NOTE]
 > - 下面的命令和脚本已在 Linux（CentOS 7.9.2009）上验证.
 
 # 2. 前置条件

From fb682e441ca9e63e4a8e36f8015793080a287219 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 23 Jan 2025 15:07:27 +0800
Subject: [PATCH 11/11] doc: add example for .env

---
 tests/script/telemetry/crash-report/README-CN.md | 16 ++++++++++++++++
 tests/script/telemetry/crash-report/README.md    | 16 ++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/tests/script/telemetry/crash-report/README-CN.md b/tests/script/telemetry/crash-report/README-CN.md
index 6570ad913d..e0deab9f5b 100644
--- a/tests/script/telemetry/crash-report/README-CN.md
+++ b/tests/script/telemetry/crash-report/README-CN.md
@@ -35,6 +35,22 @@ vim .env
 ...
 ```
 
+- .env 样例
+
+```bash
+# 过滤器排除 IP（公司网络出口 IP）
+EXCLUDE_IP="192.168.1.10"
+# 英文官网服务器 IP
+SERVER_IP="192.168.1.11"
+# 内网提供 HTTP 服务的 IP 及端口，用于提供 HTML 报告浏览
+HTTP_SERV_IP="192.168.1.12"
+HTTP_SERV_PORT=8080
+# 飞书群机器人 webhook 地址
+FEISHU_MSG_URL="https://open.feishu.cn/open-apis/bot/v2/hook/*******"
+# 负责人
+OWNER="Jayden Jia"
+```
+
 # 3. 运行
 
 在 $DIR/telemetry/crash-report 目录中，有类似文件名为 202501**.txt 的一些文件。Python 脚本会将从这些文本文件中收集崩溃信息，并将报告发送到您的飞书机器人群组中。
diff --git a/tests/script/telemetry/crash-report/README.md b/tests/script/telemetry/crash-report/README.md
index cbe70dcda4..a47c9bc8bb 100644
--- a/tests/script/telemetry/crash-report/README.md
+++ b/tests/script/telemetry/crash-report/README.md
@@ -35,6 +35,22 @@ vim .env
 ...
 ```
 
+- Example for .env
+
+```bash
+# Filter to exclude IP (Company network export IP)
+EXCLUDE_IP="192.168.1.10"
+# Official website server IP
+SERVER_IP="192.168.1.11"
+# Internal network providing HTTP service IP and port, used for HTML report browsing
+HTTP_SERV_IP="192.168.1.12"
+HTTP_SERV_PORT=8080
+# Webhook address for feiShu group bot
+FEISHU_MSG_URL="https://open.feishu.cn/open-apis/bot/v2/hook/*******"
+# Owner
+OWNER="Jayden Jia"
+```
+
 # 3. Running
 
 In `$DIR/telemetry/crash-report` directory, there are several files with names like 202501**.txt. The python script will collect crash information from these text files and send report to your Feishu bot group.