test: scan returned values in ci

Jenkinsfile2

@@ -422,13 +422,6 @@ pipeline {
                                     mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}
                                     echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt
                                     '''
-                                    sh'''
-                                    cd ${WK} || exit 0 ; rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1
-                                    '''
-                                    sh '''
-                                       cd ${WKC}/tests/ci
-                                       python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID}  -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt
-                                    '''
                                     sh '''
                                         date
                                         rm -rf ${WKC}/debug
@@ -460,6 +453,10 @@ pipeline {
                                             }
                                         }
                                     }
+                                    sh '''
+                                       cd ${WKC}/tests/ci
+                                       run_scan_container.sh -d ${WK} -b ${BRANCH_NAME}_${BUILD_ID}  -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + '''
+                                    '''
                                     sh '''
                                         cd ${WKC}/tests/parallel_test
                                         export DEFAULT_RETRY_TIME=2

tests/ci/scan_file_path.py

@@ -6,7 +6,8 @@ from datetime import datetime
 from loguru import logger
 import getopt
 
-opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:', [
+
+opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:w:', [
     'branch_name='])
 for key, value in opts:
     if key in ['-h', '--help']:
@@ -14,6 +15,7 @@ for key, value in opts:
             'Usage: python3 scan.py -b <branch_name> -f <file_list>')
         print('-b  branch name or PR ID to scan')
         print('-f  change files list')
+        print('-w  web server')
 
         sys.exit(0)
 
@@ -21,18 +23,20 @@ for key, value in opts:
         branch_name = value
     if key in ['-f', '--filesName']:
         change_file_list = value
+    if key in ['-w', '--webServer']:
+        web_server = value
 
 
 # the base source code file path
 self_path =  os.path.dirname(os.path.realpath(__file__))
 
-if ("community" in self_path):
+# if ("community" in self_path):
-    TD_project_path = self_path[:self_path.find("community")]
+#     TD_project_path = self_path[:self_path.find("community")]
-    work_path = TD_project_path[:TD_project_path.find("TDinternal")]
+#     work_path = TD_project_path[:TD_project_path.find("TDinternal")]
 
-else:
+# else:
-    TD_project_path = self_path[:self_path.find("tests")]
+#     TD_project_path = self_path[:self_path.find("tests")]
-    work_path = TD_project_path[:TD_project_path.find("TDengine")]
+#     work_path = TD_project_path[:TD_project_path.find("TDengine")]
 
 # Check if "community"  or "tests" is in self_path
 index_community = self_path.find("community")
@@ -53,12 +57,16 @@ else:
 
 
 # log file path
-log_file_path = f"{work_path}/log/{branch_name}/"
+log_file_path = f"{work_path}/{branch_name}/"
 os.makedirs(log_file_path, exist_ok=True)
 
 scan_log_file = f"{log_file_path}/scan.log"
 logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG")
-print(self_path,work_path,TD_project_path,log_file_path)
+# logging.basicConfig(level=logging.INFO,
+#                     format='%(asctime)s | %(levelname)s | %(name)s:%(lineno)d - %(message)s')
+# logger = logging.getLogger(__name__)
+
+print(self_path,work_path,TD_project_path,log_file_path,change_file_list)
 
 # scan result base path
 scan_result_base_path = f"{log_file_path}/clang_scan_result/"
@@ -81,6 +89,7 @@ print(f"compile_commands_path:{compile_commands_path}")
 # the ast parser rule for c file
 clang_scan_rules_path = f"{self_path}/filter_for_return_values"
 
+#
 # all the c files path will be checked
 all_file_path = []
 
@@ -140,7 +149,7 @@ file_res_path = ""
 
 def save_scan_res(res_base_path, file_path, out, err):
     global file_res_path
-    file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".res")
+    file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".txt")
     print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}")
     if not os.path.exists(os.path.dirname(file_res_path)):
         os.makedirs(os.path.dirname(file_res_path))
@@ -167,6 +176,7 @@ if __name__ == "__main__":
     input_files(change_file_list)
     print(f"all_file_path:{all_file_path}")
     res = []
+    web_path = []
     res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"])
     # create dir
     current_time = datetime.now().strftime("%Y%m%d%H%M%S")
@@ -179,6 +189,7 @@ if __name__ == "__main__":
         print(f"cmd:{cmd}")
         try:
             stdout, stderr = command_executor.execute(cmd)
+            print(stderr)
             lines = stdout.split("\n")
             if lines[-2].endswith("matches.") or lines[-2].endswith("match."): 
                 match_num = int(lines[-2].split(" ")[0])
@@ -186,7 +197,13 @@ if __name__ == "__main__":
                 if match_num > 0:
                     logger.info(f"scan_result_path: {scan_result_path} ,file:{file}")
                     save_scan_res(scan_result_path, file, stdout, stderr)
+                    index_tests = file_res_path.find(branch_name)
+                    if index_tests != -1:
+                        web_path_file = file_res_path[index_tests:]
+                        web_path_file = os.path.join(web_server, web_path_file)
+                        web_path.append(web_path_file)
                 res.append([file, file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail'])
+                
             else:
                 logger.warning("The result of scan is invalid for: %s" % file)            
         except Exception as e:
@@ -207,4 +224,6 @@ if __name__ == "__main__":
     logger.info("Fail files: %s" % len([item for item in res if item[3] == 'Fail']))
     if len([item for item in res if item[3] == 'Fail']) > 0:
         logger.error(f"Scan failed,please check the log file:{scan_result_log}")
+        for index, failed_result_file in enumerate(web_path):
+            logger.error(f"failed number: {index}, failed_result_file: {failed_result_file}")
         exit(1)

tests/parallel_test/run_container_scan.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -d work dir"
+    echo -e "\t -b pr and id"
+    echo -e "\t -w web server "
+    echo -e "\t -f scan file "
+    echo -e "\t -h help"
+}
+
+while getopts "d:b:w:f:h" opt; do
+    case $opt in
+        d)
+            WORKDIR=$OPTARG
+            ;;
+        b)
+            branch_name_id=$OPTARG
+            ;;
+        f)
+            scan_file_name=$OPTARG
+            ;;
+        w)
+            web_server=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$branch_name_id" ]; then
+    usage
+    exit 1
+fi
+
+if [ -z "$scan_file_name" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$WORKDIR" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$web_server" ]; then
+    usage
+    exit 1
+fi
+
+ # enterprise edition
+INTERNAL_REPDIR=$WORKDIR/TDinternal
+REPDIR_DEBUG=$WORKDIR/debugNoSan/
+
+REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/"
+REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal"
+
+CONTAINER_TESTDIR=/home/TDinternal/community
+
+#scan file log path
+scan_temp="$WORKDIR/log/${branch_name_id}/"
+docker_scan_temp="/home/${branch_name_id}/"
+mkdir -p $scan_temp
+mkdir -p $docker_scan_temp
+
+
+scan_scripts="$CONTAINER_TESTDIR/tests/ci/scan_file_path.py" 
+scan_file_name="$docker_scan_temp/docs_changed.txt"
+
+ulimit -c unlimited
+cat << EOF
+docker run \
+    -v $REP_MOUNT_PARAM \
+    -v $REP_MOUNT_DEBUG \
+    -v $scan_temp:$docker_scan_temp \
+    --rm --ulimit core=-1 taos_test:v1.0 python3  $scan_scripts -b "${branch_name_id}"  -f "${scan_file_name}" -w ${web_server}
+EOF
+docker run \
+    -v $REP_MOUNT_PARAM \
+    -v $REP_MOUNT_DEBUG \
+    -v $scan_temp:$docker_scan_temp \
+    --rm --ulimit core=-1 taos_test:v1.0 python3  $scan_scripts -b "${branch_name_id}"  -f "${scan_file_name}" -w ${web_server}
+
+
+ret=$?
+exit $ret
+