diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 1b710a967a..3204fc56fe 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -422,13 +422,6 @@ pipeline { mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' - sh''' - cd ${WK} || exit 0 ; rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1 - ''' - sh ''' - cd ${WKC}/tests/ci - python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt - ''' sh ''' date rm -rf ${WKC}/debug @@ -460,6 +453,10 @@ pipeline { } } } + sh ''' + cd ${WKC}/tests/ci + run_scan_container.sh -d ${WK} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' + ''' sh ''' cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 258c760391..b22adde427 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -6,7 +6,8 @@ from datetime import datetime from loguru import logger import getopt -opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:', [ + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:w:', [ 'branch_name=']) for key, value in opts: if key in ['-h', '--help']: @@ -14,6 +15,7 @@ for key, value in opts: 'Usage: python3 scan.py -b -f ') print('-b branch name or PR ID to scan') print('-f change files list') + print('-w web server') sys.exit(0) @@ -21,18 +23,20 @@ for key, value in opts: branch_name = value if key in ['-f', '--filesName']: change_file_list = value + if key in ['-w', '--webServer']: + web_server = value # the base source code file path self_path = os.path.dirname(os.path.realpath(__file__)) -if ("community" in self_path): - TD_project_path = self_path[:self_path.find("community")] - work_path = TD_project_path[:TD_project_path.find("TDinternal")] +# if ("community" in self_path): +# TD_project_path = self_path[:self_path.find("community")] +# work_path = TD_project_path[:TD_project_path.find("TDinternal")] -else: - TD_project_path = self_path[:self_path.find("tests")] - work_path = TD_project_path[:TD_project_path.find("TDengine")] +# else: +# TD_project_path = self_path[:self_path.find("tests")] +# work_path = TD_project_path[:TD_project_path.find("TDengine")] # Check if "community" or "tests" is in self_path index_community = self_path.find("community") @@ -53,12 +57,16 @@ else: # log file path -log_file_path = f"{work_path}/log/{branch_name}/" +log_file_path = f"{work_path}/{branch_name}/" os.makedirs(log_file_path, exist_ok=True) scan_log_file = f"{log_file_path}/scan.log" logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG") -print(self_path,work_path,TD_project_path,log_file_path) +# logging.basicConfig(level=logging.INFO, +# format='%(asctime)s | %(levelname)s | %(name)s:%(lineno)d - %(message)s') +# logger = logging.getLogger(__name__) + +print(self_path,work_path,TD_project_path,log_file_path,change_file_list) # scan result base path scan_result_base_path = f"{log_file_path}/clang_scan_result/" @@ -81,6 +89,7 @@ print(f"compile_commands_path:{compile_commands_path}") # the ast parser rule for c file clang_scan_rules_path = f"{self_path}/filter_for_return_values" +# # all the c files path will be checked all_file_path = [] @@ -140,7 +149,7 @@ file_res_path = "" def save_scan_res(res_base_path, file_path, out, err): global file_res_path - file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".res") + file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".txt") print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}") if not os.path.exists(os.path.dirname(file_res_path)): os.makedirs(os.path.dirname(file_res_path)) @@ -167,6 +176,7 @@ if __name__ == "__main__": input_files(change_file_list) print(f"all_file_path:{all_file_path}") res = [] + web_path = [] res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"]) # create dir current_time = datetime.now().strftime("%Y%m%d%H%M%S") @@ -179,6 +189,7 @@ if __name__ == "__main__": print(f"cmd:{cmd}") try: stdout, stderr = command_executor.execute(cmd) + print(stderr) lines = stdout.split("\n") if lines[-2].endswith("matches.") or lines[-2].endswith("match."): match_num = int(lines[-2].split(" ")[0]) @@ -186,7 +197,13 @@ if __name__ == "__main__": if match_num > 0: logger.info(f"scan_result_path: {scan_result_path} ,file:{file}") save_scan_res(scan_result_path, file, stdout, stderr) + index_tests = file_res_path.find(branch_name) + if index_tests != -1: + web_path_file = file_res_path[index_tests:] + web_path_file = os.path.join(web_server, web_path_file) + web_path.append(web_path_file) res.append([file, file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail']) + else: logger.warning("The result of scan is invalid for: %s" % file) except Exception as e: @@ -207,4 +224,6 @@ if __name__ == "__main__": logger.info("Fail files: %s" % len([item for item in res if item[3] == 'Fail'])) if len([item for item in res if item[3] == 'Fail']) > 0: logger.error(f"Scan failed,please check the log file:{scan_result_log}") + for index, failed_result_file in enumerate(web_path): + logger.error(f"failed number: {index}, failed_result_file: {failed_result_file}") exit(1) \ No newline at end of file diff --git a/tests/parallel_test/run_container_scan.sh b/tests/parallel_test/run_container_scan.sh new file mode 100755 index 0000000000..078757a57a --- /dev/null +++ b/tests/parallel_test/run_container_scan.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -d work dir" + echo -e "\t -b pr and id" + echo -e "\t -w web server " + echo -e "\t -f scan file " + echo -e "\t -h help" +} + +while getopts "d:b:w:f:h" opt; do + case $opt in + d) + WORKDIR=$OPTARG + ;; + b) + branch_name_id=$OPTARG + ;; + f) + scan_file_name=$OPTARG + ;; + w) + web_server=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$branch_name_id" ]; then + usage + exit 1 +fi + +if [ -z "$scan_file_name" ]; then + usage + exit 1 +fi +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi +if [ -z "$web_server" ]; then + usage + exit 1 +fi + + # enterprise edition +INTERNAL_REPDIR=$WORKDIR/TDinternal +REPDIR_DEBUG=$WORKDIR/debugNoSan/ + +REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/" +REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" + +CONTAINER_TESTDIR=/home/TDinternal/community + +#scan file log path +scan_temp="$WORKDIR/log/${branch_name_id}/" +docker_scan_temp="/home/${branch_name_id}/" +mkdir -p $scan_temp +mkdir -p $docker_scan_temp + + +scan_scripts="$CONTAINER_TESTDIR/tests/ci/scan_file_path.py" +scan_file_name="$docker_scan_temp/docs_changed.txt" + +ulimit -c unlimited +cat << EOF +docker run \ + -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_DEBUG \ + -v $scan_temp:$docker_scan_temp \ + --rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server} +EOF +docker run \ + -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_DEBUG \ + -v $scan_temp:$docker_scan_temp \ + --rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server} + + +ret=$? +exit $ret +