From ddbf300f238104c0f3318397f80ff76b5da76e11 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 12:37:36 +0800 Subject: [PATCH] test: scan returned values in ci --- Jenkinsfile2 | 11 +- source/libs/parser/src/parAstCreater.c | 2 + tests/ci/filter_for_return_values | 24 +++ tests/ci/scan_file_path.py | 199 +++++++++++++++++++++++++ tests/parallel_test/container_build.sh | 2 +- 5 files changed, 234 insertions(+), 4 deletions(-) create mode 100644 tests/ci/filter_for_return_values create mode 100644 tests/ci/scan_file_path.py diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 904c8b1651..b6275d0f6d 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -69,7 +69,9 @@ def check_docs() { echo "docs PR" docs_only=1 } else { - echo file_changed + echo file_changed + mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} + echo file_changed > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt } } } @@ -350,7 +352,6 @@ pipeline { when { allOf { not { expression { env.CHANGE_BRANCH =~ /docs\// }} - not { expression { env.CHANGE_URL =~ /\/TDinternal\// }} } } parallel { @@ -401,7 +402,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} + agent{label "slave1_47 "} options { skipDefaultCheckout() } when { changeRequest() @@ -425,6 +426,10 @@ pipeline { cd ${WKC}/tests/parallel_test time ./container_build.sh -w ${WKDIR} -e ''' + sh ''' + cd ${WKC}/tests/ci + python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + ''' def extra_param = "" def log_server_file = "/home/log_server.json" def timeout_cmd = "" diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index cd7cda01e0..a8979bbe1d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -207,6 +207,8 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { return false; } return true; + + } static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) { diff --git a/tests/ci/filter_for_return_values b/tests/ci/filter_for_return_values new file mode 100644 index 0000000000..734619a8af --- /dev/null +++ b/tests/ci/filter_for_return_values @@ -0,0 +1,24 @@ +match callExpr( + hasParent(anyOf( + compoundStmt(), + doStmt(hasCondition(expr().bind("cond")))) + ), + unless(hasType(voidType())), + unless(callee(functionDecl(hasName("memcpy")))), + unless(callee(functionDecl(hasName("strcpy")))), + unless(callee(functionDecl(hasName("strcat")))), + unless(callee(functionDecl(hasName("strncpy")))), + unless(callee(functionDecl(hasName("memset")))), + unless(callee(functionDecl(hasName("memmove")))), + unless(callee(functionDecl(hasName("sprintf")))), + unless(callee(functionDecl(hasName("snprintf")))), + unless(callee(functionDecl(hasName("scanf")))), + unless(callee(functionDecl(hasName("sncanf")))), + unless(callee(functionDecl(hasName("printf")))), + unless(callee(functionDecl(hasName("printRow")))), + unless(callee(functionDecl(hasName("puts")))), + unless(callee(functionDecl(hasName("sleep")))), + unless(callee(functionDecl(hasName("printResult")))), + unless(callee(functionDecl(hasName("getchar")))), + unless(callee(functionDecl(hasName("taos_print_row")))), + unless(callee(functionDecl(hasName("fprintf"))))) \ No newline at end of file diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py new file mode 100644 index 0000000000..92f287e797 --- /dev/null +++ b/tests/ci/scan_file_path.py @@ -0,0 +1,199 @@ +import os +import sys +import subprocess +import csv +from datetime import datetime +from loguru import logger +import getopt + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:', [ + 'branch_name=']) +for key, value in opts: + if key in ['-h', '--help']: + print( + 'Usage: python3 scan.py -b -f ') + print('-b branch name or PR ID to scan') + print('-f change files list') + + sys.exit(0) + + if key in ['-b', '--branchName']: + branch_name = value + if key in ['-f', '--filesName']: + change_file_list = value + + +# the base source code file path +self_path = os.path.dirname(os.path.realpath(__file__)) + +if ("community" in self_path): + source_path = self_path[:self_path.find("community")] + work_path = source_path[:source_path.find("TDinternal")] + +else: + source_path = self_path[:self_path.find("tests")] + work_path = source_path[:source_path.find("TDengine")] + +# Check if "community" or "tests" is in self_path +index_community = self_path.find("community") +if index_community != -1: + source_path = self_path[:index_community] + index_TDinternal = source_path.find("TDinternal") + # Check if index_TDinternal is valid and set work_path accordingly + if index_TDinternal != -1: + work_path = source_path[:index_TDinternal] +else: + index_tests = self_path.find("tests") + if index_tests != -1: + source_path = self_path[:index_tests] + # Check if index_TDengine is valid and set work_path accordingly + index_TDengine = source_path.find("TDengine") + if index_TDengine != -1: + work_path = source_path[:index_TDengine] + + +# log file path +log_file_path = f"{source_path}/../{branch_name}/" +os.makedirs(log_file_path, exist_ok=True) + +scan_log_file = f"{log_file_path}/scan.log" +logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG") +print(self_path,work_path,source_path,log_file_path) + +# scan result base path +scan_result_base_path = f"{log_file_path}/clang_scan_result/" + + +# the compile commands json file path +# compile_commands_path = f"{source_path}/../debugNoSan/compile_commands.json" +compile_commands_path = f"{source_path}/debug/compile_commands.json" + +# the ast parser rule for c file +clang_scan_rules_path = f"{self_path}/filter_for_return_values" + +# all the c files path will be checked +all_file_path = [] + +class CommandExecutor: + def __init__(self): + self._process = None + + def execute(self, command, timeout=None): + try: + self._process = subprocess.Popen(command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = self._process.communicate(timeout=timeout) + return stdout.decode('utf-8'), stderr.decode('utf-8') + except subprocess.TimeoutExpired: + self._process.kill() + self._process.communicate() + raise Exception("Command execution timeout") + except Exception as e: + raise Exception("Command execution failed: %s" % e) + +def scan_files_path(source_file_path): + # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] + scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] + scan_skip_file_list = ["/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", + "/test/", "contrib", "debug", "deps", "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + for root, dirs, files in os.walk(source_file_path): + for file in files: + if any(item in root for item in scan_dir_list): + file_path = os.path.join(root, file) + if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list): + all_file_path.append(file_path) + logger.info("Found %s files" % len(all_file_path)) + +def input_files(change_files): + # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] + scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] + scan_skip_file_list = [f"{source_path}/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", f"{source_path}/TDinternal/community/source/libs/parser/src/sql.c", f"{source_path}/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + with open(change_files, 'r') as file: + for line in file: + file_name = line.strip() + if any(dir_name in file_name for dir_name in scan_dir_list): + if (file_name.endswith(".c") or file_name.endswith(".h") or line.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list): + if "enterprise" in file_name: + file_name = os.path.join(source_path, file_name) + else: + tdc_file_path = os.path.join(source_path, "community/") + file_name = os.path.join(tdc_file_path, file_name) + all_file_path.append(file_name) + print(f"all_file_path:{all_file_path}") + # for file_path in change_files: + # if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list): + # all_file_path.append(file_path) + logger.info("Found %s files" % len(all_file_path)) +file_res_path = "" + +def save_scan_res(res_base_path, file_path, out, err): + global file_res_path + file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".res") + print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}") + if not os.path.exists(os.path.dirname(file_res_path)): + os.makedirs(os.path.dirname(file_res_path)) + logger.info("Save scan result to: %s" % file_res_path) + + # save scan result + with open(file_res_path, "w") as f: + f.write(err) + f.write(out) + logger.debug(f"file_res_file: {file_res_path}") + +def write_csv(file_path, data): + try: + with open(file_path, 'w') as f: + writer = csv.writer(f) + writer.writerows(data) + except Exception as ex: + raise Exception("Failed to write the csv file: {} with msg: {}".format(file_path, repr(ex))) + +if __name__ == "__main__": + command_executor = CommandExecutor() + # get all the c files path + # scan_files_path(source_path) + input_files(change_file_list) + print(f"all_file_path:{all_file_path}") + res = [] + res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"]) + # create dir + current_time = datetime.now().strftime("%Y%m%d%H%M%S") + scan_result_path = os.path.join(scan_result_base_path, current_time) + if not os.path.exists(scan_result_path): + os.makedirs(scan_result_path) + for file in all_file_path: + cmd = f"clang-query-10 -p {compile_commands_path} {file} -f {clang_scan_rules_path}" + print(f"cmd:{cmd}") + try: + stdout, stderr = command_executor.execute(cmd) + lines = stdout.split("\n") + if lines[-2].endswith("matches.") or lines[-2].endswith("match."): + match_num = int(lines[-2].split(" ")[0]) + logger.info("The match lines of file %s: %s" % (file, match_num)) + if match_num > 0: + logger.info(f"scan_result_path: {scan_result_path} ,file:{file}") + save_scan_res(scan_result_path, file, stdout, stderr) + res.append([file, file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail']) + else: + logger.warning("The result of scan is invalid for: %s" % file) + except Exception as e: + logger.error("Execute command failed: %s" % e) + # data = "" + # for item in res: + # data += item[0] + "," + str(item[1]) + "\n" + # logger.info("Csv data: %s" % data) + write_csv(os.path.join(scan_result_path, "scan_res.csv"), res) + scan_result_log = f"{scan_result_path}/scan_res.csv" + # delete the first element of res + res= res[1:] + logger.info("The result of scan: \n") + logger.info("Total scan files: %s" % len(res)) + logger.info("Total match lines: %s" % sum([item[2] for item in res])) + logger.info(f"scan log file : {scan_result_log}") + logger.info("Pass files: %s" % len([item for item in res if item[3] == 'Pass'])) + logger.info("Fail files: %s" % len([item for item in res if item[3] == 'Fail'])) + if len([item for item in res if item[3] == 'Fail']) > 0: + logger.error(f"Scan failed,please check the log file:{scan_result_log}") + exit(1) \ No newline at end of file diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 85e3d2ab73..26cabad107 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -83,7 +83,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then