From ddbf300f238104c0f3318397f80ff76b5da76e11 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 12:37:36 +0800 Subject: [PATCH 01/29] test: scan returned values in ci --- Jenkinsfile2 | 11 +- source/libs/parser/src/parAstCreater.c | 2 + tests/ci/filter_for_return_values | 24 +++ tests/ci/scan_file_path.py | 199 +++++++++++++++++++++++++ tests/parallel_test/container_build.sh | 2 +- 5 files changed, 234 insertions(+), 4 deletions(-) create mode 100644 tests/ci/filter_for_return_values create mode 100644 tests/ci/scan_file_path.py diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 904c8b1651..b6275d0f6d 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -69,7 +69,9 @@ def check_docs() { echo "docs PR" docs_only=1 } else { - echo file_changed + echo file_changed + mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} + echo file_changed > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt } } } @@ -350,7 +352,6 @@ pipeline { when { allOf { not { expression { env.CHANGE_BRANCH =~ /docs\// }} - not { expression { env.CHANGE_URL =~ /\/TDinternal\// }} } } parallel { @@ -401,7 +402,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} + agent{label "slave1_47 "} options { skipDefaultCheckout() } when { changeRequest() @@ -425,6 +426,10 @@ pipeline { cd ${WKC}/tests/parallel_test time ./container_build.sh -w ${WKDIR} -e ''' + sh ''' + cd ${WKC}/tests/ci + python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + ''' def extra_param = "" def log_server_file = "/home/log_server.json" def timeout_cmd = "" diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index cd7cda01e0..a8979bbe1d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -207,6 +207,8 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { return false; } return true; + + } static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) { diff --git a/tests/ci/filter_for_return_values b/tests/ci/filter_for_return_values new file mode 100644 index 0000000000..734619a8af --- /dev/null +++ b/tests/ci/filter_for_return_values @@ -0,0 +1,24 @@ +match callExpr( + hasParent(anyOf( + compoundStmt(), + doStmt(hasCondition(expr().bind("cond")))) + ), + unless(hasType(voidType())), + unless(callee(functionDecl(hasName("memcpy")))), + unless(callee(functionDecl(hasName("strcpy")))), + unless(callee(functionDecl(hasName("strcat")))), + unless(callee(functionDecl(hasName("strncpy")))), + unless(callee(functionDecl(hasName("memset")))), + unless(callee(functionDecl(hasName("memmove")))), + unless(callee(functionDecl(hasName("sprintf")))), + unless(callee(functionDecl(hasName("snprintf")))), + unless(callee(functionDecl(hasName("scanf")))), + unless(callee(functionDecl(hasName("sncanf")))), + unless(callee(functionDecl(hasName("printf")))), + unless(callee(functionDecl(hasName("printRow")))), + unless(callee(functionDecl(hasName("puts")))), + unless(callee(functionDecl(hasName("sleep")))), + unless(callee(functionDecl(hasName("printResult")))), + unless(callee(functionDecl(hasName("getchar")))), + unless(callee(functionDecl(hasName("taos_print_row")))), + unless(callee(functionDecl(hasName("fprintf"))))) \ No newline at end of file diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py new file mode 100644 index 0000000000..92f287e797 --- /dev/null +++ b/tests/ci/scan_file_path.py @@ -0,0 +1,199 @@ +import os +import sys +import subprocess +import csv +from datetime import datetime +from loguru import logger +import getopt + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:', [ + 'branch_name=']) +for key, value in opts: + if key in ['-h', '--help']: + print( + 'Usage: python3 scan.py -b -f ') + print('-b branch name or PR ID to scan') + print('-f change files list') + + sys.exit(0) + + if key in ['-b', '--branchName']: + branch_name = value + if key in ['-f', '--filesName']: + change_file_list = value + + +# the base source code file path +self_path = os.path.dirname(os.path.realpath(__file__)) + +if ("community" in self_path): + source_path = self_path[:self_path.find("community")] + work_path = source_path[:source_path.find("TDinternal")] + +else: + source_path = self_path[:self_path.find("tests")] + work_path = source_path[:source_path.find("TDengine")] + +# Check if "community" or "tests" is in self_path +index_community = self_path.find("community") +if index_community != -1: + source_path = self_path[:index_community] + index_TDinternal = source_path.find("TDinternal") + # Check if index_TDinternal is valid and set work_path accordingly + if index_TDinternal != -1: + work_path = source_path[:index_TDinternal] +else: + index_tests = self_path.find("tests") + if index_tests != -1: + source_path = self_path[:index_tests] + # Check if index_TDengine is valid and set work_path accordingly + index_TDengine = source_path.find("TDengine") + if index_TDengine != -1: + work_path = source_path[:index_TDengine] + + +# log file path +log_file_path = f"{source_path}/../{branch_name}/" +os.makedirs(log_file_path, exist_ok=True) + +scan_log_file = f"{log_file_path}/scan.log" +logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG") +print(self_path,work_path,source_path,log_file_path) + +# scan result base path +scan_result_base_path = f"{log_file_path}/clang_scan_result/" + + +# the compile commands json file path +# compile_commands_path = f"{source_path}/../debugNoSan/compile_commands.json" +compile_commands_path = f"{source_path}/debug/compile_commands.json" + +# the ast parser rule for c file +clang_scan_rules_path = f"{self_path}/filter_for_return_values" + +# all the c files path will be checked +all_file_path = [] + +class CommandExecutor: + def __init__(self): + self._process = None + + def execute(self, command, timeout=None): + try: + self._process = subprocess.Popen(command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = self._process.communicate(timeout=timeout) + return stdout.decode('utf-8'), stderr.decode('utf-8') + except subprocess.TimeoutExpired: + self._process.kill() + self._process.communicate() + raise Exception("Command execution timeout") + except Exception as e: + raise Exception("Command execution failed: %s" % e) + +def scan_files_path(source_file_path): + # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] + scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] + scan_skip_file_list = ["/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", + "/test/", "contrib", "debug", "deps", "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + for root, dirs, files in os.walk(source_file_path): + for file in files: + if any(item in root for item in scan_dir_list): + file_path = os.path.join(root, file) + if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list): + all_file_path.append(file_path) + logger.info("Found %s files" % len(all_file_path)) + +def input_files(change_files): + # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] + scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] + scan_skip_file_list = [f"{source_path}/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", f"{source_path}/TDinternal/community/source/libs/parser/src/sql.c", f"{source_path}/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + with open(change_files, 'r') as file: + for line in file: + file_name = line.strip() + if any(dir_name in file_name for dir_name in scan_dir_list): + if (file_name.endswith(".c") or file_name.endswith(".h") or line.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list): + if "enterprise" in file_name: + file_name = os.path.join(source_path, file_name) + else: + tdc_file_path = os.path.join(source_path, "community/") + file_name = os.path.join(tdc_file_path, file_name) + all_file_path.append(file_name) + print(f"all_file_path:{all_file_path}") + # for file_path in change_files: + # if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list): + # all_file_path.append(file_path) + logger.info("Found %s files" % len(all_file_path)) +file_res_path = "" + +def save_scan_res(res_base_path, file_path, out, err): + global file_res_path + file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".res") + print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}") + if not os.path.exists(os.path.dirname(file_res_path)): + os.makedirs(os.path.dirname(file_res_path)) + logger.info("Save scan result to: %s" % file_res_path) + + # save scan result + with open(file_res_path, "w") as f: + f.write(err) + f.write(out) + logger.debug(f"file_res_file: {file_res_path}") + +def write_csv(file_path, data): + try: + with open(file_path, 'w') as f: + writer = csv.writer(f) + writer.writerows(data) + except Exception as ex: + raise Exception("Failed to write the csv file: {} with msg: {}".format(file_path, repr(ex))) + +if __name__ == "__main__": + command_executor = CommandExecutor() + # get all the c files path + # scan_files_path(source_path) + input_files(change_file_list) + print(f"all_file_path:{all_file_path}") + res = [] + res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"]) + # create dir + current_time = datetime.now().strftime("%Y%m%d%H%M%S") + scan_result_path = os.path.join(scan_result_base_path, current_time) + if not os.path.exists(scan_result_path): + os.makedirs(scan_result_path) + for file in all_file_path: + cmd = f"clang-query-10 -p {compile_commands_path} {file} -f {clang_scan_rules_path}" + print(f"cmd:{cmd}") + try: + stdout, stderr = command_executor.execute(cmd) + lines = stdout.split("\n") + if lines[-2].endswith("matches.") or lines[-2].endswith("match."): + match_num = int(lines[-2].split(" ")[0]) + logger.info("The match lines of file %s: %s" % (file, match_num)) + if match_num > 0: + logger.info(f"scan_result_path: {scan_result_path} ,file:{file}") + save_scan_res(scan_result_path, file, stdout, stderr) + res.append([file, file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail']) + else: + logger.warning("The result of scan is invalid for: %s" % file) + except Exception as e: + logger.error("Execute command failed: %s" % e) + # data = "" + # for item in res: + # data += item[0] + "," + str(item[1]) + "\n" + # logger.info("Csv data: %s" % data) + write_csv(os.path.join(scan_result_path, "scan_res.csv"), res) + scan_result_log = f"{scan_result_path}/scan_res.csv" + # delete the first element of res + res= res[1:] + logger.info("The result of scan: \n") + logger.info("Total scan files: %s" % len(res)) + logger.info("Total match lines: %s" % sum([item[2] for item in res])) + logger.info(f"scan log file : {scan_result_log}") + logger.info("Pass files: %s" % len([item for item in res if item[3] == 'Pass'])) + logger.info("Fail files: %s" % len([item for item in res if item[3] == 'Fail'])) + if len([item for item in res if item[3] == 'Fail']) > 0: + logger.error(f"Scan failed,please check the log file:{scan_result_log}") + exit(1) \ No newline at end of file diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 85e3d2ab73..26cabad107 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -83,7 +83,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then From e94374f0fa626e74930d83d51026ec743c283f75 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 14:46:32 +0800 Subject: [PATCH 02/29] test: scan returned values in ci --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index b6275d0f6d..3904a62656 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -62,6 +62,7 @@ def check_docs() { script: ''' cd ${WKC} git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/" || : + mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} ''', returnStdout: true ).trim() @@ -70,7 +71,6 @@ def check_docs() { docs_only=1 } else { echo file_changed - mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} echo file_changed > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt } } From 52ead8ad20acd0022aa0c387715b1429374129f0 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 14:46:43 +0800 Subject: [PATCH 03/29] test: scan returned values in ci --- Jenkinsfile2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 3904a62656..d5a56f09fa 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -70,8 +70,8 @@ def check_docs() { echo "docs PR" docs_only=1 } else { - echo file_changed - echo file_changed > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + echo file_changed + new File("${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt").write(file_changed) } } } From 5190b7c949543625c1f3ac8dd0bce498fc08e08e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 14:57:37 +0800 Subject: [PATCH 04/29] test: scan returned values in ci --- Jenkinsfile2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index d5a56f09fa..4a70a3e80c 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -71,8 +71,12 @@ def check_docs() { docs_only=1 } else { echo file_changed - new File("${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt").write(file_changed) } + script { + sh ''' + echo ${file_changed} > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} + ''' + } } } def pre_test(){ From d82fc89b90c34850ba502bc39c6268d021f3f85f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 15:02:04 +0800 Subject: [PATCH 05/29] test: scan returned values in ci --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 4a70a3e80c..3229ffcbca 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -74,7 +74,7 @@ def check_docs() { } script { sh ''' - echo ${file_changed} > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} + echo ${file_changed} > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' } } From add5be6cea5c9f30a9c8cf31fe1d7a21567d5403 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 15:17:25 +0800 Subject: [PATCH 06/29] test: scan returned values in ci --- Jenkinsfile2 | 8 +++- tests/parallel_test/container_build.sh | 64 +++++++++++++------------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 3229ffcbca..fea43ac978 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -4,6 +4,7 @@ import jenkins.model.CauseOfInterruption docs_only=0 node { } +def file_changed def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -58,7 +59,7 @@ def check_docs() { git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD ''' - def file_changed = sh ( + file_changed = sh ( script: ''' cd ${WKC} git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/" || : @@ -76,7 +77,7 @@ def check_docs() { sh ''' echo ${file_changed} > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' - } + } } } def pre_test(){ @@ -430,6 +431,9 @@ pipeline { cd ${WKC}/tests/parallel_test time ./container_build.sh -w ${WKDIR} -e ''' + sh ''' + echo ${file_changed} > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + ''' sh ''' cd ${WKC}/tests/ci python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 26cabad107..effe8f0d5e 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -103,39 +103,39 @@ fi mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date -docker run \ - -v $REP_MOUNT_PARAM \ - -v /root/.cargo/registry:/root/.cargo/registry \ - -v /root/.cargo/git:/root/.cargo/git \ - -v /root/go/pkg/mod:/root/go/pkg/mod \ - -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.2 \ - -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ - -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ - -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ - -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ - -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ - -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ - -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ - -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ - -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ - -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ - -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ - -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ - -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ - -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ - -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ - -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ - -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ - -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ - -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ - -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ - -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " +# docker run \ +# -v $REP_MOUNT_PARAM \ +# -v /root/.cargo/registry:/root/.cargo/registry \ +# -v /root/.cargo/git:/root/.cargo/git \ +# -v /root/go/pkg/mod:/root/go/pkg/mod \ +# -v /root/.cache/go-build:/root/.cache/go-build \ +# -v /root/.cos-local.1:/root/.cos-local.2 \ +# -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ +# -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ +# -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ +# -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ +# -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ +# -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ +# -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ +# -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ +# -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ +# -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ +# -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ +# -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ +# -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ +# -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ +# -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ +# -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ +# -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ +# -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ +# -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ +# -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ +# -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ +# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ +# -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ +# --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " -mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +# mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan ret=$? exit $ret From 951299efb487f099dcddb25a977e1fa7147f8803 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 15:36:30 +0800 Subject: [PATCH 07/29] test: scan returned values in ci --- Jenkinsfile2 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index fea43ac978..ef9391a5da 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -4,8 +4,6 @@ import jenkins.model.CauseOfInterruption docs_only=0 node { } -def file_changed - def abortPreviousBuilds() { def currentJobName = env.JOB_NAME def currentBuildNumber = env.BUILD_NUMBER.toInteger() @@ -59,7 +57,7 @@ def check_docs() { git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD ''' - file_changed = sh ( + def file_changed = sh ( script: ''' cd ${WKC} git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/" || : @@ -73,9 +71,10 @@ def check_docs() { } else { echo file_changed } + env.FILE_CHANGED = file_changed script { sh ''' - echo ${file_changed} > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + echo ''' + env.FILE_CHANGED + ''' > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' } } From 81ed8c758fc9be1d2087cf3e290f2078c494d3ba Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 15:43:22 +0800 Subject: [PATCH 08/29] test: scan returned values in ci --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index ef9391a5da..8706db3baa 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -74,7 +74,7 @@ def check_docs() { env.FILE_CHANGED = file_changed script { sh ''' - echo ''' + env.FILE_CHANGED + ''' > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + echo " ''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' } } From c73de30afb331edebdbe5fb60643dde9e236b836 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 15:56:29 +0800 Subject: [PATCH 09/29] test: scan returned values in ci --- Jenkinsfile2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 8706db3baa..6d6e444dee 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -74,7 +74,7 @@ def check_docs() { env.FILE_CHANGED = file_changed script { sh ''' - echo " ''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' } } @@ -431,7 +431,8 @@ pipeline { time ./container_build.sh -w ${WKDIR} -e ''' sh ''' - echo ${file_changed} > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} + echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' sh ''' cd ${WKC}/tests/ci From 6826fc5f84f66f9fbac7dcb914f5974a5fd827f7 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 16:50:24 +0800 Subject: [PATCH 10/29] test: scan returned values in ci --- Jenkinsfile2 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 6d6e444dee..fc8d59ba89 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -61,7 +61,6 @@ def check_docs() { script: ''' cd ${WKC} git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/" || : - mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} ''', returnStdout: true ).trim() @@ -72,11 +71,6 @@ def check_docs() { echo file_changed } env.FILE_CHANGED = file_changed - script { - sh ''' - echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt - ''' - } } } def pre_test(){ From 0940299cf5f5835d18d0b56c0397dd9609d3c138 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 17:53:56 +0800 Subject: [PATCH 11/29] test: scan returned values in ci --- tests/ci/scan_file_path.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 92f287e797..91f0adc213 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -53,7 +53,7 @@ else: # log file path -log_file_path = f"{source_path}/../{branch_name}/" +log_file_path = f"{source_path}/../log/{branch_name}/" os.makedirs(log_file_path, exist_ok=True) scan_log_file = f"{log_file_path}/scan.log" @@ -65,8 +65,11 @@ scan_result_base_path = f"{log_file_path}/clang_scan_result/" # the compile commands json file path -# compile_commands_path = f"{source_path}/../debugNoSan/compile_commands.json" -compile_commands_path = f"{source_path}/debug/compile_commands.json" +compile_commands_path = f"{source_path}/../debugNoSan/compile_commands.json" +sed_command = r"sed -i 's/home/var\\lib\\jenkins\\workspace/g' compile_commands.json" +result = subprocess.run(sed_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +logger.debug(f"STDOUT: {result.stdout} STDERR: {result.stderr}") +# compile_commands_path = f"{source_path}/debug/compile_commands.json" # the ast parser rule for c file clang_scan_rules_path = f"{self_path}/filter_for_return_values" @@ -164,6 +167,7 @@ if __name__ == "__main__": if not os.path.exists(scan_result_path): os.makedirs(scan_result_path) for file in all_file_path: + cmd = f"clang-query-10 -p {compile_commands_path} {file} -f {clang_scan_rules_path}" print(f"cmd:{cmd}") try: From 7cf149ff6b2b07adb6e5ff1818b98a5176c36a48 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 18:09:55 +0800 Subject: [PATCH 12/29] test: scan returned values in ci --- tests/ci/scan_file_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 91f0adc213..24a2651058 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -65,7 +65,7 @@ scan_result_base_path = f"{log_file_path}/clang_scan_result/" # the compile commands json file path -compile_commands_path = f"{source_path}/../debugNoSan/compile_commands.json" +compile_commands_path = f"{source_path}/debugNoSan/compile_commands.json" sed_command = r"sed -i 's/home/var\\lib\\jenkins\\workspace/g' compile_commands.json" result = subprocess.run(sed_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) logger.debug(f"STDOUT: {result.stdout} STDERR: {result.stderr}") From a8c99adf8798d6f0c06ca9ac61b6254b716d7710 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 19:30:00 +0800 Subject: [PATCH 13/29] test: scan returned values in ci --- tests/ci/scan_file_path.py | 45 ++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 24a2651058..43935aa5ec 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -27,49 +27,56 @@ for key, value in opts: self_path = os.path.dirname(os.path.realpath(__file__)) if ("community" in self_path): - source_path = self_path[:self_path.find("community")] - work_path = source_path[:source_path.find("TDinternal")] + TD_project_path = self_path[:self_path.find("community")] + work_path = TD_project_path[:TD_project_path.find("TDinternal")] else: - source_path = self_path[:self_path.find("tests")] - work_path = source_path[:source_path.find("TDengine")] + TD_project_path = self_path[:self_path.find("tests")] + work_path = TD_project_path[:TD_project_path.find("TDengine")] # Check if "community" or "tests" is in self_path index_community = self_path.find("community") if index_community != -1: - source_path = self_path[:index_community] - index_TDinternal = source_path.find("TDinternal") + TD_project_path = self_path[:index_community] + index_TDinternal = TD_project_path.find("TDinternal") # Check if index_TDinternal is valid and set work_path accordingly if index_TDinternal != -1: - work_path = source_path[:index_TDinternal] + work_path = TD_project_path[:index_TDinternal] else: index_tests = self_path.find("tests") if index_tests != -1: - source_path = self_path[:index_tests] + TD_project_path = self_path[:index_tests] # Check if index_TDengine is valid and set work_path accordingly - index_TDengine = source_path.find("TDengine") + index_TDengine = TD_project_path.find("TDengine") if index_TDengine != -1: - work_path = source_path[:index_TDengine] + work_path = TD_project_path[:index_TDengine] # log file path -log_file_path = f"{source_path}/../log/{branch_name}/" +log_file_path = f"{work_path}/log/{branch_name}/" os.makedirs(log_file_path, exist_ok=True) scan_log_file = f"{log_file_path}/scan.log" logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG") -print(self_path,work_path,source_path,log_file_path) +print(self_path,work_path,TD_project_path,log_file_path) # scan result base path scan_result_base_path = f"{log_file_path}/clang_scan_result/" # the compile commands json file path -compile_commands_path = f"{source_path}/debugNoSan/compile_commands.json" -sed_command = r"sed -i 's/home/var\\lib\\jenkins\\workspace/g' compile_commands.json" +compile_commands_path = f"{work_path}/debugNoSan/compile_commands.json" +# compile_commands_path = f"{TD_project_path}/debug/compile_commands.json" +print(f"compile_commands_path:{compile_commands_path}") + +# replace the docerk worf path with real work path in compile_commands.json +docker_work_path = "home" +replace_path= work_path[1:-1] +replace_path = replace_path.replace("/", "\/") +sed_command = f"sed -i 's/{docker_work_path}/{replace_path}/g' {compile_commands_path}" +print(sed_command) result = subprocess.run(sed_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) logger.debug(f"STDOUT: {result.stdout} STDERR: {result.stderr}") -# compile_commands_path = f"{source_path}/debug/compile_commands.json" # the ast parser rule for c file clang_scan_rules_path = f"{self_path}/filter_for_return_values" @@ -112,16 +119,16 @@ def scan_files_path(source_file_path): def input_files(change_files): # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] - scan_skip_file_list = [f"{source_path}/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", f"{source_path}/TDinternal/community/source/libs/parser/src/sql.c", f"{source_path}/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + scan_skip_file_list = [f"{TD_project_path}/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", f"{TD_project_path}/TDinternal/community/source/libs/parser/src/sql.c", f"{TD_project_path}/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] with open(change_files, 'r') as file: for line in file: file_name = line.strip() if any(dir_name in file_name for dir_name in scan_dir_list): if (file_name.endswith(".c") or file_name.endswith(".h") or line.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list): if "enterprise" in file_name: - file_name = os.path.join(source_path, file_name) + file_name = os.path.join(TD_project_path, file_name) else: - tdc_file_path = os.path.join(source_path, "community/") + tdc_file_path = os.path.join(TD_project_path, "community/") file_name = os.path.join(tdc_file_path, file_name) all_file_path.append(file_name) print(f"all_file_path:{all_file_path}") @@ -156,7 +163,7 @@ def write_csv(file_path, data): if __name__ == "__main__": command_executor = CommandExecutor() # get all the c files path - # scan_files_path(source_path) + # scan_files_path(TD_project_path) input_files(change_file_list) print(f"all_file_path:{all_file_path}") res = [] From e26b9a9f1b4945a25b7e5eb2b0034bfdd2a66f48 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 20:46:49 +0800 Subject: [PATCH 14/29] test: scan returned values in ci --- Jenkinsfile2 | 15 +++--- tests/ci/scan_file_path.py | 20 ++++---- tests/parallel_test/container_build.sh | 64 +++++++++++++------------- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index fc8d59ba89..95416f4363 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -418,20 +418,23 @@ pipeline { timeout(time: 200, unit: 'MINUTES'){ pre_test() script { - sh ''' - date - rm -rf ${WKC}/debug - cd ${WKC}/tests/parallel_test - time ./container_build.sh -w ${WKDIR} -e - ''' sh ''' mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + sh''' + cd $${WK}/;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1 + ''' sh ''' cd ${WKC}/tests/ci python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + sh ''' + date + rm -rf ${WKC}/debug + cd ${WKC}/tests/parallel_test + time ./container_build.sh -w ${WKDIR} -e + ''' def extra_param = "" def log_server_file = "/home/log_server.json" def timeout_cmd = "" diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 43935aa5ec..258c760391 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -65,18 +65,18 @@ scan_result_base_path = f"{log_file_path}/clang_scan_result/" # the compile commands json file path -compile_commands_path = f"{work_path}/debugNoSan/compile_commands.json" -# compile_commands_path = f"{TD_project_path}/debug/compile_commands.json" +# compile_commands_path = f"{work_path}/debugNoSan/compile_commands.json" +compile_commands_path = f"{TD_project_path}/debug/compile_commands.json" print(f"compile_commands_path:{compile_commands_path}") -# replace the docerk worf path with real work path in compile_commands.json -docker_work_path = "home" -replace_path= work_path[1:-1] -replace_path = replace_path.replace("/", "\/") -sed_command = f"sed -i 's/{docker_work_path}/{replace_path}/g' {compile_commands_path}" -print(sed_command) -result = subprocess.run(sed_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -logger.debug(f"STDOUT: {result.stdout} STDERR: {result.stderr}") +# # replace the docerk worf path with real work path in compile_commands.json +# docker_work_path = "home" +# replace_path= work_path[1:-1] +# replace_path = replace_path.replace("/", "\/") +# sed_command = f"sed -i 's/{docker_work_path}/{replace_path}/g' {compile_commands_path}" +# print(sed_command) +# result = subprocess.run(sed_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +# logger.debug(f"STDOUT: {result.stdout} STDERR: {result.stderr}") # the ast parser rule for c file clang_scan_rules_path = f"{self_path}/filter_for_return_values" diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index effe8f0d5e..26cabad107 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -103,39 +103,39 @@ fi mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date -# docker run \ -# -v $REP_MOUNT_PARAM \ -# -v /root/.cargo/registry:/root/.cargo/registry \ -# -v /root/.cargo/git:/root/.cargo/git \ -# -v /root/go/pkg/mod:/root/go/pkg/mod \ -# -v /root/.cache/go-build:/root/.cache/go-build \ -# -v /root/.cos-local.1:/root/.cos-local.2 \ -# -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ -# -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ -# -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ -# -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ -# -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -# -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ -# -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -# -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ -# -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -# -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ -# -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ -# -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ -# -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -# -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -# -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -# -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -# -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ -# -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ -# -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ -# -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ -# -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -# -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ -# --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " +docker run \ + -v $REP_MOUNT_PARAM \ + -v /root/.cargo/registry:/root/.cargo/registry \ + -v /root/.cargo/git:/root/.cargo/git \ + -v /root/go/pkg/mod:/root/go/pkg/mod \ + -v /root/.cache/go-build:/root/.cache/go-build \ + -v /root/.cos-local.1:/root/.cos-local.2 \ + -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ + -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ + -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ + -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ + -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ + -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ + -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ + -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ + -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ + -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ + -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ + -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ + -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ + -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ + -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " -# mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan ret=$? exit $ret From 13294c9d17334feda30671f3c874cf671cbeadcb Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 1 Aug 2024 21:06:48 +0800 Subject: [PATCH 15/29] test: scan returned values in ci --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 95416f4363..1b710a967a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -423,7 +423,7 @@ pipeline { echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' sh''' - cd $${WK}/;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1 + cd ${WK} || exit 0 ; rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1 ''' sh ''' cd ${WKC}/tests/ci From 3262b61df0ebc3f6a3081203cb21417da85ade3b Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 00:48:49 +0800 Subject: [PATCH 16/29] test: scan returned values in ci --- Jenkinsfile2 | 11 +-- tests/ci/scan_file_path.py | 39 +++++++--- tests/parallel_test/run_container_scan.sh | 92 +++++++++++++++++++++++ 3 files changed, 125 insertions(+), 17 deletions(-) create mode 100755 tests/parallel_test/run_container_scan.sh diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 1b710a967a..3204fc56fe 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -422,13 +422,6 @@ pipeline { mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' - sh''' - cd ${WK} || exit 0 ; rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1 - ''' - sh ''' - cd ${WKC}/tests/ci - python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt - ''' sh ''' date rm -rf ${WKC}/debug @@ -460,6 +453,10 @@ pipeline { } } } + sh ''' + cd ${WKC}/tests/ci + run_scan_container.sh -d ${WK} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' + ''' sh ''' cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 258c760391..b22adde427 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -6,7 +6,8 @@ from datetime import datetime from loguru import logger import getopt -opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:', [ + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:w:', [ 'branch_name=']) for key, value in opts: if key in ['-h', '--help']: @@ -14,6 +15,7 @@ for key, value in opts: 'Usage: python3 scan.py -b -f ') print('-b branch name or PR ID to scan') print('-f change files list') + print('-w web server') sys.exit(0) @@ -21,18 +23,20 @@ for key, value in opts: branch_name = value if key in ['-f', '--filesName']: change_file_list = value + if key in ['-w', '--webServer']: + web_server = value # the base source code file path self_path = os.path.dirname(os.path.realpath(__file__)) -if ("community" in self_path): - TD_project_path = self_path[:self_path.find("community")] - work_path = TD_project_path[:TD_project_path.find("TDinternal")] +# if ("community" in self_path): +# TD_project_path = self_path[:self_path.find("community")] +# work_path = TD_project_path[:TD_project_path.find("TDinternal")] -else: - TD_project_path = self_path[:self_path.find("tests")] - work_path = TD_project_path[:TD_project_path.find("TDengine")] +# else: +# TD_project_path = self_path[:self_path.find("tests")] +# work_path = TD_project_path[:TD_project_path.find("TDengine")] # Check if "community" or "tests" is in self_path index_community = self_path.find("community") @@ -53,12 +57,16 @@ else: # log file path -log_file_path = f"{work_path}/log/{branch_name}/" +log_file_path = f"{work_path}/{branch_name}/" os.makedirs(log_file_path, exist_ok=True) scan_log_file = f"{log_file_path}/scan.log" logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG") -print(self_path,work_path,TD_project_path,log_file_path) +# logging.basicConfig(level=logging.INFO, +# format='%(asctime)s | %(levelname)s | %(name)s:%(lineno)d - %(message)s') +# logger = logging.getLogger(__name__) + +print(self_path,work_path,TD_project_path,log_file_path,change_file_list) # scan result base path scan_result_base_path = f"{log_file_path}/clang_scan_result/" @@ -81,6 +89,7 @@ print(f"compile_commands_path:{compile_commands_path}") # the ast parser rule for c file clang_scan_rules_path = f"{self_path}/filter_for_return_values" +# # all the c files path will be checked all_file_path = [] @@ -140,7 +149,7 @@ file_res_path = "" def save_scan_res(res_base_path, file_path, out, err): global file_res_path - file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".res") + file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".txt") print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}") if not os.path.exists(os.path.dirname(file_res_path)): os.makedirs(os.path.dirname(file_res_path)) @@ -167,6 +176,7 @@ if __name__ == "__main__": input_files(change_file_list) print(f"all_file_path:{all_file_path}") res = [] + web_path = [] res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"]) # create dir current_time = datetime.now().strftime("%Y%m%d%H%M%S") @@ -179,6 +189,7 @@ if __name__ == "__main__": print(f"cmd:{cmd}") try: stdout, stderr = command_executor.execute(cmd) + print(stderr) lines = stdout.split("\n") if lines[-2].endswith("matches.") or lines[-2].endswith("match."): match_num = int(lines[-2].split(" ")[0]) @@ -186,7 +197,13 @@ if __name__ == "__main__": if match_num > 0: logger.info(f"scan_result_path: {scan_result_path} ,file:{file}") save_scan_res(scan_result_path, file, stdout, stderr) + index_tests = file_res_path.find(branch_name) + if index_tests != -1: + web_path_file = file_res_path[index_tests:] + web_path_file = os.path.join(web_server, web_path_file) + web_path.append(web_path_file) res.append([file, file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail']) + else: logger.warning("The result of scan is invalid for: %s" % file) except Exception as e: @@ -207,4 +224,6 @@ if __name__ == "__main__": logger.info("Fail files: %s" % len([item for item in res if item[3] == 'Fail'])) if len([item for item in res if item[3] == 'Fail']) > 0: logger.error(f"Scan failed,please check the log file:{scan_result_log}") + for index, failed_result_file in enumerate(web_path): + logger.error(f"failed number: {index}, failed_result_file: {failed_result_file}") exit(1) \ No newline at end of file diff --git a/tests/parallel_test/run_container_scan.sh b/tests/parallel_test/run_container_scan.sh new file mode 100755 index 0000000000..078757a57a --- /dev/null +++ b/tests/parallel_test/run_container_scan.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -d work dir" + echo -e "\t -b pr and id" + echo -e "\t -w web server " + echo -e "\t -f scan file " + echo -e "\t -h help" +} + +while getopts "d:b:w:f:h" opt; do + case $opt in + d) + WORKDIR=$OPTARG + ;; + b) + branch_name_id=$OPTARG + ;; + f) + scan_file_name=$OPTARG + ;; + w) + web_server=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$branch_name_id" ]; then + usage + exit 1 +fi + +if [ -z "$scan_file_name" ]; then + usage + exit 1 +fi +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi +if [ -z "$web_server" ]; then + usage + exit 1 +fi + + # enterprise edition +INTERNAL_REPDIR=$WORKDIR/TDinternal +REPDIR_DEBUG=$WORKDIR/debugNoSan/ + +REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/" +REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" + +CONTAINER_TESTDIR=/home/TDinternal/community + +#scan file log path +scan_temp="$WORKDIR/log/${branch_name_id}/" +docker_scan_temp="/home/${branch_name_id}/" +mkdir -p $scan_temp +mkdir -p $docker_scan_temp + + +scan_scripts="$CONTAINER_TESTDIR/tests/ci/scan_file_path.py" +scan_file_name="$docker_scan_temp/docs_changed.txt" + +ulimit -c unlimited +cat << EOF +docker run \ + -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_DEBUG \ + -v $scan_temp:$docker_scan_temp \ + --rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server} +EOF +docker run \ + -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_DEBUG \ + -v $scan_temp:$docker_scan_temp \ + --rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server} + + +ret=$? +exit $ret + From 7065daab70371ff1a2a2ed1acb66a025f098823b Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 01:02:49 +0800 Subject: [PATCH 17/29] test: scan returned values in ci --- tests/parallel_test/container_build.sh | 64 +++++++++---------- ...ontainer_scan.sh => run_scan_container.sh} | 0 2 files changed, 32 insertions(+), 32 deletions(-) rename tests/parallel_test/{run_container_scan.sh => run_scan_container.sh} (100%) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 26cabad107..effe8f0d5e 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -103,39 +103,39 @@ fi mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date -docker run \ - -v $REP_MOUNT_PARAM \ - -v /root/.cargo/registry:/root/.cargo/registry \ - -v /root/.cargo/git:/root/.cargo/git \ - -v /root/go/pkg/mod:/root/go/pkg/mod \ - -v /root/.cache/go-build:/root/.cache/go-build \ - -v /root/.cos-local.1:/root/.cos-local.2 \ - -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ - -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ - -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ - -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ - -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ - -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ - -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ - -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ - -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ - -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ - -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ - -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ - -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ - -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ - -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ - -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ - -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ - -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ - -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ - -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ - -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " +# docker run \ +# -v $REP_MOUNT_PARAM \ +# -v /root/.cargo/registry:/root/.cargo/registry \ +# -v /root/.cargo/git:/root/.cargo/git \ +# -v /root/go/pkg/mod:/root/go/pkg/mod \ +# -v /root/.cache/go-build:/root/.cache/go-build \ +# -v /root/.cos-local.1:/root/.cos-local.2 \ +# -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ +# -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ +# -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ +# -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ +# -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ +# -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ +# -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ +# -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ +# -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ +# -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ +# -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ +# -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ +# -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ +# -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ +# -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ +# -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ +# -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ +# -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ +# -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ +# -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ +# -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ +# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ +# -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ +# --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " -mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +# mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan ret=$? exit $ret diff --git a/tests/parallel_test/run_container_scan.sh b/tests/parallel_test/run_scan_container.sh similarity index 100% rename from tests/parallel_test/run_container_scan.sh rename to tests/parallel_test/run_scan_container.sh From f81cc9353247654f75f285263dafa6fa05a83846 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 01:12:47 +0800 Subject: [PATCH 18/29] test: scan returned values in ci --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 3204fc56fe..5a19ad5441 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -454,7 +454,7 @@ pipeline { } } sh ''' - cd ${WKC}/tests/ci + cd ${WKC}/tests/parallel_test run_scan_container.sh -d ${WK} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' ''' sh ''' From 744de4f305516c9cee8e6491e627ce9eedee3fd8 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 01:36:11 +0800 Subject: [PATCH 19/29] test: scan returned values in ci --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 5a19ad5441..4636702340 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -455,7 +455,7 @@ pipeline { } sh ''' cd ${WKC}/tests/parallel_test - run_scan_container.sh -d ${WK} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' + run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' ''' sh ''' cd ${WKC}/tests/parallel_test From cc74abad596da282e78b7b6fde13d9063f53375a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 01:53:22 +0800 Subject: [PATCH 20/29] test: scan returned values in ci --- Jenkinsfile2 | 2 +- tests/ci/filter_for_return_values | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 4636702340..15e311b41e 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -455,7 +455,7 @@ pipeline { } sh ''' cd ${WKC}/tests/parallel_test - run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' + ./run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' ''' sh ''' cd ${WKC}/tests/parallel_test diff --git a/tests/ci/filter_for_return_values b/tests/ci/filter_for_return_values index 734619a8af..050ceda29b 100644 --- a/tests/ci/filter_for_return_values +++ b/tests/ci/filter_for_return_values @@ -4,7 +4,6 @@ match callExpr( doStmt(hasCondition(expr().bind("cond")))) ), unless(hasType(voidType())), - unless(callee(functionDecl(hasName("memcpy")))), unless(callee(functionDecl(hasName("strcpy")))), unless(callee(functionDecl(hasName("strcat")))), unless(callee(functionDecl(hasName("strncpy")))), From 5da0d33bf20935e177661c1ac6e345c57cb2523d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 02:12:46 +0800 Subject: [PATCH 21/29] test: scan returned values in ci --- Jenkinsfile2 | 2 +- tests/ci/filter_for_return_values | 1 + tests/parallel_test/container_build.sh | 64 +++++++++++++------------- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 15e311b41e..8d4652943d 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -400,7 +400,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 "} + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() diff --git a/tests/ci/filter_for_return_values b/tests/ci/filter_for_return_values index 050ceda29b..734619a8af 100644 --- a/tests/ci/filter_for_return_values +++ b/tests/ci/filter_for_return_values @@ -4,6 +4,7 @@ match callExpr( doStmt(hasCondition(expr().bind("cond")))) ), unless(hasType(voidType())), + unless(callee(functionDecl(hasName("memcpy")))), unless(callee(functionDecl(hasName("strcpy")))), unless(callee(functionDecl(hasName("strcat")))), unless(callee(functionDecl(hasName("strncpy")))), diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index effe8f0d5e..26cabad107 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -103,39 +103,39 @@ fi mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date -# docker run \ -# -v $REP_MOUNT_PARAM \ -# -v /root/.cargo/registry:/root/.cargo/registry \ -# -v /root/.cargo/git:/root/.cargo/git \ -# -v /root/go/pkg/mod:/root/go/pkg/mod \ -# -v /root/.cache/go-build:/root/.cache/go-build \ -# -v /root/.cos-local.1:/root/.cos-local.2 \ -# -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ -# -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ -# -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ -# -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ -# -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ -# -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ -# -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -# -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ -# -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -# -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ -# -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ -# -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ -# -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -# -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -# -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -# -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -# -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ -# -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ -# -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ -# -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ -# -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -# -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ -# --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " +docker run \ + -v $REP_MOUNT_PARAM \ + -v /root/.cargo/registry:/root/.cargo/registry \ + -v /root/.cargo/git:/root/.cargo/git \ + -v /root/go/pkg/mod:/root/go/pkg/mod \ + -v /root/.cache/go-build:/root/.cache/go-build \ + -v /root/.cos-local.1:/root/.cos-local.2 \ + -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ + -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ + -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ + -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ + -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ + -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ + -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ + -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ + -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ + -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ + -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ + -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ + -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ + -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ + -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " -# mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan ret=$? exit $ret From cef1920251080a2ae66f28643e9de26f3cf5aa10 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 09:13:33 +0800 Subject: [PATCH 22/29] test: scan returned values in ci --- tests/ci/scan_file_path.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index b22adde427..df4dc16aa4 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -62,11 +62,8 @@ os.makedirs(log_file_path, exist_ok=True) scan_log_file = f"{log_file_path}/scan.log" logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG") -# logging.basicConfig(level=logging.INFO, -# format='%(asctime)s | %(levelname)s | %(name)s:%(lineno)d - %(message)s') -# logger = logging.getLogger(__name__) - -print(self_path,work_path,TD_project_path,log_file_path,change_file_list) +#if error happens, open this to debug +# print(self_path,work_path,TD_project_path,log_file_path,change_file_list) # scan result base path scan_result_base_path = f"{log_file_path}/clang_scan_result/" @@ -75,7 +72,9 @@ scan_result_base_path = f"{log_file_path}/clang_scan_result/" # the compile commands json file path # compile_commands_path = f"{work_path}/debugNoSan/compile_commands.json" compile_commands_path = f"{TD_project_path}/debug/compile_commands.json" -print(f"compile_commands_path:{compile_commands_path}") + +#if error happens, open this to debug +# print(f"compile_commands_path:{compile_commands_path}") # # replace the docerk worf path with real work path in compile_commands.json # docker_work_path = "home" @@ -140,17 +139,14 @@ def input_files(change_files): tdc_file_path = os.path.join(TD_project_path, "community/") file_name = os.path.join(tdc_file_path, file_name) all_file_path.append(file_name) - print(f"all_file_path:{all_file_path}") - # for file_path in change_files: - # if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list): - # all_file_path.append(file_path) + # print(f"all_file_path:{all_file_path}") logger.info("Found %s files" % len(all_file_path)) file_res_path = "" def save_scan_res(res_base_path, file_path, out, err): global file_res_path file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".txt") - print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}") + # print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}") if not os.path.exists(os.path.dirname(file_res_path)): os.makedirs(os.path.dirname(file_res_path)) logger.info("Save scan result to: %s" % file_res_path) @@ -174,7 +170,7 @@ if __name__ == "__main__": # get all the c files path # scan_files_path(TD_project_path) input_files(change_file_list) - print(f"all_file_path:{all_file_path}") + # print(f"all_file_path:{all_file_path}") res = [] web_path = [] res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"]) @@ -189,7 +185,8 @@ if __name__ == "__main__": print(f"cmd:{cmd}") try: stdout, stderr = command_executor.execute(cmd) - print(stderr) + #if "error" in stderr: + # print(stderr) lines = stdout.split("\n") if lines[-2].endswith("matches.") or lines[-2].endswith("match."): match_num = int(lines[-2].split(" ")[0]) From ce006e0b91c607fb33fee67cf31c4b6ddfa550e0 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 10:30:49 +0800 Subject: [PATCH 23/29] test: scan returned values in ci --- source/libs/parser/src/parAstCreater.c | 2 -- tests/ci/scan_file_path.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a8979bbe1d..cd7cda01e0 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -207,8 +207,6 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { return false; } return true; - - } static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) { diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index df4dc16aa4..6058cbb72f 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -182,7 +182,7 @@ if __name__ == "__main__": for file in all_file_path: cmd = f"clang-query-10 -p {compile_commands_path} {file} -f {clang_scan_rules_path}" - print(f"cmd:{cmd}") + logger.debug(f"cmd:{cmd}") try: stdout, stderr = command_executor.execute(cmd) #if "error" in stderr: From 361d3634ffa85d2c5e5c6ad73568bc6278e3bdd1 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 13:21:46 +0800 Subject: [PATCH 24/29] test: scan returned values in ci --- source/libs/parser/src/parAstCreater.c | 1 + source/libs/parser/src/parAstParser.c | 2 +- tests/ci/scan.py | 106 ++++++++++++++++++++++ tests/ci/scan_file_path.py | 9 +- tests/parallel_test/run_scan_container.sh | 22 +++-- 5 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 tests/ci/scan.py diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index cd7cda01e0..f8b3cebaa9 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -207,6 +207,7 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { return false; } return true; + } static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 49be0b8d90..f0ecc14588 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -725,7 +725,7 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC strcpy(name.dbname, pStmt->dbName); strcpy(name.tname, pStmt->viewName); char dbFName[TSDB_DB_FNAME_LEN]; - (void)tNameGetFullDbName(&name, dbFName); + tNameGetFullDbName(&name, dbFName); int32_t code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->viewName, 0); if (TSDB_CODE_SUCCESS == code) { code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, diff --git a/tests/ci/scan.py b/tests/ci/scan.py new file mode 100644 index 0000000000..c542fdf448 --- /dev/null +++ b/tests/ci/scan.py @@ -0,0 +1,106 @@ +import os +import subprocess +import csv +from datetime import datetime +from loguru import logger + +# log file path +log_file_path = "/root/charles/scan.log" +logger.add(log_file_path, rotation="10MB", retention="7 days", level="DEBUG") +# scan result base path +scan_result_base_path = "/root/charles/clang_scan_result/" +# the base source code file path +source_path = "/root/charles/TDinternal/" +# the compile commands json file path +compile_commands_path = "/root/charles/TDinternal/debug/compile_commands.json" +# the ast parser rule for c file +clang_scan_rules_path = "/root/charles/clang_scan_rules" +# all the c files path will be checked +all_file_path = [] + +class CommandExecutor: + def __init__(self): + self._process = None + + def execute(self, command, timeout=None): + try: + self._process = subprocess.Popen(command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = self._process.communicate(timeout=timeout) + return stdout.decode('utf-8'), stderr.decode('utf-8') + except subprocess.TimeoutExpired: + self._process.kill() + self._process.communicate() + raise Exception("Command execution timeout") + except Exception as e: + raise Exception("Command execution failed: %s" % e) + +def scan_files_path(source_file_path): + # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"] + scan_dir_list = ["source", "include", "docs/examples", "src/plugins"] + scan_skip_file_list = ["/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", + "/test/", "contrib", "debug", "deps", "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"] + for root, dirs, files in os.walk(source_file_path): + for file in files: + if any(item in root for item in scan_dir_list): + file_path = os.path.join(root, file) + if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list): + all_file_path.append(file_path) + logger.info("Found %s files" % len(all_file_path)) + +def save_scan_res(res_base_path, file_path, out, err): + file_res_path = os.path.join(res_base_path, file_path.replace("/root/charles/", "").split(".")[0] + ".res") + if not os.path.exists(os.path.dirname(file_res_path)): + os.makedirs(os.path.dirname(file_res_path)) + logger.info("Save scan result to: %s" % file_res_path) + # save scan result + with open(file_res_path, "w") as f: + f.write(out) + f.write(err) + +def write_csv(file_path, data): + try: + with open(file_path, 'w') as f: + writer = csv.writer(f) + writer.writerows(data) + except Exception as ex: + raise Exception("Failed to write the csv file: {} with msg: {}".format(file_path, repr(ex))) + +if __name__ == "__main__": + command_executor = CommandExecutor() + # get all the c files path + scan_files_path(source_path) + res = [] + # create dir + current_time = datetime.now().strftime("%Y%m%d%H%M%S") + scan_result_path = os.path.join(scan_result_base_path, current_time) + if not os.path.exists(scan_result_path): + os.makedirs(scan_result_path) + for file in all_file_path: + cmd = "clang-query -p %s %s -f %s" % (compile_commands_path, file, clang_scan_rules_path) + try: + stdout, stderr = command_executor.execute(cmd) + lines = stdout.split("\n") + if lines[-2].endswith("matches.") or lines[-2].endswith("match."): + match_num = int(lines[-2].split(" ")[0]) + logger.info("The match lines of file %s: %s" % (file, match_num)) + if match_num > 0: + save_scan_res(scan_result_path, file, stdout, stderr) + res.append([file, match_num, 'Pass' if match_num == 0 else 'Fail']) + else: + logger.warning("The result of scan is invalid for: %s" % file) + except Exception as e: + logger.error("Execute command failed: %s" % e) + # data = "" + # for item in res: + # data += item[0] + "," + str(item[1]) + "\n" + # logger.info("Csv data: %s" % data) + write_csv(os.path.join(scan_result_path, "scan_res.csv"), res) + logger.info("The result of scan: \n") + logger.info("Total files: %s" % len(res)) + logger.info("Total match lines: %s" % sum([item[1] for item in res])) + logger.info("Pass files: %s" % len([item for item in res if item[2] == 'Pass'])) + logger.info("Fail files: %s" % len([item for item in res if item[2] == 'Fail'])) + diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 6058cbb72f..0e47387f67 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -57,7 +57,9 @@ else: # log file path -log_file_path = f"{work_path}/{branch_name}/" +current_time = datetime.now().strftime("%Y%m%d-%H%M%S") +log_file_path = f"{work_path}/scan_log/scan_{branch_name}_{current_time}/" + os.makedirs(log_file_path, exist_ok=True) scan_log_file = f"{log_file_path}/scan.log" @@ -175,8 +177,9 @@ if __name__ == "__main__": web_path = [] res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"]) # create dir - current_time = datetime.now().strftime("%Y%m%d%H%M%S") - scan_result_path = os.path.join(scan_result_base_path, current_time) + # current_time = datetime.now().strftime("%Y%m%d%H%M%S") + # scan_result_path = os.path.join(scan_result_base_path, current_time) + scan_result_path = scan_result_base_path if not os.path.exists(scan_result_path): os.makedirs(scan_result_path) for file in all_file_path: diff --git a/tests/parallel_test/run_scan_container.sh b/tests/parallel_test/run_scan_container.sh index 078757a57a..3d6d1d7c30 100755 --- a/tests/parallel_test/run_scan_container.sh +++ b/tests/parallel_test/run_scan_container.sh @@ -62,28 +62,34 @@ REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" CONTAINER_TESTDIR=/home/TDinternal/community -#scan file log path -scan_temp="$WORKDIR/log/${branch_name_id}/" -docker_scan_temp="/home/${branch_name_id}/" -mkdir -p $scan_temp -mkdir -p $docker_scan_temp +#scan change file path +scan_changefile_temp_path="$WORKDIR/tmp/${branch_name_id}/" +docker_can_changefile_temp_path="/home/tmp/${branch_name_id}/" +mkdir -p $scan_changefile_temp_path +scan_file_name="$docker_can_changefile_temp_path/docs_changed.txt" + +#scan log file path +scan_log_temp_path="$WORKDIR/scan_log/" +docker_scan_log_temp_path="/home/scan_log/" +mkdir -p $scan_log_temp_path scan_scripts="$CONTAINER_TESTDIR/tests/ci/scan_file_path.py" -scan_file_name="$docker_scan_temp/docs_changed.txt" ulimit -c unlimited cat << EOF docker run \ -v $REP_MOUNT_PARAM \ -v $REP_MOUNT_DEBUG \ - -v $scan_temp:$docker_scan_temp \ + -v $scan_changefile_temp_path:$docker_can_changefile_temp_path \ + -v $scan_log_temp_path:$docker_scan_log_temp_path \ --rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server} EOF docker run \ -v $REP_MOUNT_PARAM \ -v $REP_MOUNT_DEBUG \ - -v $scan_temp:$docker_scan_temp \ + -v $scan_changefile_temp_path:$docker_can_changefile_temp_path \ + -v $scan_log_temp_path:$docker_scan_log_temp_path \ --rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server} From 156b3d992a4acf6dd47d1c4847346ef1ccd2b937 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 13:22:10 +0800 Subject: [PATCH 25/29] test: scan returned values in ci --- Jenkinsfile2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 8d4652943d..de96a23d51 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -419,8 +419,8 @@ pipeline { pre_test() script { sh ''' - mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID} - echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt + mkdir -p ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID} + echo "''' + env.FILE_CHANGED + '''" > ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' sh ''' date @@ -455,7 +455,7 @@ pipeline { } sh ''' cd ${WKC}/tests/parallel_test - ./run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' + ./run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' ''' sh ''' cd ${WKC}/tests/parallel_test From ed1e3c304a627a3da0257691dda8ee7c18cc4808 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 14:03:07 +0800 Subject: [PATCH 26/29] test: scan returned values in ci --- tests/ci/scan_file_path.py | 18 +++++++++--------- tests/script/tsim/mnode/basic3.sim | 6 +++--- tests/system-test/2-query/tsma.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 0e47387f67..f4cf25b8f7 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -179,11 +179,11 @@ if __name__ == "__main__": # create dir # current_time = datetime.now().strftime("%Y%m%d%H%M%S") # scan_result_path = os.path.join(scan_result_base_path, current_time) - scan_result_path = scan_result_base_path - if not os.path.exists(scan_result_path): - os.makedirs(scan_result_path) - for file in all_file_path: + # scan_result_path = scan_result_base_path + # if not os.path.exists(scan_result_path): + # os.makedirs(scan_result_path) + for file in all_file_path: cmd = f"clang-query-10 -p {compile_commands_path} {file} -f {clang_scan_rules_path}" logger.debug(f"cmd:{cmd}") try: @@ -195,9 +195,9 @@ if __name__ == "__main__": match_num = int(lines[-2].split(" ")[0]) logger.info("The match lines of file %s: %s" % (file, match_num)) if match_num > 0: - logger.info(f"scan_result_path: {scan_result_path} ,file:{file}") - save_scan_res(scan_result_path, file, stdout, stderr) - index_tests = file_res_path.find(branch_name) + logger.info(f"log_file_path: {log_file_path} ,file:{file}") + save_scan_res(log_file_path, file, stdout, stderr) + index_tests = file_res_path.find("scan_log") if index_tests != -1: web_path_file = file_res_path[index_tests:] web_path_file = os.path.join(web_server, web_path_file) @@ -212,8 +212,8 @@ if __name__ == "__main__": # for item in res: # data += item[0] + "," + str(item[1]) + "\n" # logger.info("Csv data: %s" % data) - write_csv(os.path.join(scan_result_path, "scan_res.csv"), res) - scan_result_log = f"{scan_result_path}/scan_res.csv" + write_csv(os.path.join(log_file_path, "scan_res.csv"), res) + scan_result_log = f"{log_file_path}/scan_res.csv" # delete the first element of res res= res[1:] logger.info("The result of scan: \n") diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index 02650ba10d..ff7c44b67d 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -36,7 +36,7 @@ if $data(3)[4] != ready then goto step1 endi -print =============== step2: create mnode 2 +print =============== step2: create mnode 2 3 sql create mnode on dnode 2 sql create mnode on dnode 3 sql_error create mnode on dnode 4 @@ -115,7 +115,7 @@ if $data(3)[4] != ready then goto step41 endi -print =============== step5: stop dnode1 +print =============== step5: stop dnode2 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s stop @@ -154,7 +154,7 @@ if $data(3)[4] != ready then goto step51 endi -print =============== step6: stop dnode1 +print =============== step6: stop dnode3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s stop diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index 29a7562b45..fccf6291b5 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -1504,9 +1504,9 @@ class TDTestCase: # max number of list is 4093: 4096 - 3 - 2(原始表tag个数) - 1(tbname) tdSql.execute('use db4096') - self.create_tsma('tsma_4050', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4050), '5m',check_tsma_calculation=False) + self.create_tsma('tsma_4050', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4050), '5m',check_tsma_calculation=True) - self.create_tsma('tsma_4090', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4090), '6m',check_tsma_calculation=False) + self.create_tsma('tsma_4090', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4090), '6m',check_tsma_calculation=True) self.create_error_tsma('tsma_4091', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4091), '5m', -2147473856) #Too many columns From 6f5d893cf668b114a68eeb53034aee886d361c95 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 14:44:08 +0800 Subject: [PATCH 27/29] test: scan returned values in ci --- tests/parallel_test/run_scan_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/run_scan_container.sh b/tests/parallel_test/run_scan_container.sh index 3d6d1d7c30..d16d1c3017 100755 --- a/tests/parallel_test/run_scan_container.sh +++ b/tests/parallel_test/run_scan_container.sh @@ -69,7 +69,7 @@ mkdir -p $scan_changefile_temp_path scan_file_name="$docker_can_changefile_temp_path/docs_changed.txt" #scan log file path -scan_log_temp_path="$WORKDIR/scan_log/" +scan_log_temp_path="$WORKDIR/log/scan_log/" docker_scan_log_temp_path="/home/scan_log/" mkdir -p $scan_log_temp_path From 791c0c878fd8c463295c37fd40fff20af70f0fc7 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 15:04:56 +0800 Subject: [PATCH 28/29] test: scan returned values in ci --- source/libs/parser/src/parAstCreater.c | 1 - source/libs/parser/src/parAstParser.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index f8b3cebaa9..cd7cda01e0 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -207,7 +207,6 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { return false; } return true; - } static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index f0ecc14588..49be0b8d90 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -725,7 +725,7 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC strcpy(name.dbname, pStmt->dbName); strcpy(name.tname, pStmt->viewName); char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(&name, dbFName); + (void)tNameGetFullDbName(&name, dbFName); int32_t code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->viewName, 0); if (TSDB_CODE_SUCCESS == code) { code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, From 9e30aef1921dd05100e2226d96858e23c61ccd2a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 2 Aug 2024 16:54:54 +0800 Subject: [PATCH 29/29] test: scan returned values in ci --- Jenkinsfile2 | 6 +++--- tests/ci/scan_file_path.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index de96a23d51..a4f765c9fb 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -137,7 +137,7 @@ def pre_test(){ ''' } else { sh ''' - echo "unmatched reposiotry ${CHANGE_URL}" + echo "unmatched repository ${CHANGE_URL}" ''' } sh ''' @@ -247,7 +247,7 @@ def pre_test_win(){ ''' } else { bat ''' - echo "unmatched reposiotry %CHANGE_URL%" + echo "unmatched repository %CHANGE_URL%" ''' } } @@ -309,7 +309,7 @@ def pre_test_build_win() { python -m pip install taospy==2.7.13 python -m pip uninstall taos-ws-py -y python -m pip install taos-ws-py==0.3.1 - xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' return 1 } diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index f4cf25b8f7..03f2d6ee4f 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -62,7 +62,7 @@ log_file_path = f"{work_path}/scan_log/scan_{branch_name}_{current_time}/" os.makedirs(log_file_path, exist_ok=True) -scan_log_file = f"{log_file_path}/scan.log" +scan_log_file = f"{log_file_path}/scan_log.txt" logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG") #if error happens, open this to debug # print(self_path,work_path,TD_project_path,log_file_path,change_file_list) @@ -212,8 +212,8 @@ if __name__ == "__main__": # for item in res: # data += item[0] + "," + str(item[1]) + "\n" # logger.info("Csv data: %s" % data) - write_csv(os.path.join(log_file_path, "scan_res.csv"), res) - scan_result_log = f"{log_file_path}/scan_res.csv" + write_csv(os.path.join(log_file_path, "scan_res.txt"), res) + scan_result_log = f"{log_file_path}/scan_res.txt" # delete the first element of res res= res[1:] logger.info("The result of scan: \n")