test: scan returned values in ci

Jenkinsfile2

@@ -70,6 +70,8 @@ def check_docs() {
             docs_only=1
         } else {
             echo file_changed 
+            mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}
+            echo file_changed > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt
         }
     }
 }
@@ -350,7 +352,6 @@ pipeline {
             when {
                 allOf {
                     not { expression { env.CHANGE_BRANCH =~ /docs\// }}
-                    not { expression { env.CHANGE_URL =~ /\/TDinternal\// }}
                 }
             }
             parallel {
@@ -401,7 +402,7 @@ pipeline {
                     }
                 }
                 stage('linux test') {
-                    agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "}
+                    agent{label "slave1_47 "}
                     options { skipDefaultCheckout() } 
                     when {
                         changeRequest()
@@ -425,6 +426,10 @@ pipeline {
                                         cd ${WKC}/tests/parallel_test
                                         time ./container_build.sh -w ${WKDIR} -e
                                     '''
+                                    sh '''
+                                       cd ${WKC}/tests/ci
+                                       python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID}  -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt
+                                    '''
                                     def extra_param = ""
                                     def log_server_file = "/home/log_server.json"
                                     def timeout_cmd = ""

source/libs/parser/src/parAstCreater.c

@@ -207,6 +207,8 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) {
     return false;
   }
   return true;
+
+
 }
 
 static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) {

tests/ci/filter_for_return_values

@@ -0,0 +1,24 @@
+match callExpr(                                                                                                                                                                                                                                         
+  hasParent(anyOf(                                                                                                                                                                                                                                       
+      compoundStmt(),                                                                                                                                                                                                                                     
+      doStmt(hasCondition(expr().bind("cond"))))                                                                                                                                                                                                          
+  ),                                                                                                                                                                                                                                                     
+  unless(hasType(voidType())),                                                                                                                                                                                                                           
+  unless(callee(functionDecl(hasName("memcpy")))),
+  unless(callee(functionDecl(hasName("strcpy")))), 
+  unless(callee(functionDecl(hasName("strcat")))), 
+  unless(callee(functionDecl(hasName("strncpy")))), 
+  unless(callee(functionDecl(hasName("memset")))), 
+  unless(callee(functionDecl(hasName("memmove")))), 
+  unless(callee(functionDecl(hasName("sprintf")))), 
+  unless(callee(functionDecl(hasName("snprintf")))), 
+  unless(callee(functionDecl(hasName("scanf")))), 
+  unless(callee(functionDecl(hasName("sncanf")))),                                                                                                                                                                                    
+  unless(callee(functionDecl(hasName("printf")))),                                                                                                                                                                  
+  unless(callee(functionDecl(hasName("printRow")))),                                                                                                                                                                                                      
+  unless(callee(functionDecl(hasName("puts")))),                                                                                                                                                                                                          
+  unless(callee(functionDecl(hasName("sleep")))),                                                                                                                                                                                                         
+  unless(callee(functionDecl(hasName("printResult")))),                                                                                                                                                                                                   
+  unless(callee(functionDecl(hasName("getchar")))),                                                                                                                                                                                                       
+  unless(callee(functionDecl(hasName("taos_print_row")))),                                                                                                                                                                                                
+  unless(callee(functionDecl(hasName("fprintf")))))

tests/ci/scan_file_path.py

@@ -0,0 +1,199 @@
+import os
+import sys
+import subprocess
+import csv
+from datetime import datetime
+from loguru import logger
+import getopt
+
+opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:', [
+    'branch_name='])
+for key, value in opts:
+    if key in ['-h', '--help']:
+        print(
+            'Usage: python3 scan.py -b <branch_name> -f <file_list>')
+        print('-b  branch name or PR ID to scan')
+        print('-f  change files list')
+
+        sys.exit(0)
+
+    if key in ['-b', '--branchName']:
+        branch_name = value
+    if key in ['-f', '--filesName']:
+        change_file_list = value
+
+
+# the base source code file path
+self_path =  os.path.dirname(os.path.realpath(__file__))
+
+if ("community" in self_path):
+    source_path = self_path[:self_path.find("community")]
+    work_path = source_path[:source_path.find("TDinternal")]
+
+else:
+    source_path = self_path[:self_path.find("tests")]
+    work_path = source_path[:source_path.find("TDengine")]
+
+# Check if "community"  or "tests" is in self_path
+index_community = self_path.find("community")
+if index_community != -1:
+    source_path = self_path[:index_community]
+    index_TDinternal = source_path.find("TDinternal")
+    # Check if index_TDinternal is valid and set work_path accordingly
+    if index_TDinternal != -1:
+        work_path = source_path[:index_TDinternal]
+else:
+    index_tests = self_path.find("tests")
+    if index_tests != -1:
+        source_path = self_path[:index_tests]
+    # Check if index_TDengine is valid and set work_path accordingly
+        index_TDengine = source_path.find("TDengine")
+        if index_TDengine != -1:
+            work_path = source_path[:index_TDengine]
+
+
+# log file path
+log_file_path = f"{source_path}/../{branch_name}/"
+os.makedirs(log_file_path, exist_ok=True)
+
+scan_log_file = f"{log_file_path}/scan.log"
+logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG")
+print(self_path,work_path,source_path,log_file_path)
+
+# scan result base path
+scan_result_base_path = f"{log_file_path}/clang_scan_result/"
+
+
+# the compile commands json file path
+# compile_commands_path = f"{source_path}/../debugNoSan/compile_commands.json"
+compile_commands_path = f"{source_path}/debug/compile_commands.json"
+
+# the ast parser rule for c file
+clang_scan_rules_path = f"{self_path}/filter_for_return_values"
+
+# all the c files path will be checked
+all_file_path = []
+
+class CommandExecutor:
+    def __init__(self):
+        self._process = None
+
+    def execute(self, command, timeout=None):
+        try:
+            self._process = subprocess.Popen(command,
+                                            shell=True,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.PIPE)
+            stdout, stderr = self._process.communicate(timeout=timeout)
+            return stdout.decode('utf-8'), stderr.decode('utf-8')
+        except subprocess.TimeoutExpired:
+            self._process.kill()
+            self._process.communicate()
+            raise Exception("Command execution timeout")
+        except Exception as e:
+            raise Exception("Command execution failed: %s" % e)
+
+def scan_files_path(source_file_path):
+    # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"]
+    scan_dir_list = ["source", "include", "docs/examples", "src/plugins"]
+    scan_skip_file_list = ["/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl",
+                           "/test/", "contrib", "debug", "deps", "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"]
+    for root, dirs, files in os.walk(source_file_path):
+        for file in files:
+            if any(item in root for item in scan_dir_list):
+                file_path = os.path.join(root, file)
+                if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list):
+                    all_file_path.append(file_path)
+    logger.info("Found %s files" % len(all_file_path))
+
+def input_files(change_files):
+    # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"]
+    scan_dir_list = ["source", "include", "docs/examples", "src/plugins"]
+    scan_skip_file_list = [f"{source_path}/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", f"{source_path}/TDinternal/community/source/libs/parser/src/sql.c", f"{source_path}/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"]
+    with open(change_files, 'r') as file:
+        for line in file:
+            file_name = line.strip()
+            if any(dir_name in file_name for dir_name in scan_dir_list):
+                if (file_name.endswith(".c") or file_name.endswith(".h") or line.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list):
+                    if "enterprise" in file_name:
+                        file_name = os.path.join(source_path, file_name)
+                    else: 
+                        tdc_file_path = os.path.join(source_path, "community/")
+                        file_name = os.path.join(tdc_file_path, file_name)                    
+                    all_file_path.append(file_name)
+                    print(f"all_file_path:{all_file_path}")
+    # for file_path in change_files:
+    #     if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list):
+    #         all_file_path.append(file_path)
+    logger.info("Found %s files" % len(all_file_path))
+file_res_path = ""
+
+def save_scan_res(res_base_path, file_path, out, err):
+    global file_res_path
+    file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".res")
+    print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}")
+    if not os.path.exists(os.path.dirname(file_res_path)):
+        os.makedirs(os.path.dirname(file_res_path))
+    logger.info("Save scan result to: %s" % file_res_path)
+    
+    # save scan result
+    with open(file_res_path, "w") as f:
+        f.write(err)
+        f.write(out)
+    logger.debug(f"file_res_file: {file_res_path}")
+
+def write_csv(file_path, data):
+    try:
+        with open(file_path, 'w') as f:
+            writer = csv.writer(f)
+            writer.writerows(data)
+    except Exception as ex:
+        raise Exception("Failed to write the csv file: {} with msg: {}".format(file_path, repr(ex)))
+
+if __name__ == "__main__":
+    command_executor = CommandExecutor()
+    # get all the c files path
+    # scan_files_path(source_path)
+    input_files(change_file_list)
+    print(f"all_file_path:{all_file_path}")
+    res = []
+    res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"])
+    # create dir
+    current_time = datetime.now().strftime("%Y%m%d%H%M%S")
+    scan_result_path = os.path.join(scan_result_base_path, current_time)
+    if not os.path.exists(scan_result_path):
+        os.makedirs(scan_result_path)
+    for file in all_file_path:
+        cmd = f"clang-query-10 -p {compile_commands_path} {file} -f {clang_scan_rules_path}"
+        print(f"cmd:{cmd}")
+        try:
+            stdout, stderr = command_executor.execute(cmd)
+            lines = stdout.split("\n")
+            if lines[-2].endswith("matches.") or lines[-2].endswith("match."): 
+                match_num = int(lines[-2].split(" ")[0])
+                logger.info("The match lines of file %s: %s" % (file, match_num))
+                if match_num > 0:
+                    logger.info(f"scan_result_path: {scan_result_path} ,file:{file}")
+                    save_scan_res(scan_result_path, file, stdout, stderr)
+                res.append([file, file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail'])
+            else:
+                logger.warning("The result of scan is invalid for: %s" % file)            
+        except Exception as e:
+            logger.error("Execute command failed: %s" % e)
+    # data = ""
+    # for item in res:
+    #     data += item[0] + "," + str(item[1]) + "\n"
+    # logger.info("Csv data: %s" % data)
+    write_csv(os.path.join(scan_result_path, "scan_res.csv"), res)
+    scan_result_log = f"{scan_result_path}/scan_res.csv"
+    # delete the first element of res
+    res= res[1:]
+    logger.info("The result of scan: \n")
+    logger.info("Total scan files: %s" % len(res))
+    logger.info("Total match lines: %s" % sum([item[2] for item in res]))
+    logger.info(f"scan log file : {scan_result_log}")
+    logger.info("Pass files: %s" % len([item for item in res if item[3] == 'Pass']))
+    logger.info("Fail files: %s" % len([item for item in res if item[3] == 'Fail']))
+    if len([item for item in res if item[3] == 'Fail']) > 0:
+        logger.error(f"Scan failed,please check the log file:{scan_result_log}")
+        exit(1)

tests/parallel_test/container_build.sh

@@ -83,7 +83,7 @@ docker run \
     -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
     -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
     -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
-    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1"
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1"
  # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
 
 if [[ -d ${WORKDIR}/debugNoSan  ]] ;then