test: scan returned values in ci

source/libs/parser/src/parAstCreater.c

@@ -207,6 +207,7 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) {
     return false;
   }
   return true;
+
 }
 
 static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) {

source/libs/parser/src/parAstParser.c

@@ -725,7 +725,7 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC
   strcpy(name.dbname, pStmt->dbName);
   strcpy(name.tname, pStmt->viewName);
   char dbFName[TSDB_DB_FNAME_LEN];
-  (void)tNameGetFullDbName(&name, dbFName);
+  tNameGetFullDbName(&name, dbFName);
   int32_t code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->viewName, 0);
   if (TSDB_CODE_SUCCESS == code) {
     code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName,

tests/ci/scan.py

@@ -0,0 +1,106 @@
+import os
+import subprocess
+import csv
+from datetime import datetime
+from loguru import logger
+
+# log file path
+log_file_path = "/root/charles/scan.log"
+logger.add(log_file_path, rotation="10MB", retention="7 days", level="DEBUG")
+# scan result base path
+scan_result_base_path = "/root/charles/clang_scan_result/"
+# the base source code file path
+source_path = "/root/charles/TDinternal/"
+# the compile commands json file path
+compile_commands_path = "/root/charles/TDinternal/debug/compile_commands.json"
+# the ast parser rule for c file
+clang_scan_rules_path = "/root/charles/clang_scan_rules"
+# all the c files path will be checked
+all_file_path = []
+
+class CommandExecutor:
+    def __init__(self):
+        self._process = None
+
+    def execute(self, command, timeout=None):
+        try:
+            self._process = subprocess.Popen(command,
+                                            shell=True,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.PIPE)
+            stdout, stderr = self._process.communicate(timeout=timeout)
+            return stdout.decode('utf-8'), stderr.decode('utf-8')
+        except subprocess.TimeoutExpired:
+            self._process.kill()
+            self._process.communicate()
+            raise Exception("Command execution timeout")
+        except Exception as e:
+            raise Exception("Command execution failed: %s" % e)
+
+def scan_files_path(source_file_path):
+    # scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"]
+    scan_dir_list = ["source", "include", "docs/examples", "src/plugins"]
+    scan_skip_file_list = ["/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl",
+                           "/test/", "contrib", "debug", "deps", "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"]
+    for root, dirs, files in os.walk(source_file_path):
+        for file in files:
+            if any(item in root for item in scan_dir_list):
+                file_path = os.path.join(root, file)
+                if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list):
+                    all_file_path.append(file_path)
+    logger.info("Found %s files" % len(all_file_path))
+
+def save_scan_res(res_base_path, file_path, out, err):
+    file_res_path = os.path.join(res_base_path, file_path.replace("/root/charles/", "").split(".")[0] + ".res")
+    if not os.path.exists(os.path.dirname(file_res_path)):
+        os.makedirs(os.path.dirname(file_res_path))
+    logger.info("Save scan result to: %s" % file_res_path)
+    # save scan result
+    with open(file_res_path, "w") as f:
+        f.write(out)
+        f.write(err)
+
+def write_csv(file_path, data):
+    try:
+        with open(file_path, 'w') as f:
+            writer = csv.writer(f)
+            writer.writerows(data)
+    except Exception as ex:
+        raise Exception("Failed to write the csv file: {} with msg: {}".format(file_path, repr(ex)))
+
+if __name__ == "__main__":
+    command_executor = CommandExecutor()
+    # get all the c files path
+    scan_files_path(source_path)
+    res = []
+    # create dir
+    current_time = datetime.now().strftime("%Y%m%d%H%M%S")
+    scan_result_path = os.path.join(scan_result_base_path, current_time)
+    if not os.path.exists(scan_result_path):
+        os.makedirs(scan_result_path)
+    for file in all_file_path:
+        cmd = "clang-query -p %s %s -f %s" % (compile_commands_path, file, clang_scan_rules_path)
+        try:
+            stdout, stderr = command_executor.execute(cmd)
+            lines = stdout.split("\n")
+            if lines[-2].endswith("matches.") or lines[-2].endswith("match."): 
+                match_num = int(lines[-2].split(" ")[0])
+                logger.info("The match lines of file %s: %s" % (file, match_num))
+                if match_num > 0:
+                    save_scan_res(scan_result_path, file, stdout, stderr)
+                res.append([file, match_num, 'Pass' if match_num == 0 else 'Fail'])
+            else:
+                logger.warning("The result of scan is invalid for: %s" % file)            
+        except Exception as e:
+            logger.error("Execute command failed: %s" % e)
+    # data = ""
+    # for item in res:
+    #     data += item[0] + "," + str(item[1]) + "\n"
+    # logger.info("Csv data: %s" % data)
+    write_csv(os.path.join(scan_result_path, "scan_res.csv"), res)
+    logger.info("The result of scan: \n")
+    logger.info("Total files: %s" % len(res))
+    logger.info("Total match lines: %s" % sum([item[1] for item in res]))
+    logger.info("Pass files: %s" % len([item for item in res if item[2] == 'Pass']))
+    logger.info("Fail files: %s" % len([item for item in res if item[2] == 'Fail']))
+

tests/ci/scan_file_path.py

@@ -57,7 +57,9 @@ else:
 
 
 # log file path
-log_file_path = f"{work_path}/{branch_name}/"
+current_time = datetime.now().strftime("%Y%m%d-%H%M%S")
+log_file_path = f"{work_path}/scan_log/scan_{branch_name}_{current_time}/"
+
 os.makedirs(log_file_path, exist_ok=True)
 
 scan_log_file = f"{log_file_path}/scan.log"
@@ -175,8 +177,9 @@ if __name__ == "__main__":
     web_path = []
     res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"])
     # create dir
-    current_time = datetime.now().strftime("%Y%m%d%H%M%S")
-    scan_result_path = os.path.join(scan_result_base_path, current_time)
+    # current_time = datetime.now().strftime("%Y%m%d%H%M%S")
+    # scan_result_path = os.path.join(scan_result_base_path, current_time)
+    scan_result_path = scan_result_base_path
     if not os.path.exists(scan_result_path):
         os.makedirs(scan_result_path)
     for file in all_file_path:

tests/parallel_test/run_scan_container.sh

@@ -62,28 +62,34 @@ REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal"
 
 CONTAINER_TESTDIR=/home/TDinternal/community
 
-#scan file log path
-scan_temp="$WORKDIR/log/${branch_name_id}/"
-docker_scan_temp="/home/${branch_name_id}/"
-mkdir -p $scan_temp
-mkdir -p $docker_scan_temp
+#scan change file path
+scan_changefile_temp_path="$WORKDIR/tmp/${branch_name_id}/"
+docker_can_changefile_temp_path="/home/tmp/${branch_name_id}/"
+mkdir -p $scan_changefile_temp_path
+scan_file_name="$docker_can_changefile_temp_path/docs_changed.txt"
+
+#scan log file path
+scan_log_temp_path="$WORKDIR/scan_log/"
+docker_scan_log_temp_path="/home/scan_log/"
+mkdir -p $scan_log_temp_path
 
 
 scan_scripts="$CONTAINER_TESTDIR/tests/ci/scan_file_path.py" 
-scan_file_name="$docker_scan_temp/docs_changed.txt"
 
 ulimit -c unlimited
 cat << EOF
 docker run \
     -v $REP_MOUNT_PARAM \
     -v $REP_MOUNT_DEBUG \
-    -v $scan_temp:$docker_scan_temp \
+    -v $scan_changefile_temp_path:$docker_can_changefile_temp_path \
+    -v $scan_log_temp_path:$docker_scan_log_temp_path \
     --rm --ulimit core=-1 taos_test:v1.0 python3  $scan_scripts -b "${branch_name_id}"  -f "${scan_file_name}" -w ${web_server}
 EOF
 docker run \
     -v $REP_MOUNT_PARAM \
     -v $REP_MOUNT_DEBUG \
-    -v $scan_temp:$docker_scan_temp \
+    -v $scan_changefile_temp_path:$docker_can_changefile_temp_path \
+    -v $scan_log_temp_path:$docker_scan_log_temp_path \
     --rm --ulimit core=-1 taos_test:v1.0 python3  $scan_scripts -b "${branch_name_id}"  -f "${scan_file_name}" -w ${web_server}