test: scan returned values in ci
This commit is contained in:
chenhaoran
parent
13294c9d17
commit
3262b61df0
11
Jenkinsfile2
11
Jenkinsfile2
|
|
@ -422,13 +422,6 @@ pipeline {
|
|||
mkdir -p ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}
|
||||
echo "''' + env.FILE_CHANGED + '''" > ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt
|
||||
'''
|
||||
sh'''
|
||||
cd ${WK} || exit 0 ; rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1
|
||||
'''
|
||||
sh '''
|
||||
cd ${WKC}/tests/ci
|
||||
python3 scan_file_path.py -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt
|
||||
'''
|
||||
sh '''
|
||||
date
|
||||
rm -rf ${WKC}/debug
|
||||
|
|
@ -460,6 +453,10 @@ pipeline {
|
|||
}
|
||||
}
|
||||
}
|
||||
sh '''
|
||||
cd ${WKC}/tests/ci
|
||||
run_scan_container.sh -d ${WK} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WK}/../log/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + '''
|
||||
'''
|
||||
sh '''
|
||||
cd ${WKC}/tests/parallel_test
|
||||
export DEFAULT_RETRY_TIME=2
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@ from datetime import datetime
|
|||
from loguru import logger
|
||||
import getopt
|
||||
|
||||
opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:', [
|
||||
|
||||
opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:w:', [
|
||||
'branch_name='])
|
||||
for key, value in opts:
|
||||
if key in ['-h', '--help']:
|
||||
|
|
@ -14,6 +15,7 @@ for key, value in opts:
|
|||
'Usage: python3 scan.py -b <branch_name> -f <file_list>')
|
||||
print('-b branch name or PR ID to scan')
|
||||
print('-f change files list')
|
||||
print('-w web server')
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
|
@ -21,18 +23,20 @@ for key, value in opts:
|
|||
branch_name = value
|
||||
if key in ['-f', '--filesName']:
|
||||
change_file_list = value
|
||||
if key in ['-w', '--webServer']:
|
||||
web_server = value
|
||||
|
||||
|
||||
# the base source code file path
|
||||
self_path = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
if ("community" in self_path):
|
||||
TD_project_path = self_path[:self_path.find("community")]
|
||||
work_path = TD_project_path[:TD_project_path.find("TDinternal")]
|
||||
# if ("community" in self_path):
|
||||
# TD_project_path = self_path[:self_path.find("community")]
|
||||
# work_path = TD_project_path[:TD_project_path.find("TDinternal")]
|
||||
|
||||
else:
|
||||
TD_project_path = self_path[:self_path.find("tests")]
|
||||
work_path = TD_project_path[:TD_project_path.find("TDengine")]
|
||||
# else:
|
||||
# TD_project_path = self_path[:self_path.find("tests")]
|
||||
# work_path = TD_project_path[:TD_project_path.find("TDengine")]
|
||||
|
||||
# Check if "community" or "tests" is in self_path
|
||||
index_community = self_path.find("community")
|
||||
|
|
@ -53,12 +57,16 @@ else:
|
|||
|
||||
|
||||
# log file path
|
||||
log_file_path = f"{work_path}/log/{branch_name}/"
|
||||
log_file_path = f"{work_path}/{branch_name}/"
|
||||
os.makedirs(log_file_path, exist_ok=True)
|
||||
|
||||
scan_log_file = f"{log_file_path}/scan.log"
|
||||
logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG")
|
||||
print(self_path,work_path,TD_project_path,log_file_path)
|
||||
# logging.basicConfig(level=logging.INFO,
|
||||
# format='%(asctime)s | %(levelname)s | %(name)s:%(lineno)d - %(message)s')
|
||||
# logger = logging.getLogger(__name__)
|
||||
|
||||
print(self_path,work_path,TD_project_path,log_file_path,change_file_list)
|
||||
|
||||
# scan result base path
|
||||
scan_result_base_path = f"{log_file_path}/clang_scan_result/"
|
||||
|
|
@ -81,6 +89,7 @@ print(f"compile_commands_path:{compile_commands_path}")
|
|||
# the ast parser rule for c file
|
||||
clang_scan_rules_path = f"{self_path}/filter_for_return_values"
|
||||
|
||||
#
|
||||
# all the c files path will be checked
|
||||
all_file_path = []
|
||||
|
||||
|
|
@ -140,7 +149,7 @@ file_res_path = ""
|
|||
|
||||
def save_scan_res(res_base_path, file_path, out, err):
|
||||
global file_res_path
|
||||
file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".res")
|
||||
file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".txt")
|
||||
print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}")
|
||||
if not os.path.exists(os.path.dirname(file_res_path)):
|
||||
os.makedirs(os.path.dirname(file_res_path))
|
||||
|
|
@ -167,6 +176,7 @@ if __name__ == "__main__":
|
|||
input_files(change_file_list)
|
||||
print(f"all_file_path:{all_file_path}")
|
||||
res = []
|
||||
web_path = []
|
||||
res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"])
|
||||
# create dir
|
||||
current_time = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
|
|
@ -179,6 +189,7 @@ if __name__ == "__main__":
|
|||
print(f"cmd:{cmd}")
|
||||
try:
|
||||
stdout, stderr = command_executor.execute(cmd)
|
||||
print(stderr)
|
||||
lines = stdout.split("\n")
|
||||
if lines[-2].endswith("matches.") or lines[-2].endswith("match."):
|
||||
match_num = int(lines[-2].split(" ")[0])
|
||||
|
|
@ -186,7 +197,13 @@ if __name__ == "__main__":
|
|||
if match_num > 0:
|
||||
logger.info(f"scan_result_path: {scan_result_path} ,file:{file}")
|
||||
save_scan_res(scan_result_path, file, stdout, stderr)
|
||||
index_tests = file_res_path.find(branch_name)
|
||||
if index_tests != -1:
|
||||
web_path_file = file_res_path[index_tests:]
|
||||
web_path_file = os.path.join(web_server, web_path_file)
|
||||
web_path.append(web_path_file)
|
||||
res.append([file, file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail'])
|
||||
|
||||
else:
|
||||
logger.warning("The result of scan is invalid for: %s" % file)
|
||||
except Exception as e:
|
||||
|
|
@ -207,4 +224,6 @@ if __name__ == "__main__":
|
|||
logger.info("Fail files: %s" % len([item for item in res if item[3] == 'Fail']))
|
||||
if len([item for item in res if item[3] == 'Fail']) > 0:
|
||||
logger.error(f"Scan failed,please check the log file:{scan_result_log}")
|
||||
for index, failed_result_file in enumerate(web_path):
|
||||
logger.error(f"failed number: {index}, failed_result_file: {failed_result_file}")
|
||||
exit(1)
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
#!/bin/bash
|
||||
|
||||
function usage() {
|
||||
echo "$0"
|
||||
echo -e "\t -d work dir"
|
||||
echo -e "\t -b pr and id"
|
||||
echo -e "\t -w web server "
|
||||
echo -e "\t -f scan file "
|
||||
echo -e "\t -h help"
|
||||
}
|
||||
|
||||
while getopts "d:b:w:f:h" opt; do
|
||||
case $opt in
|
||||
d)
|
||||
WORKDIR=$OPTARG
|
||||
;;
|
||||
b)
|
||||
branch_name_id=$OPTARG
|
||||
;;
|
||||
f)
|
||||
scan_file_name=$OPTARG
|
||||
;;
|
||||
w)
|
||||
web_server=$OPTARG
|
||||
;;
|
||||
h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
\?)
|
||||
echo "Invalid option: -$OPTARG"
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$branch_name_id" ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$scan_file_name" ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$WORKDIR" ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$web_server" ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# enterprise edition
|
||||
INTERNAL_REPDIR=$WORKDIR/TDinternal
|
||||
REPDIR_DEBUG=$WORKDIR/debugNoSan/
|
||||
|
||||
REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/"
|
||||
REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal"
|
||||
|
||||
CONTAINER_TESTDIR=/home/TDinternal/community
|
||||
|
||||
#scan file log path
|
||||
scan_temp="$WORKDIR/log/${branch_name_id}/"
|
||||
docker_scan_temp="/home/${branch_name_id}/"
|
||||
mkdir -p $scan_temp
|
||||
mkdir -p $docker_scan_temp
|
||||
|
||||
|
||||
scan_scripts="$CONTAINER_TESTDIR/tests/ci/scan_file_path.py"
|
||||
scan_file_name="$docker_scan_temp/docs_changed.txt"
|
||||
|
||||
ulimit -c unlimited
|
||||
cat << EOF
|
||||
docker run \
|
||||
-v $REP_MOUNT_PARAM \
|
||||
-v $REP_MOUNT_DEBUG \
|
||||
-v $scan_temp:$docker_scan_temp \
|
||||
--rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server}
|
||||
EOF
|
||||
docker run \
|
||||
-v $REP_MOUNT_PARAM \
|
||||
-v $REP_MOUNT_DEBUG \
|
||||
-v $scan_temp:$docker_scan_temp \
|
||||
--rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server}
|
||||
|
||||
|
||||
ret=$?
|
||||
exit $ret
|
||||
|
||||
Loading…
Reference in New Issue