From 6a05d9e4fc8b9a9030444611027e7aa847e40032 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 2 Sep 2024 20:31:02 +0800 Subject: [PATCH 1/6] tetst:add assert check in ci --- Jenkinsfile2 | 4 + source/dnode/vnode/src/tsdb/tsdbCommit2.c | 1 + tests/ci/count_assert.py | 93 ++++++++++++++++--- .../run_check_assert_container.sh | 57 ++++++++++++ 4 files changed, 144 insertions(+), 11 deletions(-) create mode 100755 tests/parallel_test/run_check_assert_container.sh diff --git a/Jenkinsfile2 b/Jenkinsfile2 index a4f765c9fb..b074a0ee05 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -422,6 +422,10 @@ pipeline { mkdir -p ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID} echo "''' + env.FILE_CHANGED + '''" > ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + sh ''' + cd ${WKC}/tests/parallel_test + ./run_check_assert_container.sh -d ${WKDIR} + ''' sh ''' date rm -rf ${WKC}/debug diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 8ccb8b0a50..1523df912f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -213,6 +213,7 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { || TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0 // ) { return 0; + assert(10000); } SSttLvl *lvl; diff --git a/tests/ci/count_assert.py b/tests/ci/count_assert.py index 415c197b3f..72e9c461a4 100644 --- a/tests/ci/count_assert.py +++ b/tests/ci/count_assert.py @@ -1,23 +1,45 @@ import os import re +from loguru import logger # List of source directories to search + +self_path = os.path.dirname(os.path.realpath(__file__)) + +# Check if "community" or "tests" is in self_path +index_community = self_path.find("community") +if index_community != -1: + TD_project_path = self_path[:index_community] + index_TDinternal = TD_project_path.find("TDinternal") + # Check if index_TDinternal is valid and set work_path accordingly + if index_TDinternal != -1: + work_path = TD_project_path[:index_TDinternal] +else: + index_tests = self_path.find("tests") + if index_tests != -1: + TD_project_path = self_path[:index_tests] + # Check if index_TDengine is valid and set work_path accordingly + index_TDengine = TD_project_path.find("TDengine") + if index_TDengine != -1: + work_path = TD_project_path[:index_TDengine] +TD_project_path = TD_project_path.rstrip('/') +print(TD_project_path) source_dirs = [ - "community/source", - "community/include", - "enterprise/src/plugins/" + f"{TD_project_path}/community/source", + f"{TD_project_path}/community/include", + f"{TD_project_path}/enterprise/src/plugins/" ] # List of directories to exclude exclude_dirs = [ - "community/source/client/jni" + f"{TD_project_path}/community/source/client/jni" ] # List of files to exclude exclude_source_files = [ - "community/source/libs/parser/src/sql.c", - "community/source/util/src/tlog.c", - "community/include/util/tlog.h" + f"{TD_project_path}/community/source/libs/parser/src/sql.c", + f"{TD_project_path}/community/source/util/src/tlog.c", + f"{TD_project_path}/community/include/util/tlog.h" ] def grep_asserts_in_file(file_path, summary_list, detaild_list): @@ -26,10 +48,10 @@ def grep_asserts_in_file(file_path, summary_list, detaild_list): with open(file_path, 'r') as file: for line_number, line in enumerate(file, start=1): if re.search(r'\bassert\(.*\)|\bASSERT\(.*\)|\bASSERTS\(.*\)|\bASSERT_CORE\(.*\)', line): - detaild_list.append(f"{file_path}:{line_number}: {line.strip()}") + detaild_list.append(f"{file_path}:{line.strip()}:{line_number}") match_count += 1 if match_count > 0: - summary_list.append(f"Total matches in {file_path}: {match_count}") + summary_list.append(f"Total matches in {file_path}:{match_count}") def traverse_and_grep(source_dirs, exclude_dirs, exclude_source_files): """Traverse directories and grep for assert, ASSERTS, or ASSERT function calls in .h and .c files.""" @@ -47,7 +69,56 @@ def traverse_and_grep(source_dirs, exclude_dirs, exclude_source_files): grep_asserts_in_file(file_path, summary_list, detaild_list) return summary_list, detaild_list +def check_list_result(result_list,detaild_list): + logger.debug("check assert in source code") + error_message = "ERROR: do not add `assert` statements in new code." + error_message2 = "ERROR: Please check the detailed information below: assert statement with file name and line number" + remove_detail_items = [ + f"{TD_project_path}/community/source/dnode/vnode/src/tsdb/tsdbCommit2.c:ASSERT_CORE(tsdb->imem == NULL, \"imem should be null to commit mem\");", + f"{TD_project_path}/community/include/util/types.h:assert(sizeof(float) == sizeof(uint32_t));", + f"{TD_project_path}/community/include/util/types.h:assert(sizeof(double) == sizeof(uint64_t));" + ] + expected_strings = [ + f"Total matches in {TD_project_path}/community/source/dnode/vnode/src/tsdb/tsdbCommit2.c:1", + f"Total matches in {TD_project_path}/community/include/util/types.h:2" + ] + logger.debug(len(result_list)) + if len(result_list) != 2: + logger.error(f"{error_message}") + for item in expected_strings: + if item in result_list: + result_list.remove(item) + logger.error("\n" + "\n".join(result_list)) + logger.error(f"{error_message2}") + for item in remove_detail_items: + if item in detaild_list: + detaild_list.remove(item) + logger.error("\n" + "\n".join(detaild_list)) + exit(1) + else: + # check if all expected strings are in the result list + if all(item in result_list for item in expected_strings): + logger.debug(result_list) + logger.debug(detaild_list) + if all(any(remove_detail_item in detaild for remove_detail_item in remove_detail_items) for detaild in detaild_list): + logger.info("Validation successful.") + else: + logger.error(f"{error_message}") + for item in expected_strings: + if item in result_list: + logger.debug(item) + result_list.remove(item) + logger.error("\n" + "\n".join(result_list)) + logger.error(f"{error_message2}") + for item in remove_detail_items: + if item in detaild_list: + detaild_list.remove(item) + logger.error("\n" + "\n".join(detaild_list)) + exit(1) if __name__ == "__main__": summary_list, detaild_list = traverse_and_grep(source_dirs, exclude_dirs, exclude_source_files) - print("\n".join(summary_list)) - # print("\n".join(detaild_list)) \ No newline at end of file + logger.debug("\n" + "\n".join(summary_list)) + logger.debug("\n" + "\n".join(detaild_list)) + + check_list_result(summary_list,detaild_list) + diff --git a/tests/parallel_test/run_check_assert_container.sh b/tests/parallel_test/run_check_assert_container.sh new file mode 100755 index 0000000000..0a8796f617 --- /dev/null +++ b/tests/parallel_test/run_check_assert_container.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -d work dir" + echo -e "\t -h help" +} + +while getopts "d:h" opt; do + case $opt in + d) + WORKDIR=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi + + + # enterprise edition +INTERNAL_REPDIR=$WORKDIR/TDinternal +REPDIR_DEBUG=$WORKDIR/debugNoSan/ + +REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/" +REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" + +CONTAINER_TESTDIR=/home/TDinternal/community + +check_assert_scripts="$CONTAINER_TESTDIR/tests/ci/count_assert.py" + +ulimit -c unlimited +cat << EOF +docker run \ + -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_DEBUG \ + --rm --ulimit core=-1 taos_test:v1.0 python3 $check_assert_scripts +EOF +docker run \ + -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_DEBUG \ + --rm --ulimit core=-1 taos_test:v1.0 python3 $check_assert_scripts + +ret=$? +exit $ret + From cbc28491222c2b1519cd037a079a3fe548a17ee3 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 2 Sep 2024 20:32:47 +0800 Subject: [PATCH 2/6] tetst:add assert check in ci --- tests/ci/count_assert.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/ci/count_assert.py b/tests/ci/count_assert.py index 72e9c461a4..2b51596403 100644 --- a/tests/ci/count_assert.py +++ b/tests/ci/count_assert.py @@ -82,7 +82,7 @@ def check_list_result(result_list,detaild_list): f"Total matches in {TD_project_path}/community/source/dnode/vnode/src/tsdb/tsdbCommit2.c:1", f"Total matches in {TD_project_path}/community/include/util/types.h:2" ] - logger.debug(len(result_list)) + # logger.debug(len(result_list)) if len(result_list) != 2: logger.error(f"{error_message}") for item in expected_strings: @@ -98,15 +98,14 @@ def check_list_result(result_list,detaild_list): else: # check if all expected strings are in the result list if all(item in result_list for item in expected_strings): - logger.debug(result_list) - logger.debug(detaild_list) + # logger.debug(result_list) + # logger.debug(detaild_list) if all(any(remove_detail_item in detaild for remove_detail_item in remove_detail_items) for detaild in detaild_list): logger.info("Validation successful.") else: logger.error(f"{error_message}") for item in expected_strings: if item in result_list: - logger.debug(item) result_list.remove(item) logger.error("\n" + "\n".join(result_list)) logger.error(f"{error_message2}") @@ -117,8 +116,5 @@ def check_list_result(result_list,detaild_list): exit(1) if __name__ == "__main__": summary_list, detaild_list = traverse_and_grep(source_dirs, exclude_dirs, exclude_source_files) - logger.debug("\n" + "\n".join(summary_list)) - logger.debug("\n" + "\n".join(detaild_list)) - check_list_result(summary_list,detaild_list) From b1efafc437fc25d1a5e4db9dd18b92622b5722ee Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 2 Sep 2024 20:35:10 +0800 Subject: [PATCH 3/6] tetst:add assert check in ci --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index b074a0ee05..60574e2850 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -424,7 +424,7 @@ pipeline { ''' sh ''' cd ${WKC}/tests/parallel_test - ./run_check_assert_container.sh -d ${WKDIR} + python3 count_assert.py ''' sh ''' date From 932ed0c3a3c84ac1987c87527b3abe77c371acd6 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 2 Sep 2024 20:49:34 +0800 Subject: [PATCH 4/6] tetst:add assert check in ci --- Jenkinsfile2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 60574e2850..53d3b7ec54 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -423,8 +423,8 @@ pipeline { echo "''' + env.FILE_CHANGED + '''" > ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' sh ''' - cd ${WKC}/tests/parallel_test - python3 count_assert.py + cd ${WKC}/tests/ci + python3 count_assert.py ''' sh ''' date From 14674ffbba1933aa31f29954d5800e7704871ffe Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 2 Sep 2024 21:10:58 +0800 Subject: [PATCH 5/6] tetst:add assert check in ci --- Jenkinsfile2 | 4 ++-- tests/parallel_test/run_check_assert_container.sh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 53d3b7ec54..b074a0ee05 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -423,8 +423,8 @@ pipeline { echo "''' + env.FILE_CHANGED + '''" > ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' sh ''' - cd ${WKC}/tests/ci - python3 count_assert.py + cd ${WKC}/tests/parallel_test + ./run_check_assert_container.sh -d ${WKDIR} ''' sh ''' date diff --git a/tests/parallel_test/run_check_assert_container.sh b/tests/parallel_test/run_check_assert_container.sh index 0a8796f617..e8d78d62ae 100755 --- a/tests/parallel_test/run_check_assert_container.sh +++ b/tests/parallel_test/run_check_assert_container.sh @@ -33,7 +33,6 @@ fi INTERNAL_REPDIR=$WORKDIR/TDinternal REPDIR_DEBUG=$WORKDIR/debugNoSan/ -REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/" REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" CONTAINER_TESTDIR=/home/TDinternal/community @@ -44,12 +43,10 @@ ulimit -c unlimited cat << EOF docker run \ -v $REP_MOUNT_PARAM \ - -v $REP_MOUNT_DEBUG \ --rm --ulimit core=-1 taos_test:v1.0 python3 $check_assert_scripts EOF docker run \ -v $REP_MOUNT_PARAM \ - -v $REP_MOUNT_DEBUG \ --rm --ulimit core=-1 taos_test:v1.0 python3 $check_assert_scripts ret=$? From 019641e8303044af14508a05dcc54107d25baef0 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 2 Sep 2024 21:27:07 +0800 Subject: [PATCH 6/6] tetst:add assert check in ci --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 1523df912f..8ccb8b0a50 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -213,7 +213,6 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { || TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0 // ) { return 0; - assert(10000); } SSttLvl *lvl;