Merge pull request #27170 from taosdata/test/3.0/TD-31301

test:set debug in address sanitizer model
2024-08-14 19:16:27 +08:00 · 2024-08-14 19:16:27 +08:00 · ed7081eaad
parent 5c24933077 0294a2d9d6
commit ed7081eaad
5 changed files with 68 additions and 23 deletions
--- a/tests/ci/Dockerfile
+++ b/tests/ci/Dockerfile
@ -1,9 +1,14 @@
 FROM python:3.8
-RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-RUN pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro
-RUN apt-get update
-RUN apt-get install -y psmisc sudo tree libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config build-essential valgrind \
-     vim  libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common  r-base iputils-ping
+COPY sources.list /etc/apt/
+COPY id_ecdsa /root/.ssh/id_ecdsa
+RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 3B4FE6ACC0B21F32  && apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 871920D1991BC93C
+RUN apt-get update 
+RUN apt-get install -y locales psmisc sudo tree  libgeos-dev libgflags2.2 libgflags-dev  libgoogle-glog-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config build-essential valgrind rsync vim  libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common  r-base iputils-ping clang-tools-16
+RUN sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen  && locale-gen 
+RUN pip3 config set global.index-url http://admin:123456@192.168.0.212:3141/admin/dev/+simple/
+RUN pip3 config set global.trusted-host 192.168.0.212
+RUN pip3 install taospy==2.7.15 taos-ws-py==0.3.1 pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog
+ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8
 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
 RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/'
 RUN apt install -y r-base
@ -17,17 +22,16 @@ RUN apt-get install wget -y \
  && apt-get update && apt-get install -y dotnet-sdk-5.0 && apt-get install -y dotnet-sdk-6.0
 ADD node-v12.20.0-linux-x64.tar.gz /usr/local/
 RUN sh -c "rm -f /etc/localtime;ln -s /usr/share/zoneinfo/Asia/Shanghai /etc/localtime;echo \"Asia/Shanghai\" >/etc/timezone"
-COPY id_rsa /root/.ssh/id_rsa
 COPY .m2 /root/.m2
 COPY .nuget /root/.nuget
 COPY .dotnet /root/.dotnet
-COPY .cargo /root/.cargo
 COPY go /root/go
 ADD cmake-3.21.5-linux-x86_64.tar.gz /usr/local/
 RUN echo " export RUSTUP_DIST_SERVER=\"https://rsproxy.cn\" " >> /root/.bashrc
 RUN echo " export RUSTUP_UPDATE_ROOT=\"https://rsproxy.cn/rustup\" " >> /root/.bashrc
 RUN curl https://sh.rustup.rs -o /tmp/rustup-init.sh
 RUN sh /tmp/rustup-init.sh -y
+COPY .cargo/config /root/.cargo/config
 ENV PATH /usr/local/go/bin:/usr/local/node-v12.20.0-linux-x64/bin:/usr/local/apache-maven-3.8.4/bin:/usr/local/jdk1.8.0_144/bin:/usr/local/cmake-3.21.5-linux-x86_64/bin:/root/.cargo/bin:$PATH
 ENV JAVA_HOME /usr/local/jdk1.8.0_144
 RUN go env -w GOPROXY=https://goproxy.cn
@ -39,10 +43,8 @@ RUN R CMD javareconf JAVA_HOME=${JAVA_HOME} JAVA=${JAVA_HOME}/bin/java JAVAC=${J
 RUN echo "install.packages(\"RJDBC\", repos=\"http://cran.us.r-project.org\")"|R --no-save
 COPY .gitconfig /root/.gitconfig
 RUN mkdir -p /run/sshd
-COPY id_rsa.pub /root/.ssh/id_rsa.pub
-COPY id_rsa.pub /root/.ssh/authorized_keys
+COPY id_ecdsa.pub /root/.ssh/id_ecdsa.pub
+COPY id_ecdsa.pub /root/.ssh/authorized_keys
 RUN pip3 uninstall -y taostest
 COPY repository/TDinternal /home/TDinternal
-COPY repository/taos-connector-python /home/taos-connector-python
-RUN sh -c "cd /home/taos-connector-python; pip3 install ."
 COPY setup.sh /home/setup.sh
--- a/tests/ci/scan_file_path.py
+++ b/tests/ci/scan_file_path.py
@ -122,7 +122,7 @@ def scan_files_path(source_file_path):
        for file in files:
            if any(item in root for item in scan_dir_list):
                file_path = os.path.join(root, file)
-                if (file_path.endswith(".c") or file_name.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list):
+                if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list):
                    all_file_path.append(file_path)
    logger.info("Found %s files" % len(all_file_path))

@ -134,7 +134,7 @@ def input_files(change_files):
        for line in file:
            file_name = line.strip()
            if any(dir_name in file_name for dir_name in scan_dir_list):
-                if (file_name.endswith(".c")  or line.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list):
+                if (file_name.endswith(".c")  or file_name.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list):
                    if "enterprise" in file_name:
                        file_name = os.path.join(TD_project_path, file_name)
                    else: 
@ -184,12 +184,12 @@ if __name__ == "__main__":
    #     os.makedirs(scan_result_path)

    for file in all_file_path:
-        cmd = f"clang-query-10 -p {compile_commands_path} {file} -f {clang_scan_rules_path}"
+        cmd = f"clang-query-16 -p {compile_commands_path} {file} -f {clang_scan_rules_path}"
        logger.debug(f"cmd:{cmd}")
        try:
            stdout, stderr = command_executor.execute(cmd)
            #if "error" in stderr:
-            # print(stderr)
+            print(stderr)
            lines = stdout.split("\n")
            if lines[-2].endswith("matches.") or lines[-2].endswith("match."): 
                match_num = int(lines[-2].split(" ")[0])
--- a/tests/parallel_test/container_build.sh
+++ b/tests/parallel_test/container_build.sh
@ -83,7 +83,7 @@ docker run \
    -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
    -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
-    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1"
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j|| exit 1"
 # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \

 if [[ -d ${WORKDIR}/debugNoSan  ]] ;then
@ -94,13 +94,17 @@ if [[ -d ${WORKDIR}/debugSan ]] ;then
    echo "delete  ${WORKDIR}/debugSan"
    rm -rf  ${WORKDIR}/debugSan
 fi
-
-if [ "$(uname -m)" = "aarch64" ] ;then
-    CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Debug"
-else
-    CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Release"
+if [[ -d ${WORKDIR}/debugRelease ]] ;then
+    echo "delete  ${WORKDIR}/debugRelease"
+    rm -rf  ${WORKDIR}/debugRelease
 fi

+# if [ "$(uname -m)" = "aarch64" ] ;then
+#     CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Debug"
+# else
+#     CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Release"
+# fi
+
 mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugNoSan
 date
 docker run \
@ -133,10 +137,47 @@ docker run \
    -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
    -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
-    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 "
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true -DCMAKE_BUILD_TYPE=Debug -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j|| exit 1 "

 mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugSan

+date
+# docker run \
+#     -v $REP_MOUNT_PARAM \
+#     -v /root/.cargo/registry:/root/.cargo/registry \
+#     -v /root/.cargo/git:/root/.cargo/git \
+#     -v /root/go/pkg/mod:/root/go/pkg/mod \
+#     -v /root/.cache/go-build:/root/.cache/go-build \
+#     -v /root/.cos-local.1:/root/.cos-local.2 \
+#     -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \
+#     -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \
+#     -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \
+#     -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \
+#     -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
+#     -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \
+#     -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \
+#     -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
+#     -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
+#     -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \
+#     -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \
+#     -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \
+#     -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
+#     -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \
+#     -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
+#     -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
+#     -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \
+#     -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \
+#     -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \
+#     -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \
+#     -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
+#     -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
+#     -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
+#     --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DCMAKE_BUILD_TYPE=Release -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j || exit 1 "
+
+# mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugRelease
+
+
+
 ret=$?
 exit $ret

--- a/tests/parallel_test/run_scan_container.sh
+++ b/tests/parallel_test/run_scan_container.sh
@ -79,6 +79,7 @@ scan_scripts="$CONTAINER_TESTDIR/tests/ci/scan_file_path.py"
 ulimit -c unlimited
 cat << EOF
 docker run \
+    -v /root/.cos-local.1:/root/.cos-local.2 \
    -v $REP_MOUNT_PARAM \
    -v $REP_MOUNT_DEBUG \
    -v $scan_changefile_temp_path:$docker_can_changefile_temp_path \
@ -86,6 +87,7 @@ docker run \
    --rm --ulimit core=-1 taos_test:v1.0 python3  $scan_scripts -b "${branch_name_id}"  -f "${scan_file_name}" -w ${web_server}
 EOF
 docker run \
+    -v /root/.cos-local.1:/root/.cos-local.2 \
    -v $REP_MOUNT_PARAM \
    -v $REP_MOUNT_DEBUG \
    -v $scan_changefile_temp_path:$docker_can_changefile_temp_path \
--- a/tests/unit-test/test.sh
+++ b/tests/unit-test/test.sh
@ -40,7 +40,7 @@ pgrep taosd || taosd >> /dev/null 2>&1 &

 sleep 10

-ctest -j8
+ctest -E "cunit_test" -j8

 ret=$?
 exit $ret