Merge pull request #5164 from taosdata/test/jenkins

reblance server load
2021-02-06 19:07:32 +08:00 · 2021-02-06 19:07:32 +08:00 · c2215eb214
parent cf983f8c1c 9a7573561e
commit c2215eb214
9 changed files with 153 additions and 55 deletions
--- a/45
+++ b/45
@ -80,25 +80,28 @@ pipeline {
          }
          steps {
          sh'''
-          cd ${WORKSPACE}
+          cp -r ${WORKSPACE} ${WORKSPACE}.tes
+          cd ${WORKSPACE}.tes
          git checkout develop
          git pull
          git fetch origin +refs/pull/${CHANGE_ID}/merge
          git checkout -qf FETCH_HEAD
          '''
          script{
-            skipstage=sh(script:"git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 1 ",returnStdout:true) 
+            skipstage=sh(script:"git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) 
          }
+          sh'''
+          rm -rf ${WORKSPACE}.tes
+          '''
          }
      }
    
      stage('Parallel test stage') {
-        
        //only build pr
        when {
              changeRequest()
               expression {
-                    skipstage != 1
+                    skipstage != 0
              }
          }
      parallel {
@ -124,12 +127,12 @@ pipeline {
            
            pre_test()
            timeout(time: 45, unit: 'MINUTES'){
-            sh '''
-            date
-            cd ${WKC}/tests
-            find pytest -name '*'sql|xargs rm -rf
-            ./test-all.sh p2
-            date'''
+                sh '''
+                date
+                cd ${WKC}/tests
+                find pytest -name '*'sql|xargs rm -rf
+                ./test-all.sh p2
+                date'''
            }
          }
        }
@ -161,6 +164,7 @@ pipeline {

        stage('test_crash_gen_s3') {
          agent{label "b2"}
+          
          steps {
            pre_test()
            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
@ -172,17 +176,20 @@ pipeline {
            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                sh '''
                cd ${WKC}/tests/pytest
+                rm -rf /var/lib/taos/*
+                rm -rf /var/log/taos/*
                ./handle_crash_gen_val_log.sh
                '''
-            }
+            }  
            timeout(time: 45, unit: 'MINUTES'){
-              sh '''
-              date
-              cd ${WKC}/tests
-              ./test-all.sh b2fq
-              date
-              '''
-            }
+                sh '''
+                date
+                cd ${WKC}/tests
+                ./test-all.sh b2fq
+                date
+                '''
+            }         
+            
          }
        }

@ -216,6 +223,8 @@ pipeline {
              date
              cd ${WKC}/tests
              ./test-all.sh b4fq
+              cd ${WKC}/tests
+              ./test-all.sh p4
              date'''
            }
          }
--- a/tests/pytest/concurrent_inquiry.sh
+++ b/tests/pytest/concurrent_inquiry.sh
@ -0,0 +1,82 @@
+#!/bin/bash
+
+# This is the script for us to try to cause the TDengine server or client to crash
+#    
+# PREPARATION
+#
+# 1. Build an compile the TDengine source code that comes with this script, in the same directory tree
+# 2. Please follow the direction in our README.md, and build TDengine in the build/ directory
+# 3. Adjust the configuration file if needed under build/test/cfg/taos.cfg
+# 4. Run the TDengine server instance: cd build; ./build/bin/taosd -c test/cfg
+# 5. Make sure you have a working Python3 environment: run /usr/bin/python3 --version, and you should get 3.6 or above
+# 6. Make sure you have the proper Python packages: # sudo apt install python3-setuptools python3-pip python3-distutils
+#
+# RUNNING THIS SCRIPT
+# 
+# This script assumes the source code directory is intact, and that the binaries has been built in the
+# build/ directory, as such, will will load the Python libraries in the directory tree, and also load
+# the TDengine client shared library (so) file, in the build/directory, as evidenced in the env
+# variables below.
+#
+# Running the script is simple, no parameter is needed (for now, but will change in the future).
+#
+# Happy Crashing...
+
+
+# Due to the heavy path name assumptions/usage, let us require that the user be in the current directory
+EXEC_DIR=`dirname "$0"`
+if [[ $EXEC_DIR != "." ]]
+then
+    echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)"
+    exit -1
+fi
+
+CURR_DIR=`pwd`
+IN_TDINTERNAL="community"
+if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  TAOS_DIR=$CURR_DIR/../../..
+  TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
+  LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6,7|rev`/lib
+else
+  TAOS_DIR=$CURR_DIR/../..
+  TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
+  LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib
+fi
+
+# Now getting ready to execute Python
+# The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk
+PYTHON_EXEC=python3.8
+
+# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work.
+export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd)
+
+# Then let us set up the library path so that our compiled SO file can be loaded by Python
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR
+
+# Now we are all let, and let's see if we can find a crash. Note we pass all params
+CONCURRENT_INQUIRY=concurrent_inquiry.py
+if [[ $1 == '--valgrind' ]]; then
+  shift
+  export PYTHONMALLOC=malloc
+  VALGRIND_OUT=valgrind.out 
+  VALGRIND_ERR=valgrind.err
+  # How to generate valgrind suppression file: https://stackoverflow.com/questions/17159578/generating-suppressions-for-memory-leaks
+  # valgrind --leak-check=full --gen-suppressions=all --log-fd=9 python3.8 ./concurrent_inquiry.py $@ 9>>memcheck.log
+  echo Executing under VALGRIND, with STDOUT/ERR going to $VALGRIND_OUT and $VALGRIND_ERR, please watch them from a different terminal.
+  valgrind  \
+    --leak-check=yes \
+    --suppressions=crash_gen/valgrind_taos.supp \
+    $PYTHON_EXEC \
+    $CONCURRENT_INQUIRY $@ > $VALGRIND_OUT 2> $VALGRIND_ERR 
+elif [[ $1 == '--helgrind' ]]; then
+  shift
+  HELGRIND_OUT=helgrind.out 
+  HELGRIND_ERR=helgrind.err
+  valgrind  \
+    --tool=helgrind \
+    $PYTHON_EXEC \
+    $CONCURRENT_INQUIRY $@ > $HELGRIND_OUT 2> $HELGRIND_ERR
+else
+  $PYTHON_EXEC $CONCURRENT_INQUIRY $@
+fi
+
--- a/tests/pytest/pytest_2.sh
+++ b/tests/pytest/pytest_2.sh
@ -7,15 +7,10 @@ python3 ./test.py -f update/append_commit_data.py
 python3 ./test.py -f update/append_commit_last-0.py
 python3 ./test.py -f update/append_commit_last.py
 python3 ./test.py -f update/merge_commit_data.py
-python3 ./test.py -f update/merge_commit_data-0.py
+
 python3 ./test.py -f update/merge_commit_data2.py
 python3 ./test.py -f update/merge_commit_data2_update0.py
 python3 ./test.py -f update/merge_commit_last-0.py
 python3 ./test.py -f update/merge_commit_last.py
 python3 ./test.py -f update/bug_td2279.py

-# wal
-python3 ./test.py -f wal/addOldWalTest.py
-
-# function
-python3 ./test.py -f functions/all_null_value.py
--- a/tests/pytest/pytest_3.sh
+++ b/tests/pytest/pytest_3.sh
@ -70,26 +70,7 @@ python3 testCompress.py
 python3 testNoCompress.py
 python3 testMinTablesPerVnode.py

-# functions
-python3 ./test.py -f functions/function_avg.py -r 1
-python3 ./test.py -f functions/function_bottom.py -r 1
-python3 ./test.py -f functions/function_count.py -r 1
-python3 ./test.py -f functions/function_diff.py -r 1
-python3 ./test.py -f functions/function_first.py -r 1
-python3 ./test.py -f functions/function_last.py -r 1
-python3 ./test.py -f functions/function_last_row.py -r 1
-python3 ./test.py -f functions/function_leastsquares.py -r 1
-python3 ./test.py -f functions/function_max.py -r 1
-python3 ./test.py -f functions/function_min.py -r 1
-python3 ./test.py -f functions/function_operations.py -r 1 
-python3 ./test.py -f functions/function_percentile.py -r 1
-python3 ./test.py -f functions/function_spread.py -r 1
-python3 ./test.py -f functions/function_stddev.py -r 1
-python3 ./test.py -f functions/function_sum.py -r 1
-python3 ./test.py -f functions/function_top.py -r 1
-python3 ./test.py -f functions/function_twa.py -r 1
-python3 ./test.py -f functions/function_twa_test2.py
-python3 ./test.py -f functions/function_stddev_td2555.py
+
 python3 queryCount.py
 python3 ./test.py -f query/queryGroupbyWithInterval.py
 python3 client/twoClients.py
--- a/tests/pytest/pytest_4.sh
+++ b/tests/pytest/pytest_4.sh
@ -0,0 +1,26 @@
+python3 ./test.py -f update/merge_commit_data-0.py
+# wal
+python3 ./test.py -f wal/addOldWalTest.py
+
+# function
+python3 ./test.py -f functions/all_null_value.py
+# functions
+python3 ./test.py -f functions/function_avg.py -r 1
+python3 ./test.py -f functions/function_bottom.py -r 1
+python3 ./test.py -f functions/function_count.py -r 1
+python3 ./test.py -f functions/function_diff.py -r 1
+python3 ./test.py -f functions/function_first.py -r 1
+python3 ./test.py -f functions/function_last.py -r 1
+python3 ./test.py -f functions/function_last_row.py -r 1
+python3 ./test.py -f functions/function_leastsquares.py -r 1
+python3 ./test.py -f functions/function_max.py -r 1
+python3 ./test.py -f functions/function_min.py -r 1
+python3 ./test.py -f functions/function_operations.py -r 1 
+python3 ./test.py -f functions/function_percentile.py -r 1
+python3 ./test.py -f functions/function_spread.py -r 1
+python3 ./test.py -f functions/function_stddev.py -r 1
+python3 ./test.py -f functions/function_sum.py -r 1
+python3 ./test.py -f functions/function_top.py -r 1
+python3 ./test.py -f functions/function_twa.py -r 1
+python3 ./test.py -f functions/function_twa_test2.py
+python3 ./test.py -f functions/function_stddev_td2555.pyhao
--- a/tests/script/jenkins/basic_4.txt
+++ b/tests/script/jenkins/basic_4.txt
@ -1,14 +1,3 @@
-./test.sh -f unique/dnode/alternativeRole.sim
-./test.sh -f unique/dnode/balance1.sim
-./test.sh -f unique/dnode/balance2.sim
-./test.sh -f unique/dnode/balance3.sim
-./test.sh -f unique/dnode/balancex.sim
-./test.sh -f unique/dnode/offline1.sim
-./test.sh -f unique/dnode/offline2.sim
-./test.sh -f unique/dnode/reason.sim
-./test.sh -f unique/dnode/remove1.sim
-./test.sh -f unique/dnode/remove2.sim
-./test.sh -f unique/dnode/vnode_clean.sim

 ./test.sh -f unique/http/admin.sim
 ./test.sh -f unique/http/opentsdb.sim
--- a/tests/script/jenkins/basic_5.txt
+++ b/tests/script/jenkins/basic_5.txt
@ -1,3 +1,11 @@
+./test.sh -f unique/dnode/alternativeRole.sim
+./test.sh -f unique/dnode/balance1.sim
+./test.sh -f unique/dnode/balance2.sim
+./test.sh -f unique/dnode/balance3.sim
+./test.sh -f unique/dnode/balancex.sim
+./test.sh -f unique/dnode/offline1.sim
+./test.sh -f unique/dnode/offline2.sim
+
 ./test.sh -f general/stream/metrics_del.sim
 ./test.sh -f general/stream/metrics_replica1_vnoden.sim
 ./test.sh -f general/stream/restart_stream.sim
--- a/tests/script/jenkins/basic_6.txt
+++ b/tests/script/jenkins/basic_6.txt
@ -1,3 +1,8 @@
+./test.sh -f unique/dnode/reason.sim
+./test.sh -f unique/dnode/remove1.sim
+./test.sh -f unique/dnode/remove2.sim
+./test.sh -f unique/dnode/vnode_clean.sim
+
 ./test.sh -f unique/db/commit.sim
 ./test.sh -f unique/db/delete.sim
 ./test.sh -f unique/db/delete_part.sim
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@ -261,6 +261,9 @@ if [ "$2" != "sim" ]; then
  elif [ "$1" == "p3" ]; then
    echo "### run Python_3 test ###"
    runPyCaseOneByOnefq pytest_3.sh
+  elif [ "$1" == "p4" ]; then
+    echo "### run Python_4 test ###"
+    runPyCaseOneByOnefq pytest_4.sh
  elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then
    exit $(($totalFailed + $totalPyFailed))
  elif [ "$1" == "smoke" ] || [ -z "$1" ]; then