From c01e23925df17aa98715e60f36c4ad3949d6bb75 Mon Sep 17 00:00:00 2001
From: root <root@master.jzpmcbu051oebgsmotycmb3lpe.xx.internal.cloudapp.net>
Date: Sat, 27 Mar 2021 15:23:23 +0800
Subject: [PATCH 1/4] [TD-3570]<test>add taosd valgrind test

---
 Jenkinsfile                          |  6 +++
 tests/Jenkinsfile                    | 11 +++++
 tests/pytest/handle_taosd_val_log.sh | 69 ++++++++++++++++++++++++++++
 3 files changed, 86 insertions(+)
 create mode 100755 tests/pytest/handle_taosd_val_log.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index daae5a5ad0..bd7d5d700a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -185,6 +185,12 @@ pipeline {
             rm -rf /var/log/taos/*
             ./handle_crash_gen_val_log.sh
             '''
+            sh '''
+            cd ${WKC}/tests/pytest
+            rm -rf /var/lib/taos/*
+            rm -rf /var/log/taos/*
+            ./handle_taosd_val_log.sh
+            '''
             timeout(time: 45, unit: 'MINUTES'){
                 sh '''
                 date
diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile
index 6db6351442..b7b36eae07 100644
--- a/tests/Jenkinsfile
+++ b/tests/Jenkinsfile
@@ -76,9 +76,20 @@ pipeline {
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
                 cd ${WKC}/tests/pytest
+                rm -rf /var/lib/taos/*
+                rm -rf /var/log/taos/*
                 ./handle_crash_gen_val_log.sh
                 '''
             }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                rm -rf /var/lib/taos/*
+                rm -rf /var/log/taos/*
+                ./handle_taosd_val_log.sh
+                '''
+            }
+            
             sh'''
             systemctl start taosd 
             sleep 10
diff --git a/tests/pytest/handle_taosd_val_log.sh b/tests/pytest/handle_taosd_val_log.sh
new file mode 100755
index 0000000000..221dfc649f
--- /dev/null
+++ b/tests/pytest/handle_taosd_val_log.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+IN_TDINTERNAL="community"
+TDIR=`pwd`
+if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
+  cd ../..
+else
+  cd ../../..
+fi
+TOP_DIR=`pwd`
+TAOSD_DIR=`find . -name "taosd"|grep -v community|grep debug|head -n1`
+VALGRIND_OUT=taosd_valgrind.out 
+VALGRIND_ERR=taosd_valgrind.err
+rm -rf /var/lib/taos/*
+# nohup valgrind  --tool=memcheck --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR  &
+nohup valgrind  --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR  &
+sleep 5
+cd -
+./crash_gen.sh  -p -t 3 -s 200 
+ps -ef |grep valgrind|grep -v grep|awk '{print $2}'|xargs kill -term
+while true 
+do
+	monitoring=` ps -ef|grep valgrind |grep -v grep| wc -l`
+	if [ $monitoring -eq 0 ] 
+	then
+		echo "Manipulator is not running "
+    break
+	else
+		sleep 1
+	fi
+done
+
+grep 'start to execute\|ERROR SUMMARY' $VALGRIND_ERR | grep -v 'grep' | uniq | tee taosd_mem_err.log
+
+for memError in `grep 'ERROR SUMMARY' taosd_mem_err.log | awk '{print $4}'`
+do
+memError=(${memError//,/})
+echo $memError
+if [ -n "$memError" ]; then
+    if [ "$memError" -gt 12 ]; then
+    echo -e "${RED} ## Memory errors number valgrind reports is $memError.\
+                More than our threshold! ## ${NC}"
+    fi
+fi
+done
+
+grep 'start to execute\|definitely lost:' $VALGRIND_ERR|grep -v 'grep'|uniq|tee taosd-definitely-lost-out.log
+for defiMemError in `grep 'definitely lost:' taosd-definitely-lost-out.log | awk '{print $7}'`
+do
+defiMemError=(${defiMemError//,/})
+if [ -n "$defiMemError" ]; then
+    if [ "$defiMemError" -gt 0 -a "$defiMemError" -lt 1013 ]; then
+      cat $VALGRIND_ERR
+      echo -e "${RED} ## Memory errors number valgrind reports \
+                  Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
+      exit 8
+    elif [ "$defiMemError" -gt 1013 ];then           #add for azure
+      cat $VALGRIND_ERR
+      echo -e "${RED} ## Memory errors number valgrind reports \
+                Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
+      exit 8
+    fi
+fi
+done

From 81e6b7301e42959a474eb117db0da9faa0baf577 Mon Sep 17 00:00:00 2001
From: root <root@master.jzpmcbu051oebgsmotycmb3lpe.xx.internal.cloudapp.net>
Date: Sat, 27 Mar 2021 15:32:10 +0800
Subject: [PATCH 2/4] fix

---
 tests/pytest/handle_taosd_val_log.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/pytest/handle_taosd_val_log.sh b/tests/pytest/handle_taosd_val_log.sh
index 221dfc649f..6b7f669efe 100755
--- a/tests/pytest/handle_taosd_val_log.sh
+++ b/tests/pytest/handle_taosd_val_log.sh
@@ -19,9 +19,9 @@ VALGRIND_ERR=taosd_valgrind.err
 rm -rf /var/lib/taos/*
 # nohup valgrind  --tool=memcheck --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR  &
 nohup valgrind  --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR  &
-sleep 5
+sleep 20
 cd -
-./crash_gen.sh  -p -t 3 -s 200 
+./crash_gen.sh  -p -t 10 -s 200 
 ps -ef |grep valgrind|grep -v grep|awk '{print $2}'|xargs kill -term
 while true 
 do
@@ -40,7 +40,6 @@ grep 'start to execute\|ERROR SUMMARY' $VALGRIND_ERR | grep -v 'grep' | uniq | t
 for memError in `grep 'ERROR SUMMARY' taosd_mem_err.log | awk '{print $4}'`
 do
 memError=(${memError//,/})
-echo $memError
 if [ -n "$memError" ]; then
     if [ "$memError" -gt 12 ]; then
     echo -e "${RED} ## Memory errors number valgrind reports is $memError.\

From 1f4bba65332904d14b695898decdf50c2afa3568 Mon Sep 17 00:00:00 2001
From: root <root@master.jzpmcbu051oebgsmotycmb3lpe.xx.internal.cloudapp.net>
Date: Sat, 27 Mar 2021 15:34:40 +0800
Subject: [PATCH 3/4] fix

---
 Jenkinsfile | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bd7d5d700a..52236682d7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -185,12 +185,14 @@ pipeline {
             rm -rf /var/log/taos/*
             ./handle_crash_gen_val_log.sh
             '''
-            sh '''
-            cd ${WKC}/tests/pytest
-            rm -rf /var/lib/taos/*
-            rm -rf /var/log/taos/*
-            ./handle_taosd_val_log.sh
-            '''
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                rm -rf /var/lib/taos/*
+                rm -rf /var/log/taos/*
+                ./handle_crash_gen_val_log.sh
+                '''
+            }
             timeout(time: 45, unit: 'MINUTES'){
                 sh '''
                 date

From b03c0c20604a30bac3543d746d168b09c35a35b6 Mon Sep 17 00:00:00 2001
From: root <root@master.jzpmcbu051oebgsmotycmb3lpe.xx.internal.cloudapp.net>
Date: Sat, 27 Mar 2021 15:39:03 +0800
Subject: [PATCH 4/4] fix

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 52236682d7..330ac1987b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -190,7 +190,7 @@ pipeline {
                 cd ${WKC}/tests/pytest
                 rm -rf /var/lib/taos/*
                 rm -rf /var/log/taos/*
-                ./handle_crash_gen_val_log.sh
+                ./handle_taosd_val_log.sh
                 '''
             }
             timeout(time: 45, unit: 'MINUTES'){