Merge pull request #1860 from taosdata/hotfix/fix-issue-pytest-stop-by-pkill

fix issue pytest stop dnode by pkill.

.travis.yml

@@ -72,8 +72,8 @@ matrix:
             fi
           done
 
-          grep 'definitely lost' mem-error-out.txt | uniq | tee uniq-definitely-lost-out.txt
+          grep 'definitely lost:' mem-error-out.txt | uniq | tee uniq-definitely-lost-out.txt
-          for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'` 
+          for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $4}'`
           do
             if [ -n "$defiMemError" ]; then
               if [ "$defiMemError" -gt 16 ]; then

tests/pytest/insert/binary.py

@@ -14,7 +14,6 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-
         tdLog.info('=============== step1')
         tdLog.info('create table tb (ts timestamp, speed binary(5))')
         tdSql.execute('create table tb (ts timestamp, speed binary(5))')

tests/pytest/smoketest.sh

@@ -1,172 +1,116 @@
 #!/bin/bash
 python3 ./test.py $1 -f insert/basic.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/int.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/float.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/bigint.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/bool.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/double.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/smallint.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/tinyint.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/binary.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/date.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f insert/nchar.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 
 python3 ./test.py $1 -f table/column_name.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f table/column_num.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f table/db_table.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 
 python3 ./test.py $1 -f import_merge/importDataLastTO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importDataLastT.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importDataTO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importDataT.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importHeadOverlap.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importHeadPartOverlap.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importHORestart.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importHPORestart.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importHRestart.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importLastSub.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 
 python3 ./test.py $1 -f import_merge/importBlock1HO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock1HPO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock1H.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock1S.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock1Sub.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock1TO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock1TPO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock1T.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock2HO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock2HPO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock2H.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock2S.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock2Sub.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock2TO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock2TPO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlock2T.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importBlockbetween.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importCacheFileSub.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importCacheFileTO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importCacheFileT.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importDataLastSub.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importHead.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importLastTO.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importLastT.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importSpan.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importSRestart.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importSubRestart.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importTailOverlap.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importTail.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importTORestart.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importTPORestart.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1
 python3 ./test.py $1 -f import_merge/importTRestart.py
-python3 ./test.py -s $1
+python3 ./test.py $1 -s && sleep 1
-sleep 1

tests/pytest/test.py

@@ -92,20 +92,20 @@ if __name__ == "__main__":
     tdDnodes.start(1)
 
     if masterIp == "":
-        host='127.0.0.1'
+        host = '127.0.0.1'
     else:
-        host=masterIp
+        host = masterIp
 
-    tdLog.notice("Procedures for tdengine deployed in %s" % (host))
+    tdLog.info("Procedures for tdengine deployed in %s" % (host))
 
     if testCluster:
-        tdLog.notice("Procedures for testing cluster")
+        tdLog.info("Procedures for testing cluster")
         if fileName == "all":
             tdCases.runAllCluster()
         else:
             tdCases.runOneCluster(fileName)
     else:
-        tdLog.notice("Procedures for testing self-deployment")
+        tdLog.info("Procedures for testing self-deployment")
         conn = taos.connect(
             host,
             config=tdDnodes.getSimCfgPath())

tests/pytest/util/dnodes.py

@@ -205,24 +205,47 @@ class TDDnode:
         time.sleep(2)
 
     def stop(self):
+        if self.valgrind == 0:
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
         if self.running != 0:
-            cmd = "ps -ef|grep -w taosd | grep '%s' | grep -v grep | awk '{print $2}' && pkill -sigint taosd" % (
+            killCmd = "ps -ef|grep -w %s| grep '%s' | grep -v grep | awk '{print $2}' | xargs kill -INT" % (
-                self.cfgDir)
+                toBeKilled, self.cfgDir)
-            if os.system(cmd) != 0:
+
-                tdLog.exit(cmd)
+            psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
-            tdLog.debug("dnode:%d is stopped by kill -SIGINT" % (self.index))
+            processID = subprocess.check_output(psCmd, shell=True)
+
+            while(processID):
+                os.system(killCmd)
+                time.sleep(1)
+                processID = subprocess.check_output(psCmd, shell=True)
+
+            tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
             tdLog.debug(
                 "wait 2 seconds for the dnode:%d to stop." %
                 (self.index))
             time.sleep(2)
 
     def forcestop(self):
+        if self.valgrind == 0:
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
         if self.running != 0:
-            cmd = "ps -ef|grep -w taosd | grep '%s' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd" % (
+            killCmd = "ps -ef|grep -w %s| grep '%s' | grep -v grep | awk '{print $2}' | xargs kill -KILL" % (
-                self.cfgDir)
+                toBeKilled, self.cfgDir)
-            if os.system(cmd) != 0:
+            psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
-                tdLog.exit(cmd)
+            processID = subprocess.check_output(psCmd, shell=True)
-            tdLog.debug("dnode:%d is stopped by kill -9" % (self.index))
+
+            while(processID):
+                os.system(killCmd)
+                time.sleep(1)
+                processID = subprocess.check_output(psCmd, shell=True)
+
+            tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index))
             tdLog.debug(
                 "wait 2 seconds for the dnode:%d to stop." %
                 (self.index))
@@ -268,8 +291,21 @@ class TDDnodes:
         self.dnodes.append(TDDnode(10))
 
     def init(self, path):
-        cmd = "ps -ef|grep -w taosd | grep 'taosd' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd"
+        killCmd = "ps -ef|grep -w taosd | grep -v grep | awk '{print $2}' | xargs kill -KILL"
-        os.system(cmd)
+        psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True)
+        while(processID):
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
+
+        killCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}' | xargs kill -KILL"
+        psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True)
+        while(processID):
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
 
         binPath = os.path.dirname(os.path.realpath(__file__))
         binPath = binPath + "/../../../debug/"
@@ -361,8 +397,22 @@ class TDDnodes:
             os.system(cmd)
         # if os.system(cmd) != 0 :
         # tdLog.exit(cmd)
-        cmd = "ps -ef | grep -w taosd | grep 'dnode' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd"
+        killCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}' | xargs kill -KILL"
-        os.system(cmd)
+        psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True)
+        while(processID):
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
+
+        killCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}' | xargs kill -KILL"
+        psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True)
+        while(processID):
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
+
         # if os.system(cmd) != 0 :
         # tdLog.exit(cmd)