Merge remote-tracking branch 'origin/develop' into feature/wal

packaging/tools/set_core.sh

@@ -2,19 +2,39 @@
 #
 # This file is used to set config for core when taosd crash
 
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
 set -e
 # set -x
+corePath=$1
 
 csudo=""
 if command -v sudo > /dev/null; then
   csudo="sudo"
 fi
 
-#ulimit -c unlimited
+if [[ ! -n ${corePath} ]]; then
+  echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:"
+	read corePath
+  while true; do
+    if [[ ! -z "$corePath" ]]; then
+      break
+    else
+      read -p "Please enter a file directory to save the coredump file:" corePath
+    fi
+  done
+fi
+
+ulimit -c unlimited
 ${csudo} sed -i '/ulimit -c unlimited/d' /etc/profile ||:
 ${csudo} sed -i '$a\ulimit -c unlimited' /etc/profile ||:
 source /etc/profile
 
-${csudo} mkdir -p /coredump  ||:
-${csudo} sysctl -w kernel.core_pattern='/coredump/core-%e-%p'  ||:
-${csudo} echo '/coredump/core-%e-%p' | ${csudo} tee /proc/sys/kernel/core_pattern  ||:
+${csudo} mkdir -p ${corePath}  ||:
+${csudo} sysctl -w kernel.core_pattern=${corePath}/core-%e-%p  ||:
+${csudo} echo "${corePath}/core-%e-%p" | ${csudo} tee /proc/sys/kernel/core_pattern  ||:

src/kit/taosdump/taosdump.c

@@ -332,6 +332,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
       break;
     case 'N':
       arguments->data_batch = atoi(arg);
+      if (arguments->data_batch >= INT16_MAX) {
+        arguments->data_batch = INT16_MAX - 1;
+      } 
       break;
     case 'L': 
     {

src/query/inc/qHistogram.h

@@ -67,7 +67,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto);
 
 void tHistogramPrint(SHistogramInfo* pHisto);
 
-//int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
+int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
 
 SHeapEntry* tHeapCreate(int32_t numOfEntries);
 void tHeapSort(SHeapEntry* pEntry, int32_t len);

src/query/src/qHistogram.c

@@ -120,7 +120,6 @@
 //}
 
 static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val);
-static int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
 
 SHistogramInfo* tHistogramCreate(int32_t numOfEntries) {
   /* need one redundant slot */

src/query/tests/histogramTest.cpp

@@ -21,19 +21,19 @@ TEST(testCase, histogram_binary_search) {
     pHisto->elems[i].val = i;
   }
 
-  int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
+  int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
   assert(idx == 1);
 
-  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
+  idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
   assert(idx == 9);
 
-  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
+  idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
   assert(idx == 10);
 
-  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
+  idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
   assert(idx == 0);
 
-  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
+  idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
   assert(idx == 4);
 
   free(pHisto);

src/tsdb/src/tsdbCommit.c

@@ -161,6 +161,11 @@ _err:
 
 static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
   if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno);
+  SMemTable *pIMem = pRepo->imem;
+  tsdbLockRepo(pRepo);
+  pRepo->imem = NULL;
+  tsdbUnlockRepo(pRepo);
+  tsdbUnRefMemTable(pRepo, pIMem);
   sem_post(&(pRepo->readyToCommit));
 }
 

src/tsdb/src/tsdbMemTable.c

@@ -17,6 +17,7 @@
 #include "tsdbMain.h"
 
 #define TSDB_DATA_SKIPLIST_LEVEL 5
+#define TSDB_MAX_INSERT_BATCH 512
 
 static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo);
 static void        tsdbFreeMemTable(SMemTable *pMemTable);
@@ -205,7 +206,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
 int tsdbAsyncCommit(STsdbRepo *pRepo) {
   if (pRepo->mem == NULL) return 0;
 
-  SMemTable *pIMem = pRepo->imem;
+  ASSERT(pRepo->imem == NULL);
 
   sem_wait(&(pRepo->readyToCommit));
 
@@ -220,8 +221,6 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
   tsdbScheduleCommit(pRepo);
   if (tsdbUnlockRepo(pRepo) < 0) return -1;
 
-  if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
-
   return 0;
 }
 
@@ -606,19 +605,13 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
   STable *       pTable = NULL;
   SSubmitBlkIter blkIter = {0};
   SDataRow       row = NULL;
-  void **        rows = NULL;
+  void *         rows[TSDB_MAX_INSERT_BATCH] = {0};
   int            rowCounter = 0;
 
   ASSERT(pBlock->tid < pMeta->maxTables);
   pTable = pMeta->tables[pBlock->tid];
   ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
 
-  rows = (void **)calloc(pBlock->numOfRows, sizeof(void *));
-  if (rows == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
   tsdbInitSubmitBlkIter(pBlock, &blkIter);
   while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
     if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) {
@@ -632,9 +625,18 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
     if (rows[rowCounter] != NULL) {
       rowCounter++;
     }
+
+    if (rowCounter == TSDB_MAX_INSERT_BATCH) {
+      if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
+        goto _err;
+      }
+
+      rowCounter = 0;
+      memset(rows, 0, sizeof(rows));
+    }
   }
 
-  if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
+  if (rowCounter > 0 && tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
     goto _err;
   }
 
@@ -642,11 +644,9 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
   pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
   pRepo->stat.totalStorage += points * schemaVLen(pSchema);
 
-  free(rows);
   return 0;
 
 _err:
-  free(rows);
   return -1;
 }
 

tests/pytest/crash_gen/crash_gen_main.py

@@ -2224,22 +2224,25 @@ class ClientManager:
         if svcMgr: # gConfig.auto_start_service:
             svcMgr.stopTaosServices()
             svcMgr = None
+        
+
+        # Release global variables
+        gConfig = None
+        gSvcMgr = None
+        logger = None
+        
+        thPool = None
+        dbManager.cleanUp() # destructor wouldn't run in time
+        dbManager = None
+
         # Print exec status, etc., AFTER showing messages from the server
         self.conclude()
         # print("TC failed (2) = {}".format(self.tc.isFailed()))
         # Linux return code: ref https://shapeshed.com/unix-exit-codes/
         ret = 1 if self.tc.isFailed() else 0
         self.tc.cleanup()
-
-        # Release global variables
-        gConfig = None
-        gSvcMgr = None
-        logger = None
-
         # Release variables here
         self.tc = None
-        thPool = None
-        dbManager = None
 
         gc.collect() # force garbage collection
         # h = hpy()

tests/pytest/crash_gen/db.py

@@ -394,6 +394,7 @@ class DbManager():
             cType == 'native') else DbConn.createRest(dbTarget)
         try:
             self._dbConn.open()  # may throw taos.error.ProgrammingError: disconnected
+            Logging.debug("DbManager opened DB connection...")
         except taos.error.ProgrammingError as err:
             # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err))
             if (err.msg == 'client disconnected'):  # cannot open DB connection
@@ -412,6 +413,10 @@ class DbManager():
         # Moved to Database()
         # self._stateMachine = StateMechine(self._dbConn)
 
+    def __del__(self):
+        ''' Release the underlying DB connection upon deletion of DbManager '''
+        self.cleanUp()
+
     def getDbConn(self):
         return self._dbConn
 
@@ -438,5 +443,8 @@ class DbManager():
         return "table_{}".format(tblNum)
 
     def cleanUp(self):
-        self._dbConn.close()
+        if self._dbConn:
+            self._dbConn.close()
+            self._dbConn = None
+            Logging.debug("DbManager closed DB connection...")
 

tests/pytest/crash_gen/misc.py

@@ -2,6 +2,7 @@ import threading
 import random
 import logging
 import os
+import sys
 
 import taos
 
@@ -53,7 +54,7 @@ class Logging:
         # global misc.logger
         _logger = logging.getLogger('CrashGen')  # real logger
         _logger.addFilter(LoggingFilter())
-        ch = logging.StreamHandler()
+        ch = logging.StreamHandler(sys.stdout) # Ref: https://stackoverflow.com/questions/14058453/making-python-loggers-output-all-messages-to-stdout-in-addition-to-log-file
         _logger.addHandler(ch)
 
         # Logging adapter, to be used as a logger

tests/pytest/crash_gen_bootstrap.py

@@ -19,5 +19,5 @@ if __name__ == "__main__":
     mExec.init()
     exitCode = mExec.run()
 
-    print("Exiting with code: {}".format(exitCode))
+    print("\nCrash_Gen is now exiting with status code: {}".format(exitCode))
     sys.exit(exitCode)

tests/pytest/import_merge/importCacheFileH.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileHO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileHPO.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileS.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileSub.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileT.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileTO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileTPO.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataH2.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataHO.py

@@ -60,7 +60,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataHO2.py

@@ -60,7 +60,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataHPO.py

@@ -62,7 +62,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastH.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastHO.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastHPO.py

@@ -61,7 +61,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastS.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastSub.py

@@ -32,7 +32,7 @@ class TDTestCase:
         tdDnodes.stop(1)
         tdDnodes.deploy(1)
         tdDnodes.start(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
 
         tdSql.execute('reset query cache')
         tdSql.execute('drop database if exists db')
@@ -60,9 +60,9 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
 
         tdLog.info("================= step5")
         tdLog.info("import 10 data totally repetitive")

tests/pytest/import_merge/importDataLastT.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastTO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastTPO.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataS.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataSub.py

@@ -60,7 +60,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataT.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataTO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataTPO.py

@@ -57,9 +57,9 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
 
         tdLog.info("================= step5")
         tdLog.info("import 20 data later with partly overlap")

tests/pytest/import_merge/importHORestart.py

@@ -57,7 +57,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importHPORestart.py

@@ -62,7 +62,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importHRestart.py

@@ -54,7 +54,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importInsertThenImport.py

@@ -61,7 +61,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.stop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdLog.info("import 100 sequential data again")

tests/pytest/import_merge/importLastH.py

@@ -53,7 +53,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastHO.py

@@ -53,7 +53,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastHPO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastS.py

@@ -53,7 +53,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastSub.py

@@ -53,7 +53,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastT.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastTO.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastTPO.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importSRestart.py

@@ -64,7 +64,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importSubRestart.py

@@ -64,7 +64,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importTORestart.py

@@ -64,7 +64,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importTPORestart.py

@@ -68,7 +68,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importTRestart.py

@@ -61,7 +61,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/query/query.py

@@ -16,7 +16,7 @@ import taos
 from util.log import tdLog
 from util.cases import tdCases
 from util.sql import tdSql
-
+from util.dnodes import tdDnodes
 
 class TDTestCase:
     def init(self, conn, logSql):
@@ -44,6 +44,25 @@ class TDTestCase:
         tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'")
         tdSql.checkRows(1)
 
+        ## test case for https://jira.taosdata.com:18080/browse/TD-2488
+        tdSql.execute("create table m1(ts timestamp, k int) tags(a int)")
+        tdSql.execute("create table t1 using m1 tags(1)")
+        tdSql.execute("create table t2 using m1 tags(2)")
+        tdSql.execute("insert into t1 values('2020-1-1 1:1:1', 1)")
+        tdSql.execute("insert into t1 values('2020-1-1 1:10:1', 2)")
+        tdSql.execute("insert into t2 values('2020-1-1 1:5:1', 99)")
+        
+        tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/script/jenkins/basic_3.txt

@@ -1,38 +1,4 @@
 
-./test.sh -f unique/stable/balance_replica1.sim
-./test.sh -f unique/stable/dnode2_stop.sim
-./test.sh -f unique/stable/dnode2.sim
-./test.sh -f unique/stable/dnode3.sim
-./test.sh -f unique/stable/replica2_dnode4.sim
-./test.sh -f unique/stable/replica2_vnode3.sim
-./test.sh -f unique/stable/replica3_dnode6.sim
-./test.sh -f unique/stable/replica3_vnode3.sim
-
-./test.sh -f unique/mnode/mgmt20.sim
-./test.sh -f unique/mnode/mgmt21.sim
-./test.sh -f unique/mnode/mgmt22.sim
-./test.sh -f unique/mnode/mgmt23.sim
-./test.sh -f unique/mnode/mgmt24.sim
-#./test.sh -f unique/mnode/mgmt25.sim
-#./test.sh -f unique/mnode/mgmt26.sim
-./test.sh -f unique/mnode/mgmt33.sim
-./test.sh -f unique/mnode/mgmt34.sim
-./test.sh -f unique/mnode/mgmtr2.sim
-
-
-
-./test.sh -f general/parser/stream_on_sys.sim
-./test.sh -f general/stream/metrics_del.sim
-./test.sh -f general/stream/metrics_n.sim
-./test.sh -f general/stream/metrics_replica1_vnoden.sim
-./test.sh -f general/stream/restart_stream.sim
-./test.sh -f general/stream/stream_3.sim
-./test.sh -f general/stream/stream_restart.sim
-./test.sh -f general/stream/table_1.sim
-./test.sh -f general/stream/table_del.sim
-./test.sh -f general/stream/table_n.sim
-./test.sh -f general/stream/table_replica1_vnoden.sim
-
 ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim
 #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
@@ -79,3 +45,35 @@
 ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim
 ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim
 ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim
+
+./test.sh -f unique/stable/balance_replica1.sim
+./test.sh -f unique/stable/dnode2_stop.sim
+./test.sh -f unique/stable/dnode2.sim
+./test.sh -f unique/stable/dnode3.sim
+./test.sh -f unique/stable/replica2_dnode4.sim
+./test.sh -f unique/stable/replica2_vnode3.sim
+./test.sh -f unique/stable/replica3_dnode6.sim
+./test.sh -f unique/stable/replica3_vnode3.sim
+
+./test.sh -f unique/mnode/mgmt20.sim
+./test.sh -f unique/mnode/mgmt21.sim
+./test.sh -f unique/mnode/mgmt22.sim
+./test.sh -f unique/mnode/mgmt23.sim
+./test.sh -f unique/mnode/mgmt24.sim
+#./test.sh -f unique/mnode/mgmt25.sim
+#./test.sh -f unique/mnode/mgmt26.sim
+./test.sh -f unique/mnode/mgmt33.sim
+./test.sh -f unique/mnode/mgmt34.sim
+./test.sh -f unique/mnode/mgmtr2.sim
+
+./test.sh -f general/parser/stream_on_sys.sim
+./test.sh -f general/stream/metrics_del.sim
+./test.sh -f general/stream/metrics_n.sim
+./test.sh -f general/stream/metrics_replica1_vnoden.sim
+./test.sh -f general/stream/restart_stream.sim
+./test.sh -f general/stream/stream_3.sim
+./test.sh -f general/stream/stream_restart.sim
+./test.sh -f general/stream/table_1.sim
+./test.sh -f general/stream/table_del.sim
+./test.sh -f general/stream/table_n.sim
+./test.sh -f general/stream/table_replica1_vnoden.sim

tests/script/unique/arbitrator/dn3_mn1_replica_change.sim

@@ -55,7 +55,7 @@ $x = 0
 step1: 
 	$x = $x + 1
 	sleep 1000
-	if $x == 10 then
+	if $x == 40 then
 		return -1
 	endi
 
@@ -106,63 +106,20 @@ endi
 
 print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-
-$loopCnt = 0
-wait_dnode2_offline_0:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
-sql show dnodes
-if $rows != 2 then
-  sleep 2000
-  goto wait_dnode2_offline_0
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-#$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != offline then
-  sleep 2000
-  goto wait_dnode2_offline_0
-endi
-
 system sh/exec.sh -n dnode2 -s start
 
-$loopCnt = 0
-wait_dnode2_reready:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
-sql show dnodes
-if $rows != 2 then
-  sleep 2000
-  goto wait_dnode2_reready
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-#$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
+$x = 0
+a0: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
 
-if $dnode2Status != ready then
-  sleep 2000
-  goto wait_dnode2_reready
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a0
 endi
 
 sql select count(*) from $stb
@@ -179,7 +136,7 @@ $x = 0
 step2: 
 	$x = $x + 1
 	sleep 1000
-	if $x == 10 then
+	if $x == 40 then
 		return -1
 	endi
 
@@ -217,37 +174,18 @@ endi
 
 print ============== step4: stop dnode2 for checking if sync ok
 system sh/exec.sh -n dnode2 -s stop
-$loopCnt = 0
-wait_dnode2_offline:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
+$x = 0
+a2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
 
-sql show dnodes
-if $rows != 3 then
-  sleep 2000
-  goto wait_dnode2_offline
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != offline then
-  sleep 2000
-  goto wait_dnode2_offline
-endi
-if $dnode3Status != ready then
-  sleep 2000
-  goto wait_dnode2_offline
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a2
 endi
 
 # check using select
@@ -260,7 +198,7 @@ endi
 print ============== step5: restart dnode2
 system sh/exec.sh -n dnode2 -s start
 $x = 0
-a2: 
+a3: 
 	$x = $x + 1
 	sleep 1000
 	if $x == 40 then
@@ -270,7 +208,7 @@ a2:
 sql show vgroups 
 print online vnodes $data03
 if $data03 != 2 then
-	goto a2
+	goto a3
 endi
 
 # check using select
@@ -303,7 +241,7 @@ endi
 
 sql alter database $db replica 3
 $x = 0
-a3: 
+a4: 
 	$x = $x + 1
 	sleep 1000
 	if $x == 40 then
@@ -313,7 +251,7 @@ a3:
 sql show vgroups 
 print online vnodes $data03
 if $data03 != 3 then
-	goto a3
+	goto a4
 endi
 
 # check using select
@@ -325,44 +263,8 @@ endi
 
 print ============== step7: alter replica from 3 to 2, and waiting sync
 sql alter database $db replica 2
-$loopCnt = 0
-wait_vgroups_replic_to_2:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
-
-sql show vgroups
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1  $data10_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2  $data10_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3  $data10_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4  $data9_4  $data10_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-$thirdDnode_2 = $data8_1 
-$thirdDnode_3 = $data8_2 
-$thirdDnode_4 = $data8_3 
-$thirdDnode_5 = $data8_4 
-
-if $thirdDnode_2 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_2
-endi
-if $thirdDnode_3 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_2
-endi
-if $thirdDnode_4 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_2
-endi
-if $thirdDnode_5 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_2
-endi
-
 $x = 0
-a4: 
+a5: 
 	$x = $x + 1
 	sleep 1000
 	if $x == 40 then
@@ -372,7 +274,7 @@ a4:
 sql show vgroups 
 print online vnodes $data03
 if $data03 != 2 then
-	goto a4
+	goto a5
 endi
 
 # check using select
@@ -384,145 +286,6 @@ endi
 
 print ============== step8: alter replica from 2 to 1, and waiting sync
 sql alter database $db replica 1
-sleep 10000
-$loopCnt = 0
-wait_vgroups_replic_to_1:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
-
-sql show vgroups
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-$sencodDnode_2 = $data6_1 
-$sencodDnode_3 = $data6_2 
-$sencodDnode_4 = $data6_3 
-$sencodDnode_5 = $data6_4 
-
-if $sencodDnode_2 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_1
-endi
-if $sencodDnode_3 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_1
-endi
-if $sencodDnode_4 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_1
-endi
-if $sencodDnode_5 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_1
-endi
-
-$loopCnt = 0
-all_dnodes_ready:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
-
-sql show dnodes
-if $rows != 4 then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode1Status != ready then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-if $dnode2Status != ready then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-if $dnode3Status != ready then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-if $dnode4Status != ready then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-
-$x = 0
-a5: 
-	$x = $x + 1
-	sleep 1000
-	if $x == 40 then
-	  return -1
-	endi
-
-sql show vgroups 
-print online vnodes $data03
-if $data03 != 1 then
-	goto a5
-endi
-
-# check using select
-sql select count(*) from $stb
-print data00 $data00
-if $data00 != $totalRows then
-	return -1
-endi
-
-print ============== step9: drop dnode2/dnode3
-sql drop dnode $hostname2
-sql drop dnode $hostname3
-sleep $sleepTimer
-
-$loopCnt = 0
-wait_dnode23_dropped:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
-
-sql show dnodes
-if $rows != 2 then
-  sleep 2000
-  goto wait_dnode23_dropped
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-$dnode4Status = $data4_4 
-
-if $dnode2Status != null then
-  sleep 2000
-  goto wait_dnode23_dropped
-endi
-if $dnode3Status != null then
-  sleep 2000
-  goto wait_dnode23_dropped
-endi
-if $dnode4Status != ready then
-  return -1
-endi
-
 $x = 0
 a6: 
 	$x = $x + 1
@@ -537,6 +300,45 @@ if $data03 != 1 then
 	goto a6
 endi
 
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step9: drop dnode2/dnode3
+sql drop dnode $hostname2
+sql drop dnode $hostname3
+sleep $sleepTimer
+
+$x = 0
+step9: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show dnodes
+if $rows != 2 then
+  goto step9
+endi
+
+$x = 0
+a7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a7
+endi
+
 sleep $sleepTimer  #waiting move vnode from dnode3/dnode3 to dnode4
 # check using select
 sql select count(*) from $stb
@@ -544,3 +346,13 @@ print data00 $data00
 if $data00 != $totalRows then
 	return -1
 endi
+
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
+system sh/exec.sh -n dnode6 -s stop  -x SIGINT
+system sh/exec.sh -n dnode7 -s stop  -x SIGINT
+system sh/exec.sh -n dnode8 -s stop  -x SIGINT

tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim

@@ -140,3 +140,11 @@ if $data00 != $totalRows then
 	return -1
 endi
 
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
+system sh/exec.sh -n dnode6 -s stop  -x SIGINT
+system sh/exec.sh -n dnode7 -s stop  -x SIGINT
+system sh/exec.sh -n dnode8 -s stop  -x SIGINT