Merge remote-tracking branch 'origin/feature/query' into feature/query

Jenkinsfile

@@ -1,65 +1,138 @@
 pipeline {
-  agent any
+  agent none
+  environment{
+      WK = '/var/lib/jenkins/workspace/TDinternal'
+      WKC= '/var/lib/jenkins/workspace/TDinternal/community'
+  }
   stages {
     stage('build TDengine') {
+      agent{label 'master'}
       steps {
-        sh '''cd ${WORKSPACE}
+        sh '''
-export TZ=Asia/Harbin
+        cd ${WKC}
-date
+        git checkout develop
-rm -rf ${WORKSPACE}/debug
+        git pull
-mkdir debug
+        git submodule update
-cd debug
+        cd ${WK}
-cmake .. > /dev/null
+        git checkout develop
-make > /dev/null
+        git pull
-cd ${WORKSPACE}/debug'''
+        export TZ=Asia/Harbin
+        date
+        rm -rf ${WK}/debug
+        mkdir debug
+        cd debug
+        #cmake .. > /dev/null
+        #make > /dev/null
+        '''
       }
     }
 
-    stage('test_tsim') {
+    stage('Parallel test stage') {
       parallel {
-        stage('test') {
+        stage('test_b1') {
+          agent{label '184'}
           steps {
-            sh '''cd ${WORKSPACE}/tests
+            sh '''
-#./test-all.sh smoke
+            date
-sudo ./test-all.sh full'''
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            date
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests
+            #./test-all.sh smoke
+            ./test-all.sh b1
+            date'''
           }
         }
 
         stage('test_crash_gen') {
+          agent{label "185"}
           steps {
-            sh '''cd ${WORKSPACE}/tests/pytest
+            sh '''
-sudo ./crash_gen.sh -a -p -t 4 -s 2000'''
+            
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests/pytest
+            ./crash_gen.sh -a -p -t 4 -s 2000
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b2
+            date
+            '''
           }
         }
 
         stage('test_valgrind') {
+          agent{label "186"}
           steps {
-            sh '''cd ${WORKSPACE}/tests/pytest
+            sh '''
-sudo ./valgrind-test.sh 2>&1 > mem-error-out.log
+            date
-grep \'start to execute\\|ERROR SUMMARY\' mem-error-out.log|grep -v \'grep\'|uniq|tee uniq-mem-error-out.log
+            cd ${WKC}
+            git checkout develop
+            git pull
+            git submodule update
+            cd ${WK}
+            git checkout develop
+            git pull
+            export TZ=Asia/Harbin
+            date
+            rm -rf ${WK}/debug
+            mkdir debug
+            cd debug
+            cmake .. > /dev/null
+            make > /dev/null
+            cd ${WKC}/tests/pytest
+            ./valgrind-test.sh 2>&1 > mem-error-out.log
+            grep \'start to execute\\|ERROR SUMMARY\' mem-error-out.log|grep -v \'grep\'|uniq|tee uniq-mem-error-out.log
 
-for memError in `grep \'ERROR SUMMARY\' uniq-mem-error-out.log | awk \'{print $4}\'`
+            for memError in `grep \'ERROR SUMMARY\' uniq-mem-error-out.log | awk \'{print $4}\'`
-do
+            do
-  if [ -n "$memError" ]; then
+              if [ -n "$memError" ]; then
-    if [ "$memError" -gt 12 ]; then
+                if [ "$memError" -gt 12 ]; then
-      echo -e "${RED} ## Memory errors number valgrind reports is $memError.\\
+                  echo -e "${RED} ## Memory errors number valgrind reports is $memError.\\
-               More than our threshold! ## ${NC}"
+                          More than our threshold! ## ${NC}"
-      travis_terminate $memError
+                  travis_terminate $memError
-    fi
+                fi
-  fi
+              fi
-done
+            done
 
-grep \'start to execute\\|definitely lost:\' mem-error-out.log|grep -v \'grep\'|uniq|tee uniq-definitely-lost-out.log
+            grep \'start to execute\\|definitely lost:\' mem-error-out.log|grep -v \'grep\'|uniq|tee uniq-definitely-lost-out.log
-for defiMemError in `grep \'definitely lost:\' uniq-definitely-lost-out.log | awk \'{print $7}\'`
+            for defiMemError in `grep \'definitely lost:\' uniq-definitely-lost-out.log | awk \'{print $7}\'`
-do
+            do
-  if [ -n "$defiMemError" ]; then
+              if [ -n "$defiMemError" ]; then
-    if [ "$defiMemError" -gt 13 ]; then
+                if [ "$defiMemError" -gt 13 ]; then
-      echo -e "${RED} ## Memory errors number valgrind reports \\
+                  echo -e "${RED} ## Memory errors number valgrind reports \\
-               Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
+                          Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
-      travis_terminate $defiMemError
+                  travis_terminate $defiMemError
-    fi
+                fi
-  fi
+              fi
-done'''
+            done
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b3
+            date'''
           }
         }
 
@@ -67,4 +140,4 @@ done'''
     }
 
   }
-}
+}

documentation20/webdocs/markdowndocs/administrator-ch.md

@@ -381,7 +381,7 @@ KILL STREAM <stream-id>;
 
 ## 系统监控
 
-TDengine启动后，会自动创建一个监测数据库SYS，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在SYS库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
+TDengine启动后，会自动创建一个监测数据库log，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在log库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
 
 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。
 

packaging/cfg/taos.cfg

@@ -6,73 +6,76 @@
 ########################################################
 
 # first fully qualified domain name (FQDN) for TDengine system
-# firstEp               hostname1:6030
+# firstEp                   hostname:6030
-
-# second fully qualified domain name (FQDN) for TDengine system, for cluster only
-# secondEp              cluster_hostname2:6030
 
 # local fully qualified domain name (FQDN)
-# fqdn                  hostname
+# fqdn                      hostname
 
 # first port number for the connection (12 continuous UDP/TCP port number are used) 
-# serverPort            6030
+# serverPort                6030
 
 # log file's directory
-# logDir                /var/log/taos
+# logDir                    /var/log/taos
 
 # data file's directory
-# dataDir               /var/lib/taos
+# dataDir                   /var/lib/taos
 
 # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only   
-# arbitrator            arbitrator_hostname:6042     
+# arbitrator                arbitrator_hostname:6042     
 
 # number of threads per CPU core
-# numOfThreadsPerCore   1.0
+# numOfThreadsPerCore       1.0
+
+# the proportion of total threads responsible for query
+# ratioOfQueryThreads       0.5
 
 # number of management nodes in the system
-# numOfMnodes           3
+# numOfMnodes               3
 
 # enable/disable backuping vnode directory when removing dnode
-# vnodeBak              1
+# vnodeBak                  1
+
+# if report installation / use information
+# telemetryReporting        1
 
 # enable/disable load balancing
-# balance               1
+# balance                   1
 
 # role for dnode. 0 - any, 1 - mnode, 2 - dnode
-# role                  0
+# role                      0
 
 # max timer control blocks
-# maxTmrCtrl            512
+# maxTmrCtrl                512
 
 # time interval of system monitor, seconds 
-# monitorInterval       30
+# monitorInterval           30
 
 # number of seconds allowed for a dnode to be offline, for cluster only 
-# offlineThreshold      8640000
+# offlineThreshold          8640000
 
 # RPC re-try timer, millisecond
-# rpcTimer              300
+# rpcTimer                  300
 
 # RPC maximum time for ack, seconds. 
-# rpcMaxTime            600
+# rpcMaxTime                600
 
 # time interval of dnode status reporting to mnode, seconds, for cluster only 
-# statusInterval        1
+# statusInterval            1
 
 # time interval of heart beat from shell to dnode, seconds
-# shellActivityTimer    3
+# shellActivityTimer        3
 
 # time of keeping table meta data in cache, seconds
-# tableMetaKeepTimer    7200
+# tableMetaKeepTimer        7200
 
 # minimum sliding window time, milli-second
-# minSlidingTime        10
+# minSlidingTime            10
 
 # minimum time window, milli-second
-# minIntervalTime       10
+# minIntervalTime           10
 
 # maximum delay before launching a stream compution, milli-second
-# maxStreamCompDelay    20000
+# maxStreamCompDelay        20000
 
 # maximum delay before launching a stream computation for the first time, milli-second
 # maxFirstStreamCompDelay   10000
@@ -89,9 +92,6 @@
 # max number of tables per vnode
 # maxTablesPerVnode         1000000
 
-# step size of increasing table number in a vnode
-# tableIncStepPerVnode      1000
-
 # cache block size (Mbyte)
 # cache                     16
 
@@ -110,6 +110,9 @@
 # maximum rows of records in file block
 # maxRows               4096
 
+# the number of acknowledgments required for successful data writing
+# quorum                1     
+
 # enable/disable compression
 # comp                  2
 
@@ -122,15 +125,6 @@
 # number of replications, for cluster only 
 # replica               1
 
-# mqtt hostname  
-# mqttHostName          test.mosquitto.org
-
-# mqtt port  
-# mqttPort              1883
-
-# mqtt topic
-# mqttTopic             /test
-
 # the compressed rpc message, option:
 #  -1 (no compression)
 #   0 (all message compressed),
@@ -167,12 +161,12 @@
 # stop writing data when the disk size of the log folder is less than this value
 # minimalDataDirGB      0.1
 
+# One mnode is equal to the number of vnode consumed
+# mnodeEqualVnodeNum    4
+
 # enbale/disable http service
 # http                  1
 
-# enable/disable muqq service
-# mqtt                  0
-
 # enable/disable system monitor 
 # monitor               1
 
@@ -189,11 +183,12 @@
 # max number of rows per log filters
 # numOfLogLines         10000000
 
+# enable/disable async log
+# asyncLog              1
+
 # time of keeping log files, days
 # logKeepDays           0
 
-# enable/disable async log
-# asyncLog              1
 
 # The following parameters are used for debug purpose only.
 # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
@@ -231,18 +226,12 @@
 # debug flag for JNI
 # jniDebugflag          131
 
-# debug flag for ODBC
-# odbcDebugflag         131
-
 # debug flag for storage
 # uDebugflag            131
 
 # debug flag for http server
 # httpDebugFlag         131
 
-# debug flag for mqtt
-# mqttDebugFlag         131
-
 # debug flag for monitor
 # monitorDebugFlag      131
 
@@ -255,6 +244,9 @@
 # debug flag for http server
 # tsdbDebugFlag         131
 
+# debug flag for continue query
+# cqDebugFlag           131
+
 # enable/disable recording the SQL in taos client
 # tscEnableRecordSql    0
 

src/balance/src/balance.c

@@ -960,9 +960,14 @@ static void balanceMonitorDnodeModule() {
 
     mLInfo("dnode:%d, numOfMnodes:%d expect:%d, create mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes);
     mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, true);
-    
+
+#if 0
     // Only create one mnode each time
     return;
+#else
+    numOfMnodes = mnodeGetMnodesNum();
+    if (numOfMnodes >= tsNumOfMnodes) return;
+#endif
   }
 }
 

src/client/src/tscSQLParser.c

@@ -1200,6 +1200,10 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr
   return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL;
 }
 
+static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
+  SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
+  tscColumnListInsert(pQueryInfo->colList, &tsCol);
+}
 static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) {
   const char* msg1 = "invalid column name, or illegal column type";
   const char* msg2 = "invalid arithmetic expression in select clause";
@@ -1275,6 +1279,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
     addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex);
 
     insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr);
+    // add ts column
+    tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
 
     tbufCloseWriter(&bw);
     taosArrayDestroy(colList);
@@ -1317,10 +1323,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
-static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
-  SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
-  tscColumnListInsert(pQueryInfo->colList, &tsCol);
-}
 
 static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) {
   SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex);

src/common/src/tglobal.c

@@ -308,6 +308,8 @@ bool taosCfgDynamicOptions(char *msg) {
 
 static void doInitGlobalConfig(void) {
   osInit();
+  srand(taosSafeRand());
+
   SGlobalCfg cfg = {0};
   
   // ip address

tests/pytest/crash_gen.sh

@@ -35,12 +35,13 @@ CURR_DIR=`pwd`
 IN_TDINTERNAL="community"
 if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then
   TAOS_DIR=$CURR_DIR/../../..
+  TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
+  LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6,7|rev`/lib
 else
   TAOS_DIR=$CURR_DIR/../..
+  TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
+  LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib
 fi
-TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
-
-LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib
 
 # Now getting ready to execute Python
 # The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk

tests/pytest/valgrind-test.sh

@@ -1,5 +1,26 @@
 #!/bin/bash
+IN_TDINTERNAL="community"
+TDIR=`pwd`
+if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
+  cd ../..
+else
+  cd ../../..
+fi
 
+TOP_DIR=`pwd`
+TAOSLIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1`
+if [[ "$TAOSLIB_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4,5`
+else
+  LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4`
+fi
+if [ ! $LD_LIBRARY_PATH ]; then
+        export LD_LIBRARY_PATH=$TOP_DIR/$LIB_DIR
+else
+        export LD_LIBRARY_PATH=$TOP_DIR/$LIB_DIR:$LD_LIBRARY_PATH
+fi
+
+cd $TDIR
 # client
 PYTHONMALLOC=malloc python3 ./test.py -g -f client/client.py
 PYTHONMALLOC=malloc python3 ./test.py -g -s && sleep 1

tests/script/jenkins/basic.txt

@@ -272,6 +272,7 @@ cd ../../../debug; make
 ./test.sh -f unique/db/replica_part.sim
 
 ./test.sh -f unique/dnode/alternativeRole.sim
+./test.sh -f unique/dnode/simple.sim
 ./test.sh -f unique/dnode/balance1.sim
 ./test.sh -f unique/dnode/balance2.sim
 ./test.sh -f unique/dnode/balance3.sim

tests/script/jenkins/basic_1.txt

@@ -0,0 +1,202 @@
+./test.sh -f general/alter/cached_schema_after_alter.sim
+./test.sh -f general/alter/count.sim
+./test.sh -f general/alter/dnode.sim
+./test.sh -f general/alter/import.sim
+./test.sh -f general/alter/insert1.sim
+./test.sh -f general/alter/insert2.sim
+./test.sh -f general/alter/metrics.sim
+./test.sh -f general/alter/table.sim
+
+./test.sh -f general/cache/new_metrics.sim
+./test.sh -f general/cache/restart_metrics.sim
+./test.sh -f general/cache/restart_table.sim
+
+./test.sh -f general/connection/connection.sim  
+
+./test.sh -f general/column/commit.sim
+./test.sh -f general/column/metrics.sim
+./test.sh -f general/column/table.sim
+
+./test.sh -f general/compress/commitlog.sim
+./test.sh -f general/compress/compress.sim
+./test.sh -f general/compress/compress2.sim
+./test.sh -f general/compress/uncompress.sim
+
+./test.sh -f general/compute/avg.sim
+./test.sh -f general/compute/bottom.sim
+./test.sh -f general/compute/count.sim
+./test.sh -f general/compute/diff.sim
+./test.sh -f general/compute/diff2.sim
+./test.sh -f general/compute/first.sim
+./test.sh -f general/compute/interval.sim
+./test.sh -f general/compute/last.sim
+./test.sh -f general/compute/leastsquare.sim
+./test.sh -f general/compute/max.sim
+./test.sh -f general/compute/min.sim
+./test.sh -f general/compute/null.sim
+./test.sh -f general/compute/percentile.sim
+./test.sh -f general/compute/stddev.sim
+./test.sh -f general/compute/sum.sim
+./test.sh -f general/compute/top.sim
+
+./test.sh -f general/db/alter_option.sim
+./test.sh -f general/db/alter_tables_d2.sim
+./test.sh -f general/db/alter_tables_v1.sim
+./test.sh -f general/db/alter_tables_v4.sim
+./test.sh -f general/db/alter_vgroups.sim
+./test.sh -f general/db/basic.sim
+./test.sh -f general/db/basic1.sim
+./test.sh -f general/db/basic2.sim
+./test.sh -f general/db/basic3.sim
+./test.sh -f general/db/basic4.sim
+./test.sh -f general/db/basic5.sim
+./test.sh -f general/db/delete_reuse1.sim
+./test.sh -f general/db/delete_reuse2.sim
+./test.sh -f general/db/delete_reusevnode.sim
+./test.sh -f general/db/delete_reusevnode2.sim
+./test.sh -f general/db/delete_writing1.sim
+./test.sh -f general/db/delete_writing2.sim
+./test.sh -f general/db/delete.sim
+./test.sh -f general/db/len.sim
+./test.sh -f general/db/repeat.sim
+./test.sh -f general/db/tables.sim
+./test.sh -f general/db/vnodes.sim
+
+./test.sh -f general/field/2.sim
+./test.sh -f general/field/3.sim
+./test.sh -f general/field/4.sim
+./test.sh -f general/field/5.sim
+./test.sh -f general/field/6.sim
+./test.sh -f general/field/bigint.sim
+./test.sh -f general/field/binary.sim
+./test.sh -f general/field/bool.sim
+./test.sh -f general/field/single.sim
+./test.sh -f general/field/smallint.sim
+./test.sh -f general/field/tinyint.sim
+
+./test.sh -f general/http/autocreate.sim
+./test.sh -f general/http/chunked.sim
+./test.sh -f general/http/gzip.sim
+./test.sh -f general/http/restful.sim
+./test.sh -f general/http/restful_insert.sim
+./test.sh -f general/http/restful_limit.sim
+./test.sh -f general/http/restful_full.sim
+./test.sh -f general/http/prepare.sim
+./test.sh -f general/http/telegraf.sim
+./test.sh -f general/http/grafana_bug.sim
+./test.sh -f general/http/grafana.sim
+
+./test.sh -f general/import/basic.sim
+./test.sh -f general/import/commit.sim
+./test.sh -f general/import/large.sim
+./test.sh -f general/import/replica1.sim
+
+./test.sh -f general/insert/basic.sim
+./test.sh -f general/insert/insert_drop.sim
+./test.sh -f general/insert/query_block1_memory.sim
+./test.sh -f general/insert/query_block2_memory.sim
+./test.sh -f general/insert/query_block1_file.sim
+./test.sh -f general/insert/query_block2_file.sim
+./test.sh -f general/insert/query_file_memory.sim
+./test.sh -f general/insert/query_multi_file.sim
+./test.sh -f general/insert/tcp.sim
+
+./test.sh -f general/parser/alter.sim
+./test.sh -f general/parser/alter1.sim
+./test.sh -f general/parser/alter_stable.sim
+./test.sh -f general/parser/auto_create_tb.sim
+./test.sh -f general/parser/auto_create_tb_drop_tb.sim
+./test.sh -f general/parser/col_arithmetic_operation.sim
+./test.sh -f general/parser/columnValue.sim
+./test.sh -f general/parser/commit.sim
+./test.sh -f general/parser/create_db.sim
+./test.sh -f general/parser/create_mt.sim
+./test.sh -f general/parser/create_tb.sim
+./test.sh -f general/parser/dbtbnameValidate.sim
+./test.sh -f general/parser/import_commit1.sim
+./test.sh -f general/parser/import_commit2.sim
+./test.sh -f general/parser/import_commit3.sim
+./test.sh -f general/parser/insert_tb.sim
+./test.sh -f general/parser/first_last.sim
+./test.sh -f general/parser/lastrow.sim
+./test.sh -f general/parser/nchar.sim
+./test.sh -f general/parser/null_char.sim
+./test.sh -f general/parser/single_row_in_tb.sim
+./test.sh -f general/parser/select_from_cache_disk.sim
+./test.sh -f general/parser/mixed_blocks.sim
+./test.sh -f general/parser/selectResNum.sim
+./test.sh -f general/parser/limit.sim
+./test.sh -f general/parser/limit1.sim
+./test.sh -f general/parser/limit1_tblocks100.sim
+./test.sh -f general/parser/select_across_vnodes.sim
+./test.sh -f general/parser/slimit1.sim
+./test.sh -f general/parser/tbnameIn.sim
+./test.sh -f general/parser/projection_limit_offset.sim
+./test.sh -f general/parser/limit2.sim
+./test.sh -f general/parser/fill.sim
+./test.sh -f general/parser/fill_stb.sim
+./test.sh -f general/parser/where.sim
+./test.sh -f general/parser/slimit.sim
+./test.sh -f general/parser/select_with_tags.sim
+./test.sh -f general/parser/interp.sim
+./test.sh -f general/parser/tags_dynamically_specifiy.sim
+./test.sh -f general/parser/groupby.sim
+./test.sh -f general/parser/set_tag_vals.sim
+./test.sh -f general/parser/tags_filter.sim
+./test.sh -f general/parser/slimit_alter_tags.sim
+./test.sh -f general/parser/join.sim
+./test.sh -f general/parser/join_multivnode.sim
+./test.sh -f general/parser/binary_escapeCharacter.sim
+./test.sh -f general/parser/repeatAlter.sim
+./test.sh -f general/parser/union.sim
+./test.sh -f general/parser/topbot.sim
+
+./test.sh -f general/stable/disk.sim
+./test.sh -f general/stable/dnode3.sim
+./test.sh -f general/stable/metrics.sim
+./test.sh -f general/stable/refcount.sim
+./test.sh -f general/stable/show.sim
+./test.sh -f general/stable/values.sim
+./test.sh -f general/stable/vnode3.sim
+
+./test.sh -f general/table/autocreate.sim
+./test.sh -f general/table/basic1.sim
+./test.sh -f general/table/basic2.sim
+./test.sh -f general/table/basic3.sim
+./test.sh -f general/table/bigint.sim
+./test.sh -f general/table/binary.sim
+./test.sh -f general/table/bool.sim
+./test.sh -f general/table/column_name.sim
+./test.sh -f general/table/column_num.sim
+./test.sh -f general/table/column_value.sim
+./test.sh -f general/table/column2.sim
+./test.sh -f general/table/date.sim
+./test.sh -f general/table/db.table.sim
+./test.sh -f general/table/delete_reuse1.sim
+./test.sh -f general/table/delete_reuse2.sim
+./test.sh -f general/table/delete_writing.sim
+./test.sh -f general/table/describe.sim
+./test.sh -f general/table/double.sim
+./test.sh -f general/table/fill.sim
+./test.sh -f general/table/float.sim
+./test.sh -f general/table/int.sim
+./test.sh -f general/table/limit.sim
+./test.sh -f general/table/smallint.sim
+./test.sh -f general/table/table_len.sim
+./test.sh -f general/table/table.sim
+./test.sh -f general/table/tinyint.sim
+./test.sh -f general/table/vgroup.sim
+./test.sh -f unique/dnode/alternativeRole.sim
+./test.sh -f unique/dnode/balance1.sim
+./test.sh -f unique/dnode/balance2.sim
+./test.sh -f unique/dnode/balance3.sim
+./test.sh -f unique/dnode/balancex.sim
+./test.sh -f unique/dnode/offline1.sim
+./test.sh -f unique/dnode/offline2.sim
+./test.sh -f unique/dnode/reason.sim
+./test.sh -f unique/dnode/remove1.sim
+./test.sh -f unique/dnode/remove2.sim
+./test.sh -f unique/dnode/vnode_clean.sim
+
+./test.sh -f unique/http/admin.sim
+./test.sh -f unique/http/opentsdb.sim

tests/script/jenkins/basic_2.txt

@@ -0,0 +1,83 @@
+cd ../../../debug; cmake ..
+cd ../../../debug; make
+
+./test.sh -f general/tag/3.sim
+./test.sh -f general/tag/4.sim
+./test.sh -f general/tag/5.sim
+./test.sh -f general/tag/6.sim
+./test.sh -f general/tag/add.sim
+./test.sh -f general/tag/bigint.sim
+./test.sh -f general/tag/binary_binary.sim
+./test.sh -f general/tag/binary.sim
+./test.sh -f general/tag/bool_binary.sim
+./test.sh -f general/tag/bool_int.sim
+./test.sh -f general/tag/bool.sim
+./test.sh -f general/tag/change.sim
+./test.sh -f general/tag/column.sim
+./test.sh -f general/tag/commit.sim
+./test.sh -f general/tag/create.sim
+./test.sh -f general/tag/delete.sim
+./test.sh -f general/tag/double.sim
+./test.sh -f general/tag/filter.sim
+./test.sh -f general/tag/float.sim
+./test.sh -f general/tag/int_binary.sim
+./test.sh -f general/tag/int_float.sim
+./test.sh -f general/tag/int.sim
+./test.sh -f general/tag/set.sim
+./test.sh -f general/tag/smallint.sim
+./test.sh -f general/tag/tinyint.sim
+
+./test.sh -f general/user/authority.sim
+./test.sh -f general/user/monitor.sim
+./test.sh -f general/user/pass_alter.sim
+./test.sh -f general/user/pass_len.sim
+./test.sh -f general/user/user_create.sim
+./test.sh -f general/user/user_len.sim
+
+./test.sh -f general/vector/metrics_field.sim
+./test.sh -f general/vector/metrics_mix.sim
+./test.sh -f general/vector/metrics_query.sim
+./test.sh -f general/vector/metrics_tag.sim
+./test.sh -f general/vector/metrics_time.sim
+./test.sh -f general/vector/multi.sim
+./test.sh -f general/vector/single.sim
+./test.sh -f general/vector/table_field.sim
+./test.sh -f general/vector/table_mix.sim
+./test.sh -f general/vector/table_query.sim
+./test.sh -f general/vector/table_time.sim
+
+./test.sh -f unique/account/account_create.sim
+./test.sh -f unique/account/account_delete.sim
+./test.sh -f unique/account/account_len.sim
+./test.sh -f unique/account/authority.sim
+./test.sh -f unique/account/basic.sim
+./test.sh -f unique/account/paras.sim
+./test.sh -f unique/account/pass_alter.sim
+./test.sh -f unique/account/pass_len.sim
+./test.sh -f unique/account/usage.sim
+./test.sh -f unique/account/user_create.sim
+./test.sh -f unique/account/user_len.sim
+
+./test.sh -f unique/big/balance.sim
+./test.sh -f unique/big/maxvnodes.sim
+./test.sh -f unique/big/tcp.sim
+
+./test.sh -f unique/cluster/alter.sim
+./test.sh -f unique/cluster/balance1.sim
+./test.sh -f unique/cluster/balance2.sim
+./test.sh -f unique/cluster/balance3.sim
+./test.sh -f unique/cluster/cache.sim
+./test.sh -f unique/cluster/vgroup100.sim
+
+./test.sh -f unique/column/replica3.sim
+
+./test.sh -f unique/db/commit.sim
+./test.sh -f unique/db/delete.sim
+./test.sh -f unique/db/delete_part.sim
+./test.sh -f unique/db/replica_add12.sim
+./test.sh -f unique/db/replica_add13.sim
+./test.sh -f unique/db/replica_add23.sim
+./test.sh -f unique/db/replica_reduce21.sim
+./test.sh -f unique/db/replica_reduce32.sim
+./test.sh -f unique/db/replica_reduce31.sim
+./test.sh -f unique/db/replica_part.sim

tests/script/jenkins/basic_3.txt

@@ -0,0 +1,88 @@
+./test.sh -f unique/import/replica2.sim
+./test.sh -f unique/import/replica3.sim
+
+./test.sh -f unique/stable/balance_replica1.sim
+./test.sh -f unique/stable/dnode2_stop.sim
+./test.sh -f unique/stable/dnode2.sim
+./test.sh -f unique/stable/dnode3.sim
+./test.sh -f unique/stable/replica2_dnode4.sim
+./test.sh -f unique/stable/replica2_vnode3.sim
+./test.sh -f unique/stable/replica3_dnode6.sim
+./test.sh -f unique/stable/replica3_vnode3.sim
+
+./test.sh -f unique/mnode/mgmt20.sim
+./test.sh -f unique/mnode/mgmt21.sim
+./test.sh -f unique/mnode/mgmt22.sim
+./test.sh -f unique/mnode/mgmt23.sim
+./test.sh -f unique/mnode/mgmt24.sim
+#./test.sh -f unique/mnode/mgmt25.sim
+#./test.sh -f unique/mnode/mgmt26.sim
+./test.sh -f unique/mnode/mgmt33.sim
+./test.sh -f unique/mnode/mgmt34.sim
+./test.sh -f unique/mnode/mgmtr2.sim
+
+./test.sh -f unique/vnode/many.sim
+./test.sh -f unique/vnode/replica2_basic2.sim
+./test.sh -f unique/vnode/replica2_repeat.sim
+./test.sh -f unique/vnode/replica3_basic.sim
+./test.sh -f unique/vnode/replica3_repeat.sim
+./test.sh -f unique/vnode/replica3_vgroup.sim
+
+./test.sh -f general/parser/stream_on_sys.sim
+./test.sh -f general/stream/metrics_del.sim
+./test.sh -f general/stream/metrics_n.sim
+./test.sh -f general/stream/metrics_replica1_vnoden.sim
+./test.sh -f general/stream/restart_stream.sim
+./test.sh -f general/stream/stream_3.sim
+./test.sh -f general/stream/stream_restart.sim
+./test.sh -f general/stream/table_1.sim
+./test.sh -f general/stream/table_del.sim
+./test.sh -f general/stream/table_n.sim
+./test.sh -f general/stream/table_replica1_vnoden.sim
+
+./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim
+#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
+./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
+./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
+./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
+./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
+# lower the priority while file corruption
+#./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
+./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
+./test.sh -f unique/arbitrator/insert_duplicationTs.sim
+./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim
+./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim
+./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim
+./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim
+
+./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim
+./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim
+./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim
+./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim
+./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim
+./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim
+
+./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim
+./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim
+./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim
+./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim

tests/script/unique/dnode/simple.sim

@@ -0,0 +1,147 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+print ========== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql create dnode $hostname2
+system sh/exec.sh -n dnode2 -s start
+sql create dnode $hostname3
+system sh/exec.sh -n dnode3 -s start
+sleep 3000
+
+sql create database d1 replica 2  
+sql create table d1.t1 (t timestamp, i int)
+sql insert into d1.t1 values(now+1s, 15)
+sql insert into d1.t1 values(now+2s, 14)
+sql insert into d1.t1 values(now+3s, 13)
+sql insert into d1.t1 values(now+4s, 12)
+sql insert into d1.t1 values(now+5s, 11)
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+if $data2_1 != 0 then
+	return -1
+endi
+if $data2_2 != 1 then
+	return -1
+endi
+if $data2_3 != 1 then
+	return -1
+endi
+if $data2_4 != null then
+	return -1
+endi
+
+print ========== step2
+sql create dnode $hostname4
+system sh/exec.sh -n dnode4 -s start
+sleep 3000
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+if $data2_1 != 0 then
+	return -1
+endi
+if $data2_2 != 1 then
+	return -1
+endi
+if $data2_3 != 1 then
+	return -1
+endi
+if $data2_4 != 0 then
+	return -1
+endi
+
+print ========== step3
+sql drop dnode $hostname2
+
+$x = 0
+show3: 
+	$x = $x + 1
+	sleep 2000
+	if $x == 10 then
+		return -1
+	endi
+	
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+if $data2_1 != 0 then
+	goto show3
+endi
+if $data2_2 != null then
+	goto show3
+endi
+if $data2_3 != 1 then
+	goto show3
+endi
+if $data2_4 != 1 then
+	goto show3
+endi
+
+print ========== step4
+sql drop dnode $hostname3
+
+$x = 0
+show4: 
+	$x = $x + 1
+	sleep 2000
+	if $x == 10 then
+		return -1
+	endi
+	
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+if $data2_1 != 1 then
+	goto show4
+endi
+if $data2_2 != null then
+	goto show4
+endi
+if $data2_3 != null then
+	goto show4
+endi
+if $data2_4 != 1 then
+	goto show4
+endi
+
+print ========== step5
+sql select * from d1.t1 order by t desc
+print $data01 $data11 $data21 $data31 $data41 
+if $data01 != 11 then
+  return -1
+endi
+if $data11 != 12 then
+  return -1
+endi
+if $data21 != 13 then
+  return -1
+endi
+if $data31 != 14 then
+  return -1
+endi
+if $data41 != 15 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT

tests/test-all.sh

@@ -56,6 +56,15 @@ if [ "$2" != "python" ]; then
   elif [ "$1" == "full" ]; then
     echo "### run TSIM full test ###"
     runSimCaseOneByOne jenkins/basic.txt
+  elif [ "$1" == "b1" ]; then
+    echo "### run TSIM b1 test ###"
+    runSimCaseOneByOne jenkins/basic_1.txt
+  elif [ "$1" == "b2" ]; then
+    echo "### run TSIM b2 test ###"
+    runSimCaseOneByOne jenkins/basic_2.txt
+  elif [ "$1" == "b3" ]; then
+    echo "### run TSIM b3 test ###"
+    runSimCaseOneByOne jenkins/basic_3.txt
   elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
     echo "### run TSIM smoke test ###"
     runSimCaseOneByOne basicSuite.sim
@@ -112,6 +121,11 @@ if [ "$2" != "sim" ]; then
   elif [ "$1" == "full" ]; then
     echo "### run Python full test ###"
     runPyCaseOneByOne fulltest.sh
+  elif [ "$1" == "b1" ]; then
+    echo "### run Python full test ###"
+    runPyCaseOneByOne fulltest.sh
+  elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then
+    exit $(($totalFailed + $totalPyFailed))
   elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
     echo "### run Python smoke test ###"
     runPyCaseOneByOne smoketest.sh