From 0c48c361ba9f0908c9b2cdabb4fcd8540605658c Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Wed, 30 Sep 2020 13:57:21 +0800
Subject: [PATCH 001/118] [TD-1480]<test> Add test case for
 TSDB_MAX_ALLOWED_SQL_LEN

---
 tests/pytest/insert/maxSqlLength.py | 93 +++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 tests/pytest/insert/maxSqlLength.py

diff --git a/tests/pytest/insert/maxSqlLength.py b/tests/pytest/insert/maxSqlLength.py
new file mode 100644
index 0000000000..ad76d24a3f
--- /dev/null
+++ b/tests/pytest/insert/maxSqlLength.py
@@ -0,0 +1,93 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import datetime
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import string
+import random
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.ts = 1537146000000
+
+    def get_random_string(self, length):
+        letters = string.ascii_lowercase
+        result_str = ''.join(random.choice(letters) for i in range(length))
+        return result_str
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute("create table tb(ts timestamp, name1 binary(1000), name2 binary(1000), name3 binary(1000))")
+        
+        sql = "insert into tb values"
+        for i in range(21):
+            value = self.get_random_string(1000)
+            sql += "(%d, '%s', '%s', '%s')" % (self.ts + i, value, value, value)        
+        tdSql.execute(sql)
+
+        self.ts += 21
+        for i in range(22):
+            value = self.get_random_string(1000)
+            sql += "(%d, '%s', '%s', '%s')" % (self.ts + i, value, value, value)        
+        tdSql.error(sql)
+
+        tdSql.query("select * from tb")
+        tdSql.checkRows(21)
+        
+        tdDnodes.stop(1)             
+        tdDnodes.setTestCluster(False)
+        tdDnodes.setValgrind(False)
+        tdDnodes.addSimExtraCfg("maxSQLLength", "1048576")        
+        tdDnodes.deploy(1)                  
+        tdDnodes.cfg(1, "maxSQLLength", "1048576")
+        tdLog.sleep(20)
+        tdDnodes.start(1)
+        
+
+        tdSql.prepare()    
+        tdSql.execute("create table tb(ts timestamp, name1 binary(1000), name2 binary(1000), name3 binary(1000))")
+        
+        sql = "insert into tb values"
+        for i in range(22):
+            value = self.get_random_string(1000)
+            sql += "(%d, '%s', '%s', '%s')" % (self.ts + i, value, value, value)        
+        tdSql.execute(sql)
+        
+        tdSql.query("select * from tb")
+        tdSql.checkRows(43)
+
+        # self.ts += 43
+        # for i in range(330):
+        #     value = self.get_random_string(1000)
+        #     sql += "(%d, '%s', '%s', '%s')" % (self.ts + i, value, value, value)        
+        # tdSql.execute(sql)
+
+        # tdSql.query("select * from tb")
+        # tdSql.checkRows(379)
+                
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

From f349c8af87d86cd4878573c6c1a33b844b7992aa Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 26 Nov 2020 17:34:09 +0000
Subject: [PATCH 002/118] TD-2257

---
 src/client/inc/tscUtil.h   |  2 +-
 src/client/inc/tsclient.h  |  2 +-
 src/client/src/tscServer.c | 40 ++++++++++++++------------------------
 src/client/src/tscSql.c    | 24 +++++++++++++++++++----
 src/client/src/tscSystem.c |  5 -----
 src/client/src/tscUtil.c   |  8 ++++----
 6 files changed, 41 insertions(+), 40 deletions(-)

diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index bde27d2932..cfcecc6582 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -278,7 +278,7 @@ bool hasMoreClauseToTry(SSqlObj* pSql);
 void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
 void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
 void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);
-int  tscSetMgmtEpSetFromCfg(const char *first, const char *second);
+int  tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet);
 
 bool tscSetSqlOwner(SSqlObj* pSql);
 void tscClearSqlOwner(SSqlObj* pSql);
diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index c482375bff..35f9b3af62 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -336,6 +336,7 @@ typedef struct STscObj {
   struct SSqlObj *   pHb;
   struct SSqlObj *   sqlList;
   struct SSqlStream *streamList;
+  SRpcCorEpSet       *tscCorMgmtEpSet;
   void*              pDnodeConn;
   pthread_mutex_t    mutex;
   T_REF_DECLARE()
@@ -515,7 +516,6 @@ extern int       tsInsertHeadSize;
 extern int       tscNumOfThreads;
 extern int       tscRefId;
   
-extern SRpcCorEpSet tscMgmtEpSet;
 
 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
 
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 9735e26ece..3a2c673f34 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -26,7 +26,7 @@
 #include "ttimer.h"
 #include "tlockfree.h"
 
-SRpcCorEpSet  tscMgmtEpSet;
+///SRpcCorEpSet  tscMgmtEpSet;
 
 int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
 
@@ -73,10 +73,11 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) {
   assert(hasFqdn);
 }
 
-static void tscDumpMgmtEpSet(SRpcEpSet *epSet) {
-  taosCorBeginRead(&tscMgmtEpSet.version);
-  *epSet = tscMgmtEpSet.epSet;
-  taosCorEndRead(&tscMgmtEpSet.version);
+static void tscDumpMgmtEpSet(SSqlObj *pSql) {
+  SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet;
+  taosCorBeginRead(&pCorEpSet->version);
+  pSql->epSet = pCorEpSet->epSet;
+  taosCorEndRead(&pCorEpSet->version);
 }  
 static void tscEpSetHtons(SRpcEpSet *s) {
    for (int32_t i = 0; i < s->numOfEps; i++) {
@@ -94,11 +95,12 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) {
    }
    return true;
 }
-void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) {
+void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) {
   // no need to update if equal
-  taosCorBeginWrite(&tscMgmtEpSet.version);
-  tscMgmtEpSet.epSet = *pEpSet;
-  taosCorEndWrite(&tscMgmtEpSet.version);
+  SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet;
+  taosCorBeginWrite(&pCorEpSet->version);
+  pCorEpSet->epSet = *pEpSet;
+  taosCorEndWrite(&pCorEpSet->version);
 }
 static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
   if (pVgroupInfo == NULL) { return;}
@@ -133,18 +135,6 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
   taosCorEndWrite(&pVgroupInfo->version);
 }
 
-void tscPrintMgmtEp() {
-  SRpcEpSet dump;
-  tscDumpMgmtEpSet(&dump);
-  if (dump.numOfEps <= 0) {
-    tscError("invalid mnode EP list:%d", dump.numOfEps);
-  } else {
-    for (int i = 0; i < dump.numOfEps; ++i) {
-      tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]);
-    }
-  }
-}
-
 void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
   STscObj *pObj = (STscObj *)param;
   if (pObj == NULL) return;
@@ -162,7 +152,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
     SRpcEpSet *    epSet = &pRsp->epSet;
     if (epSet->numOfEps > 0) {
       tscEpSetHtons(epSet);
-      tscUpdateMgmtEpSet(epSet);
+      tscUpdateMgmtEpSet(pSql, epSet);
     }
 
     pSql->pTscObj->connId = htonl(pRsp->connId);
@@ -227,7 +217,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
 
   // set the mgmt ip list
   if (pSql->cmd.command >= TSDB_SQL_MGMT) {
-    tscDumpMgmtEpSet(&pSql->epSet);
+    tscDumpMgmtEpSet(pSql);
   }
 
   memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
@@ -289,7 +279,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
       if (pCmd->command < TSDB_SQL_MGMT) {
         tscUpdateVgroupInfo(pSql, pEpSet);
       } else {
-        tscUpdateMgmtEpSet(pEpSet);
+        tscUpdateMgmtEpSet(pSql, pEpSet);
       }
     }
   }
@@ -2070,7 +2060,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
   
   if (pConnect->epSet.numOfEps > 0) {
     tscEpSetHtons(&pConnect->epSet);
-    tscUpdateMgmtEpSet(&pConnect->epSet);
+    tscUpdateMgmtEpSet(pSql, &pConnect->epSet);
   } 
 
   strcpy(pObj->sversion, pConnect->serverVersion);
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index de01776472..761a222b26 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -58,6 +58,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH;
     return NULL;
   }
+  SRpcCorEpSet corMgmtEpSet;
 
   char secretEncrypt[32] = {0};
   int  secretEncryptLen = 0;
@@ -82,11 +83,13 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     }
     secretEncryptLen = outlen;
   }
-
+  
   if (ip) {
-    if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL;
-    if (port) tscMgmtEpSet.epSet.port[0] = port;
-  } 
+    if (tscSetMgmtEpSetFromCfg(ip, NULL, &corMgmtEpSet) < 0) return NULL;
+    if (port) corMgmtEpSet.epSet.port[0] = port;
+  } else {
+    if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond, &corMgmtEpSet) < 0) return NULL;
+  }
  
   void *pDnodeConn = NULL;
   if (tscInitRpc(user, secretEncrypt, &pDnodeConn) != 0) {
@@ -100,10 +103,20 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     rpcClose(pDnodeConn);
     return NULL;
   }
+  // set up tscObj's mgmtEpSet
+  pObj->tscCorMgmtEpSet = (SRpcCorEpSet *)malloc(sizeof(SRpcCorEpSet));
+  if (NULL == pObj->tscCorMgmtEpSet) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    rpcClose(pDnodeConn);
+    free(pObj->tscCorMgmtEpSet);
+    free(pObj);
+  }
+  memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(SRpcCorEpSet));
 
   pObj->signature = pObj;
   pObj->pDnodeConn = pDnodeConn;
   T_REF_INIT_VAL(pObj, 1);
+  
 
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass));
@@ -115,6 +128,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     if (len >= TSDB_DB_NAME_LEN) {
       terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
       rpcClose(pDnodeConn);
+      free(pObj->tscCorMgmtEpSet);
       free(pObj);
       return NULL;
     }
@@ -132,6 +146,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
   if (NULL == pSql) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     rpcClose(pDnodeConn);
+    free(pObj->tscCorMgmtEpSet);
     free(pObj);
     return NULL;
   }
@@ -149,6 +164,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     rpcClose(pDnodeConn);
     free(pSql);
+    free(pObj->tscCorMgmtEpSet);
     free(pObj);
     return NULL;
   }
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 9e9a00550a..a5bc1ae7ad 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -116,11 +116,6 @@ void taos_init_imp(void) {
     taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
   }
 
-  if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond) < 0) {
-    tscError("failed to init mnode EP list");
-    return;
-  } 
-
   tscInitMsgsFp();
   int queueSize = tsMaxConnections*2;
 
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index d22c2588cc..077fd8aa00 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -867,7 +867,7 @@ void tscCloseTscObj(void *param) {
     rpcClose(pObj->pDnodeConn);
     pObj->pDnodeConn = NULL;
   }
-
+  tfree(pObj->tscCorMgmtEpSet);
   pthread_mutex_destroy(&pObj->mutex);
 
   tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p);
@@ -2459,10 +2459,10 @@ char* strdup_throw(const char* str) {
   return p;
 }
 
-int tscSetMgmtEpSetFromCfg(const char *first, const char *second) {
+int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corMgmtEpSet) {
+  corMgmtEpSet->version = 0;
   // init mgmt ip set 
-  tscMgmtEpSet.version = 0;
-  SRpcEpSet *mgmtEpSet = &(tscMgmtEpSet.epSet);
+  SRpcEpSet *mgmtEpSet = &(corMgmtEpSet->epSet);
   mgmtEpSet->numOfEps = 0;
   mgmtEpSet->inUse = 0;
 

From dcb0bc49e846890780044873cf7913df8539e1c7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 1 Dec 2020 13:20:53 +0800
Subject: [PATCH 003/118] [TD-2129]

---
 src/client/src/tscFunctionImpl.c | 218 +++++++++++++++-----
 src/query/src/qExecutor.c        | 339 ++++++++++++++++++++-----------
 2 files changed, 394 insertions(+), 163 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 3867032d46..56b7f052f7 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -426,8 +426,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   }
   
   SET_VAL(pCtx, 1, 1);
-  
-  *((int64_t *)pCtx->aOutputBuf) += 1;
+  *((int64_t *)pCtx->aOutputBuf) += pCtx->size;
   
   // do not need it actually
   SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
@@ -3632,114 +3631,119 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) {
   return true;
 }
 
-static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) {
+static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) {
   int32_t notNullElems = 0;
   TSKEY *primaryKey = pCtx->ptsList;
   
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
 
   STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
   int32_t i = index;
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
 
   if (pCtx->start.key != INT64_MIN) {
-    assert(pCtx->start.key < primaryKey[index] && pInfo->lastKey == INT64_MIN);
+    assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) ||
+               (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC));
 
-    pInfo->lastKey = primaryKey[index];
-    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0));
+    assert(pInfo->lastKey == INT64_MIN);
+
+    pInfo->lastKey = primaryKey[tsIndex + i];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
 
     pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key);
 
     pInfo->hasResult = DATA_SET_FLAG;
-    pInfo->win.skey  = pCtx->start.key;
+    pInfo->win.skey = pCtx->start.key;
     notNullElems++;
-    i += 1;
+    i += step;
   } else if (pInfo->lastKey == INT64_MIN) {
-    pInfo->lastKey = primaryKey[index];
-    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0));
+    pInfo->lastKey = primaryKey[tsIndex + i];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
 
     pInfo->hasResult = DATA_SET_FLAG;
-    pInfo->win.skey  = pInfo->lastKey;
+    pInfo->win.skey = pInfo->lastKey;
     notNullElems++;
-    i += 1;
+    i += step;
   }
 
   // calculate the value of
   switch(pCtx->inputType) {
     case TSDB_DATA_TYPE_TINYINT: {
-      int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
-      for (; i < size; i++) {
+      int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
         if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
           continue;
         }
 
-        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey);
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
         pInfo->lastValue = val[i];
-        pInfo->lastKey = primaryKey[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
       }
       break;
     }
     case TSDB_DATA_TYPE_SMALLINT: {
-      int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
-      for (; i < size; i++) {
+      int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
         if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
           continue;
         }
 
-        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey);
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
         pInfo->lastValue = val[i];
-        pInfo->lastKey = primaryKey[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
       }
       break;
     }
     case TSDB_DATA_TYPE_INT: {
-      int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
-      for (; i < size; i++) {
+      int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
         if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
           continue;
         }
 
-        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey);
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
         pInfo->lastValue = val[i];
-        pInfo->lastKey = primaryKey[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
       }
       break;
     }
     case TSDB_DATA_TYPE_BIGINT: {
-      int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
-      for (; i < size; i++) {
+      int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
         if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
           continue;
         }
 
-        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey);
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
         pInfo->lastValue = (double) val[i];
-        pInfo->lastKey = primaryKey[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
       }
       break;
     }
     case TSDB_DATA_TYPE_FLOAT: {
-      float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index);
-      for (; i < size; i++) {
+      float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
         if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
           continue;
         }
 
-        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey);
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
         pInfo->lastValue = val[i];
-        pInfo->lastKey = primaryKey[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
       }
       break;
     }
     case TSDB_DATA_TYPE_DOUBLE: {
-      double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index);
-      for (; i < size; i++) {
+      double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
         if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
           continue;
         }
 
-        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey);
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
         pInfo->lastValue = val[i];
-        pInfo->lastKey = primaryKey[i];
+        pInfo->lastKey = primaryKey[i + tsIndex];
       }
       break;
     }
@@ -3764,16 +3768,13 @@ static void twa_function(SQLFunctionCtx *pCtx) {
   STwaInfo *   pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   
   // skip null value
-  int32_t i = 0;
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
+  int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1);
   while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) {
-    i++;
+    i += step;
   }
-  
-  if (i >= pCtx->size) {
-    return;
-  }
-  
-  int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, pCtx->size);
+
+  int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size);
   SET_VAL(pCtx, notNullElems, 1);
   
   if (notNullElems > 0) {
@@ -3791,11 +3792,136 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
     return;
   }
 
-  int32_t notNullElems = twa_function_impl(pCtx, index, 1);
+  int32_t notNullElems = 0;
+  TSKEY *primaryKey = pCtx->ptsList;
+
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+
+  STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  int32_t i = pCtx->startOffset;
+  int32_t size = pCtx->size;
+
+  if (pCtx->start.key != INT64_MIN) {
+    assert(pInfo->lastKey == INT64_MIN);
+
+    pInfo->lastKey = primaryKey[index];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
+
+    pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key);
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pCtx->start.key;
+    notNullElems++;
+    i += 1;
+  } else if (pInfo->lastKey == INT64_MIN) {
+    pInfo->lastKey = primaryKey[index];
+    GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pInfo->lastKey;
+    notNullElems++;
+    i += 1;
+  }
+
+  // calculate the value of
+  switch(pCtx->inputType) {
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = (double) val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index);
+      for (; i < size; i++) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+        pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
+        pInfo->lastValue = val[i];
+        pInfo->lastKey = primaryKey[i + index];
+      }
+      break;
+    }
+    default: assert(0);
+  }
+
+  // the last interpolated time window value
+  if (pCtx->end.key != INT64_MIN) {
+    pInfo->dOutput  += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey);
+    pInfo->lastValue = pCtx->end.val;
+    pInfo->lastKey   = pCtx->end.key;
+  }
+
+  pInfo->win.ekey  = pInfo->lastKey;
+
   SET_VAL(pCtx, notNullElems, 1);
 
+  if (notNullElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+
   if (pCtx->stableQuery) {
-    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
     memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo));
   }
 }
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index ec70280c7f..0133035a7b 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -196,6 +196,7 @@ static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo*
 static int32_t checkForQueryBuf(size_t numOfTables);
 static void releaseQueryBuf(size_t numOfTables);
 static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order);
+static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type);
 
 bool doFilterData(SQuery *pQuery, int32_t elemPos) {
   for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) {
@@ -660,7 +661,7 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
   }
 }
 
-static bool isResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) {
+static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) {
   assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP));
   if (type == RESULT_ROW_START_INTERP) {
     return pResult->startInterp == true;
@@ -989,7 +990,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
   if (functionId == TSDB_FUNC_ARITHM) {
     sas->pArithExpr = &pQuery->pExpr1[col];
 
-    sas->offset    = (QUERY_IS_ASC_QUERY(pQuery)) ? pQuery->pos : pQuery->pos - (size - 1);
+    sas->offset    = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1);
     sas->colList   = pQuery->colList;
     sas->numOfCols = pQuery->numOfCols;
     sas->data      = calloc(pQuery->numOfCols, POINTER_BYTES);
@@ -1032,85 +1033,89 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
   return dataBlock;
 }
 
-// window start key interpolation
-static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) {
-  SQuery* pQuery = pRuntimeEnv->pQuery;
-
-  TSKEY start  = tsCols[pos];
-  TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0];
-  TSKEY prevTs = (pos == 0)? lastTs : tsCols[pos - 1];
-
-  // if lastTs == INT64_MIN, it is the first block, no need to do the start time interpolation
-  if (((lastTs != INT64_MIN && pos >= 0) || (lastTs == INT64_MIN && pos > 0)) && win->skey > lastTs &&
-       win->skey < start) {
-
-    for (int32_t k = 0; k < pQuery->numOfCols; ++k) {
-      SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k);
-      if (k == 0 && pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
-        assert(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
-        continue;
-      }
-
-      double v1 = 0, v2 = 0, v = 0;
-
-      char *prevVal = pos == 0 ? pRuntimeEnv->prevRow[k] : ((char*)pColInfo->pData) + (pos - 1) * pColInfo->info.bytes;
-
-      GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal);
-      GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes);
-
-      SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
-      SPoint point2 = (SPoint){.key = start, .val = &v2};
-      SPoint point  = (SPoint){.key = win->skey, .val = &v};
-      taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point);
-      pRuntimeEnv->pCtx[k].start.key = point.key;
-      pRuntimeEnv->pCtx[k].start.val = v;
+static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) {
+  if (type == RESULT_ROW_START_INTERP) {
+    for (int32_t k = 0; k < numOfOutput; ++k) {
+      pCtx[k].start.key = INT64_MIN;
     }
-
-    return true;
   } else {
-    for (int32_t k = 0; k < pQuery->numOfCols; ++k) {
-      pRuntimeEnv->pCtx[k].start.key = INT64_MIN;
+    for (int32_t k = 0; k < numOfOutput; ++k) {
+      pCtx[k].end.key = INT64_MIN;
     }
-
-    return false;
   }
 }
 
-static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, SArray* pDataBlock, TSKEY* tsCols, TSKEY ekey, STimeWindow* win) {
+//static double getTSWindowInterpoVal(SColumnInfoData* pColInfo, int16_t srcColIndex, int16_t rowIndex, TSKEY key, char** prevRow, TSKEY* tsCols, int32_t step) {
+//  TSKEY start  = tsCols[rowIndex];
+//  TSKEY prevTs = (rowIndex == 0)? *(TSKEY *) prevRow[0] : tsCols[rowIndex - step];
+//
+//  double v1 = 0, v2 = 0, v = 0;
+//  char *prevVal = (rowIndex == 0)? prevRow[srcColIndex] : ((char*)pColInfo->pData) + (rowIndex - step) * pColInfo->info.bytes;
+//
+//  GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal);
+//  GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + rowIndex * pColInfo->info.bytes);
+//
+//  SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
+//  SPoint point2 = (SPoint){.key = start,  .val = &v2};
+//  SPoint point  = (SPoint){.key = key,    .val = &v};
+//  taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point);
+//
+//  return v;
+//}
+
+// window start key interpolation
+static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) {
   SQuery* pQuery = pRuntimeEnv->pQuery;
-  TSKEY   trueEndKey = tsCols[pos];
 
-  if (win->ekey < ekey && win->ekey != trueEndKey) {
-    int32_t nextIndex = pos + 1;
-    TSKEY   next = tsCols[nextIndex];
-
-    for (int32_t k = 0; k < pQuery->numOfCols; ++k) {
-      SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k);
-      if (k == 0 && pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP &&
-          pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
-        continue;
-      }
-
-      double v1 = 0, v2 = 0, v = 0;
-      GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes);
-      GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + nextIndex * pColInfo->info.bytes);
-
-      SPoint point1 = (SPoint){.key = trueEndKey, .val = &v1};
-      SPoint point2 = (SPoint){.key = next, .val = &v2};
-      SPoint point  = (SPoint){.key = win->ekey, .val = &v};
-      taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point);
-      pRuntimeEnv->pCtx[k].end.key = point.key;
-      pRuntimeEnv->pCtx[k].end.val = v;
-    }
+  TSKEY curTs  = tsCols[pos];
+  TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0];
 
+  // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
+  // start exactly from this point, no need to do interpolation
+  TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey;
+  if (key == curTs) {
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
     return true;
-  } else {  // current time window does not ended in current data block, do nothing
-    for (int32_t k = 0; k < pQuery->numOfCols; ++k) {
-      pRuntimeEnv->pCtx[k].end.key = INT64_MIN;
-    }
+  }
 
+  if (lastTs == INT64_MIN && ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))) {
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
+    return true;
+  }
+
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
+  TSKEY   prevTs = ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))?
+      lastTs:tsCols[pos - step];
+
+  doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP);
+  return true;
+}
+
+static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t endRowIndex, SArray* pDataBlock, TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) {
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+  TSKEY   actualEndKey = tsCols[endRowIndex];
+
+  TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey;
+
+  // not ended in current data block, do not invoke interpolation
+  if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) {
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP);
     return false;
   }
+
+  // there is actual end point of current time window, no interpolation need
+  if (key == actualEndKey) {
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP);
+    return true;
+  }
+
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
+  int32_t nextRowIndex = endRowIndex + step;
+  assert(nextRowIndex >= 0);
+
+  TSKEY nextKey = tsCols[nextRowIndex];
+  doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, RESULT_ROW_END_INTERP);
+  return true;
 }
 
 static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock) {
@@ -1119,10 +1124,10 @@ static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo*
   }
 
   SQuery* pQuery = pRuntimeEnv->pQuery;
+  int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0;
   for (int32_t k = 0; k < pQuery->numOfCols; ++k) {
     SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k);
-    memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * (pDataBlockInfo->rows - 1)),
-           pColInfo->info.bytes);
+    memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes);
   }
 }
 
@@ -1174,11 +1179,13 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
 
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
   if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
+    int32_t prevIndex = curTimeWindowIndex(pWindowResInfo);
+
     TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step);
+    STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
 
     bool hasTimeWindow  = false;
     SResultRow* pResult = NULL;
-    STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
     int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult);
     if (ret != TSDB_CODE_SUCCESS) {
       tfree(sasArray);
@@ -1193,20 +1200,47 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
       TSKEY ekey = reviseWindowEkey(pQuery, &win);
       forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true);
 
+      // prev time window not interpolation yet.
+      int32_t curIndex = curTimeWindowIndex(pWindowResInfo);
+      if (prevIndex != -1 && prevIndex < curIndex) {
+        for(int32_t j = prevIndex; j < curIndex; ++j) {
+          SResultRow *pRes = pWindowResInfo->pResult[j];
+
+          STimeWindow w = pRes->win;
+          ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult);
+          assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP));
+
+          int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1;
+          doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1,  tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP);
+          setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
+          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
+
+          bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
+          doBlockwiseApplyFunctions(pRuntimeEnv, closed, &w, startPos, 0, tsCols, pDataBlockInfo->rows);
+        }
+
+        // restore current time window
+        ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult);
+        assert (ret == TSDB_CODE_SUCCESS);  // null data, too many state code
+      }
+
       // window start key interpolation
       if (pRuntimeEnv->timeWindowInterpo) {
-        bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-        if (!alreadyInterp) {
-          bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, pQuery->pos, pDataBlock, tsCols, &win);
+        bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
+        if (!done) {
+          int32_t startRowIndex = pQuery->pos;
+          bool    interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &win);
           if (interp) {
             setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
           }
         }
 
-        alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
-        if (!alreadyInterp) {
-          bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, pQuery->pos + forwardStep - 1, pDataBlock, tsCols,
-                                                   pDataBlockInfo->window.ekey, &win);
+        done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
+        if (!done) {
+          int32_t endRowIndex = pQuery->pos + (forwardStep - 1) * step;
+
+          TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey;
+          bool  interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &win);
           if (interp) {
             setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
           }
@@ -1243,17 +1277,20 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
 
       // window start(end) key interpolation
       if (pRuntimeEnv->timeWindowInterpo) {
-        bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-        if (!alreadyInterp) {
-          bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startPos, pDataBlock, tsCols, &nextWin);
+        bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
+        if (!done) {
+          int32_t startRowIndex = startPos;
+          bool    interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &nextWin);
           if (interp) {
             setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
           }
         }
 
-        alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
-        if (!alreadyInterp) {
-          bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, startPos + forwardStep - 1, pDataBlock, tsCols, pDataBlockInfo->window.ekey, &nextWin);
+        done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
+        if (!done) {
+          int32_t endRowIndex = startPos + (forwardStep - 1)*step;
+          TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey;
+          bool  interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &nextWin);
           if (interp) {
             setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
           }
@@ -1459,6 +1496,45 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
   return true;
 }
 
+void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey,  int32_t type) {
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+  for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
+    int32_t functionId = pQuery->pExpr1[k].base.functionId;
+    if (functionId != TSDB_FUNC_TWA) {
+      pRuntimeEnv->pCtx[k].start.key = INT64_MIN;
+      continue;
+    }
+
+    SColIndex* pColIndex = &pQuery->pExpr1[k].base.colInfo;
+    int16_t index = pColIndex->colIndex;
+    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index);
+
+    assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey);
+    double v1 = 0, v2 = 0, v = 0;
+
+    if (prevRowIndex == -1) {
+      GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[k]);
+    } else {
+      GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes);
+    }
+
+    GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes);
+
+    SPoint point1 = (SPoint){.key = prevTs, .val = &v1};
+    SPoint point2 = (SPoint){.key = curTs, .val = &v2};
+    SPoint point  = (SPoint){.key = windowKey, .val = &v};
+    taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point);
+
+    if (type == RESULT_ROW_START_INTERP) {
+      pRuntimeEnv->pCtx[k].start.key = point.key;
+      pRuntimeEnv->pCtx[k].start.val = v;
+    } else {
+      pRuntimeEnv->pCtx[k].end.key = point.key;
+      pRuntimeEnv->pCtx[k].end.val = v;
+    }
+  }
+}
+
 static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo,
     SWindowResInfo *pWindowResInfo, SArray *pDataBlock) {
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
@@ -1489,6 +1565,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
     setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId);
+    pCtx[k].size = 1;
   }
 
   // set the input column data
@@ -1508,7 +1585,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
   }
 
   int32_t offset = -1;
-//  TSKEY   prev   = -1;
+  TSKEY   prevTs   = *(TSKEY*) pRuntimeEnv->prevRow[0];
+  int32_t prevRowIndex = -1;
 
   for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) {
     offset = GET_COL_DATA_POS(pQuery, j, step);
@@ -1530,7 +1608,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
     // interval window query, decide the time window according to the primary timestamp
     if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
-      int64_t     ts = tsCols[offset];
+      int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo);
+
+      int64_t     ts  = tsCols[offset];
       STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
 
       bool hasTimeWindow  = false;
@@ -1543,27 +1623,58 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
       if (!hasTimeWindow) {
         continue;
       }
-/*
+
       // window start key interpolation
       if (pRuntimeEnv->timeWindowInterpo) {
-        bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-        if (!alreadyInterp) {
-          bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, pos, pDataBlock, tsCols, &win);
-          if (interp) {
-            setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
+        // check for the time window end time interpolation
+        int32_t curIndex = curTimeWindowIndex(pWindowResInfo);
+        if (prevWindowIndex != -1 && prevWindowIndex < curIndex) {
+          for (int32_t k = prevWindowIndex; k < curIndex; ++k) {
+            SResultRow *pRes = pWindowResInfo->pResult[k];
+            STimeWindow w = pRes->win;
+            ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult);
+            assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP));
+
+            TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? w.ekey:w.skey;
+            doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP);
+            setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
+
+            setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
+            for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
+              pRuntimeEnv->pCtx[i].size = 0;
+            }
+
+            bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
+            doRowwiseApplyFunctions(pRuntimeEnv, closed, &w, offset);
+          }
+
+          // restore current time window
+          ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow,
+                                        &pResult);
+          if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
+            continue;
           }
         }
 
-        alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
-        if (!alreadyInterp) {
-          bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, pQuery->pos + forwardStep - 1, pDataBlock, tsCols,
-                                                        pDataBlockInfo->window.ekey, &win);
-          if (interp) {
-            setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
+        bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
+        if (!done) {
+          TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win.skey:win.ekey;
+          if (prevTs != INT64_MIN && ts != key) {
+            doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP);
+            setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
+          } else {
+            setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
           }
+
+          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP);
+          for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
+            pRuntimeEnv->pCtx[k].size = 1;
+          }
+        } else {
+          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
         }
       }
-*/
+
       bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
       doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset);
 
@@ -1588,26 +1699,19 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
         }
 
         if (hasTimeWindow) {
-/*
-          // window start(end) key interpolation
-          if (pRuntimeEnv->timeWindowInterpo) {
-            bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-            if (!alreadyInterp) {
-              bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startPos, pDataBlock, tsCols, &nextWin);
-              if (interp) {
-                setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-              }
+          bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
+          if (!done) {
+            if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && (prevTs < nextWin.skey)) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > nextWin.ekey))) {
+              TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? nextWin.skey:nextWin.ekey;
+              doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP);
+              setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
+            } else {
+              setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
             }
 
-            alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
-            if (!alreadyInterp) {
-              bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, startPos + forwardStep - 1, pDataBlock, tsCols, pDataBlockInfo->window.ekey, &nextWin);
-              if (interp) {
-                setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
-              }
-            }
+            setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP);
           }
-*/
+
           closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
           doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset);
         }
@@ -1633,7 +1737,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
       }
     }
 
-//    prev = tsCols[offset];
+    prevTs = tsCols[offset];
+    prevRowIndex = offset;
 
     if (pRuntimeEnv->pTSBuf != NULL) {
       // if timestamp filter list is empty, quit current query

From 114aef8e7eea8f7e7b844bd4d5af7b3764cc122d Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Tue, 1 Dec 2020 16:27:34 +0800
Subject: [PATCH 004/118] [TD-2295] feature: add test case to support hivemq
 extension.

---
 tests/pytest/hivemq-extension-test.py | 166 ++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)
 create mode 100644 tests/pytest/hivemq-extension-test.py

diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py
new file mode 100644
index 0000000000..22773a6581
--- /dev/null
+++ b/tests/pytest/hivemq-extension-test.py
@@ -0,0 +1,166 @@
+#!/usr/bin/python
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+# install pip
+# pip install src/connector/python/linux/python2/
+import sys
+import os
+import os.path
+import time
+import glob
+import getopt
+import subprocess
+from shutil import which
+from multipledispatch import dispatch
+
+@dispatch(str, str)
+def v_print(msg: str, arg: str):
+    if verbose:
+        print(msg % arg)
+
+
+@dispatch(str, int)
+def v_print(msg: str, arg: int):
+    if verbose:
+        print(msg % int(arg))
+
+
+@dispatch(str, int, int)
+def v_print(msg: str, arg1: int, arg2: int):
+    if verbose:
+        print(msg % (int(arg1), int(arg2)))
+
+
+@dispatch(str, int, int, int)
+def v_print(msg: str, arg1: int, arg2: int, arg3: int):
+    if verbose:
+        print(msg % (int(arg1), int(arg2), int(arg3)))
+
+
+@dispatch(str, int, int, int, int)
+def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int):
+    if verbose:
+        print(msg % (int(arg1), int(arg2), int(arg3), int(arg4)))
+
+def isHiveMQInstalled():
+    v_print("%s", "Check if HiveMQ installed")
+    defaultHiveMQPath = "/opt/hivemq*"
+    hiveMQDir = glob.glob(defaultHiveMQPath)
+    if (len(hiveMQDir) == 0):
+        return False
+    else:
+        v_print("HiveMQ installed at %s", hiveMQDir[0])
+    return True
+
+def isMosquittoInstalled():
+    v_print("%s", "Check if mosquitto installed")
+    if not which('mosquitto_pub'):
+        v_print("%s", "mosquitto is not installed")
+        return False
+    else:
+        return True
+
+def installExtension():
+    currentDir = os.getcwd()
+    os.chdir('../../src/connector/hivemq-tdengine-extension')
+    v_print("%s", "build extension..")
+    os.system('mvn clean package')
+
+    tdExtensionZip = 'target/hivemq-tdengine-extension*.zip'
+    tdExtensionZipDir = glob.glob(tdExtensionZip)
+
+    defaultHiveMQPath = "/opt/hivemq*"
+    hiveMQDir = glob.glob(defaultHiveMQPath)
+    extPath = hiveMQDir[0] + '/extensions'
+
+    tdExtDir = glob.glob(extPath + '/hivemq-tdengine-extension')
+    if len(tdExtDir):
+        v_print("%s", "delete exist extension..")
+        os.system('rm -rf %s' % tdExtDir[0])
+
+    v_print("%s", "unzip extension..")
+    os.system('unzip %s -d %s' % (tdExtensionZipDir[0], extPath))
+
+    os.chdir(currentDir)
+
+def stopHiveMQ():
+    toBeKilled = "hivemq.jar"
+    psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % toBeKilled
+
+    processID = subprocess.check_output(
+        psCmd, shell=True).decode("utf-8")
+
+    while(processID):
+        killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
+        os.system(killCmd)
+        time.sleep(1)
+        processID = subprocess.check_output(
+            psCmd, shell=True).decode("utf-8")
+        print("hiveMQ processID: %s" % processID)
+    v_print("%s", "hivemq is NOT running")
+
+def runHiveMQ():
+    defaultHiveMQPath = "/opt/hivemq*"
+    hiveMQDir = glob.glob(defaultHiveMQPath)
+    runPath = hiveMQDir[0] + '/bin/run.sh > /dev/null &'
+    os.system(runPath)
+    time.sleep(10)
+    v_print("%s", "hivemq is running")
+
+def runTDengine():
+    pass
+
+def reCreateDatabase():
+    os.system('taos -s "DROP DATABASE IF EXISTS hivemq"')
+    os.system('taos -s "CREATE DATABASE IF NOT EXISTS hivemq"')
+
+def sendMqttMsg(topic: str, payload: str):
+    testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload)
+    os.system(testStr)
+    time.sleep(3)
+
+def checkTDengineData(topic: str, payload: str):
+    output = subprocess.check_output('taos -s "select * from hivemq.mqtt_payload"', shell=True).decode('utf-8')
+    if (topic in output) and (payload in output):
+        v_print("%s", output)
+        return True
+    else:
+        v_print("%s", "ERROR: mqtt topic or payload NOT found")
+        return False
+
+
+if __name__ == "__main__":
+    verbose = True
+    testTopic = 'test'
+    testPayload = 'hello world'
+
+    if not isHiveMQInstalled():
+        sys.exit(1)
+
+    if not isMosquittoInstalled():
+        sys.exit(1)
+
+    stopHiveMQ()
+
+    installExtension()
+
+    runTDengine()
+
+    reCreateDatabase()
+
+    runHiveMQ()
+
+    sendMqttMsg(testTopic, testPayload)
+
+    if not checkTDengineData(testTopic, testPayload):
+        sys.exit(1)
+
+    sys.exit(0)

From 2558e74bc6ad4c7d93cc1693cae98c0a6eb16f8d Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Tue, 1 Dec 2020 16:58:57 +0800
Subject: [PATCH 005/118] launch taosd from build directory instead of system.

---
 tests/pytest/hivemq-extension-test.py | 53 +++++++++++++++++++++++----
 1 file changed, 45 insertions(+), 8 deletions(-)

diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py
index 22773a6581..daff763f72 100644
--- a/tests/pytest/hivemq-extension-test.py
+++ b/tests/pytest/hivemq-extension-test.py
@@ -91,9 +91,8 @@ def installExtension():
 
     os.chdir(currentDir)
 
-def stopHiveMQ():
-    toBeKilled = "hivemq.jar"
-    psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % toBeKilled
+def stopProgram(prog: str):
+    psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog
 
     processID = subprocess.check_output(
         psCmd, shell=True).decode("utf-8")
@@ -104,7 +103,10 @@ def stopHiveMQ():
         time.sleep(1)
         processID = subprocess.check_output(
             psCmd, shell=True).decode("utf-8")
-        print("hiveMQ processID: %s" % processID)
+    pass
+
+def stopHiveMQ():
+    stopProgram("hivemq.jar")
     v_print("%s", "hivemq is NOT running")
 
 def runHiveMQ():
@@ -115,12 +117,44 @@ def runHiveMQ():
     time.sleep(10)
     v_print("%s", "hivemq is running")
 
+def getBuildPath():
+    selfPath = os.path.dirname(os.path.realpath(__file__))
+
+    if ("community" in selfPath):
+        projPath = selfPath[:selfPath.find("community")]
+    else:
+        projPath = selfPath[:selfPath.find("tests")]
+
+    for root, dirs, files in os.walk(projPath):
+        if ("taosd" in files):
+            rootRealPath = os.path.dirname(os.path.realpath(root))
+            if ("packaging" not in rootRealPath):
+                buildPath = root[:len(root)-len("/build/bin")]
+                break
+    return buildPath
+
 def runTDengine():
-    pass
+    stopProgram("taosd")
+
+    buildPath = getBuildPath()
+
+    if (buildPath == ""):
+        v_print("%s", "taosd NOT found!")
+        os.exit(1)
+    else:
+        v_print("%s", "taosd found in %s" % buildPath)
+
+    binPath = buildPath + "/build/bin/taosd"
+
+    os.system('%s > /dev/null &' % binPath)
+    time.sleep(10)
 
 def reCreateDatabase():
-    os.system('taos -s "DROP DATABASE IF EXISTS hivemq"')
-    os.system('taos -s "CREATE DATABASE IF NOT EXISTS hivemq"')
+    buildPath = getBuildPath()
+    binPath = buildPath + "/build/bin/taos"
+
+    os.system('%s -s "DROP DATABASE IF EXISTS hivemq"' % binPath)
+    os.system('%s -s "CREATE DATABASE IF NOT EXISTS hivemq"' % binPath)
 
 def sendMqttMsg(topic: str, payload: str):
     testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload)
@@ -128,7 +162,10 @@ def sendMqttMsg(topic: str, payload: str):
     time.sleep(3)
 
 def checkTDengineData(topic: str, payload: str):
-    output = subprocess.check_output('taos -s "select * from hivemq.mqtt_payload"', shell=True).decode('utf-8')
+    buildPath = getBuildPath()
+    binPath = buildPath + "/build/bin/taos"
+
+    output = subprocess.check_output('%s -s "select * from hivemq.mqtt_payload"' % binPath, shell=True).decode('utf-8')
     if (topic in output) and (payload in output):
         v_print("%s", output)
         return True

From c3c7643fac71a9b844234df2585f768eefe078e8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 1 Dec 2020 16:59:25 +0800
Subject: [PATCH 006/118] [TD-2277]<fix>: fix invalid column type in stable
 query.

---
 src/client/src/tscLocalMerge.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c
index 2cd37013c5..9fdadfa957 100644
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -721,10 +721,16 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
 
   // final result depends on the fields number
   memset(pSchema, 0, sizeof(SSchema) * size);
+
   for (int32_t i = 0; i < size; ++i) {
     SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
 
-    SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
+    SSchema p1 = {0};
+    if (pExpr->colInfo.colIndex != TSDB_TBNAME_COLUMN_INDEX) {
+      p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
+    } else {
+      p1 = tGetTableNameColumnSchema();
+    }
 
     int32_t inter = 0;
     int16_t type = -1;
@@ -743,7 +749,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
         functionId = TSDB_FUNC_LAST;
       }
 
-      getResultDataInfo(p1->type, p1->bytes, functionId, 0, &type, &bytes, &inter, 0, false);
+      int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false);
+      assert(ret == TSDB_CODE_SUCCESS);
     }
 
     pSchema[i].type = (uint8_t)type;

From 43a3f513c4bd14f3ce702941451f641704bbfd51 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 1 Dec 2020 17:01:44 +0800
Subject: [PATCH 007/118] [TD-2129]

---
 src/query/src/qExecutor.c                |  96 +++++-----
 tests/script/general/parser/function.sim | 228 +++++++++++++++++++++++
 2 files changed, 277 insertions(+), 47 deletions(-)
 create mode 100644 tests/script/general/parser/function.sim

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 0133035a7b..6de57b32fd 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -1074,12 +1074,12 @@ static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int
   // start exactly from this point, no need to do interpolation
   TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey;
   if (key == curTs) {
-    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
     return true;
   }
 
   if (lastTs == INT64_MIN && ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))) {
-    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
     return true;
   }
 
@@ -1099,13 +1099,13 @@ static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32
 
   // not ended in current data block, do not invoke interpolation
   if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) {
-    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP);
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP);
     return false;
   }
 
   // there is actual end point of current time window, no interpolation need
   if (key == actualEndKey) {
-    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP);
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP);
     return true;
   }
 
@@ -1535,6 +1535,43 @@ void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDa
   }
 }
 
+static void setTimeWindowSKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) {
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+
+  bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
+  if (!done) {
+    TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey;
+    if (key == ts) {
+      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
+    } else if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && prevTs < key) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > key))) {
+      doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP);
+      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
+    } else {
+      setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
+    }
+
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP);
+    for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
+      pRuntimeEnv->pCtx[k].size = 1;
+    }
+  } else {
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
+  }
+}
+
+static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) {
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+
+  TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey;
+  doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP);
+  setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
+
+  setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
+  for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
+    pRuntimeEnv->pCtx[i].size = 0;
+  }
+}
+
 static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo,
     SWindowResInfo *pWindowResInfo, SArray *pDataBlock) {
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
@@ -1585,7 +1622,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
   }
 
   int32_t offset = -1;
-  TSKEY   prevTs   = *(TSKEY*) pRuntimeEnv->prevRow[0];
+  TSKEY   prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0];
   int32_t prevRowIndex = -1;
 
   for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) {
@@ -1609,8 +1646,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
     // interval window query, decide the time window according to the primary timestamp
     if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
       int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo);
+      int64_t ts  = tsCols[offset];
 
-      int64_t     ts  = tsCols[offset];
       STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
 
       bool hasTimeWindow  = false;
@@ -1631,21 +1668,14 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
         if (prevWindowIndex != -1 && prevWindowIndex < curIndex) {
           for (int32_t k = prevWindowIndex; k < curIndex; ++k) {
             SResultRow *pRes = pWindowResInfo->pResult[k];
-            STimeWindow w = pRes->win;
-            ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult);
+
+            ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &pRes->win, masterScan, &hasTimeWindow, &pResult);
             assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP));
 
-            TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? w.ekey:w.skey;
-            doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP);
-            setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
-
-            setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
-            for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-              pRuntimeEnv->pCtx[i].size = 0;
-            }
+            setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win);
 
             bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
-            doRowwiseApplyFunctions(pRuntimeEnv, closed, &w, offset);
+            doRowwiseApplyFunctions(pRuntimeEnv, closed, &pRes->win, offset);
           }
 
           // restore current time window
@@ -1656,23 +1686,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
           }
         }
 
-        bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
-        if (!done) {
-          TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win.skey:win.ekey;
-          if (prevTs != INT64_MIN && ts != key) {
-            doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP);
-            setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-          } else {
-            setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
-          }
-
-          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP);
-          for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
-            pRuntimeEnv->pCtx[k].size = 1;
-          }
-        } else {
-          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
-        }
+        setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win);
       }
 
       bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
@@ -1699,19 +1713,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
         }
 
         if (hasTimeWindow) {
-          bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
-          if (!done) {
-            if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && (prevTs < nextWin.skey)) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > nextWin.ekey))) {
-              TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? nextWin.skey:nextWin.ekey;
-              doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP);
-              setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-            } else {
-              setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP);
-            }
-
-            setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP);
-          }
-
+          setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin);
           closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
           doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset);
         }
diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim
new file mode 100644
index 0000000000..34e9844f71
--- /dev/null
+++ b/tests/script/general/parser/function.sim
@@ -0,0 +1,228 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3
+system sh/exec.sh -n dnode1 -s start
+sleep 500
+sql connect
+
+$dbPrefix = m_func_db
+$tbPrefix = m_func_tb
+$mtPrefix = m_func_mt
+
+$tbNum = 10
+$rowNum = 5
+$totalNum = $tbNum * $rowNum
+$ts0 = 1537146000000
+$delta = 600000
+print ========== alter.sim
+$i = 0
+$db = $dbPrefix . $i
+$mt = $mtPrefix . $i
+
+sql drop database if exists $db
+sql create database $db
+sql use $db
+
+print =====================================> test case for twa in single block
+
+sql create table t1 (ts timestamp, k float);
+sql insert into t1 values('2015-08-18 00:00:00', 2.064);
+sql insert into t1 values('2015-08-18 00:06:00', 2.116);
+sql insert into t1 values('2015-08-18 00:12:00', 2.028);
+sql insert into t1 values('2015-08-18 00:18:00', 2.126);
+sql insert into t1 values('2015-08-18 00:24:00', 2.041);
+sql insert into t1 values('2015-08-18 00:30:00', 2.051);
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:05:00'
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 2.063999891 then
+  return -1
+endi
+
+if $data01 != 2.063999891 then
+  return -1
+endi
+
+if $data02 != 1 then
+  return -1
+endi
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00'
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 2.089999914 then
+  return -1
+endi
+
+if $data01 != 2.089999914 then
+  return -1
+endi
+
+if $data02 != 2 then
+  return -1
+endi
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00' interval(1m) order by ts asc
+if $rows != 2 then
+  return -1
+endi
+
+if $data00 != @15-08-18 00:00:00.000@ then
+  return -1
+endi
+
+if $data01 != 2.068333156 then
+  return -1
+endi
+
+if $data02 != 2.063999891 then
+  return -1
+endi
+
+if $data03 != 1 then
+  return -1
+endi
+
+if $data10 != @15-08-18 00:06:00.000@ then
+  return -1
+endi
+
+if $data11 != 2.115999937 then
+  return -1
+endi
+
+if $data12 != 2.115999937 then
+  return -1
+endi
+
+if $data13 != 1 then
+  return -1
+endi
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00' interval(1m) order by ts desc;
+if $rows != 2 then
+ return -1
+endi
+
+if $data00 != @15-08-18 00:06:00.00@ then
+  return -1
+endi
+
+if $data01 != 2.115999937 then
+  return -1
+endi
+
+if $data02 != 2.115999937 then
+  return -1
+endi
+
+if $data03 != 1 then
+  return -1
+endi
+
+if $data11 != 2.068333156 then
+  return -1
+endi
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:27:00' interval(10m) order by ts asc
+if $rows != 3 then
+ return -1
+endi
+
+if $data01 != 2.088666666 then
+ return -1
+endi
+
+if $data02 != 2.089999914 then
+ return -1
+endi
+
+if $data03 != 2 then
+  return -1
+endi
+
+if $data11 != 2.077099980 then
+  return -1
+endi
+
+if $data12 != 2.077000022 then
+  return -1
+endi
+
+if $data13 != 2 then
+  return -1
+endi
+
+if $data21 != 2.069333235 then
+  return -1
+endi
+
+if $data22 != 2.040999889 then
+  return -1
+endi
+
+if $data23 != 1 then
+  return -1
+endi
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:27:00' interval(10m) order by ts desc
+if $rows != 3 then
+  return -1
+endi
+
+if $data01 != 2.069333235 then
+  return -1
+endi
+
+if $data11 != 2.077099980 then
+  return -1
+endi
+
+if $data21 != 2.088666666 then
+  return -1
+endi
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00'  order by ts asc
+if $data00 != 2.073699975 then
+  return -1
+endi
+
+if $data01 != 2.070999980 then
+  return -1
+endi
+
+if $data02 != 6 then
+  return -1
+endi
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00'  order by ts desc
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 2.073699975 then
+  return -1
+endi
+
+if $data01 != 2.070999980 then
+  return -1
+endi
+
+if $data02 != 6 then
+  return -1
+endi
+
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts asc
+sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts desc
+
+
+#todo add test case while column filte exists.
+
+select count(*),TWA(k) from tm0 where ts>='1970-1-1 13:43:00' and ts<='1970-1-1 13:44:10' interval(9s)

From 6526ea44a9d6bd531346ef056750d5d644126ad4 Mon Sep 17 00:00:00 2001
From: Xiaowei Su <46439638+Shawshank-Smile@users.noreply.github.com>
Date: Tue, 1 Dec 2020 17:24:51 +0800
Subject: [PATCH 008/118] Update cluster-ch.md

---
 documentation20/webdocs/markdowndocs/cluster-ch.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md
index 10c28c284c..db479417c5 100644
--- a/documentation20/webdocs/markdowndocs/cluster-ch.md
+++ b/documentation20/webdocs/markdowndocs/cluster-ch.md
@@ -37,7 +37,7 @@ fqdn                  h1.taosdata.com
 // 配置本数据节点的端口号，缺省是6030
 serverPort            6030
 
-// 服务端节点数为偶数的时候，需要配置，请参考《Arbitrator的使用》的部分
+// 使用场景，请参考《Arbitrator的使用》的部分
 arbitrator            ha.taosdata.com:6042
 ```
 

From 28985c1fdbbb1d606445c31f2d41b680c65c2612 Mon Sep 17 00:00:00 2001
From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com>
Date: Tue, 1 Dec 2020 17:26:00 +0800
Subject: [PATCH 009/118] Update readme.md

---
 tests/examples/JDBC/JDBCDemo/readme.md | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/tests/examples/JDBC/JDBCDemo/readme.md b/tests/examples/JDBC/JDBCDemo/readme.md
index 9b8790adad..e348e458fe 100644
--- a/tests/examples/JDBC/JDBCDemo/readme.md
+++ b/tests/examples/JDBC/JDBCDemo/readme.md
@@ -6,10 +6,24 @@ TDengine's JDBC demo project is organized in a Maven way so that users can easil
 Make sure you have already installed a tdengine client on your current develop environment.
 Download the tdengine package on our website: ``https://www.taosdata.com/cn/all-downloads/`` and install the client.
 
+## How to run jdbcChecker
+<pre>mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcChecker" -Dexec.args="-host localhost"</pre>
+
+## How to run jdbcTaosDemo
+run command:
+<pre> mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo"</pre>
+and run with your customed args
+<pre>mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo" -Dexec.args="-host localhost"</pre>
+
 ## Compile the Demo Code and Run It
+
 To compile the demo project, go to the source directory ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute
-<pre>mvn clean package assembly:single</pre>
+
+<pre>
+mvn clean package assembly:single
+</pre>
+
 The ``pom.xml`` is configured to package all the dependencies into one executable jar file.
 
 To run it, go to ``examples/JDBC/JDBCDemo/target`` and execute
-<pre>java -jar jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host localhost</pre>
\ No newline at end of file
+<pre>java -jar jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host localhost</pre>

From 773101a247d7548aa21f050b4491d8664a272de2 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Tue, 1 Dec 2020 17:50:09 +0800
Subject: [PATCH 010/118] change to python3 explicitly.

---
 tests/pytest/hivemq-extension-test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py
index daff763f72..eb27f7a7d9 100644
--- a/tests/pytest/hivemq-extension-test.py
+++ b/tests/pytest/hivemq-extension-test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 ###################################################################
 #           Copyright (c) 2016 by TAOS Technologies, Inc.
 #                     All rights reserved.
@@ -55,6 +55,7 @@ def isHiveMQInstalled():
     defaultHiveMQPath = "/opt/hivemq*"
     hiveMQDir = glob.glob(defaultHiveMQPath)
     if (len(hiveMQDir) == 0):
+        v_print("%s", "ERROR: hivemq not found!")
         return False
     else:
         v_print("HiveMQ installed at %s", hiveMQDir[0])

From d4c6aaf2af44f59a8b5bd18d78bdea3dd52282b2 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 1 Dec 2020 18:15:26 +0800
Subject: [PATCH 011/118] [TD-2178]<test>add table per batch

---
 tests/pytest/insert/restfulInsert.py | 28 ++++++----------------------
 1 file changed, 6 insertions(+), 22 deletions(-)

diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py
index df253040f3..da797f788f 100644
--- a/tests/pytest/insert/restfulInsert.py
+++ b/tests/pytest/insert/restfulInsert.py
@@ -44,22 +44,9 @@ class RestfulInsert:
             if response.status_code != 200:
                     print(response.content)
 
-    def insertData(self, threadID):        
-        print("thread %d started" % threadID)
-        tablesPerThread = int (self.numOfTables / self.numOfThreads)        
-        for i in range(tablesPerThread):
-            tableID = i + threadID * tablesPerThread
-            start = self.ts
-            for j in range(int(self.recordsPerTable / self.batchSize)):
-                data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID)
-                values = []
-                for k in range(self.batchSize):
-                    data +=  "(%d, %d, %d, %d)" %  (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))                                
-                response = requests.post(self.url, data, headers = self.header)
-                if response.status_code != 200:
-                    print(response.content)
+
    
-    def insertnData(self, threadID):        
+    def insertData(self, threadID):        
         print("thread %d started" % threadID)
         tablesPerThread = int (self.numOfTables / self.numOfThreads)   
         loop = int(self.recordsPerTable / self.batchSize)   
@@ -81,11 +68,9 @@ class RestfulInsert:
                     if self.outOfOrder :
                         random.shuffle(values)
                     data+=''.join(values)
-                    
                     response = requests.post(self.url, data, headers = self.header)
                     if response.status_code != 200:
                         print(response.content)
-                print('----------------',loop,time.time()-start1)
         else:
             for i in range(0,tablesPerThread+self.tablePerbatch,self.tablePerbatch): 
                 for k in range(loop):
@@ -101,8 +86,7 @@ class RestfulInsert:
                             values.append("(%d, %d, %d, %d)" %  (start + k * self.batchSize + l, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)))    
                         if self.outOfOrder :
                             random.shuffle(values)
-                        data+=''.join(values)  
-                    print('------------------',len(data))                        
+                        data+=''.join(values)                          
                     if len(data) > 1024*1024 : 
                         print ('batch size is larger than 1M')
                         exit(-1)
@@ -169,7 +153,7 @@ class RestfulInsert:
         
         for i in range(self.numOfThreads):
             threads[i].join()
-        print("inserting %d records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime)))
+        print("inserting %s records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime)))
 
 parser = argparse.ArgumentParser()
 parser.add_argument(
@@ -204,14 +188,14 @@ parser.add_argument(
     '-T',
     '--number-of-tables',
     action='store',
-    default=1000,
+    default=10000,
     type=int,
     help='Number of tables to be created (default: 1000)')
 parser.add_argument(
     '-r',
     '--number-of-records',
     action='store',
-    default=1000,
+    default=10000,
     type=int,
     help='Number of record to be created for each table  (default: 1000, -1 for unlimited records)')
 parser.add_argument(

From 31e747239113ea1e5d5f7dae01e153a65f292276 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Tue, 1 Dec 2020 18:59:50 +0800
Subject: [PATCH 012/118] [TD-2279] <test> add test case

---
 tests/pytest/fulltest.sh          |  2 +
 tests/pytest/update/bug_td2279.py | 67 +++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 tests/pytest/update/bug_td2279.py

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index 0e3b482e3d..042fd826e8 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -19,6 +19,7 @@ python3 ./test.py -f insert/randomNullCommit.py
 python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
+python3 ./test.py -f insert/before_1970.py
 
 python3 ./test.py -f table/alter_wal0.py
 python3 ./test.py -f table/column_name.py
@@ -228,6 +229,7 @@ python3 ./test.py -f update/merge_commit_data2.py
 python3 ./test.py -f update/merge_commit_data2_update0.py
 python3 ./test.py -f update/merge_commit_last-0.py
 python3 ./test.py -f update/merge_commit_last.py
+python3 ./test.py -f update/bug_td2279.py
 
 # wal
 python3 ./test.py -f wal/addOldWalTest.py
\ No newline at end of file
diff --git a/tests/pytest/update/bug_td2279.py b/tests/pytest/update/bug_td2279.py
new file mode 100644
index 0000000000..7e8640dfa0
--- /dev/null
+++ b/tests/pytest/update/bug_td2279.py
@@ -0,0 +1,67 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.ts = 1606700000000
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.startWithoutSleep(1)
+        tdSql.execute("use db")
+
+    def run(self):            
+        tdSql.prepare()
+
+        print("==============step1")        
+        tdSql.execute("create table t (ts timestamp, a int)")
+
+        for i in range(3276):
+            tdSql.execute("insert into t values(%d, 0)" % (self.ts + i))
+
+        newTs = 1606700010000
+        for i in range(3275):
+            tdSql.execute("insert into t values(%d, 0)" % (self.ts + i))
+        tdSql.execute("insert into t values(%d, 0)" % 1606700013280)
+        
+        self.restartTaosd()
+        
+        for i in range(1606700003275, 1606700006609):
+            tdSql.execute("insert into t values(%d, 0)" % i)
+        tdSql.execute("insert into t values(%d, 0)" % 1606700006612)
+        
+        self.restartTaosd()
+
+        tdSql.execute("insert into t values(%d, 0)" % 1606700006610)        
+        tdSql.query("select * from t")
+        tdSql.checkRows(6612)
+
+        tdDnodes.stop(1)
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

From 8072b26a2f26383c4ffda0d6e409f1f791038894 Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Tue, 1 Dec 2020 19:04:05 +0800
Subject: [PATCH 013/118] [TD-2263]<enhance>: debug log epSet.fqdn when connect
 success

---
 src/client/src/tscServer.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 5d3b86caf3..9e8d3d339e 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -2058,6 +2058,10 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
   if (pConnect->epSet.numOfEps > 0) {
     tscEpSetHtons(&pConnect->epSet);
     tscUpdateMgmtEpSet(&pConnect->epSet);
+
+    for (int i = 0; i < pConnect->epSet.numOfEps; ++i) {
+      tscDebug("%p epSet.fqdn[%d]: %s, pObj:%p", pSql, i, pConnect->epSet.fqdn[i], pObj);
+    }
   } 
 
   strcpy(pObj->sversion, pConnect->serverVersion);

From bd6e5f2c418557018b958d7389caca59f1ca5792 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Wed, 2 Dec 2020 10:13:11 +0800
Subject: [PATCH 014/118] <test>compile arm version while full test

---
 Jenkinsfile | 42 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 940815febe..edbe11d428 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -175,7 +175,47 @@ pipeline {
           
          }
        }
-
+       stage('arm64_build'){
+                agent{label 'arm64'}
+                steps{
+                    sh '''
+                    cd ${WK}
+                    git fetch
+                    git checkout develop
+                    git pull
+                    cd ${WKC}
+                    git fetch
+                    git checkout develop
+                    git pull
+                    git submodule update                    
+                    cd ${WKC}/packaging
+                    ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
+                    
+                    '''
+                }
+            }
+            stage('arm32_build'){
+                agent{label 'arm32'}
+                steps{
+                    catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                        sh '''
+                        cd ${WK}
+                        git fetch
+                        git checkout develop
+                        git pull
+                        cd ${WKC}
+                        git fetch
+                        git checkout develop
+                        git pull
+                        git submodule update
+                        cd ${WKC}/packaging
+                        ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
+                        
+                        '''
+                    }
+                    
+                }
+            }
       }
     }
 

From 982e6465ac6af2a803bd196deb13c01bab00e95e Mon Sep 17 00:00:00 2001
From: Steven Li <stevenli@taosdata.com>
Date: Wed, 2 Dec 2020 02:15:46 +0000
Subject: [PATCH 015/118] Refactored crash_gen tool, plus ensured proper
 termination upon trasntion failure

---
 tests/pytest/crash_gen/crash_gen_main.py  | 241 ++++++++++++++--------
 tests/pytest/crash_gen/db.py              |   9 +-
 tests/pytest/crash_gen/misc.py            |  22 +-
 tests/pytest/crash_gen/service_manager.py |  28 ++-
 4 files changed, 192 insertions(+), 108 deletions(-)

diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py
index 8d68457ec8..e2ce4b26fa 100755
--- a/tests/pytest/crash_gen/crash_gen_main.py
+++ b/tests/pytest/crash_gen/crash_gen_main.py
@@ -352,6 +352,12 @@ class ThreadCoordinator:
                 self._execStats.registerFailure("Broken DB Connection")
                 # continue # don't do that, need to tap all threads at
                 # end, and maybe signal them to stop
+            if isinstance(err, CrashGenError): # our own transition failure
+                Logging.info("State transition error")
+                traceback.print_stack()
+                transitionFailed = True
+                self._te = None  # Not running any more
+                self._execStats.registerFailure("State transition error")
             else:
                 raise
         # return transitionFailed # Why did we have this??!!
@@ -388,12 +394,20 @@ class ThreadCoordinator:
                 self._syncAtBarrier() # For now just cross the barrier
                 Progress.emit(Progress.END_THREAD_STEP)
             except threading.BrokenBarrierError as err:
-                Logging.info("Main loop aborted, caused by worker thread(s) time-out")
                 self._execStats.registerFailure("Aborted due to worker thread timeout")
-                print("\n\nWorker Thread time-out detected, TAOS related threads are:")
+                Logging.error("\n")
+                Logging.error("Main loop aborted, caused by worker thread(s) time-out of {} seconds".format(
+                    ThreadCoordinator.WORKER_THREAD_TIMEOUT))
+                Logging.error("TAOS related threads blocked at (stack frames top-to-bottom):")
                 ts = ThreadStacks()
                 ts.print(filterInternal=True)
                 workerTimeout = True
+
+                # Enable below for deadlock debugging, using gdb to attach to process
+                # while True:
+                #     Logging.error("Deadlock detected")
+                #     time.sleep(60.0)
+
                 break
 
             # At this point, all threads should be pass the overall "barrier" and before the per-thread "gate"
@@ -701,7 +715,7 @@ class AnyState:
                 # task.logDebug("Task success found")
                 sCnt += 1
                 if (sCnt >= 2):
-                    raise RuntimeError(
+                    raise CrashGenError(
                         "Unexpected more than 1 success with task: {}".format(cls))
 
     def assertIfExistThenSuccess(self, tasks, cls):
@@ -714,7 +728,7 @@ class AnyState:
             if task.isSuccess():
                 sCnt += 1
         if (exists and sCnt <= 0):
-            raise RuntimeError("Unexpected zero success for task type: {}, from tasks: {}"
+            raise CrashGenError("Unexpected zero success for task type: {}, from tasks: {}"
                 .format(cls, tasks))
 
     def assertNoTask(self, tasks, cls):
@@ -727,7 +741,7 @@ class AnyState:
         for task in tasks:
             if isinstance(task, cls):
                 if task.isSuccess():
-                    raise RuntimeError(
+                    raise CrashGenError(
                         "Unexpected successful task: {}".format(cls))
 
     def hasSuccess(self, tasks, cls):
@@ -926,8 +940,9 @@ class StateMechine:
             Logging.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time()))
             return StateDbOnly()
 
+        # For sure we have tables, which means we must have the super table. # TODO: are we sure?
         sTable = self._db.getFixedSuperTable()
-        if sTable.hasRegTables(dbc, dbName):  # no regular tables
+        if sTable.hasRegTables(dbc):  # no regular tables
             Logging.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time()))
             return StateSuperTableOnly()
         else:  # has actual tables
@@ -1050,9 +1065,8 @@ class Database:
     def getFixedSuperTableName(cls):
         return "fs_table"
 
-    @classmethod
-    def getFixedSuperTable(cls) -> TdSuperTable:
-        return TdSuperTable(cls.getFixedSuperTableName())
+    def getFixedSuperTable(self) -> TdSuperTable:
+        return TdSuperTable(self.getFixedSuperTableName(), self.getName())
 
     # We aim to create a starting time tick, such that, whenever we run our test here once
     # We should be able to safely create 100,000 records, which will not have any repeated time stamp
@@ -1107,6 +1121,11 @@ class Database:
         # print("Float obtained: {}".format(ret))
         return ret
 
+    ALL_COLORS = ['red', 'white', 'blue', 'green', 'purple']
+
+    def getNextColor(self):
+        return random.choice(self.ALL_COLORS)
+
 
 class TaskExecutor():
     class BoundedList:
@@ -1240,7 +1259,7 @@ class Task():
         if errno in [
                 0x05,  # TSDB_CODE_RPC_NOT_READY
                 0x0B,  # Unable to establish connection, more details in TD-1648
-                0x200, # invalid SQL， TODO: re-examine with TD-934
+                # 0x200, # invalid SQL， TODO: re-examine with TD-934
                 0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776
                 0x213, # "Disconnected from service", result of "kill connection ???"
                 0x217, # "db not selected", client side defined error code
@@ -1569,8 +1588,8 @@ class TaskCreateSuperTable(StateTransitionTask):
         sTable = self._db.getFixedSuperTable() # type: TdSuperTable
         # wt.execSql("use db")    # should always be in place
 
-        sTable.create(wt.getDbConn(), self._db.getName(), 
-            {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'},
+        sTable.create(wt.getDbConn(), 
+            {'ts':'TIMESTAMP', 'speed':'INT', 'color':'BINARY(16)'}, {'b':'BINARY(200)', 'f':'FLOAT'},
             dropIfExists = True
             )
         # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
@@ -1579,30 +1598,33 @@ class TaskCreateSuperTable(StateTransitionTask):
 
 
 class TdSuperTable:
-    def __init__(self, stName):
+    def __init__(self, stName, dbName):
         self._stName = stName
+        self._dbName = dbName
 
     def getName(self):
         return self._stName
 
-    def drop(self, dbc, dbName, skipCheck = False):
-        if self.exists(dbc, dbName) : # if myself exists
+    def drop(self, dbc, skipCheck = False):
+        dbName = self._dbName
+        if self.exists(dbc) : # if myself exists
             fullTableName = dbName + '.' + self._stName                
             dbc.execute("DROP TABLE {}".format(fullTableName))
         else:
             if not skipCheck:
                 raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName))
 
-    def exists(self, dbc, dbName):
-        dbc.execute("USE " + dbName)
+    def exists(self, dbc):
+        dbc.execute("USE " + self._dbName)
         return dbc.existsSuperTable(self._stName)
 
     # TODO: odd semantic, create() method is usually static?
-    def create(self, dbc, dbName, cols: dict, tags: dict,
+    def create(self, dbc, cols: dict, tags: dict,
         dropIfExists = False
         ):
-
         '''Creating a super table'''
+
+        dbName = self._dbName
         dbc.execute("USE " + dbName)
         fullTableName = dbName + '.' + self._stName       
         if dbc.existsSuperTable(self._stName):
@@ -1623,7 +1645,8 @@ class TdSuperTable:
             )
         dbc.execute(sql)        
 
-    def getRegTables(self, dbc: DbConn, dbName: str):
+    def getRegTables(self, dbc: DbConn):
+        dbName = self._dbName
         try:
             dbc.query("select TBNAME from {}.{}".format(dbName, self._stName))  # TODO: analyze result set later            
         except taos.error.ProgrammingError as err:                    
@@ -1634,10 +1657,11 @@ class TdSuperTable:
         qr = dbc.getQueryResult()
         return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation
 
-    def hasRegTables(self, dbc: DbConn, dbName: str):
-        return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0
+    def hasRegTables(self, dbc: DbConn):
+        return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0
 
-    def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str):
+    def ensureTable(self, task: Task, dbc: DbConn, regTableName: str):
+        dbName = self._dbName
         sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName)
         if dbc.query(sql) >= 1 : # reg table exists already
             return
@@ -1650,15 +1674,15 @@ class TdSuperTable:
         # print("(" + fullTableName[-3:] + ")", end="", flush=True)  
         try:
             sql = "CREATE TABLE {} USING {}.{} tags ({})".format(
-                fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName)
+                fullTableName, dbName, self._stName, self._getTagStrForSql(dbc)
             )
             dbc.execute(sql)
         finally:
             if task is not None:
                 task.unlockTable(fullTableName) # no matter what
 
-    def _getTagStrForSql(self, dbc, dbName: str) :
-        tags = self._getTags(dbc, dbName)
+    def _getTagStrForSql(self, dbc) :
+        tags = self._getTags(dbc)
         tagStrs = []
         for tagName in tags: 
             tagType = tags[tagName]
@@ -1672,36 +1696,86 @@ class TdSuperTable:
                 raise RuntimeError("Unexpected tag type: {}".format(tagType))
         return ", ".join(tagStrs)
 
-    def _getTags(self, dbc, dbName) -> dict:
-        dbc.query("DESCRIBE {}.{}".format(dbName, self._stName))
+    def _getTags(self, dbc) -> dict:
+        dbc.query("DESCRIBE {}.{}".format(self._dbName, self._stName))
         stCols = dbc.getQueryResult()
         # print(stCols)
         ret = {row[0]:row[1] for row in stCols if row[3]=='TAG'} # name:type
         # print("Tags retrieved: {}".format(ret))
         return ret
 
-    def addTag(self, dbc, dbName, tagName, tagType):
-        if tagName in self._getTags(dbc, dbName): # already 
+    def addTag(self, dbc, tagName, tagType):
+        if tagName in self._getTags(dbc): # already 
             return
         # sTable.addTag("extraTag", "int")
-        sql = "alter table {}.{} add tag {} {}".format(dbName, self._stName, tagName, tagType)
+        sql = "alter table {}.{} add tag {} {}".format(
+            self._dbName, self._stName, tagName, tagType)
         dbc.execute(sql)
 
-    def dropTag(self, dbc, dbName, tagName):
-        if not tagName in self._getTags(dbc, dbName): # don't have this tag
+    def dropTag(self, dbc, tagName):
+        if not tagName in self._getTags(dbc): # don't have this tag
             return
-        sql = "alter table {}.{} drop tag {}".format(dbName, self._stName, tagName)
+        sql = "alter table {}.{} drop tag {}".format(self._dbName, self._stName, tagName)
         dbc.execute(sql)
 
-    def changeTag(self, dbc, dbName, oldTag, newTag):
-        tags = self._getTags(dbc, dbName)
+    def changeTag(self, dbc, oldTag, newTag):
+        tags = self._getTags(dbc)
         if not oldTag in tags: # don't have this tag
             return
         if newTag in tags: # already have this tag
             return
-        sql = "alter table {}.{} change tag {} {}".format(dbName, self._stName, oldTag, newTag)
+        sql = "alter table {}.{} change tag {} {}".format(self._dbName, self._stName, oldTag, newTag)
         dbc.execute(sql)
 
+    def generateQueries(self, dbc: DbConn) -> List[SqlQuery]:
+        ''' Generate queries to test/exercise this super table '''
+        ret = [] # type: List[SqlQuery]
+
+        for rTbName in self.getRegTables(dbc):  # regular tables
+            
+            filterExpr = Dice.choice([ # TODO: add various kind of WHERE conditions
+                None
+            ])
+
+            # Run the query against the regular table first
+            doAggr = (Dice.throw(2) == 0) # 1 in 2 chance
+            if not doAggr: # don't do aggregate query, just simple one
+                ret.append(SqlQuery( # reg table
+                    "select {} from {}.{}".format('*', self._dbName, rTbName)))
+                ret.append(SqlQuery( # super table
+                    "select {} from {}.{}".format('*', self._dbName, self.getName())))
+            else: # Aggregate query
+                aggExpr = Dice.choice([                
+                    'count(*)',
+                    'avg(speed)',
+                    # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable
+                    'sum(speed)', 
+                    'stddev(speed)', 
+                    # SELECTOR functions
+                    'min(speed)', 
+                    'max(speed)', 
+                    'first(speed)', 
+                    'last(speed)',
+                    'top(speed, 50)', # TODO: not supported?
+                    'bottom(speed, 50)', # TODO: not supported?
+                    'apercentile(speed, 10)', # TODO: TD-1316
+                    'last_row(speed)',
+                    # Transformation Functions
+                    # 'diff(speed)', # TODO: no supported?!
+                    'spread(speed)'
+                    ]) # TODO: add more from 'top'
+
+            
+                if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?!
+                    sql = "select {} from {}.{}".format(aggExpr, self._dbName, self.getName())
+                    if Dice.throw(3) == 0: # 1 in X chance
+                        sql = sql + ' GROUP BY color'
+                        Progress.emit(Progress.QUERY_GROUP_BY)
+                        # Logging.info("Executing GROUP-BY query: " + sql)
+                    ret.append(SqlQuery(sql))
+
+        return ret        
+
 class TaskReadData(StateTransitionTask):
     @classmethod
     def getEndState(cls):
@@ -1716,10 +1790,8 @@ class TaskReadData(StateTransitionTask):
     #         return True # always
     #     return gSvcMgr.isActive() # only if it's running TODO: race condition here
 
-    def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
-        sTable = self._db.getFixedSuperTable()
-
-        # 1 in 5 chance, simulate a broken connection, only if service stable (not restarting)
+    def _reconnectIfNeeded(self, wt):
+        # 1 in 20 chance, simulate a broken connection, only if service stable (not restarting)
         if random.randrange(20)==0: # and self._canRestartService():  # TODO: break connection in all situations
             # Logging.info("Attempting to reconnect to server") # TODO: change to DEBUG
             Progress.emit(Progress.SERVICE_RECONNECT_START) 
@@ -1744,43 +1816,36 @@ class TaskReadData(StateTransitionTask):
             return # TODO: fix server restart status race condtion
 
 
+    def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
+        self._reconnectIfNeeded(wt)
+
         dbc = wt.getDbConn()
-        dbName = self._db.getName()
-        for rTbName in sTable.getRegTables(dbc, dbName):  # regular tables
-            aggExpr = Dice.choice([
-                '*',
-                'count(*)',
-                'avg(speed)',
-                # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable
-                'sum(speed)', 
-                'stddev(speed)', 
-                # SELECTOR functions
-                'min(speed)', 
-                'max(speed)', 
-                'first(speed)', 
-                'last(speed)',
-                'top(speed, 50)', # TODO: not supported?
-                'bottom(speed, 50)', # TODO: not supported?
-                'apercentile(speed, 10)', # TODO: TD-1316
-                'last_row(speed)',
-                # Transformation Functions
-                # 'diff(speed)', # TODO: no supported?!
-                'spread(speed)'
-                ]) # TODO: add more from 'top'
-            filterExpr = Dice.choice([ # TODO: add various kind of WHERE conditions
-                None
-            ])
+        sTable = self._db.getFixedSuperTable()
+        
+        for q in sTable.generateQueries(dbc):  # regular tables            
             try:
-                # Run the query against the regular table first
-                dbc.execute("select {} from {}.{}".format(aggExpr, dbName, rTbName))
-                # Then run it against the super table
-                if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?!
-                    dbc.execute("select {} from {}.{}".format(aggExpr, dbName, sTable.getName()))
+                sql = q.getSql()
+                # if 'GROUP BY' in sql:
+                #     Logging.info("Executing GROUP-BY query: " + sql)
+                dbc.execute(sql)
             except taos.error.ProgrammingError as err:                    
                 errno2 = Helper.convertErrno(err.errno)
                 Logging.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql()))
                 raise
 
+class SqlQuery:
+    @classmethod
+    def buildRandom(cls, db: Database):
+        '''Build a random query against a certain database'''
+        
+        dbName = db.getName()
+
+    def __init__(self, sql:str = None):
+        self._sql = sql
+
+    def getSql(self):
+        return self._sql
+    
 class TaskDropSuperTable(StateTransitionTask):
     @classmethod
     def getEndState(cls):
@@ -1837,19 +1902,18 @@ class TaskAlterTags(StateTransitionTask):
         # tblName = self._dbManager.getFixedSuperTableName()
         dbc = wt.getDbConn()
         sTable = self._db.getFixedSuperTable()
-        dbName = self._db.getName()
         dice = Dice.throw(4)
         if dice == 0:
-            sTable.addTag(dbc, dbName, "extraTag", "int")
+            sTable.addTag(dbc, "extraTag", "int")
             # sql = "alter table db.{} add tag extraTag int".format(tblName)
         elif dice == 1:
-            sTable.dropTag(dbc, dbName, "extraTag")
+            sTable.dropTag(dbc, "extraTag")
             # sql = "alter table db.{} drop tag extraTag".format(tblName)
         elif dice == 2:
-            sTable.dropTag(dbc, dbName, "newTag")
+            sTable.dropTag(dbc, "newTag")
             # sql = "alter table db.{} drop tag newTag".format(tblName)
         else:  # dice == 3
-            sTable.changeTag(dbc, dbName, "extraTag", "newTag")
+            sTable.changeTag(dbc, "extraTag", "newTag")
             # sql = "alter table db.{} change tag extraTag newTag".format(tblName)
 
 class TaskRestartService(StateTransitionTask):
@@ -1920,15 +1984,17 @@ class TaskAddData(StateTransitionTask):
         for j in range(numRecords):  # number of records per table
             nextInt = db.getNextInt()
             nextTick = db.getNextTick()
-            sql += "('{}', {});".format(nextTick, nextInt)
+            nextColor = db.getNextColor()
+            sql += "('{}', {}, '{}');".format(nextTick, nextInt, nextColor)
         dbc.execute(sql)
 
-    def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches
+    def _addData(self, db: Database, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches
         numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS        
 
         for j in range(numRecords):  # number of records per table
             nextInt = db.getNextInt()
             nextTick = db.getNextTick()
+            nextColor = db.getNextColor()
             if gConfig.record_ops:
                 self.prepToRecordOps()
                 self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName))
@@ -1942,11 +2008,11 @@ class TaskAddData(StateTransitionTask):
                 # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written
 
             try:
-                sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {})
+                sql = "insert into {} values ('{}', {}, '{}');".format( # removed: tags ('{}', {})
                     fullTableName,
                     # ds.getFixedSuperTableName(),
                     # ds.getNextBinary(), ds.getNextFloat(),
-                    nextTick, nextInt)
+                    nextTick, nextInt, nextColor)
                 dbc.execute(sql)
             except: # Any exception at all
                 if gConfig.verify_data:
@@ -1964,10 +2030,10 @@ class TaskAddData(StateTransitionTask):
                             .format(nextInt, readBack), 0x999)
                 except taos.error.ProgrammingError as err:
                     errno = Helper.convertErrno(err.errno)
-                    if errno in [0x991, 0x992]  : # not a single result
+                    if errno in [CrashGenError.INVALID_EMPTY_RESULT, CrashGenError.INVALID_MULTIPLE_RESULT]  : # not a single result
                         raise taos.error.ProgrammingError(
                             "Failed to read back same data for tick: {}, wrote: {}, read: {}"
-                            .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"),
+                            .format(nextTick, nextInt, "Empty Result" if errno == CrashGenError.INVALID_EMPTY_RESULT else "Multiple Result"),
                             errno)
                     elif errno in [0x218, 0x362]: # table doesn't exist
                         # do nothing
@@ -2000,11 +2066,12 @@ class TaskAddData(StateTransitionTask):
             else:
                 self.activeTable.add(i)  # marking it active
             
+            dbName = db.getName()
             sTable = db.getFixedSuperTable()
             regTableName = self.getRegTableName(i)  # "db.reg_table_{}".format(i)            
-            fullTableName = db.getName() + '.' + regTableName
+            fullTableName = dbName + '.' + regTableName
             # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked"
-            sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName)  # Ensure the table exists           
+            sTable.ensureTable(self, wt.getDbConn(), regTableName)  # Ensure the table exists           
             # self._unlockTable(fullTableName)
            
             if Dice.throw(1) == 0: # 1 in 2 chance
@@ -2024,7 +2091,7 @@ class ThreadStacks: # stack info for all threads
             self._allStacks[th.native_id] = stack
 
     def print(self, filteredEndName = None, filterInternal = False):
-        for thNid, stack in self._allStacks.items(): # for each thread            
+        for thNid, stack in self._allStacks.items(): # for each thread, stack frames top to bottom
             lastFrame = stack[-1]
             if filteredEndName: # we need to filter out stacks that match this name                
                 if lastFrame.name == filteredEndName : # end did not match
@@ -2036,9 +2103,9 @@ class ThreadStacks: # stack info for all threads
                     '__init__']: # the thread that extracted the stack
                     continue # ignore
             # Now print
-            print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid))
+            print("\n<----- Thread Info for LWP/ID: {} (most recent call last) <-----".format(thNid))
             stackFrame = 0
-            for frame in stack:
+            for frame in stack: # was using: reversed(stack)
                 # print(frame)
                 print("[{sf}] File {filename}, line {lineno}, in {name}".format(
                     sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name))
diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py
index 2a4b362f82..855e18be55 100644
--- a/tests/pytest/crash_gen/db.py
+++ b/tests/pytest/crash_gen/db.py
@@ -78,7 +78,7 @@ class DbConn:
         if nRows != 1:
             raise taos.error.ProgrammingError(
                 "Unexpected result for query: {}, rows = {}".format(sql, nRows), 
-                (0x991 if nRows==0 else 0x992)
+                (CrashGenError.INVALID_EMPTY_RESULT if nRows==0 else CrashGenError.INVALID_MULTIPLE_RESULT)
             )
         if self.getResultRows() != 1 or self.getResultCols() != 1:
             raise RuntimeError("Unexpected result set for query: {}".format(sql))
@@ -349,7 +349,8 @@ class DbConnNative(DbConn):
 
     def execute(self, sql):
         if (not self.isOpen):
-            raise RuntimeError("Cannot execute database commands until connection is open")
+            raise CrashGenError(
+                "Cannot exec SQL unless db connection is open", CrashGenError.DB_CONNECTION_NOT_OPEN)
         Logging.debug("[SQL] Executing SQL: {}".format(sql))
         self._lastSql = sql
         nRows = self._tdSql.execute(sql)
@@ -360,8 +361,8 @@ class DbConnNative(DbConn):
 
     def query(self, sql):  # return rows affected
         if (not self.isOpen):
-            raise RuntimeError(
-                "Cannot query database until connection is open")
+            raise CrashGenError(
+                "Cannot query database until connection is open, restarting?", CrashGenError.DB_CONNECTION_NOT_OPEN)
         Logging.debug("[SQL] Executing SQL: {}".format(sql))
         self._lastSql = sql
         nRows = self._tdSql.query(sql)
diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py
index 2d2ce99d95..a374ed943b 100644
--- a/tests/pytest/crash_gen/misc.py
+++ b/tests/pytest/crash_gen/misc.py
@@ -3,14 +3,20 @@ import random
 import logging
 import os
 
+import taos
 
-class CrashGenError(Exception):
-    def __init__(self, msg=None, errno=None):
-        self.msg = msg
-        self.errno = errno
 
-    def __str__(self):
-        return self.msg
+class CrashGenError(taos.error.ProgrammingError):
+    INVALID_EMPTY_RESULT    = 0x991
+    INVALID_MULTIPLE_RESULT = 0x992
+    DB_CONNECTION_NOT_OPEN  = 0x993
+    # def __init__(self, msg=None, errno=None):
+    #     self.msg = msg
+    #     self.errno = errno
+
+    # def __str__(self):
+    #     return self.msg
+    pass
 
 
 class LoggingFilter(logging.Filter):
@@ -168,6 +174,7 @@ class Progress:
     SERVICE_RECONNECT_FAILURE   = 6
     SERVICE_START_NAP           = 7
     CREATE_TABLE_ATTEMPT        = 8
+    QUERY_GROUP_BY              = 9
 
     tokens = {
         STEP_BOUNDARY:      '.',
@@ -178,7 +185,8 @@ class Progress:
         SERVICE_RECONNECT_SUCCESS:  '.r>',
         SERVICE_RECONNECT_FAILURE:  '.xr>',
         SERVICE_START_NAP:           '_zz',
-        CREATE_TABLE_ATTEMPT: '_c',
+        CREATE_TABLE_ATTEMPT:       'c',
+        QUERY_GROUP_BY:             'g',
     }
 
     @classmethod
diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py
index d249abc439..ae6f8d5d3a 100644
--- a/tests/pytest/crash_gen/service_manager.py
+++ b/tests/pytest/crash_gen/service_manager.py
@@ -51,10 +51,12 @@ class TdeInstance():
     def prepareGcovEnv(cls, env):
         # Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html
         bPath = cls._getBuildPath() # build PATH
-        numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3
-        numSegments = numSegments - 1 # DEBUG only
-        env['GCOV_PREFIX'] = bPath + '/svc_gcov'
+        numSegments = len(bPath.split('/')) # "/x/TDengine/build" should yield 3
+        # numSegments += 2 # cover "/src" after build
+        # numSegments = numSegments - 1 # DEBUG only
+        env['GCOV_PREFIX'] = bPath + '/src_s' # Server side source
         env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings
+        # VERY VERY important note: GCOV data collection NOT effective upon SIG_KILL
         Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format(
             numSegments, env['GCOV_PREFIX'] ))
 
@@ -258,14 +260,15 @@ class TdeSubProcess:
         TdeInstance.prepareGcovEnv(myEnv)
 
         # print(myEnv)
-        # print(myEnv.items())
+        # print("Starting TDengine with env: ", myEnv.items())
         # print("Starting TDengine via Shell: {}".format(cmdLineStr))
 
         useShell = True    
         self.subProcess = subprocess.Popen(
-            ' '.join(cmdLine) if useShell else cmdLine,
-            shell=useShell,
-            # svcCmdSingle, shell=True, # capture core dump?
+            # ' '.join(cmdLine) if useShell else cmdLine,
+            # shell=useShell,
+            ' '.join(cmdLine),
+            shell=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             # bufsize=1, # not supported in binary mode
@@ -273,7 +276,8 @@ class TdeSubProcess:
             env=myEnv
             )  # had text=True, which interferred with reading EOF
 
-    STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process?
+    STOP_SIGNAL = signal.SIGKILL # signal.SIGKILL/SIGINT # What signal to use (in kill) to stop a taosd process?
+    SIG_KILL_RETCODE = 137 # ref: https://stackoverflow.com/questions/43268156/process-finished-with-exit-code-137-in-pycharm
 
     def stop(self):
         """
@@ -320,8 +324,12 @@ class TdeSubProcess:
         retCode = self.subProcess.returncode # should always be there
         # May throw subprocess.TimeoutExpired exception above, therefore
         # The process is guranteed to have ended by now
-        self.subProcess = None        
-        if retCode != 0: # != (- signal.SIGINT):
+        self.subProcess = None       
+        if retCode == self.SIG_KILL_RETCODE:
+            Logging.info("TSP.stop(): sub proc KILLED, as expected")
+        elif retCode == (- self.STOP_SIGNAL):
+            Logging.info("TSP.stop(), sub process STOPPED, as expected")
+        elif retCode != 0: # != (- signal.SIGINT):
             Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(
                 self.STOP_SIGNAL, retCode))
         else:

From 5907b8e8b96816a6045c96af57f169d7b7b02714 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Wed, 2 Dec 2020 11:06:50 +0800
Subject: [PATCH 016/118] fix test case failure

---
 tests/pytest/client/alterDatabase.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/pytest/client/alterDatabase.py b/tests/pytest/client/alterDatabase.py
index fa397d16c5..8191312cc0 100644
--- a/tests/pytest/client/alterDatabase.py
+++ b/tests/pytest/client/alterDatabase.py
@@ -32,9 +32,9 @@ class TDTestCase:
         tdSql.query("show databases")
         tdSql.checkData(0, 14, 2)
 
-        tdSql.execute("alter database db keep 365")
+        tdSql.execute("alter database db keep 365,365,365")
         tdSql.query("show databases")
-        tdSql.checkData(0, 7, "3650,3650,365")
+        tdSql.checkData(0, 7, "365,365,365")
 
         tdSql.execute("alter database db quorum 2")
         tdSql.query("show databases")

From 19a410f94b5d72a136b850fef0b159e477e4e64a Mon Sep 17 00:00:00 2001
From: zyyang <zyyang@taosdata.com>
Date: Wed, 2 Dec 2020 11:36:56 +0800
Subject: [PATCH 017/118] [TD-2301]<hotfix>: change

---
 tests/examples/JDBC/JDBCDemo/pom.xml                            | 2 +-
 .../com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml
index 121a3b5cd6..98f908b77e 100644
--- a/tests/examples/JDBC/JDBCDemo/pom.xml
+++ b/tests/examples/JDBC/JDBCDemo/pom.xml
@@ -56,7 +56,7 @@
         <dependency>
             <groupId>com.taosdata.jdbc</groupId>
             <artifactId>taos-jdbcdriver</artifactId>
-            <version>2.0.8</version>
+            <version>2.0.12</version>
         </dependency>
         <dependency>
             <groupId>log4j</groupId>
diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java
index a35628bb58..644de52dd3 100644
--- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java
+++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java
@@ -41,7 +41,7 @@ public class InsertTableTask implements Runnable {
                 long ts = start.toEpochMilli() + (j * timeGap);
                 // insert data into echo table
                 for (int i = startTbIndex; i < startTbIndex + tableNumber; i++) {
-                    String sql = SqlSpeller.insertOneRowSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts);
+                    String sql = SqlSpeller.insertBatchSizeRowsSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts, config.getNumberOfRecordsPerRequest());
                     logger.info(Thread.currentThread().getName() + ">>> " + sql);
                     Statement statement = connection.createStatement();
                     statement.execute(sql);

From 6db661976cae0b71cf2c239d2033cf647854e52d Mon Sep 17 00:00:00 2001
From: haojun Liao <hjxilinx@users.noreply.github.com>
Date: Wed, 2 Dec 2020 13:45:48 +0800
Subject: [PATCH 018/118] Update connector-ch.md

---
 documentation20/webdocs/markdowndocs/connector-ch.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md
index c5a955f43f..69c560bbc4 100644
--- a/documentation20/webdocs/markdowndocs/connector-ch.md
+++ b/documentation20/webdocs/markdowndocs/connector-ch.md
@@ -142,7 +142,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
   获取最近一次API调用失败的原因，返回值为错误代码。
 
 
-**注意**：对于单个数据库连接，在同一时刻只能有一个线程使用该连接调用API，否则会有未定义的行为出现并可能导致客户端crash。客户端应用可以通过建立多个连接进行多线程的数据写入或查询处理。
+**注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。
 
 
 ### 异步查询API

From e690846ce80c15986623534a36f7e8b808c819a7 Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Wed, 2 Dec 2020 06:34:51 +0000
Subject: [PATCH 019/118] fix td-2290

---
 src/client/src/tscSub.c | 91 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 83 insertions(+), 8 deletions(-)

diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 043e78a9f3..52b74f7502 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -157,7 +157,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
 
   registerSqlObj(pSql);
 
-  code = tsParseSql(pSql, false);
+  code = tsParseSql(pSql, true);
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     tsem_wait(&pSub->sem);
     code = pSql->res.code;
@@ -168,7 +168,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
     goto fail;
   }
 
-  if (pSql->cmd.command != TSDB_SQL_SELECT) {
+  if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
     line = __LINE__;
     code = TSDB_CODE_TSC_INVALID_SQL;
     goto fail;
@@ -182,7 +182,7 @@ fail:
     if (pSql->self != 0) {
       taosReleaseRef(tscObjRef, pSql->self);
     } else {
-      tscFreeSqlObj(pSql);
+    tscFreeSqlObj(pSql);
     }
 
     pSql = NULL;
@@ -401,9 +401,11 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
     tscLoadSubscriptionProgress(pSub);
   }
 
-  if (!tscUpdateSubscription(pObj, pSub)) {
-    taos_unsubscribe(pSub, 1);
-    return NULL;
+  if (pSub->pSql->cmd.command == TSDB_SQL_SELECT) {
+    if (!tscUpdateSubscription(pObj, pSub)) {
+      taos_unsubscribe(pSub, 1);
+      return NULL;
+    }
   }
 
   pSub->interval = interval;
@@ -417,10 +419,80 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
   return pSub;
 }
 
+SSqlObj* recreateSqlObj(SSub* pSub) {
+  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  if (pSql == NULL) {
+    return NULL;
+  }
+
+  pSql->signature = pSql;
+  pSql->pTscObj = pSub->taos;
+
+  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlRes* pRes = &pSql->res;
+  if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  pSql->param = pSub;
+  pSql->maxRetry = TSDB_MAX_REPLICA;
+  pSql->fp = asyncCallback;
+  pSql->fetchFp = asyncCallback;
+  pSql->sqlstr = strdup(pSub->pSql->sqlstr);
+  if (pSql->sqlstr == NULL) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  pRes->qhandle = 0;
+  pRes->numOfRows = 1;
+
+  int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
+  if (code != TSDB_CODE_SUCCESS) {
+    tscFreeSqlObj(pSql);
+    return NULL;
+  }
+
+  registerSqlObj(pSql);
+
+  code = tsParseSql(pSql, true);
+  if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+    tsem_wait(&pSub->sem);
+    code = pSql->res.code;
+  }
+
+  if (code != TSDB_CODE_SUCCESS) {
+    taosReleaseRef(tscObjRef, pSql->self);
+    return NULL;
+  }
+
+  if (pSql->cmd.command != TSDB_SQL_SELECT) {
+    taosReleaseRef(tscObjRef, pSql->self);
+    return NULL;
+  }
+
+  return pSql;
+}
+
 TAOS_RES *taos_consume(TAOS_SUB *tsub) {
   SSub *pSub = (SSub *)tsub;
   if (pSub == NULL) return NULL;
 
+  if (pSub->pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
+    SSqlObj* pSql = recreateSqlObj(pSub);
+    if (pSql == NULL) {
+      return NULL;
+    }
+    if (pSub->pSql->self != 0) {
+      taosReleaseRef(tscObjRef, pSub->pSql->self);
+    } else {
+      tscFreeSqlObj(pSub->pSql);
+    }
+    pSub->pSql = pSql;
+    pSql->pSubscription = pSub;
+  }
+
   tscSaveSubscriptionProgress(pSub);
 
   SSqlObj *pSql = pSub->pSql;
@@ -512,10 +584,13 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
   }
 
   if (pSub->pSql != NULL) {
-    taos_free_result(pSub->pSql);
+    if (pSub->pSql->self != 0) {
+      taosReleaseRef(tscObjRef, pSub->pSql->self);
+    } else {
+      tscFreeSqlObj(pSub->pSql);
+    }
   }
 
-  tscFreeSqlObj(pSub->pSql);
   taosArrayDestroy(pSub->progress);
   tsem_destroy(&pSub->sem);
   memset(pSub, 0, sizeof(*pSub));

From b55548782bda709da4a5a7d99a40644eb82734d8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 2 Dec 2020 15:23:34 +0800
Subject: [PATCH 020/118] [TD-2293]<enhance>: improve projection query
 performance for super table.

---
 src/client/src/tscSQLParser.c |   7 +-
 src/query/inc/qExecutor.h     |   4 +-
 src/query/src/qExecutor.c     | 318 ++++++++++++++++++++++------------
 src/query/src/qUtil.c         |   2 -
 src/util/src/hash.c           |   5 +-
 5 files changed, 217 insertions(+), 119 deletions(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 309d354849..0c9ad78cd9 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5311,9 +5311,10 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
 
     if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
       /*
-       * the limitation/offset value should be removed during retrieve data from virtual node,
-       * since the global order are done in client side, so the limitation should also
-       * be done at the client side.
+       * the offset value should be removed during retrieve data from virtual node, since the
+       * global order are done in client side, so the offset is applied at the client side
+       * However, note that the maximum allowed number of result for each table should be less
+       * than or equal to the value of limit.
        */
       if (pQueryInfo->limit.limit > 0) {
         pQueryInfo->limit.limit = -1;
diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h
index 71fc01923a..4ec6ed30ba 100644
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@@ -190,6 +190,7 @@ typedef struct SQueryRuntimeEnv {
   bool                 groupbyNormalCol; // denote if this is a groupby normal column query
   bool                 hasTagResults;    // if there are tag values in final result or not
   bool                 timeWindowInterpo;// if the time window start/end required interpolation
+  bool                 queryWindowIdentical; // all query time windows are identical for all tables in one group
   int32_t              interBufSize;     // intermediate buffer sizse
   int32_t              prevGroupId;      // previous executed group id
   SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
@@ -217,7 +218,8 @@ typedef struct SQInfo {
   STableGroupInfo  tableGroupInfo;       // table <tid, last_key> list  SArray<STableKeyInfo>
   STableGroupInfo  tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
   SQueryRuntimeEnv runtimeEnv;
-  SArray*          arrTableIdInfo;
+//  SArray*          arrTableIdInfo;
+  SHashObj*        arrTableIdInfo;
   int32_t          groupIndex;
 
   /*
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 6de57b32fd..7e6950a735 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -197,6 +197,8 @@ static int32_t checkForQueryBuf(size_t numOfTables);
 static void releaseQueryBuf(size_t numOfTables);
 static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order);
 static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type);
+static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery);
+static STableIdInfo createTableIdInfo(SQuery* pQuery);
 
 bool doFilterData(SQuery *pQuery, int32_t elemPos) {
   for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) {
@@ -2781,6 +2783,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa
   pQuery->rec.capacity = capacity;
 }
 
+// TODO merge with enuserOutputBufferSimple
 static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) {
   // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block
   SQuery* pQuery = pRuntimeEnv->pQuery;
@@ -3916,13 +3919,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
 
   SET_REVERSE_SCAN_FLAG(pRuntimeEnv);
 
-  STsdbQueryCond cond = {
-      .order   = pQuery->order.order,
-      .colList = pQuery->colList,
-      .numOfCols = pQuery->numOfCols,
-  };
-
-  TIME_WINDOW_COPY(cond.twindow, pQuery->window);
+  STsdbQueryCond cond = createTsdbQueryCond(pQuery);
 
   setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
   switchCtxOrder(pRuntimeEnv);
@@ -4005,18 +4002,11 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
       break;
     }
 
-    STsdbQueryCond cond = {
-        .order   = pQuery->order.order,
-        .colList = pQuery->colList,
-        .numOfCols = pQuery->numOfCols,
-    };
-
-    TIME_WINDOW_COPY(cond.twindow, qstatus.curWindow);
-
     if (pRuntimeEnv->pSecQueryHandle != NULL) {
       tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
     }
 
+    STsdbQueryCond cond = createTsdbQueryCond(pQuery);
     restoreTimeWindow(&pQInfo->tableGroupInfo, &cond);
     pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
     if (pRuntimeEnv->pSecQueryHandle == NULL) {
@@ -4541,16 +4531,19 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
     }
   }
 
-  int32_t numOfTables = (int32_t)taosArrayGetSize(pQInfo->arrTableIdInfo);
+  int32_t numOfTables = taosHashGetSize(pQInfo->arrTableIdInfo);
   *(int32_t*)data = htonl(numOfTables);
   data += sizeof(int32_t);
-  for(int32_t i = 0; i < numOfTables; i++) {
-    STableIdInfo* pSrc = taosArrayGet(pQInfo->arrTableIdInfo, i);
+
+  STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL);
+  while(item) {
     STableIdInfo* pDst = (STableIdInfo*)data;
-    pDst->uid = htobe64(pSrc->uid);
-    pDst->tid = htonl(pSrc->tid);
-    pDst->key = htobe64(pSrc->key);
+    pDst->uid = htobe64(item->uid);
+    pDst->tid = htonl(item->tid);
+    pDst->key = htobe64(item->key);
+
     data += sizeof(STableIdInfo);
+    item = taosHashIterate(pQInfo->arrTableIdInfo, item);
   }
 
   // Check if query is completed or not for stable query or normal table query respectively.
@@ -4877,13 +4870,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
     return TSDB_CODE_SUCCESS;
   }
 
-  STsdbQueryCond cond = {
-    .order   = pQuery->order.order,
-    .colList = pQuery->colList,
-    .numOfCols = pQuery->numOfCols,
-  };
-
-  TIME_WINDOW_COPY(cond.twindow, pQuery->window);
+  STsdbQueryCond cond = createTsdbQueryCond(pQuery);
 
   if (!isSTableQuery
     && (pQInfo->tableqinfoGroupInfo.numOfTables == 1)
@@ -5276,6 +5263,41 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
   return true;
 }
 
+STsdbQueryCond createTsdbQueryCond(SQuery* pQuery) {
+  STsdbQueryCond cond = {
+      .colList   = pQuery->colList,
+      .order     = pQuery->order.order,
+      .numOfCols = pQuery->numOfCols,
+  };
+
+  TIME_WINDOW_COPY(cond.twindow, pQuery->window);
+  return cond;
+}
+
+static STableIdInfo createTableIdInfo(SQuery* pQuery) {
+  assert(pQuery != NULL && pQuery->current != NULL);
+
+  STableIdInfo tidInfo;
+  STableId* id = TSDB_TABLEID(pQuery->current->pTable);
+
+  tidInfo.uid = id->uid;
+  tidInfo.tid = id->tid;
+  tidInfo.key = pQuery->current->lastKey;
+
+  return tidInfo;
+}
+
+static void updateTableIdInfo(SQuery* pQuery, SHashObj* pTableIdInfo) {
+  STableIdInfo tidInfo = createTableIdInfo(pQuery);
+  STableIdInfo* idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid));
+  if (idinfo != NULL) {
+    assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid);
+    idinfo->key = tidInfo.key;
+  } else {
+    taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo));
+  }
+}
+
 /**
  * super table query handler
  * 1. super table projection query, group-by on normal columns query, ts-comp query
@@ -5295,18 +5317,11 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0);
 
     while (pQInfo->groupIndex < numOfGroups) {
-      SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex);
+      SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex);
 
-      qDebug("QInfo:%p last_row query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, pQInfo->groupIndex,
-             numOfGroups, group);
-
-      STsdbQueryCond cond = {
-          .colList = pQuery->colList,
-          .order   = pQuery->order.order,
-          .numOfCols = pQuery->numOfCols,
-      };
-
-      TIME_WINDOW_COPY(cond.twindow, pQuery->window);
+      qDebug("QInfo:%p point interpolation query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo,
+             pQInfo->groupIndex, numOfGroups, group);
+      STsdbQueryCond cond = createTsdbQueryCond(pQuery);
 
       SArray *g1 = taosArrayInit(1, POINTER_BYTES);
       SArray *tx = taosArrayClone(group);
@@ -5330,14 +5345,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       initCtxOutputBuf(pRuntimeEnv);
 
-      SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
+      SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
       assert(taosArrayGetSize(s) >= 1);
 
       setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb);
       taosArrayDestroy(s);
 
       // here we simply set the first table as current table
-      SArray* first = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex);
+      SArray *first = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex);
       pQuery->current = taosArrayGetP(first, 0);
 
       scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
@@ -5359,19 +5374,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         break;
       }
     }
-  } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query
+  } else if (pRuntimeEnv->groupbyNormalCol) {  // group-by on normal columns query
     while (pQInfo->groupIndex < numOfGroups) {
-      SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex);
+      SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex);
 
-      qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, numOfGroups);
+      qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex,
+             numOfGroups);
 
-      STsdbQueryCond cond = {
-          .colList = pQuery->colList,
-          .order   = pQuery->order.order,
-          .numOfCols = pQuery->numOfCols,
-      };
-
-      TIME_WINDOW_COPY(cond.twindow, pQuery->window);
+      STsdbQueryCond cond = createTsdbQueryCond(pQuery);
 
       SArray *g1 = taosArrayInit(1, POINTER_BYTES);
       SArray *tx = taosArrayClone(group);
@@ -5394,7 +5404,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         longjmp(pRuntimeEnv->env, terrno);
       }
 
-      SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
+      SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
       assert(taosArrayGetSize(s) >= 1);
 
       setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb);
@@ -5405,24 +5415,24 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
-        // no results generated for current group, continue to try the next group
+      // no results generated for current group, continue to try the next group
       taosArrayDestroy(s);
       if (pWindowResInfo->size <= 0) {
         continue;
       }
 
       for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
-        pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns
+        pWindowResInfo->pResult[i]->closed = true;  // enable return all results for group by normal columns
 
         SResultRow *pResult = pWindowResInfo->pResult[i];
         for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
-          SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j);
+          SResultRowCellInfo *pCell = getResultCell(pRuntimeEnv, pResult, j);
           pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes));
         }
       }
 
       qDebug("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size,
-          pQInfo->groupIndex);
+             pQInfo->groupIndex);
       int32_t currentGroupIndex = pQInfo->groupIndex;
 
       pQuery->rec.rows = 0;
@@ -5431,16 +5441,119 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size);
       copyFromWindowResToSData(pQInfo, pWindowResInfo);
 
-      pQInfo->groupIndex = currentGroupIndex;  //restore the group index
+      pQInfo->groupIndex = currentGroupIndex;  // restore the group index
       assert(pQuery->rec.rows == pWindowResInfo->size);
 
       clearClosedTimeWindow(pRuntimeEnv);
       break;
     }
+  } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) {
+    //super table projection query with identical query time range for all tables.
+    SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
+
+    resetDefaultResInfoOutputBuf(pRuntimeEnv);
+
+    SArray *group = GET_TABLEGROUP(pQInfo, 0);
+    assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables &&
+           1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList));
+
+    void *pQueryHandle = pRuntimeEnv->pQueryHandle;
+    if (pQueryHandle == NULL) {
+      STsdbQueryCond con = createTsdbQueryCond(pQuery);
+      pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &con, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
+      pQueryHandle = pRuntimeEnv->pQueryHandle;
+    }
+
+    // skip blocks without load the actual data block from file if no filter condition present
+    //    skipBlocks(&pQInfo->runtimeEnv);
+    //    if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) {
+    //      setQueryStatus(pQuery, QUERY_COMPLETED);
+    //      return;
+    //    }
+
+    bool hasMoreBlock = true;
+    SQueryCostInfo *summary = &pRuntimeEnv->summary;
+    while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) {
+      summary->totalBlocks += 1;
+
+      if (IS_QUERY_KILLED(pQInfo)) {
+        longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+      }
+
+      tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo);
+      STableQueryInfo **pTableQueryInfo =
+          (STableQueryInfo **)taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid));
+      if (pTableQueryInfo == NULL) {
+        break;
+      }
+
+      pQuery->current = *pTableQueryInfo;
+      if (QUERY_IS_ASC_QUERY(pQuery)) {
+        assert(((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) &&
+               ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) &&
+               ((*pTableQueryInfo)->win.skey >= pQuery->window.skey &&
+                (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey));
+      } else {
+        assert(((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) &&
+               ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) &&
+               ((*pTableQueryInfo)->win.skey <= pQuery->window.skey &&
+                (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey));
+      }
+
+      if (pRuntimeEnv->hasTagResults) {
+        setTagVal(pRuntimeEnv, pQuery->current->pTable, pQInfo->tsdb);
+      }
+
+      uint32_t     status = 0;
+      SDataStatis *pStatis = NULL;
+      SArray *     pDataBlock = NULL;
+
+      int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo,
+                                          &pStatis, &pDataBlock, &status);
+      if (ret != TSDB_CODE_SUCCESS) {
+        break;
+      }
+
+      assert(status != BLK_DATA_DISCARD);
+      ensureOutputBuffer(pRuntimeEnv, &blockInfo);
+
+      pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1;
+      int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock);
+
+      summary->totalRows += blockInfo.rows;
+      qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64,
+             GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes,
+             pQuery->current->lastKey);
+
+      pQuery->rec.rows = getNumOfResult(pRuntimeEnv);
+
+      // the flag may be set by tableApplyFunctionsOnBlock, clear it here
+      CLEAR_QUERY_STATUS(pQuery, QUERY_COMPLETED);
+
+      updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo);
+      skipResults(pRuntimeEnv);
+
+      // the limitation of output result is reached, set the query completed
+      if (limitResults(pRuntimeEnv)) {
+        SET_STABLE_QUERY_OVER(pQInfo);
+        break;
+      }
+
+      // while the output buffer is full or limit/offset is applied, query may be paused here
+      if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL|QUERY_COMPLETED)) {
+        break;
+      }
+    }
+
+    if (!hasMoreBlock) {
+      setQueryStatus(pQuery, QUERY_COMPLETED);
+      SET_STABLE_QUERY_OVER(pQInfo);
+    }
   } else {
     /*
-     * 1. super table projection query, 2. ts-comp query
-     * if the subgroup index is larger than 0, results generated by group by tbname,k is existed.
+     * the following two cases handled here.
+     * 1. ts-comp query, and 2. the super table projection query with different query time range for each table.
+     * If the subgroup index is larger than 0, results generated by group by tbname,k is existed.
      * we need to return it to client in the first place.
      */
     if (pQInfo->groupIndex > 0) {
@@ -5503,14 +5616,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
          * to ensure that, we can reset the query range once query on a meter is completed.
          */
         pQInfo->tableIndex++;
-
-        STableIdInfo tidInfo = {0};
-
-        STableId* id = TSDB_TABLEID(pQuery->current->pTable);
-        tidInfo.uid = id->uid;
-        tidInfo.tid = id->tid;
-        tidInfo.key = pQuery->current->lastKey;
-        taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
+        updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo);
 
         // if the buffer is full or group by each table, we need to jump out of the loop
         if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) {
@@ -5537,31 +5643,31 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) {
       setQueryStatus(pQuery, QUERY_COMPLETED);
     }
-  }
 
-  /*
-   * 1. super table projection query, group-by on normal columns query, ts-comp query
-   * 2. point interpolation query, last row query
-   *
-   * group-by on normal columns query and last_row query do NOT invoke the finalizer here,
-   * since the finalize stage will be done at the client side.
-   *
-   * projection query, point interpolation query do not need the finalizer.
-   *
-   * Only the ts-comp query requires the finalizer function to be executed here.
-   */
-  if (isTSCompQuery(pQuery)) {
-    finalizeQueryResult(pRuntimeEnv);
-  }
+    /*
+     * 1. super table projection query, group-by on normal columns query, ts-comp query
+     * 2. point interpolation query, last row query
+     *
+     * group-by on normal columns query and last_row query do NOT invoke the finalizer here,
+     * since the finalize stage will be done at the client side.
+     *
+     * projection query, point interpolation query do not need the finalizer.
+     *
+     * Only the ts-comp query requires the finalizer function to be executed here.
+     */
+    if (isTSCompQuery(pQuery)) {
+      finalizeQueryResult(pRuntimeEnv);
+    }
 
-  if (pRuntimeEnv->pTSBuf != NULL) {
-    pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur;
-  }
+    if (pRuntimeEnv->pTSBuf != NULL) {
+      pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur;
+    }
 
-  qDebug(
-      "QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 " points returned, total:%" PRId64 ", offset:%" PRId64,
-      pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
-      pQuery->limit.offset);
+    qDebug("QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64
+           " points returned, total:%" PRId64 ", offset:%" PRId64,
+           pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows,
+           pQuery->rec.total, pQuery->limit.offset);
+  }
 }
 
 static void doSaveContext(SQInfo *pQInfo) {
@@ -5576,13 +5682,7 @@ static void doSaveContext(SQInfo *pQInfo) {
     SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order);
   }
 
-  STsdbQueryCond cond = {
-      .order   = pQuery->order.order,
-      .colList = pQuery->colList,
-      .numOfCols = pQuery->numOfCols,
-  };
-
-  TIME_WINDOW_COPY(cond.twindow, pQuery->window);
+  STsdbQueryCond cond = createTsdbQueryCond(pQuery);
 
   // clean unused handle
   if (pRuntimeEnv->pSecQueryHandle != NULL) {
@@ -5855,13 +5955,8 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
     qDebug("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo,
         pQuery->current->lastKey, pQuery->window.ekey);
   } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-    STableIdInfo tidInfo;
-    STableId* id = TSDB_TABLEID(pQuery->current->pTable);
-
-    tidInfo.uid = id->uid;
-    tidInfo.tid = id->tid;
-    tidInfo.key = pQuery->current->lastKey;
-    taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo);
+    STableIdInfo tidInfo = createTableIdInfo(pQuery);
+    taosHashPut(pQInfo->arrTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo));
   }
 
   if (!isTSCompQuery(pQuery)) {
@@ -6859,8 +6954,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
                                                    taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
   }
 
-  int tableIndex = 0;
-
   pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery);
   pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo));
 
@@ -6882,7 +6975,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
   }
 
   // NOTE: pTableCheckInfo need to update the query time range and the lastKey info
-  pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));
+  pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
   pQInfo->dataReady = QUERY_RESULT_NOT_READY;
   pQInfo->rspContext = NULL;
   pthread_mutex_init(&pQInfo->lock, NULL);
@@ -6892,10 +6985,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
   pQuery->window = pQueryMsg->window;
   changeExecuteScanOrder(pQInfo, pQueryMsg, stableQuery);
 
+  pQInfo->runtimeEnv.queryWindowIdentical = true;
   STimeWindow window = pQuery->window;
 
   int32_t index = 0;
-
   for(int32_t i = 0; i < numOfGroups; ++i) {
     SArray* pa = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i);
 
@@ -6910,9 +7003,12 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
     for(int32_t j = 0; j < s; ++j) {
       STableKeyInfo* info = taosArrayGet(pa, j);
 
-      void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo);
-
       window.skey = info->lastKey;
+      if (info->lastKey != pQuery->window.skey) {
+        pQInfo->runtimeEnv.queryWindowIdentical = false;
+      }
+
+      void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo);
       STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf);
       if (item == NULL) {
         goto _cleanup;
@@ -7126,7 +7222,7 @@ static void freeQInfo(SQInfo *pQInfo) {
 
   tfree(pQInfo->pBuf);
   tsdbDestroyTableGroup(&pQInfo->tableGroupInfo);
-  taosArrayDestroy(pQInfo->arrTableIdInfo);
+  taosHashCleanup(pQInfo->arrTableIdInfo);
 
   pQInfo->signature = 0;
 
@@ -7506,7 +7602,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   size_t  size = getResultSize(pQInfo, &pQuery->rec.rows);
 
   size += sizeof(int32_t);
-  size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo);
+  size += sizeof(STableIdInfo) * taosHashGetSize(pQInfo->arrTableIdInfo);
 
   *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp));
 
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 2c41d9bfc5..f5f2da9f25 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -83,8 +83,6 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
     return;
   }
 
-//  assert(pWindowResInfo->size == 1);
-
   for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
     SResultRow *pWindowRes = pWindowResInfo->pResult[i];
     clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
diff --git a/src/util/src/hash.c b/src/util/src/hash.c
index 3ac54eedd8..0e3e0d3e24 100644
--- a/src/util/src/hash.c
+++ b/src/util/src/hash.c
@@ -151,8 +151,9 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode);
  */
 
 SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) {
-  if (capacity == 0 || fn == NULL) {
-    return NULL;
+  assert(fn != NULL);
+  if (capacity == 0) {
+    capacity = 4;
   }
 
   SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj));

From 5746fdabe71d3c97381bdb8dab114d4d3d66e15c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 2 Dec 2020 15:28:34 +0800
Subject: [PATCH 021/118] [TD-2129] refactor codes.

---
 src/query/src/qExecutor.c | 38 ++++++++++++++++----------------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 7e6950a735..d65b2e8f0a 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -5078,6 +5078,20 @@ static FORCE_INLINE void setEnvForEachBlock(SQInfo* pQInfo, STableQueryInfo* pTa
   }
 }
 
+static void doTableQueryInfoTimeWindowCheck(SQuery* pQuery, STableQueryInfo* pTableQueryInfo) {
+  if (QUERY_IS_ASC_QUERY(pQuery)) {
+    assert(
+        (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) &&
+        (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) &&
+        (pTableQueryInfo->win.skey >= pQuery->window.skey && pTableQueryInfo->win.ekey <= pQuery->window.ekey));
+  } else {
+    assert(
+        (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) &&
+        (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) &&
+        (pTableQueryInfo->win.skey <= pQuery->window.skey && pTableQueryInfo->win.ekey >= pQuery->window.ekey));
+  }
+}
+
 static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery*           pQuery = pRuntimeEnv->pQuery;
@@ -5104,17 +5118,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
     }
 
     pQuery->current = *pTableQueryInfo;
-    if (QUERY_IS_ASC_QUERY(pQuery)) {
-      assert(
-          ((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) &&
-          ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) &&
-          ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey));
-    } else {
-      assert(
-          ((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) &&
-          ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) &&
-          ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey));
-    }
+    doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo);
 
     if (!pRuntimeEnv->groupbyNormalCol) {
       setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo);
@@ -5488,17 +5492,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       }
 
       pQuery->current = *pTableQueryInfo;
-      if (QUERY_IS_ASC_QUERY(pQuery)) {
-        assert(((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) &&
-               ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) &&
-               ((*pTableQueryInfo)->win.skey >= pQuery->window.skey &&
-                (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey));
-      } else {
-        assert(((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) &&
-               ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) &&
-               ((*pTableQueryInfo)->win.skey <= pQuery->window.skey &&
-                (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey));
-      }
+      doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo);
 
       if (pRuntimeEnv->hasTagResults) {
         setTagVal(pRuntimeEnv, pQuery->current->pTable, pQInfo->tsdb);

From 6b2d676ac1628fc2357f3033040fae9a135ec738 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 2 Dec 2020 15:43:50 +0800
Subject: [PATCH 022/118] [TD-2293]add parameters in query message.

---
 src/inc/taosmsg.h         | 16 +++++++++-------
 src/query/src/qExecutor.c |  2 +-
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index b77db69c46..b4d3bec958 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -476,19 +476,21 @@ typedef struct {
   int16_t     numOfGroupCols;   // num of group by columns
   int16_t     orderByIdx;
   int16_t     orderType;        // used in group by xx order by xxx
+  int64_t     tableLimit;       // limit the number of rows for each table, used in order by + limit in stable projection query.
+  int16_t     prjOrder;         // global order in super table projection query.
   int64_t     limit;
   int64_t     offset;
   uint32_t    queryType;        // denote another query process
   int16_t     numOfOutput;      // final output columns numbers
   int16_t     tagNameRelType;   // relation of tag criteria and tbname criteria
-  int16_t     fillType;      // interpolate type
-  uint64_t    fillVal;       // default value array list
+  int16_t     fillType;         // interpolate type
+  uint64_t    fillVal;          // default value array list
   int32_t     secondStageOutput;
-  int32_t     tsOffset;       // offset value in current msg body, NOTE: ts list is compressed
-  int32_t     tsLen;          // total length of ts comp block
-  int32_t     tsNumOfBlocks;  // ts comp block numbers
-  int32_t     tsOrder;        // ts comp block order
-  int32_t     numOfTags;      // number of tags columns involved
+  int32_t     tsOffset;         // offset value in current msg body, NOTE: ts list is compressed
+  int32_t     tsLen;            // total length of ts comp block
+  int32_t     tsNumOfBlocks;    // ts comp block numbers
+  int32_t     tsOrder;          // ts comp block order
+  int32_t     numOfTags;        // number of tags columns involved
   SColumnInfo colList[];
 } SQueryTableMsg;
 
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index d65b2e8f0a..8c705c893c 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -5500,7 +5500,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       uint32_t     status = 0;
       SDataStatis *pStatis = NULL;
-      SArray *     pDataBlock = NULL;
+      SArray      *pDataBlock = NULL;
 
       int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo,
                                           &pStatis, &pDataBlock, &status);

From e9c5859e41703801e35b4dfcfaf8b6ddf9c05353 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Wed, 2 Dec 2020 15:48:09 +0800
Subject: [PATCH 023/118] auto pull submodule code and add more check.

---
 tests/pytest/hivemq-extension-test.py | 69 ++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 12 deletions(-)

diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py
index eb27f7a7d9..3d0b1ef83f 100644
--- a/tests/pytest/hivemq-extension-test.py
+++ b/tests/pytest/hivemq-extension-test.py
@@ -21,6 +21,7 @@ import subprocess
 from shutil import which
 from multipledispatch import dispatch
 
+
 @dispatch(str, str)
 def v_print(msg: str, arg: str):
     if verbose:
@@ -50,28 +51,34 @@ def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int):
     if verbose:
         print(msg % (int(arg1), int(arg2), int(arg3), int(arg4)))
 
+
 def isHiveMQInstalled():
     v_print("%s", "Check if HiveMQ installed")
     defaultHiveMQPath = "/opt/hivemq*"
     hiveMQDir = glob.glob(defaultHiveMQPath)
     if (len(hiveMQDir) == 0):
-        v_print("%s", "ERROR: hivemq not found!")
+        v_print("%s", "ERROR: HiveMQ NOT found")
         return False
     else:
         v_print("HiveMQ installed at %s", hiveMQDir[0])
     return True
 
+
 def isMosquittoInstalled():
     v_print("%s", "Check if mosquitto installed")
     if not which('mosquitto_pub'):
-        v_print("%s", "mosquitto is not installed")
+        v_print("%s", "ERROR: mosquitto is NOT installed")
         return False
     else:
         return True
 
+
 def installExtension():
     currentDir = os.getcwd()
-    os.chdir('../../src/connector/hivemq-tdengine-extension')
+    extDir = 'src/connector/hivemq-tdengine-extension'
+    os.chdir('../..')
+    os.system('git submodule update --init -- %s' % extDir)
+    os.chdir(extDir)
     v_print("%s", "build extension..")
     os.system('mvn clean package')
 
@@ -92,6 +99,7 @@ def installExtension():
 
     os.chdir(currentDir)
 
+
 def stopProgram(prog: str):
     psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog
 
@@ -106,9 +114,24 @@ def stopProgram(prog: str):
             psCmd, shell=True).decode("utf-8")
     pass
 
+
 def stopHiveMQ():
     stopProgram("hivemq.jar")
-    v_print("%s", "hivemq is NOT running")
+    v_print("%s", "ERROR: HiveMQ is NOT running")
+
+
+def checkProgramRunning(prog: str):
+    psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog
+
+    processID = subprocess.check_output(
+        psCmd, shell=True).decode("utf-8")
+
+    if not processID:
+        v_print("ERROR: %s is NOT running", prog)
+        return False
+    else:
+        return True
+
 
 def runHiveMQ():
     defaultHiveMQPath = "/opt/hivemq*"
@@ -116,11 +139,19 @@ def runHiveMQ():
     runPath = hiveMQDir[0] + '/bin/run.sh > /dev/null &'
     os.system(runPath)
     time.sleep(10)
-    v_print("%s", "hivemq is running")
+
+    if not checkProgramRunning("hivemq.jar"):
+        return False
+    else:
+        v_print("%s", "hivemq is running")
+        return True
+
 
 def getBuildPath():
     selfPath = os.path.dirname(os.path.realpath(__file__))
 
+    binPath = ''
+
     if ("community" in selfPath):
         projPath = selfPath[:selfPath.find("community")]
     else:
@@ -130,9 +161,10 @@ def getBuildPath():
         if ("taosd" in files):
             rootRealPath = os.path.dirname(os.path.realpath(root))
             if ("packaging" not in rootRealPath):
-                buildPath = root[:len(root)-len("/build/bin")]
+                binPath = root[:len(root) - len("/build/bin")]
                 break
-    return buildPath
+    return binPath
+
 
 def runTDengine():
     stopProgram("taosd")
@@ -140,8 +172,8 @@ def runTDengine():
     buildPath = getBuildPath()
 
     if (buildPath == ""):
-        v_print("%s", "taosd NOT found!")
-        os.exit(1)
+        v_print("%s", "ERROR: taosd NOT found!")
+        sys.exit(1)
     else:
         v_print("%s", "taosd found in %s" % buildPath)
 
@@ -149,6 +181,13 @@ def runTDengine():
 
     os.system('%s > /dev/null &' % binPath)
     time.sleep(10)
+    if not checkProgramRunning("taosd"):
+        return False
+    else:
+        v_print("%s", "TDengine is running")
+        return True
+
+
 
 def reCreateDatabase():
     buildPath = getBuildPath()
@@ -157,16 +196,20 @@ def reCreateDatabase():
     os.system('%s -s "DROP DATABASE IF EXISTS hivemq"' % binPath)
     os.system('%s -s "CREATE DATABASE IF NOT EXISTS hivemq"' % binPath)
 
+
 def sendMqttMsg(topic: str, payload: str):
     testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload)
     os.system(testStr)
     time.sleep(3)
 
+
 def checkTDengineData(topic: str, payload: str):
     buildPath = getBuildPath()
     binPath = buildPath + "/build/bin/taos"
 
-    output = subprocess.check_output('%s -s "select * from hivemq.mqtt_payload"' % binPath, shell=True).decode('utf-8')
+    output = subprocess.check_output(
+        '%s -s "select * from hivemq.mqtt_payload"' %
+        binPath, shell=True).decode('utf-8')
     if (topic in output) and (payload in output):
         v_print("%s", output)
         return True
@@ -190,11 +233,13 @@ if __name__ == "__main__":
 
     installExtension()
 
-    runTDengine()
+    if not runTDengine():
+        sys.exit(1)
 
     reCreateDatabase()
 
-    runHiveMQ()
+    if not runHiveMQ():
+        sys.exit(1)
 
     sendMqttMsg(testTopic, testPayload)
 

From 0c38e1d9d3b339294ae2290e632bc9f03b5cc0f0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 2 Dec 2020 18:37:28 +0800
Subject: [PATCH 024/118] [TD-2298]<fix>: fix taosd crash during table interval
 query.

---
 src/query/src/qExecutor.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 8c705c893c..cda9455d6f 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -3597,7 +3597,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *
 
   if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) {
     // do nothing, no results
-  } else {
+  } else {// even win.skey != lastKey, the results may not generated.
     pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step;
   }
 
@@ -3986,11 +3986,13 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
       qstatus.status = pQuery->status;
 
       // do nothing if no data blocks are found qualified during scan
+      qstatus.lastKey = pTableQueryInfo->lastKey;
       if (qstatus.lastKey != pTableQueryInfo->lastKey) {
         qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step;
+      } else { // the lastkey does not increase, which means no data checked yet
+        qDebug("QInfo:%p no results generated in this scan, abort", pQInfo);
+        return;
       }
-
-      qstatus.lastKey = pTableQueryInfo->lastKey;
     }
 
     if (!needScanDataBlocksAgain(pRuntimeEnv)) {

From 81036205759f1558abc056e38a416fd1f0a6c4ab Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 2 Dec 2020 18:43:23 +0800
Subject: [PATCH 025/118] [TD-225] refactor codes.

---
 src/query/inc/qExecutor.h | 15 +++++++--------
 src/query/inc/qUtil.h     |  2 +-
 src/query/src/qExecutor.c | 28 ++++++----------------------
 src/query/src/qUtil.c     |  5 ++---
 4 files changed, 16 insertions(+), 34 deletions(-)

diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h
index 4ec6ed30ba..aa018e75d4 100644
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@@ -84,14 +84,13 @@ typedef struct SResultRec {
 } SResultRec;
 
 typedef struct SWindowResInfo {
-  SResultRow**   pResult;    // result list
-  int16_t        type:8;     // data type for hash key
-  int32_t        size:24;    // number of result set
-  int32_t        threshold;  // threshold to halt query and return the generated results.
-  int32_t        capacity;   // max capacity
-  int32_t        curIndex;   // current start active index
-  int64_t        startTime;  // start time of the first time window for sliding query
-  int64_t        prevSKey;   // previous (not completed) sliding window start key
+  SResultRow** pResult;    // result list
+  int16_t      type:8;     // data type for hash key
+  int32_t      size:24;    // number of result set
+  int32_t      capacity;   // max capacity
+  int32_t      curIndex;   // current start active index
+  int64_t      startTime;  // start time of the first time window for sliding query
+  int64_t      prevSKey;   // previous (not completed) sliding window start key
 } SWindowResInfo;
 
 typedef struct SColumnFilterElem {
diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h
index 8b84ac0182..f15f1f33eb 100644
--- a/src/query/inc/qUtil.h
+++ b/src/query/inc/qUtil.h
@@ -30,7 +30,7 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t typ
 void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type);
 SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
 
-int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type);
+int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int16_t type);
 
 void    cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo);
 void    resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index cda9455d6f..bfdcd6d28a 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -761,7 +761,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
     pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey;
 
     // the number of completed slots are larger than the threshold, return current generated results to client.
-    if (numOfClosed > pWindowResInfo->threshold) {
+    if (numOfClosed > pQuery->rec.threshold) {
       qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return",
           GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold);
 
@@ -3587,17 +3587,9 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *
 
   // order has changed already
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
-
-  // TODO validate the assertion
-//  if (!QUERY_IS_ASC_QUERY(pQuery)) {
-//    assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step);
-//  } else {
-//    assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step);
-//  }
-
   if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) {
     // do nothing, no results
-  } else {// even win.skey != lastKey, the results may not generated.
+  } else {// NOTE: even win.skey != lastKey, the results may not generated.
     pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step;
   }
 
@@ -4102,9 +4094,8 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void
 
   // set more initial size of interval/groupby query
   if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) {
-    int32_t initialSize = 16;
-    int32_t initialThreshold = 100;
-    int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, initialThreshold, TSDB_DATA_TYPE_INT);
+    int32_t initialSize = 128;
+    int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT);
     if (code != TSDB_CODE_SUCCESS) {
       return NULL;
     }
@@ -4989,20 +4980,13 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
 
     if (!QUERY_IS_INTERVAL_QUERY(pQuery)) {
       int16_t type = TSDB_DATA_TYPE_NULL;
-      int32_t threshold = 0;
-
       if (pRuntimeEnv->groupbyNormalCol) {  // group by columns not tags;
         type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr);
-        threshold = 4000;
       } else {
         type = TSDB_DATA_TYPE_INT;  // group id
-        threshold = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo));
-        if (threshold < 8) {
-          threshold = 8;
-        }
       }
 
-      code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, threshold, type);
+      code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, type);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }
@@ -5022,7 +5006,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
       type = TSDB_DATA_TYPE_TIMESTAMP;
     }
 
-    code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, 1024, type);
+    code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, type);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index f5f2da9f25..3f010a579d 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -43,10 +43,9 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
   return size;
 }
 
-int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) {
+int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t type) {
   pWindowResInfo->capacity = size;
-  pWindowResInfo->threshold = threshold;
-  
+
   pWindowResInfo->type = type;
   pWindowResInfo->curIndex = -1;
   pWindowResInfo->size     = 0;

From c121f5b7b91e5680a63a7c35b4bb5e3cd46f1fe5 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 2 Dec 2020 18:44:32 +0800
Subject: [PATCH 026/118] [TD-225] refactor codes.

---
 src/query/inc/qExecutor.h |  8 +++----
 src/query/inc/qUtil.h     | 18 +++++++-------
 src/query/src/qExecutor.c | 50 +++++++++++++++++++--------------------
 src/query/src/qUtil.c     | 20 ++++++++--------
 4 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h
index aa018e75d4..b73f7ce3f5 100644
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@@ -83,7 +83,7 @@ typedef struct SResultRec {
   int32_t threshold;  // result size threshold in rows.
 } SResultRec;
 
-typedef struct SWindowResInfo {
+typedef struct SResultRowInfo {
   SResultRow** pResult;    // result list
   int16_t      type:8;     // data type for hash key
   int32_t      size:24;    // number of result set
@@ -91,7 +91,7 @@ typedef struct SWindowResInfo {
   int32_t      curIndex;   // current start active index
   int64_t      startTime;  // start time of the first time window for sliding query
   int64_t      prevSKey;   // previous (not completed) sliding window start key
-} SWindowResInfo;
+} SResultRowInfo;
 
 typedef struct SColumnFilterElem {
   int16_t           bytes;  // column length
@@ -114,7 +114,7 @@ typedef struct STableQueryInfo {
   STimeWindow win;
   STSCursor   cur;
   void*       pTable;         // for retrieve the page id list
-  SWindowResInfo windowResInfo;
+  SResultRowInfo windowResInfo;
 } STableQueryInfo;
 
 typedef struct SQueryCostInfo {
@@ -178,7 +178,7 @@ typedef struct SQueryRuntimeEnv {
   uint16_t*            offset;
   uint16_t             scanFlag;         // denotes reversed scan of data or not
   SFillInfo*           pFillInfo;
-  SWindowResInfo       windowResInfo;
+  SResultRowInfo       windowResInfo;
   STSBuf*              pTSBuf;
   STSCursor            cur;
   SQueryCostInfo       summary;
diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h
index f15f1f33eb..fb71c8a5fe 100644
--- a/src/query/inc/qUtil.h
+++ b/src/query/inc/qUtil.h
@@ -30,19 +30,19 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t typ
 void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type);
 SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
 
-int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int16_t type);
+int32_t initWindowResInfo(SResultRowInfo* pWindowResInfo, int32_t size, int16_t type);
 
-void    cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo);
-void    resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
+void    cleanupTimeWindowInfo(SResultRowInfo* pWindowResInfo);
+void    resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pWindowResInfo);
 void    clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num);
 
 void    clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv);
-int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo);
-void    closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
-void    closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
-void    removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
+int32_t numOfClosedTimeWindow(SResultRowInfo* pWindowResInfo);
+void    closeTimeWindow(SResultRowInfo* pWindowResInfo, int32_t slot);
+void    closeAllTimeWindow(SResultRowInfo* pWindowResInfo);
+void    removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
 
-static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) {
+static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pWindowResInfo, int32_t slot) {
   assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
   return pWindowResInfo->pResult[slot];
 }
@@ -50,7 +50,7 @@ static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int
 #define curTimeWindowIndex(_winres)        ((_winres)->curIndex)
 #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1)
 
-bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
+bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot);
 
 int32_t initResultRow(SResultRow *pResultRow);
 
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index bfdcd6d28a..3e683609d1 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -461,7 +461,7 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis
   return true;
 }
 
-static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData,
+static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, char *pData,
                                              int16_t bytes, bool masterscan, uint64_t uid) {
   SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid);
   int32_t *p1 =
@@ -518,7 +518,7 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
 }
 
 // get the correct time window according to the handled timestamp
-static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) {
+static STimeWindow getActiveTimeWindow(SResultRowInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) {
   STimeWindow w = {0};
 
  if (pWindowResInfo->curIndex == -1) {  // the first window, from the previous stored value
@@ -611,7 +611,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf
   return 0;
 }
 
-static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo,
+static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, SDataBlockInfo* pBockInfo,
                                        STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) {
   assert(win->skey <= win->ekey);
   SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
@@ -644,7 +644,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes
   return TSDB_CODE_SUCCESS;
 }
 
-static bool getResultRowStatus(SWindowResInfo *pWindowResInfo, int32_t slot) {
+static bool getResultRowStatus(SResultRowInfo *pWindowResInfo, int32_t slot) {
   assert(slot >= 0 && slot < pWindowResInfo->size);
   return pWindowResInfo->pResult[slot]->closed;
 }
@@ -703,7 +703,7 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se
 /**
  * NOTE: the query status only set for the first scan of master scan.
  */
-static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) {
+static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
   if (pRuntimeEnv->scanFlag != MASTER_SCAN) {
     return pWindowResInfo->size;
@@ -1157,7 +1157,7 @@ static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY
  *                          such as count/min/max etc.
  */
 static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo,
-                                    SWindowResInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) {
+                                    SResultRowInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) {
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
   bool            masterScan = IS_MASTER_SCAN(pRuntimeEnv);
 
@@ -1575,7 +1575,7 @@ static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pData
 }
 
 static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo,
-    SWindowResInfo *pWindowResInfo, SArray *pDataBlock) {
+    SResultRowInfo *pWindowResInfo, SArray *pDataBlock) {
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
   bool masterScan = IS_MASTER_SCAN(pRuntimeEnv);
 
@@ -1781,7 +1781,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   STableQueryInfo* pTableQInfo = pQuery->current;
-  SWindowResInfo*  pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo*  pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
   if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) {
     rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock);
@@ -2605,7 +2605,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
   return false;
 }
 
-int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) {
+int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   *status = BLK_DATA_NO_NEEDED;
@@ -2828,7 +2828,7 @@ static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo
 
   if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) {
     STimeWindow w = TSWINDOW_INITIALIZER;
-    SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+    SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
     if (QUERY_IS_ASC_QUERY(pQuery)) {
       getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w);
@@ -3192,14 +3192,14 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param)
     return -1;
   }
 
-  SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo;
+  SResultRowInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo;
   SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos);
   tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId);
 
   char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1);
   TSKEY leftTimestamp = GET_INT64_VAL(b1);
 
-  SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo;
+  SResultRowInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo;
   SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos);
   tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId);
 
@@ -3439,7 +3439,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
 
     int32_t pos = pTree->pNode[0].index;
 
-    SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo;
+    SResultRowInfo *pWindowResInfo = &pTableList[pos]->windowResInfo;
     SResultRow  *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]);
     tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
 
@@ -3603,7 +3603,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *
   pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1;
 }
 
-static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t order) {
+static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo *pWindowResInfo, int32_t order) {
   SQuery* pQuery = pRuntimeEnv->pQuery;
 
   for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
@@ -3635,7 +3635,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
   int32_t order = pQuery->order.order;
 
   // group by normal columns and interval query on normal table
-  SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
   if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
     disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order);
   } else {  // for simple result of table query,
@@ -3826,7 +3826,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) {
   bool toContinue = false;
   if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
     // for each group result, call the finalize function for each column
-    SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+    SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
     for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
       SResultRow *pResult = getResultRow(pWindowResInfo, i);
@@ -4038,7 +4038,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) {
 
   if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
     // for each group result, call the finalize function for each column
-    SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+    SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
     if (pRuntimeEnv->groupbyNormalCol) {
       closeAllTimeWindow(pWindowResInfo);
     }
@@ -4122,7 +4122,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) {
 void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   STableQueryInfo  *pTableQueryInfo = pRuntimeEnv->pQuery->current;
-  SWindowResInfo   *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo   *pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
   // lastKey needs to be updated
   pTableQueryInfo->lastKey = nextKey;
@@ -4290,7 +4290,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
      * operations involve.
      */
     STimeWindow     w = TSWINDOW_INITIALIZER;
-    SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo;
+    SResultRowInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo;
 
     TSKEY sk = MIN(win.skey, win.ekey);
     TSKEY ek = MAX(win.skey, win.ekey);
@@ -4334,7 +4334,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) {
   return loadPrimaryTS;
 }
 
-static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_t orderType) {
+static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_t orderType) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *          pQuery = pRuntimeEnv->pQuery;
 
@@ -4411,7 +4411,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
  * @param pQInfo
  * @param result
  */
-void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo) {
+void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
   int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC;
@@ -4450,7 +4450,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc
   SQuery *         pQuery = pRuntimeEnv->pQuery;
   STableQueryInfo* pTableQueryInfo = pQuery->current;
 
-  SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo;
+  SResultRowInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo;
   pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1;
 
   if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) {
@@ -4698,7 +4698,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
 
 static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
-  SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
   assert(pQuery->limit.offset == 0);
   STimeWindow tw = *win;
@@ -4764,7 +4764,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
 
   STimeWindow w = TSWINDOW_INITIALIZER;
 
-  SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
   STableQueryInfo *pTableQueryInfo = pQuery->current;
 
   SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
@@ -5403,7 +5403,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       scanMultiTableDataBlocks(pQInfo);
       pQInfo->groupIndex += 1;
 
-      SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+      SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
 
       // no results generated for current group, continue to try the next group
       taosArrayDestroy(s);
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 3f010a579d..8260d8bc15 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -43,7 +43,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
   return size;
 }
 
-int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t type) {
+int32_t initWindowResInfo(SResultRowInfo *pWindowResInfo, int32_t size, int16_t type) {
   pWindowResInfo->capacity = size;
 
   pWindowResInfo->type = type;
@@ -59,7 +59,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t
   return TSDB_CODE_SUCCESS;
 }
 
-void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
+void cleanupTimeWindowInfo(SResultRowInfo *pWindowResInfo) {
   if (pWindowResInfo == NULL) {
     return;
   }
@@ -77,7 +77,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
   tfree(pWindowResInfo->pResult);
 }
 
-void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) {
+void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo) {
   if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) {
     return;
   }
@@ -101,7 +101,7 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
 }
 
 void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
-  SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
   if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) {
     return;
   }
@@ -160,7 +160,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
 }
 
 void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) {
-  SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
   if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) {
     return;
   }
@@ -169,7 +169,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) {
   clearFirstNWindowRes(pRuntimeEnv, numOfClosed);
 }
 
-int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) {
+int32_t numOfClosedTimeWindow(SResultRowInfo *pWindowResInfo) {
   int32_t i = 0;
   while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) {
     ++i;
@@ -178,7 +178,7 @@ int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) {
   return i;
 }
 
-void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
+void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) {
   assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
   
   for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
@@ -195,7 +195,7 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
  * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time.
  * NOTE: remove redundant, only when the result set order equals to traverse order
  */
-void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order) {
+void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order) {
   assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
   if (pWindowResInfo->size <= 1) {
     return;
@@ -224,11 +224,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
   }
 }
 
-bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) {
+bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot) {
   return (getResultRow(pWindowResInfo, slot)->closed == true);
 }
 
-void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) {
+void closeTimeWindow(SResultRowInfo *pWindowResInfo, int32_t slot) {
   getResultRow(pWindowResInfo, slot)->closed = true;
 }
 

From 597dd6341cc0c5eba5bf3c6dfbd27fd6f9b9bd4e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 2 Dec 2020 18:46:28 +0800
Subject: [PATCH 027/118] [TD-225]

---
 src/query/src/qExecutor.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 3e683609d1..f47cd34729 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -5515,6 +5515,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       // the limitation of output result is reached, set the query completed
       if (limitResults(pRuntimeEnv)) {
+        setQueryStatus(pQuery, QUERY_COMPLETED);
         SET_STABLE_QUERY_OVER(pQInfo);
         break;
       }

From 8cc7daa929e2e0d079c92ad80aed67abbe048400 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Wed, 2 Dec 2020 19:39:31 +0800
Subject: [PATCH 028/118] [TD-2308] doc: add more items to faq document.

---
 documentation20/webdocs/markdowndocs/faq-ch.md | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md
index 757b6d9929..42f9761c95 100644
--- a/documentation20/webdocs/markdowndocs/faq-ch.md
+++ b/documentation20/webdocs/markdowndocs/faq-ch.md
@@ -101,7 +101,20 @@ Connection = DriverManager.getConnection(url, properties);
   <version>2.0.4</version>
 </dependency>
 ```
-## 14. 怎么报告问题？
+## 14. taos connect failed, reason: invalid timestamp 
+
+常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。
+
+
+
+## 15. 表名显示不全
+
+由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。
+
+
+
+## 16. 怎么报告问题？
+
 如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包:
 1. /var/log/taos
 2. /etc/taos 

From 551827eb2bda040c34f97c0d03a9f42dcb1e5be7 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 2 Dec 2020 22:53:52 +0800
Subject: [PATCH 029/118] TD-2270

---
 src/balance/inc/bnThread.h                   |  32 ++++++
 src/balance/src/{balance.c => balanceMain.c} |   0
 src/balance/src/bnThread.c                   | 107 +++++++++++++++++++
 3 files changed, 139 insertions(+)
 create mode 100644 src/balance/inc/bnThread.h
 rename src/balance/src/{balance.c => balanceMain.c} (100%)
 create mode 100644 src/balance/src/bnThread.c

diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h
new file mode 100644
index 0000000000..d337e963f1
--- /dev/null
+++ b/src/balance/inc/bnThread.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_H
+#define TDENGINE_BALANCE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t bnThreadInit();
+void    bnThreadCleanup();
+void    bnThreadSyncNotify();
+void    bnThreadAsyncNotify();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/balance/src/balance.c b/src/balance/src/balanceMain.c
similarity index 100%
rename from src/balance/src/balance.c
rename to src/balance/src/balanceMain.c
diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c
new file mode 100644
index 0000000000..12cb2cf992
--- /dev/null
+++ b/src/balance/src/bnThread.c
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tutil.h"
+#include "tbalance.h"
+#include "tsync.h"
+#include "tsync.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "tdataformat.h"
+#include "dnode.h"
+#include "mnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeDnode.h"
+#include "mnodeDb.h"
+#include "mnodeMnode.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "mnodeUser.h"
+#include "mnodeVgroup.h"
+
+typedef struct {
+  bool            stop;
+  pthread_mutex_t mutex;
+  pthread_cond_t  cond;
+  pthread_t       thread;
+} SBalanceThread;
+
+static SBalanceThread tsBnThread;
+
+static void *bnThreadFunc(void *arg) {
+  while (1) {
+    pthread_mutex_lock(&tsBnThread.mutex);
+    if (tsBnThread.stop) {
+      pthread_mutex_unlock(&tsBnThread.mutex);
+      break;
+    }
+
+    pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
+
+    pthread_mutex_unlock(&(tsBnThread.mutex));
+  }
+
+  return NULL;
+}
+
+int32_t bnThreadInit() {
+  tsBnThread.stop = false;
+  pthread_mutex_init(&tsBnThread.mutex, NULL);
+  pthread_cond_init(&tsBnThread.cond, NULL);
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
+  int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    mError("failed to create balance thread since %s", strerror(errno));
+    return -1;
+  }
+
+  mDebug("balance thread is created");
+  return 0;
+}
+
+void bnThreadCleanup() {
+  mDebug("balance thread will be cleanup");
+
+  pthread_mutex_lock(&tsBnThread.mutex);
+  tsBnThread.stop = true;
+  pthread_cond_signal(&tsBnThread.cond);
+  pthread_mutex_unlock(&(tsBnThread.mutex));
+  pthread_join(tsBnThread.thread, NULL);
+
+  pthread_cond_destroy(&tsBnThread.cond);
+  pthread_mutex_destroy(&tsBnThread.mutex);
+}
+
+void bnThreadSyncNotify() {
+  mDebug("balance thread sync notify");
+  pthread_mutex_lock(&tsBnThread.mutex);
+  pthread_cond_signal(&tsBnThread.cond);
+  pthread_mutex_unlock(&(tsBnThread.mutex));
+}
+
+void bnThreadAsyncNotify() {
+  mDebug("balance thread async notify");
+  pthread_mutex_lock(&tsBnThread.mutex);
+  pthread_cond_signal(&tsBnThread.cond);
+  pthread_mutex_unlock(&(tsBnThread.mutex));
+}

From d59b8f18da1c940c4df942b6082bb05c280fff5b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 10:32:55 +0800
Subject: [PATCH 030/118] [TD-2298]

---
 src/query/src/qExecutor.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index f47cd34729..1b2d3f12f5 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -3978,13 +3978,14 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
       qstatus.status = pQuery->status;
 
       // do nothing if no data blocks are found qualified during scan
-      qstatus.lastKey = pTableQueryInfo->lastKey;
       if (qstatus.lastKey != pTableQueryInfo->lastKey) {
         qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step;
       } else { // the lastkey does not increase, which means no data checked yet
         qDebug("QInfo:%p no results generated in this scan, abort", pQInfo);
         return;
       }
+
+      qstatus.lastKey = pTableQueryInfo->lastKey;
     }
 
     if (!needScanDataBlocksAgain(pRuntimeEnv)) {

From caeabf382148fc8453ccb2422c563468a47d3e60 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 10:33:40 +0800
Subject: [PATCH 031/118] [TD-225] refactor codes.

---
 src/client/inc/tsclient.h     |   5 +-
 src/client/src/tscSQLParser.c |   4 +-
 src/client/src/tscServer.c    |   1 +
 src/query/src/qExecutor.c     |   4 +-
 src/query/src/qUtil.c         | 126 +++++++++++++++++-----------------
 5 files changed, 72 insertions(+), 68 deletions(-)

diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index c482375bff..29841035f7 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -246,11 +246,14 @@ typedef struct SQueryInfo {
   int16_t          fillType;      // final result fill type
   int16_t          numOfTables;
   STableMetaInfo **pTableMetaInfo;
-  struct STSBuf *  tsBuf;
+  struct STSBuf   *tsBuf;
   int64_t *        fillVal;       // default value for fill
   char *           msg;           // pointer to the pCmd->payload to keep error message temporarily
   int64_t          clauseLimit;   // limit for current sub clause
+
   int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
+  int64_t          tableLimit;    // table limit in case of super table projection query + global order + limit
+
   int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
   int16_t          resColumnId;   // result column id
 } SQueryInfo;
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 0c9ad78cd9..abbe26204d 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5307,7 +5307,8 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
 
     // keep original limitation value in globalLimit
     pQueryInfo->clauseLimit = pQueryInfo->limit.limit;
-    pQueryInfo->prjOffset = pQueryInfo->limit.offset;
+    pQueryInfo->prjOffset   = pQueryInfo->limit.offset;
+    pQueryInfo->tableLimit  = -1;
 
     if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
       /*
@@ -5317,6 +5318,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
        * than or equal to the value of limit.
        */
       if (pQueryInfo->limit.limit > 0) {
+        pQueryInfo->tableLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset;
         pQueryInfo->limit.limit = -1;
       }
 
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 9735e26ece..5fc09c0586 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -697,6 +697,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
   pQueryMsg->numOfTags      = htonl(numOfTags);
   pQueryMsg->queryType      = htonl(pQueryInfo->type);
+  pQueryMsg->tableLimit     = htobe64(pQueryInfo->tableLimit);
   
   size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
   pQueryMsg->numOfOutput = htons((int16_t)numOfOutput);  // this is the stage one output column number
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 1b2d3f12f5..03de6aea1f 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -6260,11 +6260,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
   pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval);
   pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding);
   pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset);
-  // pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit;
-  // pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit;
-  // pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit;
   pQueryMsg->limit = htobe64(pQueryMsg->limit);
   pQueryMsg->offset = htobe64(pQueryMsg->offset);
+  pQueryMsg->tableLimit = htobe64(pQueryMsg->tableLimit);
 
   pQueryMsg->order = htons(pQueryMsg->order);
   pQueryMsg->orderColId = htons(pQueryMsg->orderColId);
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 8260d8bc15..6c845b012f 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -43,48 +43,48 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
   return size;
 }
 
-int32_t initWindowResInfo(SResultRowInfo *pWindowResInfo, int32_t size, int16_t type) {
-  pWindowResInfo->capacity = size;
+int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) {
+  pResultRowInfo->capacity = size;
 
-  pWindowResInfo->type = type;
-  pWindowResInfo->curIndex = -1;
-  pWindowResInfo->size     = 0;
-  pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
+  pResultRowInfo->type = type;
+  pResultRowInfo->curIndex = -1;
+  pResultRowInfo->size     = 0;
+  pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
 
-  pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES);
-  if (pWindowResInfo->pResult == NULL) {
+  pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES);
+  if (pResultRowInfo->pResult == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
-void cleanupTimeWindowInfo(SResultRowInfo *pWindowResInfo) {
-  if (pWindowResInfo == NULL) {
+void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) {
+  if (pResultRowInfo == NULL) {
     return;
   }
-  if (pWindowResInfo->capacity == 0) {
-    assert(pWindowResInfo->pResult == NULL);
+  if (pResultRowInfo->capacity == 0) {
+    assert(pResultRowInfo->pResult == NULL);
     return;
   }
 
-  if (pWindowResInfo->type == TSDB_DATA_TYPE_BINARY || pWindowResInfo->type == TSDB_DATA_TYPE_NCHAR) {
-    for(int32_t i = 0; i < pWindowResInfo->size; ++i) {
-      tfree(pWindowResInfo->pResult[i]->key);
+  if (pResultRowInfo->type == TSDB_DATA_TYPE_BINARY || pResultRowInfo->type == TSDB_DATA_TYPE_NCHAR) {
+    for(int32_t i = 0; i < pResultRowInfo->size; ++i) {
+      tfree(pResultRowInfo->pResult[i]->key);
     }
   }
   
-  tfree(pWindowResInfo->pResult);
+  tfree(pResultRowInfo->pResult);
 }
 
-void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo) {
-  if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) {
+void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) {
+  if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) {
     return;
   }
 
-  for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
-    SResultRow *pWindowRes = pWindowResInfo->pResult[i];
-    clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
+  for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
+    SResultRow *pWindowRes = pResultRowInfo->pResult[i];
+    clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type);
 
     int32_t groupIndex = 0;
     int64_t uid = 0;
@@ -93,30 +93,30 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowR
     taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(groupIndex)));
   }
   
-  pWindowResInfo->curIndex = -1;
-  pWindowResInfo->size = 0;
+  pResultRowInfo->curIndex = -1;
+  pResultRowInfo->size = 0;
   
-  pWindowResInfo->startTime = TSKEY_INITIAL_VAL;
-  pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
+  pResultRowInfo->startTime = TSKEY_INITIAL_VAL;
+  pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
 }
 
 void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
-  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
-  if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) {
+  SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo;
+  if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0 || num == 0) {
     return;
   }
   
-  int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
+  int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo);
   assert(num >= 0 && num <= numOfClosed);
 
-  int16_t type = pWindowResInfo->type;
+  int16_t type = pResultRowInfo->type;
   int64_t uid = getResultInfoUId(pRuntimeEnv);
 
   char    *key  = NULL;
   int16_t  bytes = -1;
 
   for (int32_t i = 0; i < num; ++i) {
-    SResultRow *pResult = pWindowResInfo->pResult[i];
+    SResultRow *pResult = pResultRowInfo->pResult[i];
     if (pResult->closed) {  // remove the window slot from hash table
       getResultRowKeyInfo(pResult, type, &key, &bytes);
       SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
@@ -126,23 +126,23 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
     }
   }
   
-  int32_t remain = pWindowResInfo->size - num;
+  int32_t remain = pResultRowInfo->size - num;
   
   // clear all the closed windows from the window list
   for (int32_t k = 0; k < remain; ++k) {
-    copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type);
+    copyResultRow(pRuntimeEnv, pResultRowInfo->pResult[k], pResultRowInfo->pResult[num + k], type);
   }
   
   // move the unclosed window in the front of the window list
-  for (int32_t k = remain; k < pWindowResInfo->size; ++k) {
-    SResultRow *pWindowRes = pWindowResInfo->pResult[k];
-    clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
+  for (int32_t k = remain; k < pResultRowInfo->size; ++k) {
+    SResultRow *pWindowRes = pResultRowInfo->pResult[k];
+    clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type);
   }
   
-  pWindowResInfo->size = remain;
+  pResultRowInfo->size = remain;
 
-  for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
-    SResultRow *pResult = pWindowResInfo->pResult[k];
+  for (int32_t k = 0; k < pResultRowInfo->size; ++k) {
+    SResultRow *pResult = pResultRowInfo->pResult[k];
     getResultRowKeyInfo(pResult, type, &key, &bytes);
     SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
 
@@ -150,43 +150,43 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
     assert(p != NULL); 
 
     int32_t  v = (*p - num);
-    assert(v >= 0 && v <= pWindowResInfo->size);
+    assert(v >= 0 && v <= pResultRowInfo->size);
 
     SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
     taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t));
   }
   
-  pWindowResInfo->curIndex = -1;
+  pResultRowInfo->curIndex = -1;
 }
 
 void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) {
-  SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
-  if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) {
+  SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo;
+  if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0) {
     return;
   }
   
-  int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
+  int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo);
   clearFirstNWindowRes(pRuntimeEnv, numOfClosed);
 }
 
-int32_t numOfClosedTimeWindow(SResultRowInfo *pWindowResInfo) {
+int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) {
   int32_t i = 0;
-  while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) {
+  while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) {
     ++i;
   }
   
   return i;
 }
 
-void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) {
-  assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
+void closeAllTimeWindow(SResultRowInfo *pResultRowInfo) {
+  assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
   
-  for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
-    if (pWindowResInfo->pResult[i]->closed) {
+  for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
+    if (pResultRowInfo->pResult[i]->closed) {
       continue;
     }
     
-    pWindowResInfo->pResult[i]->closed = true;
+    pResultRowInfo->pResult[i]->closed = true;
   }
 }
 
@@ -195,41 +195,41 @@ void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) {
  * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time.
  * NOTE: remove redundant, only when the result set order equals to traverse order
  */
-void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order) {
-  assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
-  if (pWindowResInfo->size <= 1) {
+void removeRedundantWindow(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order) {
+  assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
+  if (pResultRowInfo->size <= 1) {
     return;
   }
 
   // get the result order
-  int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1;
+  int32_t resultOrder = (pResultRowInfo->pResult[0]->win.skey < pResultRowInfo->pResult[1]->win.skey)? 1:-1;
   if (order != resultOrder) {
     return;
   }
 
   int32_t i = 0;
   if (order == QUERY_ASC_FORWARD_STEP) {
-    TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey;
-    while (i < pWindowResInfo->size && (ekey < lastKey)) {
+    TSKEY ekey = pResultRowInfo->pResult[i]->win.ekey;
+    while (i < pResultRowInfo->size && (ekey < lastKey)) {
       ++i;
     }
   } else if (order == QUERY_DESC_FORWARD_STEP) {
-    while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) {
+    while (i < pResultRowInfo->size && (pResultRowInfo->pResult[i]->win.skey > lastKey)) {
       ++i;
     }
   }
 
-  if (i < pWindowResInfo->size) {
-    pWindowResInfo->size = (i + 1);
+  if (i < pResultRowInfo->size) {
+    pResultRowInfo->size = (i + 1);
   }
 }
 
-bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot) {
-  return (getResultRow(pWindowResInfo, slot)->closed == true);
+bool isWindowResClosed(SResultRowInfo *pResultRowInfo, int32_t slot) {
+  return (getResultRow(pResultRowInfo, slot)->closed == true);
 }
 
-void closeTimeWindow(SResultRowInfo *pWindowResInfo, int32_t slot) {
-  getResultRow(pWindowResInfo, slot)->closed = true;
+void closeTimeWindow(SResultRowInfo *pResultRowInfo, int32_t slot) {
+  getResultRow(pResultRowInfo, slot)->closed = true;
 }
 
 void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) {

From ef901f1077da070c6f555eb5041c9a99a0316c02 Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Thu, 3 Dec 2020 03:29:10 +0000
Subject: [PATCH 032/118] [TD-2309]<feature>: add config option to
 enable/disable stream (continuous query)

---
 .../webdocs/markdowndocs/administrator-ch.md  |  1 +
 packaging/cfg/taos.cfg                        |  5 ++++-
 src/common/inc/tglobal.h                      |  3 +++
 src/common/src/tglobal.c                      | 13 +++++++++++
 src/cq/src/cqMain.c                           | 18 +++++++++++++++
 src/vnode/src/vnodeMain.c                     | 22 ++++++++++---------
 6 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md
index 36466d2b7e..f54c6b91a1 100644
--- a/documentation20/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation20/webdocs/markdowndocs/administrator-ch.md
@@ -95,6 +95,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 - logKeepDays：日志文件的最长保存时间。大于0时，日志文件会被重命名为taosdlog.xxx，其中xxx为日志文件最后修改的时间戳，单位为秒。默认值：0天。
 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。
 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。
+- stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。
 
 **注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。
 
diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg
index ca88bca3c8..7662d49280 100644
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@@ -260,4 +260,7 @@
 # maxBinaryDisplayWidth 30
 
 # enable/disable telemetry reporting
-# telemetryReporting    1
\ No newline at end of file
+# telemetryReporting    1
+
+# enable/disable stream (continuous query)
+# stream                1
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index efe3d7678a..6e4274b358 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -125,6 +125,9 @@ extern char    tsMonitorDbName[];
 extern char    tsInternalPass[];
 extern int32_t tsMonitorInterval;
 
+// stream
+extern int32_t tsEnableStream;
+
 // internal
 extern int32_t tsPrintAuth;
 extern int32_t tscEmbedded;
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index d26746a5d1..18aa0ae6e3 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -161,6 +161,9 @@ char    tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
 char    tsInternalPass[] = "secretkey";
 int32_t tsMonitorInterval = 30;  // seconds
 
+// stream
+int32_t tsEnableStream = 1;
+
 // internal
 int32_t tsPrintAuth = 0;
 int32_t tscEmbedded = 0;
@@ -1015,6 +1018,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "stream";
+  cfg.ptr = &tsEnableStream;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "httpEnableRecordSql";
   cfg.ptr = &tsHttpEnableRecordSql;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c
index b1977fd5d9..efb8795962 100644
--- a/src/cq/src/cqMain.c
+++ b/src/cq/src/cqMain.c
@@ -69,6 +69,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row);
 static void cqCreateStream(SCqContext *pContext, SCqObj *pObj);
 
 void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
+  if (tsEnableStream == 0) {
+    return NULL;
+  }
   SCqContext *pContext = calloc(sizeof(SCqContext), 1);
   if (pContext == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -99,6 +102,9 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
 }
 
 void cqClose(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   if (handle == NULL) return;
 
@@ -129,6 +135,9 @@ void cqClose(void *handle) {
 }
 
 void cqStart(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   if (pContext->dbConn || pContext->master) return;
 
@@ -147,6 +156,9 @@ void cqStart(void *handle) {
 }
 
 void cqStop(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqContext *pContext = handle;
   cInfo("vgId:%d, stop all CQs", pContext->vgId);
   if (pContext->dbConn == NULL || pContext->master == 0) return;
@@ -174,6 +186,9 @@ void cqStop(void *handle) {
 }
 
 void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
+  if (tsEnableStream == 0) {
+    return NULL;
+  }
   SCqContext *pContext = handle;
 
   SCqObj *pObj = calloc(sizeof(SCqObj), 1);
@@ -203,6 +218,9 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *
 }
 
 void cqDrop(void *handle) {
+  if (tsEnableStream == 0) {
+    return;
+  }
   SCqObj *pObj = handle;
   SCqContext *pContext = pObj->pContext;
 
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 204dd2f3dd..b516c9d90e 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -267,16 +267,18 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
     return terrno;
   }
 
-  SCqCfg cqCfg = {0};
-  sprintf(cqCfg.user, "_root");
-  strcpy(cqCfg.pass, tsInternalPass);
-  strcpy(cqCfg.db, pVnode->db);
-  cqCfg.vgId = vnode;
-  cqCfg.cqWrite = vnodeWriteToCache;
-  pVnode->cq = cqOpen(pVnode, &cqCfg);
-  if (pVnode->cq == NULL) {
-    vnodeCleanUp(pVnode);
-    return terrno;
+  if (tsEnableStream) {
+    SCqCfg cqCfg = {0};
+    sprintf(cqCfg.user, "_root");
+    strcpy(cqCfg.pass, tsInternalPass);
+    strcpy(cqCfg.db, pVnode->db);
+    cqCfg.vgId = vnode;
+    cqCfg.cqWrite = vnodeWriteToCache;
+    pVnode->cq = cqOpen(pVnode, &cqCfg);
+    if (pVnode->cq == NULL) {
+      vnodeCleanUp(pVnode);
+      return terrno;
+    }
   }
 
   STsdbAppH appH = {0};

From ca87041503c9187f5ada13b658381e500abf9abe Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 3 Dec 2020 13:00:13 +0800
Subject: [PATCH 033/118] TD-2270

---
 src/balance/inc/bnMain.h      |  56 ++++
 src/balance/inc/bnScore.h     |  34 +++
 src/balance/inc/bnThread.h    |  13 +-
 src/balance/src/balanceMain.c | 555 +++++++---------------------------
 src/balance/src/bnScore.c     | 328 ++++++++++++++++++++
 src/balance/src/bnThread.c    |  72 +++--
 src/inc/tbalance.h            |  16 +-
 src/mnode/src/mnodeDb.c       |   2 +-
 src/mnode/src/mnodeDnode.c    |  24 +-
 src/mnode/src/mnodeMain.c     |   2 +-
 src/mnode/src/mnodeSdb.c      |   2 +-
 src/mnode/src/mnodeVgroup.c   |   2 +-
 12 files changed, 611 insertions(+), 495 deletions(-)
 create mode 100644 src/balance/inc/bnMain.h
 create mode 100644 src/balance/inc/bnScore.h
 create mode 100644 src/balance/src/bnScore.c

diff --git a/src/balance/inc/bnMain.h b/src/balance/inc/bnMain.h
new file mode 100644
index 0000000000..101f5d5fd1
--- /dev/null
+++ b/src/balance/inc/bnMain.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_MAIN_H
+#define TDENGINE_BALANCE_MAIN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "mnodeInt.h"
+#include "mnodeDnode.h"
+
+typedef struct {
+  int32_t     size;
+  int32_t     maxSize;
+  SDnodeObj **list;
+} SBnDnodes;
+
+typedef struct {
+  void *          timer;
+  bool            stop;
+  pthread_mutex_t mutex;
+  pthread_cond_t  cond;
+  pthread_t       thread;
+} SBnThread;
+
+typedef struct {
+  pthread_mutex_t mutex;
+} SBnMgmt;
+
+int32_t bnInit();
+void    bnCleanUp();
+bool    bnStart();
+void    bnCheckStatus();
+void    bnCheckModules();
+
+extern SBnDnodes tsBnDnodes;
+extern void *tsMnodeTmr;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/balance/inc/bnScore.h b/src/balance/inc/bnScore.h
new file mode 100644
index 0000000000..48e423b562
--- /dev/null
+++ b/src/balance/inc/bnScore.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_BALANCE_SCORE_H
+#define TDENGINE_BALANCE_SCORE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "bnMain.h"
+
+void  bnInitDnodes();
+void  bnCleanupDnodes();
+void  bnAccquireDnodes();
+void  bnReleaseDnodes();
+float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h
index d337e963f1..df9b06cf87 100644
--- a/src/balance/inc/bnThread.h
+++ b/src/balance/inc/bnThread.h
@@ -13,17 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_BALANCE_H
-#define TDENGINE_BALANCE_H
+#ifndef TDENGINE_BALANCE_THREAD_H
+#define TDENGINE_BALANCE_THREAD_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "bnMain.h"
 
-int32_t bnThreadInit();
-void    bnThreadCleanup();
-void    bnThreadSyncNotify();
-void    bnThreadAsyncNotify();
+int32_t bnInitThread();
+void    bnCleanupThread();
+void    bnNotify();
+void    bnStartTimer(int64_t mseconds);
 
 #ifdef __cplusplus
 }
diff --git a/src/balance/src/balanceMain.c b/src/balance/src/balanceMain.c
index b172867929..9f9d5103be 100644
--- a/src/balance/src/balanceMain.c
+++ b/src/balance/src/balanceMain.c
@@ -33,36 +33,23 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-/*
- * once sdb work as mater, then tsAccessSquence reset to zero
- * increase tsAccessSquence every balance interval
- */
-extern void *       tsMnodeTmr;
-static void *       tsBalanceTimer = NULL;
-static int32_t      tsBalanceDnodeListSize = 0;
-static SDnodeObj ** tsBalanceDnodeList = NULL;
-static int32_t      tsBalanceDnodeListMallocSize = 16;
-static pthread_mutex_t tsBalanceMutex;
+#include "bnMain.h"
+#include "bnScore.h"
+#include "bnThread.h"
 
-static void  balanceStartTimer(int64_t mseconds);
-static void  balanceInitDnodeList();
-static void  balanceCleanupDnodeList();
-static void  balanceAccquireDnodeList();
-static void  balanceReleaseDnodeList();
-static void  balanceMonitorDnodeModule();
-static float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
-static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static SBnMgmt tsBnMgmt;;
 
-static void balanceLock() {
-  pthread_mutex_lock(&tsBalanceMutex);
+static void  bnMonitorDnodeModule();
+
+static void bnLock() {
+  pthread_mutex_lock(&tsBnMgmt.mutex);
 }
 
-static void balanceUnLock() {
-  pthread_mutex_unlock(&tsBalanceMutex);
+static void bnUnLock() {
+  pthread_mutex_unlock(&tsBnMgmt.mutex);
 }
 
-static bool balanceCheckFree(SDnodeObj *pDnode) {
+static bool bnCheckFree(SDnodeObj *pDnode) {
   if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
     mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
     return false;
@@ -86,7 +73,7 @@ static bool balanceCheckFree(SDnodeObj *pDnode) {
   return true;
 }
 
-static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
+static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
   mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
 
   SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId);
@@ -111,27 +98,26 @@ static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
   mnodeUpdateVgroup(pVgroup);
 }
 
-static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
+static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
   // SVnodeGid tmp = *pVnodeGid1;
   // *pVnodeGid1 = *pVnodeGid2;
   // *pVnodeGid2 = tmp;
 }
 
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+int32_t bnAllocVnodes(SVgObj *pVgroup) {
   static int32_t randIndex = 0;
   int32_t dnode = 0;
   int32_t vnodes = 0;
 
-  balanceLock();
-
-  balanceAccquireDnodeList();
+  bnLock();
+  bnAccquireDnodes();
 
   mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
-         mnodeGetDnodesNum(), tsBalanceDnodeListSize);
+         mnodeGetDnodesNum(), tsBnDnodes.size);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    for (; dnode < tsBalanceDnodeListSize; ++dnode) {
-      SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
-      if (balanceCheckFree(pDnode)) {
+    for (; dnode < tsBnDnodes.size; ++dnode) {
+      SDnodeObj *pDnode = tsBnDnodes.list[dnode];
+      if (bnCheckFree(pDnode)) {
         SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i;
         pVnodeGid->dnodeId = pDnode->dnodeId;
         pVnodeGid->pDnode = pDnode;
@@ -148,8 +134,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   }
 
   if (vnodes != pVgroup->numOfVnodes) {
-    balanceReleaseDnodeList();
-    balanceUnLock();
+    bnReleaseDnodes();
+    bnUnLock();
 
     mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
 
@@ -179,33 +165,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   if (pVgroup->numOfVnodes == 1) {
   } else if (pVgroup->numOfVnodes == 2) {
     if (randIndex++ % 2 == 0) {
-      balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
     }
   } else {
     int32_t randVal = randIndex++ % 6;
     if (randVal == 1) {  // 1, 0, 2
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
     } else if (randVal == 2) {  // 1, 2, 0
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     } else if (randVal == 3) {  // 2, 1, 0
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
     } else if (randVal == 4) {  // 2, 0, 1
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     }
     if (randVal == 5) {  // 0, 2, 1
-      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+      bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
     } else {
     }  // 0, 1, 2
   }
 
-  balanceReleaseDnodeList();
-  balanceUnLock();
+  bnReleaseDnodes();
+  bnUnLock();
   return TSDB_CODE_SUCCESS;
 }
 
-static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
+static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
   if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) {
     return false;
   }
@@ -232,7 +218,7 @@ static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
  * desc: remove one vnode from vgroup
  * all vnodes in vgroup should in ready state, except the balancing one
  **/
-static int32_t balanceRemoveVnode(SVgObj *pVgroup) {
+static int32_t bnRemoveVnode(SVgObj *pVgroup) {
   if (pVgroup->numOfVnodes <= 1) return -1;
 
   SVnodeGid *pRmVnode = NULL;
@@ -274,17 +260,17 @@ static int32_t balanceRemoveVnode(SVgObj *pVgroup) {
     pSelVnode = pRmVnode;
   }
 
-  if (!balanceCheckVgroupReady(pVgroup, pSelVnode)) {
+  if (!bnCheckVgroupReady(pVgroup, pSelVnode)) {
     mDebug("vgId:%d, is not ready", pVgroup->vgId);
     return -1;
   } else {
     mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
-    balanceDiscardVnode(pVgroup, pSelVnode);
+    bnDiscardVnode(pVgroup, pSelVnode);
     return TSDB_CODE_SUCCESS;
   }
 }
 
-static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
+static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SVnodeGid *pGid = &pVgroup->vnodeGid[i];
     if (pGid->dnodeId == 0) break;
@@ -299,13 +285,13 @@ static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
 /**
  * desc: add vnode to vgroup, find a new one if dest dnode is null
  **/
-static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
+static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
   if (pDestDnode == NULL) {
-    for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
-      SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
+      SDnodeObj *pDnode = tsBnDnodes.list[i];
       if (pDnode == pSrcDnode) continue;
-      if (balanceCheckDnodeInVgroup(pDnode, pVgroup)) continue;
-      if (!balanceCheckFree(pDnode)) continue;
+      if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue;
+      if (!bnCheckFree(pDnode)) continue;
       
       pDestDnode = pDnode;
       mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
@@ -333,25 +319,25 @@ static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj
   return TSDB_CODE_SUCCESS;
 }
 
-static bool balanceMonitorBalance() {
-  if (tsBalanceDnodeListSize < 2) return false;
+static bool bnMonitorBalance() {
+  if (tsBnDnodes.size < 2) return false;
 
-  for (int32_t src = tsBalanceDnodeListSize - 1; src >= 0; --src) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[src];
-    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBalanceDnodeListSize - src - 1,
+  for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) {
+    SDnodeObj *pDnode = tsBnDnodes.list[src];
+    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1,
            pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
            pDnode->openVnodes);
   }
 
-  float scoresDiff = tsBalanceDnodeList[tsBalanceDnodeListSize - 1]->score - tsBalanceDnodeList[0]->score;
+  float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score;
   if (scoresDiff < 0.01) {
-    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBalanceDnodeListSize, scoresDiff);
+    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff);
     return false;
   }
 
-  for (int32_t src = tsBalanceDnodeListSize - 1; src > 0; --src) {
-    SDnodeObj *pSrcDnode = tsBalanceDnodeList[src];
-    float srcScore = balanceTryCalcDnodeScore(pSrcDnode, -1);
+  for (int32_t src = tsBnDnodes.size - 1; src > 0; --src) {
+    SDnodeObj *pSrcDnode = tsBnDnodes.list[src];
+    float srcScore = bnTryCalcDnodeScore(pSrcDnode, -1);
     if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) {
       continue;
     }
@@ -362,19 +348,19 @@ static bool balanceMonitorBalance() {
       pIter = mnodeGetNextVgroup(pIter, &pVgroup);
       if (pVgroup == NULL) break;
 
-      if (balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+      if (bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
         for (int32_t dest = 0; dest < src; dest++) {
-          SDnodeObj *pDestDnode = tsBalanceDnodeList[dest];
-          if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
+          SDnodeObj *pDestDnode = tsBnDnodes.list[dest];
+          if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
 
-          float destScore = balanceTryCalcDnodeScore(pDestDnode, 1);
+          float destScore = bnTryCalcDnodeScore(pDestDnode, 1);
           if (srcScore + 0.0001 < destScore) continue;
-          if (!balanceCheckFree(pDestDnode)) continue;
+          if (!bnCheckFree(pDestDnode)) continue;
           
           mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f",
                  pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score,
                  srcScore, pDestDnode->score, destScore);
-          balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+          bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
           mnodeDecVgroupRef(pVgroup);
           mnodeCancelGetNextVgroup(pIter);
           return true;
@@ -392,7 +378,7 @@ static bool balanceMonitorBalance() {
 // 1. reset balanceAccessSquence to zero
 // 2. reset state of dnodes to offline
 // 3. reset lastAccess of dnodes to zero
-void balanceReset() {
+void bnReset() {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
   while (1) {
@@ -413,7 +399,7 @@ void balanceReset() {
   tsAccessSquence = 0;
 }
 
-static int32_t balanceMonitorVgroups() {
+static int32_t bnMonitorVgroups() {
   void *  pIter = NULL;
   SVgObj *pVgroup = NULL;
   bool    hasUpdatingVgroup = false;
@@ -429,11 +415,11 @@ static int32_t balanceMonitorVgroups() {
     if (vgReplica > dbReplica) {
       mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica);
       hasUpdatingVgroup = true;
-      code = balanceRemoveVnode(pVgroup);
+      code = bnRemoveVnode(pVgroup);
     } else if (vgReplica < dbReplica) {
       mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
       hasUpdatingVgroup = true;
-      code = balanceAddVnode(pVgroup, NULL, NULL);
+      code = bnAddVnode(pVgroup, NULL, NULL);
     }
 
     mnodeDecVgroupRef(pVgroup);
@@ -446,7 +432,7 @@ static int32_t balanceMonitorVgroups() {
   return hasUpdatingVgroup;
 }
 
-static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
+static bool bnMonitorDnodeDropping(SDnodeObj *pDnode) {
   mDebug("dnode:%d, in dropping state", pDnode->dnodeId);
 
   void *  pIter = NULL;
@@ -456,7 +442,7 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
     pIter = mnodeGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
 
-    hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup);
+    hasThisDnode = bnCheckDnodeInVgroup(pDnode, pVgroup);
     mnodeDecVgroupRef(pVgroup);
 
     if (hasThisDnode) {
@@ -474,7 +460,7 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
   return false;
 }
 
-static bool balanceMontiorDropping() {
+static bool bnMontiorDropping() {
   void *pIter = NULL;
   SDnodeObj *pDnode = NULL;
 
@@ -499,7 +485,7 @@ static bool balanceMontiorDropping() {
     }
 
     if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
-      bool ret = balanceMonitorDnodeDropping(pDnode);
+      bool ret = bnMonitorDnodeDropping(pDnode);
       mnodeDecDnodeRef(pDnode);
       mnodeCancelGetNextDnode(pIter);
       return ret;
@@ -509,33 +495,32 @@ static bool balanceMontiorDropping() {
   return false;
 }
 
-static bool balanceStart() {
+bool bnStart() {
   if (!sdbIsMaster()) return false;
 
-  balanceLock();
+  bnLock();
+  bnAccquireDnodes();
 
-  balanceAccquireDnodeList();
+  bnMonitorDnodeModule();
 
-  balanceMonitorDnodeModule();
-
-  bool updateSoon = balanceMontiorDropping();
+  bool updateSoon = bnMontiorDropping();
 
   if (!updateSoon) {
-    updateSoon = balanceMonitorVgroups();
+    updateSoon = bnMonitorVgroups();
   }
 
   if (!updateSoon) {
-    updateSoon = balanceMonitorBalance();
+    updateSoon = bnMonitorBalance();
   }
  
-  balanceReleaseDnodeList();
+  bnReleaseDnodes();
 
-  balanceUnLock();
+  bnUnLock();
 
   return updateSoon;
 }
 
-static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
+static void bnSetVgroupOffline(SDnodeObj* pDnode) {
   void *pIter = NULL;
   while (1) {
     SVgObj *pVgroup;
@@ -551,7 +536,7 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
   }
 }
 
-static void balanceCheckDnodeAccess() {
+void bnCheckStatus() {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
 
@@ -564,84 +549,39 @@ static void balanceCheckDnodeAccess() {
         pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
         mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
               pDnode->lastAccess, pDnode->status);
-        balanceSetVgroupOffline(pDnode);
+        bnSetVgroupOffline(pDnode);
       }
     }
     mnodeDecDnodeRef(pDnode);
   }
 }
 
-static void balanceProcessBalanceTimer(void *handle, void *tmrId) {
-  if (!sdbIsMaster()) return;
-
-  tsBalanceTimer = NULL;
-  tsAccessSquence ++;
-
-  balanceCheckDnodeAccess();  
-  bool updateSoon = false;
-
-  if (handle == NULL) {
-    if (tsAccessSquence % tsBalanceInterval == 0) {
-      mDebug("balance function is scheduled by timer");
-      updateSoon = balanceStart();
-    }
-  } else {
-    int64_t mseconds = (int64_t)handle;
-    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
-    updateSoon = balanceStart();
-  }
-
-  if (updateSoon) {
-    balanceStartTimer(1000);
-  } else {
-    taosTmrReset(balanceProcessBalanceTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBalanceTimer);
-  }
-}
-
-static void balanceStartTimer(int64_t mseconds) {
-  taosTmrReset(balanceProcessBalanceTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBalanceTimer);
-}
-
-void balanceSyncNotify() {
+void bnCheckModules() {
   if (sdbIsMaster()) {
-    balanceLock();
-    balanceAccquireDnodeList();
-    balanceMonitorDnodeModule();
-    balanceReleaseDnodeList();
-    balanceUnLock();
+    bnLock();
+    bnAccquireDnodes();
+    bnMonitorDnodeModule();
+    bnReleaseDnodes();
+    bnUnLock();
   }
 }
 
-void balanceAsyncNotify() {
-  balanceStartTimer(500); 
-}
-
-int32_t balanceInit() {
-  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta);
-  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores);
-  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode);
-  
-  pthread_mutex_init(&tsBalanceMutex, NULL);
-  balanceInitDnodeList();
-  balanceStartTimer(2000);
-  mDebug("balance start fp:%p initialized", balanceProcessBalanceTimer);
-
-  balanceReset();
+int32_t bnInit() {
+  pthread_mutex_init(&tsBnMgmt.mutex, NULL);
+  bnInitDnodes();
+  bnInitThread();
+  bnReset();
   
   return 0;
 }
 
-void balanceCleanUp() {
-  if (tsBalanceTimer != NULL) {
-    taosTmrStopA(&tsBalanceTimer);
-    pthread_mutex_destroy(&tsBalanceMutex);
-    tsBalanceTimer = NULL;
-    mDebug("stop balance timer");
-  }
-  balanceCleanupDnodeList();
+void bnCleanUp() {
+  bnCleanupThread();
+  pthread_mutex_destroy(&tsBnMgmt.mutex);
+  bnCleanupDnodes();
 }
 
-int32_t balanceDropDnode(SDnodeObj *pDnode) {
+int32_t bnDropDnode(SDnodeObj *pDnode) {
   int32_t    totalFreeVnodes = 0;
   void *     pIter = NULL;
   SDnodeObj *pTempDnode = NULL;
@@ -650,7 +590,7 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
     pIter = mnodeGetNextDnode(pIter, &pTempDnode);
     if (pTempDnode == NULL) break;
 
-    if (pTempDnode != pDnode && balanceCheckFree(pTempDnode)) {
+    if (pTempDnode != pDnode && bnCheckFree(pTempDnode)) {
       totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes);
     }
 
@@ -665,298 +605,19 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
   pDnode->status = TAOS_DN_STATUS_DROPPING;
   mnodeUpdateDnode(pDnode);
   
-  balanceStartTimer(1100);
+  bnStartTimer(1100);
 
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t balanceCalcCpuScore(SDnodeObj *pDnode) {
-  if (pDnode->cpuAvgUsage < 80)
-    return 0;
-  else if (pDnode->cpuAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
 
-static int32_t balanceCalcMemoryScore(SDnodeObj *pDnode) {
-  if (pDnode->memoryAvgUsage < 80)
-    return 0;
-  else if (pDnode->memoryAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
 
-static int32_t balanceCalcDiskScore(SDnodeObj *pDnode) {
-  if (pDnode->diskAvgUsage < 80)
-    return 0;
-  else if (pDnode->diskAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t balanceCalcBandwidthScore(SDnodeObj *pDnode) {
-  if (pDnode->bandwidthUsage < 30)
-    return 0;
-  else if (pDnode->bandwidthUsage < 80)
-    return 10;
-  else
-    return 50;
-}
-
-static float balanceCalcModuleScore(SDnodeObj *pDnode) {
-  if (pDnode->numOfCores <= 0) return 0;
-  if (pDnode->isMgmt) {
-     return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
-  }
-  return 0;
-}
-
-static float balanceCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
-  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
-  if (pDnode->numOfCores <= 0) return 0;
-  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
-}
-
-/**
- * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
- * 1. get the score config
- * 2. if the value is out of range, use border data
- * 3. otherwise use interpolation method
- **/
-void balanceCalcDnodeScore(SDnodeObj *pDnode) {
-  pDnode->score = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                  balanceCalcBandwidthScore(pDnode) + balanceCalcModuleScore(pDnode) +
-                  balanceCalcVnodeScore(pDnode, 0) + pDnode->customScore;
-}
-
-float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
-  int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                        balanceCalcBandwidthScore(pDnode);
-  float moduleScore = balanceCalcModuleScore(pDnode);
-  float vnodeScore = balanceCalcVnodeScore(pDnode, extra);
-
-  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
-  return score;
-}
-
-static void balanceInitDnodeList() {
-  tsBalanceDnodeList = calloc(tsBalanceDnodeListMallocSize, sizeof(SDnodeObj *));
-}
-
-static void balanceCleanupDnodeList() {
-  if (tsBalanceDnodeList != NULL) {
-    free(tsBalanceDnodeList);
-    tsBalanceDnodeList = NULL;
-  }
-}
-
-static void balanceCheckDnodeListSize(int32_t dnodesNum) {
-  if (tsBalanceDnodeListMallocSize <= dnodesNum) {
-    tsBalanceDnodeListMallocSize = dnodesNum * 2;
-    tsBalanceDnodeList = realloc(tsBalanceDnodeList, tsBalanceDnodeListMallocSize * sizeof(SDnodeObj *));
-  }
-}
-
-void balanceAccquireDnodeList() {
-  int32_t dnodesNum = mnodeGetDnodesNum();
-  balanceCheckDnodeListSize(dnodesNum);
-
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  int32_t    dnodeIndex = 0;
-
-  while (1) {
-    if (dnodeIndex >= dnodesNum) {
-      mnodeCancelGetNextDnode(pIter);
-      break;
-    }
-
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
-      mnodeDecDnodeRef(pDnode);
-      continue;
-    }
-
-    balanceCalcDnodeScore(pDnode);
-    
-    int32_t orderIndex = dnodeIndex;
-    for (; orderIndex > 0; --orderIndex) {
-      if (pDnode->score > tsBalanceDnodeList[orderIndex - 1]->score) {
-        break;
-      }
-      tsBalanceDnodeList[orderIndex] = tsBalanceDnodeList[orderIndex - 1];
-    }
-    tsBalanceDnodeList[orderIndex] = pDnode;
-    dnodeIndex++;
-  }
-
-  tsBalanceDnodeListSize = dnodeIndex;
-}
-
-void balanceReleaseDnodeList() {
-  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[i];
-    if (pDnode != NULL) {
-      mnodeDecDnodeRef(pDnode);
-    }
-  }
-}
-
-static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
-  SUserObj *pUser = mnodeGetUserFromConn(pConn);
-  if (pUser == NULL) return 0;
-
-  if (strcmp(pUser->pAcct->user, "root") != 0)  {
-    mnodeDecUserRef(pUser);
-    return TSDB_CODE_MND_NO_RIGHTS;
-  }
-
-  int32_t cols = 0;
-  SSchema *pSchema = pMeta->schema;
-
-  pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-  strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "system scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "custom scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "module scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "vnode scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "total scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "open vnodes");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "cpu cores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
-  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "balance state");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pMeta->numOfColumns = htons(cols);
-  pShow->numOfColumns = cols;
-
-  pShow->offset[0] = 0;
-  for (int32_t i = 1; i < cols; ++i) {
-    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
-  }
-
-  pShow->numOfRows = mnodeGetDnodesNum();
-  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pIter = NULL;
-
-  mnodeDecUserRef(pUser);
-
-  return 0;
-}
-
-static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
-  int32_t    numOfRows = 0;
-  SDnodeObj *pDnode = NULL;
-  char *     pWrite;
-  int32_t    cols = 0;
-  
-  while (numOfRows < rows) {
-    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
-                      balanceCalcBandwidthScore(pDnode);
-    float moduleScore = balanceCalcModuleScore(pDnode);
-    float vnodeScore = balanceCalcVnodeScore(pDnode, 0);
-
-    cols = 0;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pDnode->dnodeId;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = systemScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = pDnode->customScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)moduleScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)vnodeScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = pDnode->openVnodes;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = pDnode->numOfCores;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
-    cols++;
-
-    numOfRows++;
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
-  pShow->numOfReads += numOfRows;
-  return numOfRows;
-}
-
-static void balanceMonitorDnodeModule() {
+static void bnMonitorDnodeModule() {
   int32_t numOfMnodes = mnodeGetMnodesNum();
   if (numOfMnodes >= tsNumOfMnodes) return;
 
-  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
-    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
+    SDnodeObj *pDnode = tsBnDnodes.list[i];
     if (pDnode == NULL) break;
 
     if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
@@ -980,7 +641,7 @@ static void balanceMonitorDnodeModule() {
   }
 }
 
-int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
+int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
   if (!sdbIsMaster()) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId);
     return TSDB_CODE_MND_DNODE_NOT_EXIST;
@@ -1004,29 +665,29 @@ int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t
     return TSDB_CODE_MND_DNODE_NOT_EXIST;
   }
 
-  balanceLock();
-  balanceAccquireDnodeList();
+  bnLock();
+  bnAccquireDnodes();
 
   int32_t code = TSDB_CODE_SUCCESS;
-  if (!balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+  if (!bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId,
            dnodeId, pSrcDnode->dnodeId);
     code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE;
-  } else if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) {
+  } else if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId,
            dnodeId, dnodeId);
     code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE;
-  } else if (!balanceCheckFree(pDestDnode)) {
+  } else if (!bnCheckFree(pDestDnode)) {
     mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId,
            dnodeId);
     code = TSDB_CODE_MND_DNODE_NOT_FREE;
   } else {
-    code = balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+    code = bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
     mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code));
   }
 
-  balanceReleaseDnodeList();
-  balanceUnLock();
+  bnReleaseDnodes();
+  bnUnLock();
 
   mnodeDecVgroupRef(pVgroup);
   mnodeDecDnodeRef(pDestDnode);
diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c
new file mode 100644
index 0000000000..db5d515fbf
--- /dev/null
+++ b/src/balance/src/bnScore.c
@@ -0,0 +1,328 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tutil.h"
+#include "tbalance.h"
+#include "tsync.h"
+#include "tsync.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "tdataformat.h"
+#include "dnode.h"
+#include "mnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeDnode.h"
+#include "mnodeDb.h"
+#include "mnodeMnode.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "mnodeUser.h"
+#include "mnodeVgroup.h"
+
+#include "bnScore.h"
+
+SBnDnodes tsBnDnodes;
+
+static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+
+static int32_t bnCalcCpuScore(SDnodeObj *pDnode) {
+  if (pDnode->cpuAvgUsage < 80)
+    return 0;
+  else if (pDnode->cpuAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcMemoryScore(SDnodeObj *pDnode) {
+  if (pDnode->memoryAvgUsage < 80)
+    return 0;
+  else if (pDnode->memoryAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcDiskScore(SDnodeObj *pDnode) {
+  if (pDnode->diskAvgUsage < 80)
+    return 0;
+  else if (pDnode->diskAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t bnCalcBandScore(SDnodeObj *pDnode) {
+  if (pDnode->bandwidthUsage < 30)
+    return 0;
+  else if (pDnode->bandwidthUsage < 80)
+    return 10;
+  else
+    return 50;
+}
+
+static float bnCalcModuleScore(SDnodeObj *pDnode) {
+  if (pDnode->numOfCores <= 0) return 0;
+  if (pDnode->isMgmt) {
+    return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
+  }
+  return 0;
+}
+
+static float bnCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
+  if (pDnode->numOfCores <= 0) return 0;
+  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
+}
+
+/**
+ * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
+ * 1. get the score config
+ * 2. if the value is out of range, use border data
+ * 3. otherwise use interpolation method
+ **/
+static void bnCalcDnodeScore(SDnodeObj *pDnode) {
+  pDnode->score = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
+                  bnCalcBandScore(pDnode) + bnCalcModuleScore(pDnode) + bnCalcVnodeScore(pDnode, 0) +
+                  pDnode->customScore;
+}
+
+float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
+                        bnCalcBandScore(pDnode);
+  float moduleScore = bnCalcModuleScore(pDnode);
+  float vnodeScore = bnCalcVnodeScore(pDnode, extra);
+
+  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
+  return score;
+}
+
+void bnInitDnodes() {
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, bnGetScoresMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, bnRetrieveScores);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode);
+
+  memset(&tsBnDnodes, 0, sizeof(SBnDnodes));
+  tsBnDnodes.maxSize = 16;
+  tsBnDnodes.list = calloc(tsBnDnodes.maxSize, sizeof(SDnodeObj *));
+}
+
+void bnCleanupDnodes() {
+  if (tsBnDnodes.list != NULL) {
+    free(tsBnDnodes.list);
+    tsBnDnodes.list = NULL;
+  }
+}
+
+static void bnCheckDnodesSize(int32_t dnodesNum) {
+  if (tsBnDnodes.maxSize <= dnodesNum) {
+    tsBnDnodes.maxSize = dnodesNum * 2;
+    tsBnDnodes.list = realloc(tsBnDnodes.list, tsBnDnodes.maxSize * sizeof(SDnodeObj *));
+  }
+}
+
+void bnAccquireDnodes() {
+  int32_t dnodesNum = mnodeGetDnodesNum();
+  bnCheckDnodesSize(dnodesNum);
+
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  int32_t    dnodeIndex = 0;
+
+  while (1) {
+    if (dnodeIndex >= dnodesNum) {
+      mnodeCancelGetNextDnode(pIter);
+      break;
+    }
+
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      mnodeDecDnodeRef(pDnode);
+      continue;
+    }
+
+    bnCalcDnodeScore(pDnode);
+    
+    int32_t orderIndex = dnodeIndex;
+    for (; orderIndex > 0; --orderIndex) {
+      if (pDnode->score > tsBnDnodes.list[orderIndex - 1]->score) {
+        break;
+      }
+      tsBnDnodes.list[orderIndex] = tsBnDnodes.list[orderIndex - 1];
+    }
+    tsBnDnodes.list[orderIndex] = pDnode;
+    dnodeIndex++;
+  }
+
+  tsBnDnodes.size = dnodeIndex;
+}
+
+void bnReleaseDnodes() {
+  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
+    SDnodeObj *pDnode = tsBnDnodes.list[i];
+    if (pDnode != NULL) {
+      mnodeDecDnodeRef(pDnode);
+    }
+  }
+}
+
+static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+  SUserObj *pUser = mnodeGetUserFromConn(pConn);
+  if (pUser == NULL) return 0;
+
+  if (strcmp(pUser->pAcct->user, "root") != 0) {
+    mnodeDecUserRef(pUser);
+    return TSDB_CODE_MND_NO_RIGHTS;
+  }
+
+  int32_t  cols = 0;
+  SSchema *pSchema = pMeta->schema;
+
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "id");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "system scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "custom scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "module scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "vnode scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "total scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "open vnodes");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "cpu cores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "balance state");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = mnodeGetDnodesNum();
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  pShow->pIter = NULL;
+
+  mnodeDecUserRef(pUser);
+
+  return 0;
+}
+
+static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+  int32_t    numOfRows = 0;
+  SDnodeObj *pDnode = NULL;
+  char *     pWrite;
+  int32_t    cols = 0;
+
+  while (numOfRows < rows) {
+    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + bnCalcBandScore(pDnode);
+    float moduleScore = bnCalcModuleScore(pDnode);
+    float vnodeScore = bnCalcVnodeScore(pDnode, 0);
+
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = pDnode->dnodeId;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = systemScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = pDnode->customScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)moduleScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)vnodeScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->openVnodes;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->numOfCores;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
+    cols++;
+
+    numOfRows++;
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+  pShow->numOfReads += numOfRows;
+  return numOfRows;
+}
diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c
index 12cb2cf992..39eb6d0e31 100644
--- a/src/balance/src/bnThread.c
+++ b/src/balance/src/bnThread.c
@@ -34,14 +34,10 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-typedef struct {
-  bool            stop;
-  pthread_mutex_t mutex;
-  pthread_cond_t  cond;
-  pthread_t       thread;
-} SBalanceThread;
 
-static SBalanceThread tsBnThread;
+#include "bnThread.h"
+
+static SBnThread tsBnThread;
 
 static void *bnThreadFunc(void *arg) {
   while (1) {
@@ -52,14 +48,16 @@ static void *bnThreadFunc(void *arg) {
     }
 
     pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
-
+    bool updateSoon = bnStart();
+    bnStartTimer(updateSoon ? 1000 : -1);
     pthread_mutex_unlock(&(tsBnThread.mutex));
   }
 
   return NULL;
 }
 
-int32_t bnThreadInit() {
+int32_t bnInitThread() {
+  memset(&tsBnThread, 0, sizeof(SBnThread));
   tsBnThread.stop = false;
   pthread_mutex_init(&tsBnThread.mutex, NULL);
   pthread_cond_init(&tsBnThread.cond, NULL);
@@ -75,13 +73,20 @@ int32_t bnThreadInit() {
     return -1;
   }
 
+  bnStartTimer(2000);
   mDebug("balance thread is created");
   return 0;
 }
 
-void bnThreadCleanup() {
+void bnCleanupThread() {
   mDebug("balance thread will be cleanup");
 
+  if (tsBnThread.timer != NULL) {
+    taosTmrStopA(&tsBnThread.timer);
+    tsBnThread.timer = NULL;
+    mDebug("stop balance timer");
+  }
+
   pthread_mutex_lock(&tsBnThread.mutex);
   tsBnThread.stop = true;
   pthread_cond_signal(&tsBnThread.cond);
@@ -92,16 +97,47 @@ void bnThreadCleanup() {
   pthread_mutex_destroy(&tsBnThread.mutex);
 }
 
-void bnThreadSyncNotify() {
-  mDebug("balance thread sync notify");
-  pthread_mutex_lock(&tsBnThread.mutex);
-  pthread_cond_signal(&tsBnThread.cond);
-  pthread_mutex_unlock(&(tsBnThread.mutex));
-}
-
-void bnThreadAsyncNotify() {
+static void bnPostSignal() {
   mDebug("balance thread async notify");
   pthread_mutex_lock(&tsBnThread.mutex);
   pthread_cond_signal(&tsBnThread.cond);
   pthread_mutex_unlock(&(tsBnThread.mutex));
 }
+
+/*
+ * once sdb work as mater, then tsAccessSquence reset to zero
+ * increase tsAccessSquence every balance interval
+ */
+
+static void bnProcessTimer(void *handle, void *tmrId) {
+  if (!sdbIsMaster()) return;
+
+  tsBnThread.timer = NULL;
+  tsAccessSquence++;
+
+  bnCheckStatus();
+
+  if (handle == NULL) {
+    if (tsAccessSquence % tsBalanceInterval == 0) {
+      mDebug("balance function is scheduled by timer");
+      bnPostSignal();
+    }
+  } else {
+    int64_t mseconds = (int64_t)handle;
+    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
+    bnPostSignal();
+  }
+}
+
+void bnStartTimer(int64_t mseconds) {
+  bool updateSoon = (mseconds != -1);
+  if (updateSoon) {
+    taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
+  } else {
+    taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);
+  }
+}
+
+void bnNotify() {
+  bnStartTimer(500); 
+}
diff --git a/src/inc/tbalance.h b/src/inc/tbalance.h
index f0da4a3747..b9f4e3c608 100644
--- a/src/inc/tbalance.h
+++ b/src/inc/tbalance.h
@@ -23,14 +23,14 @@ extern "C" {
 struct SVgObj;
 struct SDnodeObj;
 
-int32_t balanceInit();
-void    balanceCleanUp();
-void    balanceAsyncNotify();
-void    balanceSyncNotify();
-void    balanceReset();
-int32_t balanceAllocVnodes(struct SVgObj *pVgroup);
-int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId);
-int32_t balanceDropDnode(struct SDnodeObj *pDnode);
+int32_t bnInit();
+void    bnCleanUp();
+void    bnNotify();
+void    bnCheckModules();
+void    bnReset();
+int32_t bnAllocVnodes(struct SVgObj *pVgroup);
+int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId);
+int32_t bnDropDnode(struct SDnodeObj *pDnode);
 
 #ifdef __cplusplus
 }
diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index 69e8f076e9..63bd066c36 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -1004,7 +1004,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
   mDebug("db:%s, all vgroups is altered", pDb->name);
   mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
 
-  balanceAsyncNotify();
+  bnNotify();
 
   return TSDB_CODE_SUCCESS;
 }
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 65f4060392..0d429f5797 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -115,7 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) {
   mnodeDropAllDnodeVgroups(pDnode);
 #endif  
   mnodeDropMnodeLocal(pDnode->dnodeId);
-  balanceAsyncNotify();
+  bnNotify();
   mnodeUpdateDnodeEps();
 
   mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
@@ -347,7 +347,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
       return TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION;
     }
 
-    int32_t code = balanceAlterDnode(pDnode, vnodeId, dnodeId);
+    int32_t code = bnAlterDnode(pDnode, vnodeId, dnodeId);
     mnodeDecDnodeRef(pDnode);
     return code;
   } else {
@@ -591,8 +591,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
     mInfo("dnode:%d, from offline to online", pDnode->dnodeId);
     pDnode->status = TAOS_DN_STATUS_READY;
     pDnode->offlineReason = TAOS_DN_OFF_ONLINE;
-    balanceSyncNotify();
-    balanceAsyncNotify();
+    bnCheckModules();
+    bnNotify();
   }
 
   if (openVnodes != pDnode->openVnodes) {
@@ -708,7 +708,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
 #ifndef _SYNC
   int32_t code = mnodeDropDnode(pDnode, pMsg);
 #else
-  int32_t code = balanceDropDnode(pDnode);
+  int32_t code = bnDropDnode(pDnode);
 #endif  
   mnodeDecDnodeRef(pDnode);
   return code;
@@ -1182,12 +1182,12 @@ static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) {
 
 #ifndef _SYNC
 
-int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
-void    balanceCleanUp() {}
-void    balanceAsyncNotify() {}
-void    balanceSyncNotify() {}
-void    balanceReset() {}
-int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
+int32_t bnInit() { return TSDB_CODE_SUCCESS; }
+void    bnCleanUp() {}
+void    bnNotify() {}
+void    bnCheckModules() {}
+void    bnReset() {}
+int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
 
 char* syncRole[] = {
   "offline",
@@ -1197,7 +1197,7 @@ char* syncRole[] = {
   "master"
 };
 
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+int32_t bnAllocVnodes(SVgObj *pVgroup) {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
   SDnodeObj *pSelDnode = NULL;
diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c
index d15b32da54..9c8bc6c35b 100644
--- a/src/mnode/src/mnodeMain.c
+++ b/src/mnode/src/mnodeMain.c
@@ -58,7 +58,7 @@ static const SMnodeComponent tsMnodeComponents[] = {
   {"tables",  mnodeInitTables,  mnodeCleanupTables},  
   {"mnodes",  mnodeInitMnodes,  mnodeCleanupMnodes},
   {"sdb",     sdbInit,          sdbCleanUp},
-  {"balance", balanceInit,      balanceCleanUp},
+  {"balance", bnInit,           bnCleanUp},
   {"grant",   grantInit,        grantCleanUp},
   {"show",    mnodeInitShow,    mnodeCleanUpShow}
 };
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 40e2e1cfcc..07b3ef09ec 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -244,7 +244,7 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) {
   sdbInfo("vgId:1, mnode role changed from %s to %s", syncRole[tsSdbMgmt.role], syncRole[role]);
 
   if (role == TAOS_SYNC_ROLE_MASTER && tsSdbMgmt.role != TAOS_SYNC_ROLE_MASTER) {
-    balanceReset();
+    bnReset();
   }
   tsSdbMgmt.role = role;
 
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 3e974f417f..7cf9e35bd2 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -563,7 +563,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) {
   pVgroup->numOfVnodes = pDb->cfg.replications;
   pVgroup->createdTime = taosGetTimestampMs();
   pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
-  int32_t code = balanceAllocVnodes(pVgroup);
+  int32_t code = bnAllocVnodes(pVgroup);
   if (code != TSDB_CODE_SUCCESS) {
     mError("db:%s, no enough dnode to alloc %d vnodes to vgroup, reason:%s", pDb->name, pVgroup->numOfVnodes,
            tstrerror(code));

From b39b84e6d8ad8723f2bc6b35b4da1adb45b587d4 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 3 Dec 2020 13:02:06 +0800
Subject: [PATCH 034/118] TD-2270 rename files

---
 src/balance/inc/{bnMain.h => bnInt.h}       | 0
 src/balance/inc/bnScore.h                   | 2 +-
 src/balance/inc/bnThread.h                  | 2 +-
 src/balance/src/{balanceMain.c => bnMain.c} | 2 +-
 4 files changed, 3 insertions(+), 3 deletions(-)
 rename src/balance/inc/{bnMain.h => bnInt.h} (100%)
 rename src/balance/src/{balanceMain.c => bnMain.c} (99%)

diff --git a/src/balance/inc/bnMain.h b/src/balance/inc/bnInt.h
similarity index 100%
rename from src/balance/inc/bnMain.h
rename to src/balance/inc/bnInt.h
diff --git a/src/balance/inc/bnScore.h b/src/balance/inc/bnScore.h
index 48e423b562..a28c4459dd 100644
--- a/src/balance/inc/bnScore.h
+++ b/src/balance/inc/bnScore.h
@@ -19,7 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "bnMain.h"
+#include "bnInt.h"
 
 void  bnInitDnodes();
 void  bnCleanupDnodes();
diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h
index df9b06cf87..8f54b66028 100644
--- a/src/balance/inc/bnThread.h
+++ b/src/balance/inc/bnThread.h
@@ -19,7 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "bnMain.h"
+#include "bnInt.h"
 
 int32_t bnInitThread();
 void    bnCleanupThread();
diff --git a/src/balance/src/balanceMain.c b/src/balance/src/bnMain.c
similarity index 99%
rename from src/balance/src/balanceMain.c
rename to src/balance/src/bnMain.c
index 9f9d5103be..92f5ceb950 100644
--- a/src/balance/src/balanceMain.c
+++ b/src/balance/src/bnMain.c
@@ -33,7 +33,7 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-#include "bnMain.h"
+#include "bnInt.h"
 #include "bnScore.h"
 #include "bnThread.h"
 

From 1993fbc480575d23eed679476edf5b3eaab0ca5f Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 3 Dec 2020 13:19:28 +0800
Subject: [PATCH 035/118] TD-2270

---
 src/balance/inc/bnInt.h    |  5 +++--
 src/balance/src/bnMain.c   | 19 +++----------------
 src/balance/src/bnScore.c  | 16 ----------------
 src/balance/src/bnThread.c | 17 -----------------
 4 files changed, 6 insertions(+), 51 deletions(-)

diff --git a/src/balance/inc/bnInt.h b/src/balance/inc/bnInt.h
index 101f5d5fd1..e924776ff1 100644
--- a/src/balance/inc/bnInt.h
+++ b/src/balance/inc/bnInt.h
@@ -13,13 +13,14 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_BALANCE_MAIN_H
-#define TDENGINE_BALANCE_MAIN_H
+#ifndef TDENGINE_BALANCE_INT_H
+#define TDENGINE_BALANCE_INT_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "mnodeInt.h"
+#include "mnodeDef.h"
 #include "mnodeDnode.h"
 
 typedef struct {
diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c
index 92f5ceb950..54e957a864 100644
--- a/src/balance/src/bnMain.c
+++ b/src/balance/src/bnMain.c
@@ -15,17 +15,12 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tutil.h"
-#include "tbalance.h"
 #include "tsync.h"
-#include "ttimer.h"
 #include "tglobal.h"
-#include "tdataformat.h"
 #include "dnode.h"
-#include "mnode.h"
-#include "mnodeDef.h"
-#include "mnodeInt.h"
-#include "mnodeDnode.h"
+#include "bnInt.h"
+#include "bnScore.h"
+#include "bnThread.h"
 #include "mnodeDb.h"
 #include "mnodeMnode.h"
 #include "mnodeSdb.h"
@@ -33,12 +28,7 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-#include "bnInt.h"
-#include "bnScore.h"
-#include "bnThread.h"
-
 static SBnMgmt tsBnMgmt;;
-
 static void  bnMonitorDnodeModule();
 
 static void bnLock() {
@@ -514,7 +504,6 @@ bool bnStart() {
   }
  
   bnReleaseDnodes();
-
   bnUnLock();
 
   return updateSoon;
@@ -610,8 +599,6 @@ int32_t bnDropDnode(SDnodeObj *pDnode) {
   return TSDB_CODE_SUCCESS;
 }
 
-
-
 static void bnMonitorDnodeModule() {
   int32_t numOfMnodes = mnodeGetMnodesNum();
   if (numOfMnodes >= tsNumOfMnodes) return;
diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c
index db5d515fbf..e5ad7a2119 100644
--- a/src/balance/src/bnScore.c
+++ b/src/balance/src/bnScore.c
@@ -15,25 +15,9 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tutil.h"
-#include "tbalance.h"
-#include "tsync.h"
-#include "tsync.h"
-#include "ttimer.h"
 #include "tglobal.h"
-#include "tdataformat.h"
-#include "dnode.h"
-#include "mnode.h"
-#include "mnodeDef.h"
-#include "mnodeInt.h"
-#include "mnodeDnode.h"
-#include "mnodeDb.h"
-#include "mnodeMnode.h"
-#include "mnodeSdb.h"
 #include "mnodeShow.h"
 #include "mnodeUser.h"
-#include "mnodeVgroup.h"
-
 #include "bnScore.h"
 
 SBnDnodes tsBnDnodes;
diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c
index 39eb6d0e31..08a045b94b 100644
--- a/src/balance/src/bnThread.c
+++ b/src/balance/src/bnThread.c
@@ -15,26 +15,9 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tutil.h"
-#include "tbalance.h"
-#include "tsync.h"
-#include "tsync.h"
 #include "ttimer.h"
 #include "tglobal.h"
-#include "tdataformat.h"
-#include "dnode.h"
-#include "mnode.h"
-#include "mnodeDef.h"
-#include "mnodeInt.h"
-#include "mnodeDnode.h"
-#include "mnodeDb.h"
-#include "mnodeMnode.h"
 #include "mnodeSdb.h"
-#include "mnodeShow.h"
-#include "mnodeUser.h"
-#include "mnodeVgroup.h"
-
-
 #include "bnThread.h"
 
 static SBnThread tsBnThread;

From 1408a331f194049cec5d8b9754216a87a2a516f9 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 3 Dec 2020 13:24:18 +0800
Subject: [PATCH 036/118] TD-2270 rename files

---
 src/dnode/src/dnodeMgmt.c     | 2 +-
 src/inc/{tbalance.h => tbn.h} | 0
 src/mnode/src/mnodeDb.c       | 2 +-
 src/mnode/src/mnodeDnode.c    | 4 ++--
 src/mnode/src/mnodeMain.c     | 2 +-
 src/mnode/src/mnodeMnode.c    | 2 +-
 src/mnode/src/mnodePeer.c     | 2 +-
 src/mnode/src/mnodeRead.c     | 2 +-
 src/mnode/src/mnodeSdb.c      | 2 +-
 src/mnode/src/mnodeVgroup.c   | 2 +-
 src/mnode/src/mnodeWrite.c    | 2 +-
 11 files changed, 11 insertions(+), 11 deletions(-)
 rename src/inc/{tbalance.h => tbn.h} (100%)

diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 0d1a56cfc4..5c01f64716 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -24,7 +24,7 @@
 #include "tqueue.h"
 #include "tsync.h"
 #include "ttimer.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tglobal.h"
 #include "dnode.h"
 #include "vnode.h"
diff --git a/src/inc/tbalance.h b/src/inc/tbn.h
similarity index 100%
rename from src/inc/tbalance.h
rename to src/inc/tbn.h
diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index 63bd066c36..c971a945aa 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -20,7 +20,7 @@
 #include "tgrant.h"
 #include "tglobal.h"
 #include "tname.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tdataformat.h"
 #include "mnode.h"
 #include "mnodeDef.h"
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 0d429f5797..0ff50b2307 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -16,12 +16,12 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tgrant.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tglobal.h"
 #include "tconfig.h"
 #include "tutil.h"
 #include "tsocket.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tsync.h"
 #include "tdataformat.h"
 #include "mnode.h"
diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c
index 9c8bc6c35b..86f2c821f9 100644
--- a/src/mnode/src/mnodeMain.c
+++ b/src/mnode/src/mnodeMain.c
@@ -17,7 +17,7 @@
 #include "os.h"
 #include "taosdef.h"
 #include "tsched.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tgrant.h"
 #include "ttimer.h"
 #include "tglobal.h"
diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c
index 7428e8d2c8..d20d51f82b 100644
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@@ -19,7 +19,7 @@
 #include "tglobal.h"
 #include "trpc.h"
 #include "tsync.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tutil.h"
 #include "tsocket.h"
 #include "tdataformat.h"
diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c
index ef1b819018..cfb7b7781b 100644
--- a/src/mnode/src/mnodePeer.c
+++ b/src/mnode/src/mnodePeer.c
@@ -20,7 +20,7 @@
 #include "tsystem.h"
 #include "tutil.h"
 #include "tgrant.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tglobal.h"
 #include "mnode.h"
 #include "dnode.h"
diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c
index a39d35506f..c2a70bc01d 100644
--- a/src/mnode/src/mnodeRead.c
+++ b/src/mnode/src/mnodeRead.c
@@ -17,7 +17,7 @@
 #include "os.h"
 #include "taosdef.h"
 #include "tsched.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tgrant.h"
 #include "ttimer.h"
 #include "tglobal.h"
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 07b3ef09ec..2ef758baf1 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -19,7 +19,7 @@
 #include "hash.h"
 #include "tutil.h"
 #include "tref.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tqueue.h"
 #include "twal.h"
 #include "tsync.h"
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 7cf9e35bd2..d3020de6bd 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -20,7 +20,7 @@
 #include "tsocket.h"
 #include "tidpool.h"
 #include "tsync.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tglobal.h"
 #include "tdataformat.h"
 #include "dnode.h"
diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c
index f0cfe1aedc..53981238a7 100644
--- a/src/mnode/src/mnodeWrite.c
+++ b/src/mnode/src/mnodeWrite.c
@@ -17,7 +17,7 @@
 #include "os.h"
 #include "taosdef.h"
 #include "tsched.h"
-#include "tbalance.h"
+#include "tbn.h"
 #include "tgrant.h"
 #include "tglobal.h"
 #include "trpc.h"

From 51a8b29a8f4efdfaed88e685a318d34e3108ca1f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 13:29:13 +0800
Subject: [PATCH 037/118] [TD-2298]

---
 src/query/src/qExecutor.c | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 03de6aea1f..6ce40b9204 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -3981,8 +3981,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
       if (qstatus.lastKey != pTableQueryInfo->lastKey) {
         qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step;
       } else { // the lastkey does not increase, which means no data checked yet
-        qDebug("QInfo:%p no results generated in this scan, abort", pQInfo);
-        return;
+        qDebug("QInfo:%p no results generated in this scan", pQInfo);
       }
 
       qstatus.lastKey = pTableQueryInfo->lastKey;
@@ -4749,7 +4748,23 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w
 
 static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
-  *start = pQuery->current->lastKey;
+
+  // get the first unclosed time window
+  bool assign = false;
+  for(int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) {
+    if (pRuntimeEnv->windowResInfo.pResult[i]->closed) {
+      continue;
+    }
+
+    assign = true;
+    *start = pRuntimeEnv->windowResInfo.pResult[i]->win.skey;
+  }
+
+  if (!assign) {
+    *start = pQuery->current->lastKey;
+  }
+
+  assert(*start <= pQuery->current->lastKey);
 
   // if queried with value filter, do NOT forward query start position
   if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->pFillInfo != NULL) {
@@ -5441,7 +5456,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
   } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) {
     //super table projection query with identical query time range for all tables.
     SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
-
     resetDefaultResInfoOutputBuf(pRuntimeEnv);
 
     SArray *group = GET_TABLEGROUP(pQInfo, 0);

From 26b32bd9e7d16b420b7e742788b5da29ba156dd9 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 3 Dec 2020 14:02:10 +0800
Subject: [PATCH 038/118] TD-2270

---
 src/balance/src/bnMain.c   | 2 +-
 src/balance/src/bnThread.c | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c
index 54e957a864..383f981913 100644
--- a/src/balance/src/bnMain.c
+++ b/src/balance/src/bnMain.c
@@ -566,8 +566,8 @@ int32_t bnInit() {
 
 void bnCleanUp() {
   bnCleanupThread();
-  pthread_mutex_destroy(&tsBnMgmt.mutex);
   bnCleanupDnodes();
+  pthread_mutex_destroy(&tsBnMgmt.mutex);
 }
 
 int32_t bnDropDnode(SDnodeObj *pDnode) {
diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c
index 08a045b94b..a11bc61b01 100644
--- a/src/balance/src/bnThread.c
+++ b/src/balance/src/bnThread.c
@@ -81,7 +81,6 @@ void bnCleanupThread() {
 }
 
 static void bnPostSignal() {
-  mDebug("balance thread async notify");
   pthread_mutex_lock(&tsBnThread.mutex);
   pthread_cond_signal(&tsBnThread.cond);
   pthread_mutex_unlock(&(tsBnThread.mutex));
@@ -94,11 +93,13 @@ static void bnPostSignal() {
 
 static void bnProcessTimer(void *handle, void *tmrId) {
   if (!sdbIsMaster()) return;
+  if (tsBnThread.stop) return;
 
   tsBnThread.timer = NULL;
   tsAccessSquence++;
 
   bnCheckStatus();
+  bnStartTimer(-1);
 
   if (handle == NULL) {
     if (tsAccessSquence % tsBalanceInterval == 0) {
@@ -113,6 +114,8 @@ static void bnProcessTimer(void *handle, void *tmrId) {
 }
 
 void bnStartTimer(int64_t mseconds) {
+  if (tsBnThread.stop) return;
+
   bool updateSoon = (mseconds != -1);
   if (updateSoon) {
     taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);

From 024397f265f5115caa83808890d101f0776debcb Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 3 Dec 2020 06:57:49 +0000
Subject: [PATCH 039/118] fix TD-2314

---
 src/tsdb/src/tsdbFile.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index 03c50d42f7..5d8933d141 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -248,8 +248,13 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
   if (pGroup == NULL) {  // if not exists, create one
     pFGroup->fileId = fid;
     for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
-      if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0)
-        goto _err;
+      if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) {
+        for (int i = type; i >= 0; i--) {
+          remove(pFGroup->files[i].fname);
+        }
+
+        return NULL;
+      }
     }
 
     pthread_rwlock_wrlock(&pFileH->fhlock);
@@ -261,10 +266,6 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
   }
 
   return pGroup;
-
-_err:
-  for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]);
-  return NULL;
 }
 
 void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) {

From b6d30543d82c5f7c900e839a11d79a9ede653c82 Mon Sep 17 00:00:00 2001
From: zyyang <zyyang@taosdata.com>
Date: Thu, 3 Dec 2020 15:06:11 +0800
Subject: [PATCH 040/118] change

---
 .../example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java
index 82613037db..36745a9394 100644
--- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java
+++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java
@@ -14,9 +14,9 @@ public final class JdbcTaosdemoConfig {
     //Destination database. Default is 'test'
     private String dbName = "test";
     //keep
-    private int keep = 3650;
+    private int keep = 36500;
     //days
-    private int days = 10;
+    private int days = 120;
 
     //Super table Name. Default is 'meters'
     private String stbName = "meters";

From 9887f533c6e27f03e4e5c7687a97cbfddd2e99dd Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 15:25:43 +0800
Subject: [PATCH 041/118] [TD-2298] add test cases.

---
 tests/script/general/parser/first_last.sim    | 42 ++++++++++++++++++-
 .../general/parser/first_last_query.sim       |  4 +-
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim
index a934d3bcab..773f92afcf 100644
--- a/tests/script/general/parser/first_last.sim
+++ b/tests/script/general/parser/first_last.sim
@@ -46,7 +46,8 @@ while $i < $tbNum
   endw 
  
   $i = $i + 1
-endw 
+endw
+
 $ts = $ts + 60000
 $tb = $tbPrefix . 0
 sql insert into $tb (ts) values ( $ts )
@@ -84,4 +85,43 @@ sleep 500
 
 run general/parser/first_last_query.sim
 
+print =================> insert data regression test
+sql create database test keep 36500
+sql use test
+sql create table tm0 (ts timestamp, k int)
+
+print =========================> td-2298
+$ts0 = 1537146000000
+$xs  = 6000
+
+$x = 0
+while $x < 5000
+    $ts = $ts0 + $xs
+    $ts1 = $ts + $xs
+    $x1 = $x + 1
+
+    sql insert into tm0 values ( $ts , $x ) ( $ts1 , $x1 )
+    $x = $x1
+    $ts0 = $ts1
+endw
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 3000
+system sh/exec.sh -n dnode1 -s start
+print ================== server restart completed
+sql connect
+sleep 500
+
+sql use test
+sql select count(*), last(ts) from tm0 interval(1s)
+if $rows != 10000 then
+  print expect 10000, actual: $rows
+  return -1
+endi
+
+sql select last(ts) from tm0 interval(1s)
+if $rows != 10000 then
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/general/parser/first_last_query.sim b/tests/script/general/parser/first_last_query.sim
index a982f10362..52d888b04d 100644
--- a/tests/script/general/parser/first_last_query.sim
+++ b/tests/script/general/parser/first_last_query.sim
@@ -266,4 +266,6 @@ endi
 if $data14 != @test2@ then
   print expect test2 , actual: $data14
   return -1
-endi
\ No newline at end of file
+endi
+
+sql drop table stest
\ No newline at end of file

From a837f33731213ee8f5fa9a860e3d762f15ee0cf7 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Thu, 3 Dec 2020 15:43:21 +0800
Subject: [PATCH 042/118] [TD-2298] <test> add test case for interval

---
 tests/pytest/fulltest.sh            |  3 ++-
 tests/pytest/query/queryInterval.py | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index 042fd826e8..448a0413e0 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -159,7 +159,8 @@ python3 ./test.py -f query/bug1471.py
 python3 ./test.py -f query/bug1874.py
 python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
-python3 ./test.py -f query/bug2218.py 
+python3 ./test.py -f query/bug2218.py
+python3 ./test.py -f query/sliding.py 
 
 #stream
 python3 ./test.py -f stream/metric_1.py
diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py
index 98b8c9fbef..871c076c08 100644
--- a/tests/pytest/query/queryInterval.py
+++ b/tests/pytest/query/queryInterval.py
@@ -16,6 +16,7 @@ import taos
 from util.log import tdLog
 from util.cases import tdCases
 from util.sql import tdSql
+from util.dnodes import tdDnodes
 
 
 class TDTestCase:
@@ -72,6 +73,19 @@ class TDTestCase:
         tdSql.checkData(6, 0, "2020-09-16 00:00:00")
         tdSql.checkData(6, 1, 222.0)
 
+        # test case for https://jira.taosdata.com:18080/browse/TD-2298
+        tdSql.execute("create database test keep 36500")
+        tdSql.execute("use test")
+        tdSql.execute("create table t (ts timestamp, voltage int)")
+        for i in range(10000):
+            tdSql.execute("insert into t values(%d, 0)" % (1000000 + i * 6000))
+        
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select last(*) from t interval(1s)")
+        tdSql.checkRows(10000)
+        
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

From 92ca5ded0d222f4914876b745a6428d0dc276f75 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Thu, 3 Dec 2020 16:19:56 +0800
Subject: [PATCH 043/118] [TD-785]Improve the robustness of python connector

---
 src/connector/python/linux/python2/taos/__init__.py     | 2 +-
 src/connector/python/linux/python2/taos/cinterface.py   | 2 ++
 src/connector/python/linux/python3/taos/__init__.py     | 4 ++--
 src/connector/python/linux/python3/taos/cinterface.py   | 2 ++
 src/connector/python/windows/python2/taos/__init__.py   | 2 +-
 src/connector/python/windows/python2/taos/cinterface.py | 2 ++
 src/connector/python/windows/python3/taos/__init__.py   | 2 +-
 src/connector/python/windows/python3/taos/cinterface.py | 2 ++
 8 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/connector/python/linux/python2/taos/__init__.py b/src/connector/python/linux/python2/taos/__init__.py
index 4894488bc8..5d21671eed 100644
--- a/src/connector/python/linux/python2/taos/__init__.py
+++ b/src/connector/python/linux/python2/taos/__init__.py
@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0'
+apilevel = '2.0.2'
 threadsafety = 0
 paramstyle = 'pyformat'
 
diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py
index 6f0435722f..2b1b29ee31 100644
--- a/src/connector/python/linux/python2/taos/cinterface.py
+++ b/src/connector/python/linux/python2/taos/cinterface.py
@@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     if num_of_rows > 0:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
@@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     res=[]
     if num_of_rows > 0:
         for i in range(abs(num_of_rows)):
diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py
index 4894488bc8..c46131eeaf 100644
--- a/src/connector/python/linux/python3/taos/__init__.py
+++ b/src/connector/python/linux/python3/taos/__init__.py
@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0'
+apilevel = '2.0.2'
 threadsafety = 0
 paramstyle = 'pyformat'
 
@@ -21,4 +21,4 @@ def connect(*args, **kwargs):
 
     @rtype: TDengineConnector
     """
-    return TDengineConnection(*args, **kwargs)
\ No newline at end of file
+    return TDengineConnection(*args, **kwargs)
diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py
index bbf5a0c714..fdebe349fe 100644
--- a/src/connector/python/linux/python3/taos/cinterface.py
+++ b/src/connector/python/linux/python3/taos/cinterface.py
@@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     if num_of_rows > 0:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
@@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     res=[]
     if num_of_rows > 0:
         for i in range(abs(num_of_rows)):
diff --git a/src/connector/python/windows/python2/taos/__init__.py b/src/connector/python/windows/python2/taos/__init__.py
index 4894488bc8..5d21671eed 100644
--- a/src/connector/python/windows/python2/taos/__init__.py
+++ b/src/connector/python/windows/python2/taos/__init__.py
@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0'
+apilevel = '2.0.2'
 threadsafety = 0
 paramstyle = 'pyformat'
 
diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py
index d4bf1a0350..14f4f49be8 100644
--- a/src/connector/python/windows/python2/taos/cinterface.py
+++ b/src/connector/python/windows/python2/taos/cinterface.py
@@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     if num_of_rows > 0:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
@@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     res=[]
     if num_of_rows > 0:
         for i in range(abs(num_of_rows)):
diff --git a/src/connector/python/windows/python3/taos/__init__.py b/src/connector/python/windows/python3/taos/__init__.py
index 2fec471a1a..0708b9544f 100644
--- a/src/connector/python/windows/python3/taos/__init__.py
+++ b/src/connector/python/windows/python3/taos/__init__.py
@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0'
+apilevel = '2.0.2'
 threadsafety = 0
 paramstyle = 'pyformat'
 
diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py
index fb1b65fc15..42b820ca80 100644
--- a/src/connector/python/windows/python3/taos/cinterface.py
+++ b/src/connector/python/windows/python3/taos/cinterface.py
@@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     if num_of_rows > 0:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
@@ -108,6 +109,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
 def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
+    assert(nbytes is not None)
     res=[]
     if num_of_rows > 0:
         for i in range(abs(num_of_rows)):

From d79d3058ae9cf061e5217d8bc91d6d08f63307aa Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 16:39:33 +0800
Subject: [PATCH 044/118] [TD-225] fix compiler error.

---
 src/query/src/qExecutor.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 6ce40b9204..0c07149e8e 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -4524,7 +4524,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
     }
   }
 
-  int32_t numOfTables = taosHashGetSize(pQInfo->arrTableIdInfo);
+  int32_t numOfTables = (int32_t) taosHashGetSize(pQInfo->arrTableIdInfo);
   *(int32_t*)data = htonl(numOfTables);
   data += sizeof(int32_t);
 

From fa810a383a1252bb13dc8b09eaa57f38f2dffbd1 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 17:25:59 +0800
Subject: [PATCH 045/118] [TD-2265]<fix>: add the total number of rows in
 submit block during parse insert sql.

---
 src/client/src/tscParseInsert.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 6de8195d73..6ff1cdcb79 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -630,11 +630,17 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
   return TSDB_CODE_SUCCESS;
 }
 
-static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
+static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
   pBlocks->tid = pTableMeta->id.tid;
   pBlocks->uid = pTableMeta->id.uid;
   pBlocks->sversion = pTableMeta->sversion;
-  pBlocks->numOfRows += numOfRows;
+
+  if (pBlocks->numOfRows + numOfRows >= INT16_MAX) {
+    return TSDB_CODE_TSC_INVALID_SQL;
+  } else {
+    pBlocks->numOfRows += numOfRows;
+    return TSDB_CODE_SUCCESS;
+  }
 }
 
 // data block is disordered, sort it in ascending order
@@ -722,7 +728,11 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
   }
 
   SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData);
-  tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
+  code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
+  if (code != TSDB_CODE_SUCCESS) {
+    tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", *str);
+    return code;
+  }
 
   dataBuf->vgId = pTableMeta->vgroupInfo.vgId;
   dataBuf->numOfTables = 1;

From a3fb9a1a8e13c83b783cede3460650743f0e53e9 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 17:28:02 +0800
Subject: [PATCH 046/118] [TD-2265]

---
 src/client/src/tscParseInsert.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 6ff1cdcb79..08935adf8b 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -1394,7 +1394,10 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
   STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta;
 
   SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData);
-  tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
+  code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
+  if (code != TSDB_CODE_SUCCESS) {
+    return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", "");
+  }
 
   if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
     return code;

From 5015365172cfd9cc91344eeadc2a266e0df19be4 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 17:28:38 +0800
Subject: [PATCH 047/118] [TD-2265]

---
 src/client/src/tscParseInsert.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 08935adf8b..18e5b6f074 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -1396,7 +1396,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
   SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData);
   code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
   if (code != TSDB_CODE_SUCCESS) {
-    return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", "");
+    return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", NULL);
   }
 
   if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {

From 418f15e2d1ca92fbaf887d59d91d6c7a7b4babc9 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 3 Dec 2020 18:04:18 +0800
Subject: [PATCH 048/118] T-2270 change monitordebugflag

---
 packaging/cfg/taos.cfg                        |   2 +-
 src/client/src/tscSQLParser.c                 |   2 +-
 src/common/inc/tglobal.h                      |   2 +-
 src/common/src/tglobal.c                      |  26 +--
 src/dnode/src/dnodeModule.c                   |   8 +-
 src/inc/monitor.h                             |  14 +-
 src/mnode/inc/mnodeInt.h                      |   6 +-
 .../monitor/src/{monitorMain.c => monMain.c}  | 150 +++++++++---------
 tests/script/general/alter/dnode.sim          |   8 +-
 tests/script/sh/deploy.sh                     |   2 +-
 10 files changed, 110 insertions(+), 110 deletions(-)
 rename src/plugins/monitor/src/{monitorMain.c => monMain.c} (70%)

diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg
index ca88bca3c8..6e9a17dbe4 100644
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@@ -236,7 +236,7 @@
 # httpDebugFlag         131
 
 # debug flag for monitor
-# monitorDebugFlag      131
+# monDebugFlag          131
 
 # debug flag for query
 # qDebugflag            131
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 309d354849..110e306ce7 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5103,7 +5103,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
   const int tokenDebugFlagEnd = 20;
   const SDNodeDynConfOption cfgOptions[] = {
       {"resetLog", 8},    {"resetQueryCache", 15},  {"balance", 7},     {"monitor", 7},
-      {"debugFlag", 9},   {"monitorDebugFlag", 16}, {"vDebugFlag", 10}, {"mDebugFlag", 10},
+      {"debugFlag", 9},   {"monDebugFlag", 11},     {"vDebugFlag", 10}, {"mDebugFlag", 10},
       {"cDebugFlag", 10}, {"httpDebugFlag", 13},    {"qDebugflag", 10}, {"sdbDebugFlag", 12},
       {"uDebugFlag", 10}, {"tsdbDebugFlag", 13},    {"sDebugflag", 10}, {"rpcDebugFlag", 12},
       {"dDebugFlag", 10}, {"mqttDebugFlag", 13},    {"wDebugFlag", 10}, {"tmrDebugFlag", 12},
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index efe3d7678a..bdd9b8ce9f 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -176,7 +176,7 @@ extern int32_t tmrDebugFlag;
 extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
 extern int32_t mqttDebugFlag;
-extern int32_t monitorDebugFlag;
+extern int32_t monDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;
 extern int32_t odbcDebugFlag;
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index d26746a5d1..2a84184f8d 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -206,7 +206,7 @@ int32_t jniDebugFlag = 131;
 int32_t odbcDebugFlag = 131;
 int32_t httpDebugFlag = 131;
 int32_t mqttDebugFlag = 131;
-int32_t monitorDebugFlag = 131;
+int32_t monDebugFlag = 131;
 int32_t qDebugFlag = 131;
 int32_t rpcDebugFlag = 131;
 int32_t uDebugFlag = 131;
@@ -216,9 +216,9 @@ int32_t wDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 135;
 
-int32_t (*monitorStartSystemFp)() = NULL;
-void (*monitorStopSystemFp)() = NULL;
-void (*monitorExecuteSQLFp)(char *sql) = NULL;
+int32_t (*monStartSystemFp)() = NULL;
+void (*monStopSystemFp)() = NULL;
+void (*monExecuteSQLFp)(char *sql) = NULL;
 
 char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
 
@@ -235,7 +235,7 @@ void taosSetAllDebugFlag() {
     odbcDebugFlag = debugFlag;
     httpDebugFlag = debugFlag;
     mqttDebugFlag = debugFlag;
-    monitorDebugFlag = debugFlag;
+    monDebugFlag = debugFlag;
     qDebugFlag = debugFlag;    
     rpcDebugFlag = debugFlag;
     uDebugFlag = debugFlag;
@@ -276,15 +276,15 @@ bool taosCfgDynamicOptions(char *msg) {
 
     if (strncasecmp(cfg->option, "monitor", olen) == 0) {
       if (1 == vint) {
-        if (monitorStartSystemFp) {
-          (*monitorStartSystemFp)();
+        if (monStartSystemFp) {
+          (*monStartSystemFp)();
           uInfo("monitor is enabled");
         } else {
           uError("monitor can't be updated, for monitor not initialized");
         }
       } else {
-        if (monitorStopSystemFp) {
-          (*monitorStopSystemFp)();
+        if (monStopSystemFp) {
+          (*monStopSystemFp)();
           uInfo("monitor is disabled");
         } else {
           uError("monitor can't be updated, for monitor not initialized");
@@ -307,8 +307,8 @@ bool taosCfgDynamicOptions(char *msg) {
   }
 
   if (strncasecmp(option, "resetQueryCache", 15) == 0) {
-    if (monitorExecuteSQLFp) {
-      (*monitorExecuteSQLFp)("resetQueryCache");
+    if (monExecuteSQLFp) {
+      (*monExecuteSQLFp)("resetQueryCache");
       uInfo("resetquerycache is executed");
     } else {
       uError("resetquerycache can't be executed, for monitor not started");
@@ -1227,8 +1227,8 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "monitorDebugFlag";
-  cfg.ptr = &monitorDebugFlag;
+  cfg.option = "monDebugFlag";
+  cfg.ptr = &monDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
   cfg.minValue = 0;
diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c
index bd9500ba51..7faa3c8913 100644
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@@ -78,10 +78,10 @@ static void dnodeAllocModules() {
 
   tsModule[TSDB_MOD_MONITOR].enable    = (tsEnableMonitorModule == 1);
   tsModule[TSDB_MOD_MONITOR].name      = "monitor";
-  tsModule[TSDB_MOD_MONITOR].initFp    = monitorInitSystem;
-  tsModule[TSDB_MOD_MONITOR].cleanUpFp = monitorCleanUpSystem;
-  tsModule[TSDB_MOD_MONITOR].startFp   = monitorStartSystem;
-  tsModule[TSDB_MOD_MONITOR].stopFp    = monitorStopSystem;
+  tsModule[TSDB_MOD_MONITOR].initFp    = monInitSystem;
+  tsModule[TSDB_MOD_MONITOR].cleanUpFp = monCleanupSystem;
+  tsModule[TSDB_MOD_MONITOR].startFp   = monStartSystem;
+  tsModule[TSDB_MOD_MONITOR].stopFp    = monStopSystem;
   if (tsEnableMonitorModule) {
     dnodeSetModuleStatus(TSDB_MOD_MONITOR);
   }
diff --git a/src/inc/monitor.h b/src/inc/monitor.h
index b1229cca6b..1aefb0b848 100644
--- a/src/inc/monitor.h
+++ b/src/inc/monitor.h
@@ -47,13 +47,13 @@ typedef struct {
   int8_t  accessState;
 } SAcctMonitorObj;
 
-int32_t monitorInitSystem();
-int32_t monitorStartSystem();
-void    monitorStopSystem();
-void    monitorCleanUpSystem();
-void    monitorSaveAcctLog(SAcctMonitorObj *pMonObj);
-void    monitorSaveLog(int32_t level, const char *const format, ...);
-void    monitorExecuteSQL(char *sql);
+int32_t monInitSystem();
+int32_t monStartSystem();
+void    monStopSystem();
+void    monCleanupSystem();
+void    monSaveAcctLog(SAcctMonitorObj *pMonObj);
+void    monSaveLog(int32_t level, const char *const format, ...);
+void    monExecuteSQL(char *sql);
 
 #ifdef __cplusplus
 }
diff --git a/src/mnode/inc/mnodeInt.h b/src/mnode/inc/mnodeInt.h
index 44626fd167..7a791d76e6 100644
--- a/src/mnode/inc/mnodeInt.h
+++ b/src/mnode/inc/mnodeInt.h
@@ -41,9 +41,9 @@ extern int32_t sdbDebugFlag;
 #define sdbDebug(...) { if (sdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }}
 #define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }}
 
-#define mLError(...) { monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) }
-#define mLWarn(...)  { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__)  }
-#define mLInfo(...)  { monitorSaveLog(0, __VA_ARGS__); mInfo(__VA_ARGS__) }
+#define mLError(...) { monSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) }
+#define mLWarn(...)  { monSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__)  }
+#define mLInfo(...)  { monSaveLog(0, __VA_ARGS__); mInfo(__VA_ARGS__) }
 
 #ifdef __cplusplus
 }
diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monMain.c
similarity index 70%
rename from src/plugins/monitor/src/monitorMain.c
rename to src/plugins/monitor/src/monMain.c
index 24998b54cd..9443b1ce12 100644
--- a/src/plugins/monitor/src/monitorMain.c
+++ b/src/plugins/monitor/src/monMain.c
@@ -27,12 +27,12 @@
 #include "monitor.h"
 #include "taoserror.h"
 
-#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
-#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
-#define mnWarn(...)  { if (monitorDebugFlag & DEBUG_WARN)  { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
-#define mnInfo(...)  { if (monitorDebugFlag & DEBUG_INFO)  { taosPrintLog("MON ", 255, __VA_ARGS__); }}
-#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
-#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define monFatal(...) { if (monDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
+#define monError(...) { if (monDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
+#define monWarn(...)  { if (monDebugFlag & DEBUG_WARN)  { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
+#define monInfo(...)  { if (monDebugFlag & DEBUG_INFO)  { taosPrintLog("MON ", 255, __VA_ARGS__); }}
+#define monDebug(...) { if (monDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
+#define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
 
 #define SQL_LENGTH     1030
 #define LOG_LEN_STR    100
@@ -48,12 +48,12 @@ typedef enum {
   MON_CMD_CREATE_TB_ACCT_ROOT,
   MON_CMD_CREATE_TB_SLOWQUERY,
   MON_CMD_MAX
-} EMonitorCommand;
+} EMonCmd;
 
 typedef enum {
   MON_STATE_NOT_INIT,
   MON_STATE_INITED
-} EMonitorState;
+} EMonState;
 
 typedef struct {
   pthread_t thread;
@@ -64,17 +64,17 @@ typedef struct {
   int8_t    start;   // enable/disable by mnode
   int8_t    quiting; // taosd is quiting 
   char      sql[SQL_LENGTH + 1];
-} SMonitorConn;
+} SMonConn;
 
-static SMonitorConn tsMonitor = {0};
-static void  monitorSaveSystemInfo();
-static void *monitorThreadFunc(void *param);
-static void  monitorBuildMonitorSql(char *sql, int32_t cmd);
-extern int32_t (*monitorStartSystemFp)();
-extern void    (*monitorStopSystemFp)();
-extern void    (*monitorExecuteSQLFp)(char *sql);
+static SMonConn tsMonitor = {0};
+static void  monSaveSystemInfo();
+static void *monThreadFunc(void *param);
+static void  monBuildMonitorSql(char *sql, int32_t cmd);
+extern int32_t (*monStartSystemFp)();
+extern void    (*monStopSystemFp)();
+extern void    (*monExecuteSQLFp)(char *sql);
 
-int32_t monitorInitSystem() {
+int32_t monInitSystem() {
   if (tsMonitor.ep[0] == 0) {
     strcpy(tsMonitor.ep, tsLocalEp);
   }
@@ -90,29 +90,29 @@ int32_t monitorInitSystem() {
   pthread_attr_init(&thAttr);
   pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-  if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) {
-    mnError("failed to create thread to for monitor module, reason:%s", strerror(errno));
+  if (pthread_create(&tsMonitor.thread, &thAttr, monThreadFunc, NULL)) {
+    monError("failed to create thread to for monitor module, reason:%s", strerror(errno));
     return -1;
   }
 
   pthread_attr_destroy(&thAttr);
-  mnDebug("monitor thread is launched");
+  monDebug("monitor thread is launched");
 
-  monitorStartSystemFp = monitorStartSystem;
-  monitorStopSystemFp = monitorStopSystem;
+  monStartSystemFp = monStartSystem;
+  monStopSystemFp = monStopSystem;
   return 0;
 }
 
-int32_t monitorStartSystem() {
+int32_t monStartSystem() {
   taos_init();
   tsMonitor.start = 1;
-  monitorExecuteSQLFp = monitorExecuteSQL;
-  mnInfo("monitor module start");
+  monExecuteSQLFp = monExecuteSQL;
+  monInfo("monitor module start");
   return 0;
 }
 
-static void *monitorThreadFunc(void *param) {
-  mnDebug("starting to initialize monitor module ...");
+static void *monThreadFunc(void *param) {
+  monDebug("starting to initialize monitor module ...");
 
   while (1) {
     static int32_t accessTimes = 0;
@@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) {
 
     if (tsMonitor.quiting) {
       tsMonitor.state = MON_STATE_NOT_INIT;
-      mnInfo("monitor thread will quit, for taosd is quiting");
+      monInfo("monitor thread will quit, for taosd is quiting");
       break;
     } else {
       taosGetDisk();
@@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) {
     }
     
     if (dnodeGetDnodeId() <= 0) {
-      mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
+      monDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
       continue;
     }
 
@@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) {
       tsMonitor.state = MON_STATE_NOT_INIT;
       tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
       if (tsMonitor.conn == NULL) {
-        mnError("failed to connect to database, reason:%s", tstrerror(terrno));
+        monError("failed to connect to database, reason:%s", tstrerror(terrno));
         continue;
       } else {
-        mnDebug("connect to database success");
+        monDebug("connect to database success");
       }
     }
 
@@ -151,16 +151,16 @@ static void *monitorThreadFunc(void *param) {
       int code = 0;
 
       for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) {
-        monitorBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex);
+        monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex);
         void *res = taos_query(tsMonitor.conn, tsMonitor.sql);
         code = taos_errno(res);
         taos_free_result(res);
 
         if (code != 0) {
-          mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
+          monError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
           break;
         } else {
-          mnDebug("successfully to exec sql:%s", tsMonitor.sql);
+          monDebug("successfully to exec sql:%s", tsMonitor.sql);
         }
       }
 
@@ -171,16 +171,16 @@ static void *monitorThreadFunc(void *param) {
 
     if (tsMonitor.state == MON_STATE_INITED) {
       if (accessTimes % tsMonitorInterval == 0) {
-        monitorSaveSystemInfo();
+        monSaveSystemInfo();
       }
     }
   }
 
-  mnInfo("monitor thread is stopped");
+  monInfo("monitor thread is stopped");
   return NULL;
 }
 
-static void monitorBuildMonitorSql(char *sql, int32_t cmd) {
+static void monBuildMonitorSql(char *sql, int32_t cmd) {
   memset(sql, 0, SQL_LENGTH);
 
   if (cmd == MON_CMD_CREATE_DB) {
@@ -236,47 +236,47 @@ static void monitorBuildMonitorSql(char *sql, int32_t cmd) {
   sql[SQL_LENGTH] = 0;
 }
 
-void monitorStopSystem() {
+void monStopSystem() {
   tsMonitor.start = 0;
   tsMonitor.state = MON_STATE_NOT_INIT;
-  monitorExecuteSQLFp = NULL;
-  mnInfo("monitor module stopped");
+  monExecuteSQLFp = NULL;
+  monInfo("monitor module stopped");
 }
 
-void monitorCleanUpSystem() {
+void monCleanupSystem() {
   tsMonitor.quiting = 1;
-  monitorStopSystem();
+  monStopSystem();
   pthread_join(tsMonitor.thread, NULL);
   if (tsMonitor.conn != NULL) {
     taos_close(tsMonitor.conn);
     tsMonitor.conn = NULL;
   }
-  mnInfo("monitor module is cleaned up");
+  monInfo("monitor module is cleaned up");
 }
 
 // unit is MB
-static int32_t monitorBuildMemorySql(char *sql) {
+static int32_t monBuildMemorySql(char *sql) {
   float sysMemoryUsedMB = 0;
   bool  suc = taosGetSysMemory(&sysMemoryUsedMB);
   if (!suc) {
-    mnDebug("failed to get sys memory info");
+    monDebug("failed to get sys memory info");
   }
 
   float procMemoryUsedMB = 0;
   suc = taosGetProcMemory(&procMemoryUsedMB);
   if (!suc) {
-    mnDebug("failed to get proc memory info");
+    monDebug("failed to get proc memory info");
   }
 
   return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB);
 }
 
 // unit is %
-static int32_t monitorBuildCpuSql(char *sql) {
+static int32_t monBuildCpuSql(char *sql) {
   float sysCpuUsage = 0, procCpuUsage = 0;
   bool  suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
   if (!suc) {
-    mnDebug("failed to get cpu usage");
+    monDebug("failed to get cpu usage");
   }
 
   if (sysCpuUsage <= procCpuUsage) {
@@ -287,72 +287,72 @@ static int32_t monitorBuildCpuSql(char *sql) {
 }
 
 // unit is GB
-static int32_t monitorBuildDiskSql(char *sql) {
+static int32_t monBuildDiskSql(char *sql) {
   return sprintf(sql, ", %f, %d", (tsTotalDataDirGB - tsAvailDataDirGB), (int32_t)tsTotalDataDirGB);
 }
 
 // unit is Kb
-static int32_t monitorBuildBandSql(char *sql) {
+static int32_t monBuildBandSql(char *sql) {
   float bandSpeedKb = 0;
   bool  suc = taosGetBandSpeed(&bandSpeedKb);
   if (!suc) {
-    mnDebug("failed to get bandwidth speed");
+    monDebug("failed to get bandwidth speed");
   }
 
   return sprintf(sql, ", %f", bandSpeedKb);
 }
 
-static int32_t monitorBuildReqSql(char *sql) {
+static int32_t monBuildReqSql(char *sql) {
   SStatisInfo info = dnodeGetStatisInfo();
   return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum);
 }
 
-static int32_t monitorBuildIoSql(char *sql) {
+static int32_t monBuildIoSql(char *sql) {
   float readKB = 0, writeKB = 0;
   bool  suc = taosGetProcIO(&readKB, &writeKB);
   if (!suc) {
-    mnDebug("failed to get io info");
+    monDebug("failed to get io info");
   }
 
   return sprintf(sql, ", %f, %f", readKB, writeKB);
 }
 
-static void monitorSaveSystemInfo() {
+static void monSaveSystemInfo() {
   int64_t ts = taosGetTimestampUs();
   char *  sql = tsMonitor.sql;
   int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn%d values(%" PRId64, tsMonitorDbName, dnodeGetDnodeId(), ts);
 
-  pos += monitorBuildCpuSql(sql + pos);
-  pos += monitorBuildMemorySql(sql + pos);
-  pos += monitorBuildDiskSql(sql + pos);
-  pos += monitorBuildBandSql(sql + pos);
-  pos += monitorBuildIoSql(sql + pos);
-  pos += monitorBuildReqSql(sql + pos);
+  pos += monBuildCpuSql(sql + pos);
+  pos += monBuildMemorySql(sql + pos);
+  pos += monBuildDiskSql(sql + pos);
+  pos += monBuildBandSql(sql + pos);
+  pos += monBuildIoSql(sql + pos);
+  pos += monBuildReqSql(sql + pos);
 
   void *res = taos_query(tsMonitor.conn, tsMonitor.sql);
   int   code = taos_errno(res);
   taos_free_result(res);
 
   if (code != 0) {
-    mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
+    monError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
   } else {
-    mnDebug("successfully to save system info, sql:%s", tsMonitor.sql);
+    monDebug("successfully to save system info, sql:%s", tsMonitor.sql);
   }
 }
 
-static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
+static void monExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
   int32_t c = taos_errno(result);
   if (c != TSDB_CODE_SUCCESS) {
-    mnError("save %s failed, reason:%s", (char *)param, tstrerror(c));
+    monError("save %s failed, reason:%s", (char *)param, tstrerror(c));
   } else {
     int32_t rows = taos_affected_rows(result);
-    mnDebug("save %s succ, rows:%d", (char *)param, rows);
+    monDebug("save %s succ, rows:%d", (char *)param, rows);
   }
 
   taos_free_result(result);
 }
 
-void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
+void monSaveAcctLog(SAcctMonitorObj *pMon) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 
   char sql[1024] = {0};
@@ -382,11 +382,11 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
           pMon->totalConns, pMon->maxConns,
           pMon->accessState);
 
-  mnDebug("save account info, sql:%s", sql);
-  taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info");
+  monDebug("save account info, sql:%s", sql);
+  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "account info");
 }
 
-void monitorSaveLog(int32_t level, const char *const format, ...) {
+void monSaveLog(int32_t level, const char *const format, ...) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 
   va_list argpointer;
@@ -403,13 +403,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
   len += sprintf(sql + len, "', '%s')", tsLocalEp);
   sql[len++] = 0;
 
-  mnDebug("save log, sql: %s", sql);
-  taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log");
+  monDebug("save log, sql: %s", sql);
+  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "log");
 }
 
-void monitorExecuteSQL(char *sql) {
+void monExecuteSQL(char *sql) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 
-  mnDebug("execute sql:%s", sql);
-  taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql");
+  monDebug("execute sql:%s", sql);
+  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql");
 }
diff --git a/tests/script/general/alter/dnode.sim b/tests/script/general/alter/dnode.sim
index 20ce879979..73a095ec05 100644
--- a/tests/script/general/alter/dnode.sim
+++ b/tests/script/general/alter/dnode.sim
@@ -24,7 +24,7 @@ sql alter dnode 1 debugFlag 135
 sql alter dnode 1 debugFlag 131
 sql alter dnode 1 monitor 0
 sql alter dnode 1 debugFlag 135
-sql alter dnode 1 monitorDebugFlag 135
+sql alter dnode 1 monDebugFlag 135
 sql alter dnode 1 vDebugFlag 135
 sql alter dnode 1 mDebugFlag 135
 sql alter dnode 1 cDebugFlag 135
@@ -44,15 +44,15 @@ sql_error alter dnode 2 tmrDebugFlag 135
 
 print ======== step3
 sql_error alter $hostname1 debugFlag 135
-sql_error alter $hostname1 monitorDebugFlag 135
+sql_error alter $hostname1 monDebugFlag 135
 sql_error alter $hostname1 vDebugFlag 135
 sql_error alter $hostname1 mDebugFlag 135
 sql_error alter dnode $hostname2 debugFlag 135
-sql_error alter dnode $hostname2 monitorDebugFlag 135
+sql_error alter dnode $hostname2 monDebugFlag 135
 sql_error alter dnode $hostname2 vDebugFlag 135
 sql_error alter dnode $hostname2 mDebugFlag 135
 sql alter dnode $hostname1 debugFlag 135
-sql alter dnode $hostname1 monitorDebugFlag 135
+sql alter dnode $hostname1 monDebugFlag 135
 sql alter dnode $hostname1 vDebugFlag 135
 sql alter dnode $hostname1 tmrDebugFlag 131
 
diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh
index e26778e86b..cd2f3772eb 100755
--- a/tests/script/sh/deploy.sh
+++ b/tests/script/sh/deploy.sh
@@ -120,7 +120,7 @@ echo "cDebugFlag             143"                >> $TAOS_CFG
 echo "jnidebugFlag           143"                >> $TAOS_CFG
 echo "odbcdebugFlag          143"                >> $TAOS_CFG
 echo "httpDebugFlag          143"                >> $TAOS_CFG
-echo "monitorDebugFlag       143"                >> $TAOS_CFG
+echo "monDebugFlag           143"                >> $TAOS_CFG
 echo "mqttDebugFlag          143"                >> $TAOS_CFG
 echo "qdebugFlag             143"                >> $TAOS_CFG
 echo "rpcDebugFlag           143"                >> $TAOS_CFG

From 5ab5299316eaed66ffe5e0fd789700bc6100722c Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 3 Dec 2020 18:10:46 +0800
Subject: [PATCH 049/118] minor changes

---
 src/client/src/tscSQLParser.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 110e306ce7..e002592de1 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5103,7 +5103,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
   const int tokenDebugFlagEnd = 20;
   const SDNodeDynConfOption cfgOptions[] = {
       {"resetLog", 8},    {"resetQueryCache", 15},  {"balance", 7},     {"monitor", 7},
-      {"debugFlag", 9},   {"monDebugFlag", 11},     {"vDebugFlag", 10}, {"mDebugFlag", 10},
+      {"debugFlag", 9},   {"monDebugFlag", 12},     {"vDebugFlag", 10}, {"mDebugFlag", 10},
       {"cDebugFlag", 10}, {"httpDebugFlag", 13},    {"qDebugflag", 10}, {"sdbDebugFlag", 12},
       {"uDebugFlag", 10}, {"tsdbDebugFlag", 13},    {"sDebugflag", 10}, {"rpcDebugFlag", 12},
       {"dDebugFlag", 10}, {"mqttDebugFlag", 13},    {"wDebugFlag", 10}, {"tmrDebugFlag", 12},

From 0fe731af0fccb4eb1cd31865d19b591a81a8f0cf Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 22:01:55 +0800
Subject: [PATCH 050/118] [TD-2325]<feature>: enable configure if 50% cpu will
 be used in query processing.

---
 packaging/cfg/taos.cfg    |  5 ++-
 src/common/inc/tglobal.h  |  1 +
 src/common/src/tglobal.c  | 15 +++++++-
 src/util/src/tconfig.c    |  2 --
 src/vnode/src/vnodeRead.c | 75 ++++++++++++++++++++-------------------
 5 files changed, 57 insertions(+), 41 deletions(-)

diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg
index 7662d49280..014ebc37d2 100644
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@@ -251,7 +251,7 @@
 # cqDebugFlag           131
 
 # enable/disable recording the SQL in taos client
-# tscEnableRecordSql    0
+# enableRecordSql    0
 
 # generate core file when service crash
 # enableCoreFile        1
@@ -264,3 +264,6 @@
 
 # enable/disable stream (continuous query)
 # stream                1
+
+# only 50% CPU resources will be used in query processing
+# halfCoresForQuery     0
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index 6e4274b358..10bcfcb82b 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -56,6 +56,7 @@ extern char     tsTempDir[];
 
 //query buffer management
 extern int32_t  tsQueryBufferSize;      // maximum allowed usage buffer for each data node during query processing
+extern int32_t  tsHalfCoresForQuery;         // only 50% will be used in query processing
 
 // client
 extern int32_t tsTableMetaKeepTimer;
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index 18aa0ae6e3..026cd67a81 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -107,6 +107,9 @@ int64_t tsMaxRetentWindow = 24 * 3600L;  // maximum time window tolerance
 // positive value (in MB)
 int32_t tsQueryBufferSize = -1;
 
+// only 50% cpu will be used in query processing in dnode
+int32_t tsHalfCoresForQuery = 0;
+
 // db parameters
 int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
 int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
@@ -884,6 +887,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_BYTE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "halfCoresForQuery";
+  cfg.ptr = &tsHalfCoresForQuery;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   // locale & charset
   cfg.option = "timezone";
   cfg.ptr = tsTimezone;
@@ -1290,7 +1303,7 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "tscEnableRecordSql";
+  cfg.option = "enableRecordSql";
   cfg.ptr = &tsTscEnableRecordSql;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index e89dea5a24..173de294cf 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -19,9 +19,7 @@
 #include "taoserror.h"
 #include "tconfig.h"
 #include "tglobal.h"
-#include "tkey.h"
 #include "tulog.h"
-#include "tsocket.h"
 #include "tsystem.h"
 #include "tutil.h"
 
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index 5ef79cfbf0..ed6d29505f 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -275,41 +275,40 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
 
     vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
 
-
-#if _NON_BLOCKING_RETRIEVE
-    bool freehandle = false;
-    bool buildRes = qTableQuery(*qhandle);  // do execute query
-
-    // build query rsp, the retrieve request has reached here already
-    if (buildRes) {
-      // update the connection info according to the retrieve connection
-      pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle);
-      assert(pRead->rpcHandle != NULL);
-
-      vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
-             pRead->rpcHandle);
-
-      // set the real rsp error code
-      pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle);
-
-      // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
-      code = TSDB_CODE_QRY_HAS_RSP;
+    // In the retrieve blocking model, only 50% CPU will be used in query processing
+    if (tsHalfCoresForQuery) {
+      qTableQuery(*qhandle);  // do execute query
+      qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false);
     } else {
-      void* h1 = qGetResultRetrieveMsg(*qhandle);
-      assert(h1 == NULL);
+      bool freehandle = false;
+      bool buildRes = qTableQuery(*qhandle);  // do execute query
 
-      freehandle = qQueryCompleted(*qhandle);
-    }
+      // build query rsp, the retrieve request has reached here already
+      if (buildRes) {
+        // update the connection info according to the retrieve connection
+        pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle);
+        assert(pRead->rpcHandle != NULL);
 
-    // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle.
-    // If the building of result is not required, simply free it. Otherwise, mandatorily free the qhandle
-    if (freehandle || (!buildRes)) {
-      qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle);
+        vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
+               pRead->rpcHandle);
+
+        // set the real rsp error code
+        pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle);
+
+        // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
+        code = TSDB_CODE_QRY_HAS_RSP;
+      } else {
+        void *h1 = qGetResultRetrieveMsg(*qhandle);
+        assert(h1 == NULL);
+        freehandle = qQueryCompleted(*qhandle);
+      }
+
+      // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle.
+      // If the building of result is not required, simply free it. Otherwise, mandatorily free the qhandle
+      if (freehandle || (!buildRes)) {
+        qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle);
+      }
     }
-#else
-    qTableQuery(*qhandle);  // do execute query
-    qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false);
-#endif
   }
 
   return code;
@@ -375,14 +374,16 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
     freeHandle = true;
   } else {  // result is not ready, return immediately
     assert(buildRes == true);
-#if _NON_BLOCKING_RETRIEVE
-    if (!buildRes) {
-      assert(pRead->rpcHandle != NULL);
 
-      qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
-      return TSDB_CODE_QRY_NOT_READY;
+    // Only effects in the non-blocking model
+    if (!tsHalfCoresForQuery) {
+      if (!buildRes) {
+        assert(pRead->rpcHandle != NULL);
+
+        qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
+        return TSDB_CODE_QRY_NOT_READY;
+      }
     }
-#endif
 
     // ahandle is the sqlObj pointer
     code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle);

From 12a6fb04990d712badf97fb402556b440bc54c25 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Thu, 3 Dec 2020 22:48:08 +0800
Subject: [PATCH 051/118] crash while quit balance module

---
 src/balance/src/bnThread.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c
index a11bc61b01..bf046a9fae 100644
--- a/src/balance/src/bnThread.c
+++ b/src/balance/src/bnThread.c
@@ -36,6 +36,7 @@ static void *bnThreadFunc(void *arg) {
     pthread_mutex_unlock(&(tsBnThread.mutex));
   }
 
+  mDebug("balance thread is stopped");
   return NULL;
 }
 
@@ -47,7 +48,7 @@ int32_t bnInitThread() {
 
   pthread_attr_t thattr;
   pthread_attr_init(&thattr);
-  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
   int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL);
   pthread_attr_destroy(&thattr);
 
@@ -81,6 +82,8 @@ void bnCleanupThread() {
 }
 
 static void bnPostSignal() {
+  if (tsBnThread.stop) return;
+
   pthread_mutex_lock(&tsBnThread.mutex);
   pthread_cond_signal(&tsBnThread.cond);
   pthread_mutex_unlock(&(tsBnThread.mutex));

From 7090b3feb0bea56433e4d10565b18c74bf1f5e7b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 3 Dec 2020 22:52:17 +0800
Subject: [PATCH 052/118] [TD-2169]<fix>: fix memory leaks in group by query
 processing.

---
 src/query/src/qExecutor.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 0c07149e8e..bcce7c50a8 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -1372,8 +1372,12 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
   }
 
   if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-    pResultRow->key = malloc(varDataTLen(pData));
-    varDataCopy(pResultRow->key, pData);
+    if (pResultRow->key == NULL) {
+      pResultRow->key = malloc(varDataTLen(pData));
+      varDataCopy(pResultRow->key, pData);
+    } else {
+      assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0);
+    }
   } else {
     pResultRow->win.skey = v;
     pResultRow->win.ekey = v;

From 1be5c3f88e75d07186c48ccbdfc9d7ecb8701d8d Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Thu, 3 Dec 2020 21:17:50 +0000
Subject: [PATCH 053/118] TD-2257

---
 src/client/src/tscProfile.c |  2 ++
 src/client/src/tscSystem.c  |  2 +-
 src/util/src/tref.c         | 19 ++++++++++---------
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c
index acc5acd786..18fc79c474 100644
--- a/src/client/src/tscProfile.c
+++ b/src/client/src/tscProfile.c
@@ -39,6 +39,7 @@ void tscInitConnCb(void *param, TAOS_RES *result, int code) {
     tscSlowQueryConnInitialized = true;
     tscSaveSlowQueryFp(sql, NULL);
   }
+  taos_free_result(result);
 }
 
 void tscAddIntoSqlList(SSqlObj *pSql) {
@@ -69,6 +70,7 @@ void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) {
   } else {
     tscDebug("success to save slow query, code:%d", code);
   }
+  taos_free_result(result);
 }
 
 void tscSaveSlowQueryFp(void *handle, void *tmrId) {
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 91eabda78b..6aea03d214 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -144,7 +144,7 @@ void taos_init_imp(void) {
   int64_t refreshTime = 10; // 10 seconds by default
   if (tscMetaCache == NULL) {
     tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta");
-    tscObjRef = taosOpenRef(4096, tscFreeRegisteredSqlObj);
+    tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj);
   }
 
   tscRefId = taosOpenRef(200, tscCloseTscObj);
diff --git a/src/util/src/tref.c b/src/util/src/tref.c
index 4c1a87c960..1f83abcb84 100644
--- a/src/util/src/tref.c
+++ b/src/util/src/tref.c
@@ -329,7 +329,7 @@ void *taosIterateRef(int rsetId, int64_t rid) {
     pNode->count++;  // acquire it
     newP = pNode->p;  
     taosUnlockList(pSet->lockedBy+hash);
-    uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid);
+    uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); 
   } else {
     uTrace("rsetId:%d the list is over", rsetId);
   }
@@ -423,24 +423,25 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
       if (pNode->next) {
         pNode->next->prev = pNode->prev; 
       } 
-		
-      (*pSet->fp)(pNode->p); 
-
-      uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
-      free(pNode);
       released = 1;
     } else {
-      uTrace("rsetId:%d p:%p rid:%" PRId64 " is released, count:%d", rsetId, pNode->p, rid, pNode->count);
+       uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); 
     }
   } else {
-    uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);
+    uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); 
     terrno = TSDB_CODE_REF_NOT_EXIST;
     code = -1;
   }
 
   taosUnlockList(pSet->lockedBy+hash);
 
-  if (released) taosDecRsetCount(pSet);
+  if (released) {
+    uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
+    (*pSet->fp)(pNode->p); 
+    free(pNode);
+
+    taosDecRsetCount(pSet);
+  } 
 
   return code;
 }

From 0a0080903baf427b20769a4247176607ddd3936c Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Fri, 4 Dec 2020 01:45:53 +0000
Subject: [PATCH 054/118] [TD-2313]<fix>: ensure length of nchar cols not
 exceed limit

---
 src/query/src/qParserImpl.c | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c
index 5cd9d3c77a..7d71d9f7f1 100644
--- a/src/query/src/qParserImpl.c
+++ b/src/query/src/qParserImpl.c
@@ -405,14 +405,29 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
         if (type->type == 0) {
           pField->bytes = 0;
         } else {
-          pField->bytes = (int16_t)(-(int32_t)type->type * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
+          int32_t bytes = -(int32_t)(type->type);
+          if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
+            // we have to postpone reporting the error because it cannot be done here
+            // as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow
+            bytes = TSDB_MAX_NCHAR_LEN + 1;
+          } else {
+            bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
+          }
+          pField->bytes = (int16_t)bytes;
         }
       } else if (i == TSDB_DATA_TYPE_BINARY) {
         /* for binary, the TOKENTYPE is the length of binary */
         if (type->type == 0) {
           pField->bytes = 0;
         } else {
-          pField->bytes = (int16_t) (-(int32_t) type->type + VARSTR_HEADER_SIZE);
+          int32_t bytes = -(int32_t)(type->type);
+          if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
+            // refer comment for NCHAR above
+            bytes = TSDB_MAX_BINARY_LEN + 1;
+          } else {
+            bytes += VARSTR_HEADER_SIZE;
+          }
+          pField->bytes = (int16_t)bytes;
         } 
       }
       break;

From 573fcb4e9c5e0cbed4139506c36b5af50f41ec52 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 4 Dec 2020 10:07:13 +0800
Subject: [PATCH 055/118] TD-2299

---
 src/dnode/src/dnodePeer.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c
index 6bf22cee4e..6b5b28622b 100644
--- a/src/dnode/src/dnodePeer.c
+++ b/src/dnode/src/dnodePeer.c
@@ -182,6 +182,8 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
   SRpcEpSet epSet = {0};
   dnodeGetEpSetForPeer(&epSet);
+
+  assert(tsClientRpc != 0);
   rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
 }
 

From 1993ad245fb7adc8f3af3671c8917f3d776355ea Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Fri, 4 Dec 2020 11:16:42 +0800
Subject: [PATCH 056/118] [TD-785]Improve the robustness of python connector

---
 src/connector/python/linux/python2/taos/cursor.py   | 4 ++--
 src/connector/python/linux/python3/taos/cursor.py   | 4 ++--
 src/connector/python/windows/python2/taos/cursor.py | 4 ++--
 src/connector/python/windows/python3/taos/cursor.py | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py
index eee5e39488..82a01be671 100644
--- a/src/connector/python/linux/python2/taos/cursor.py
+++ b/src/connector/python/linux/python2/taos/cursor.py
@@ -26,7 +26,7 @@ class TDengineCursor(object):
     """
 
     def __init__(self, connection=None):
-        self._description = None
+        self._description = []
         self._rowcount = -1
         self._connection = None
         self._result = None
@@ -234,7 +234,7 @@ class TDengineCursor(object):
     def _reset_result(self):
         """Reset the result to unused version.
         """
-        self._description = None
+        self._description = []
         self._rowcount = -1
         if self._result is not None:
             CTaosInterface.freeResult(self._result)
diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py
index d8184668c8..0ce20f0eda 100644
--- a/src/connector/python/linux/python3/taos/cursor.py
+++ b/src/connector/python/linux/python3/taos/cursor.py
@@ -27,7 +27,7 @@ class TDengineCursor(object):
     """
 
     def __init__(self, connection=None):
-        self._description = None
+        self._description = []
         self._rowcount = -1
         self._connection = None
         self._result = None
@@ -242,7 +242,7 @@ class TDengineCursor(object):
     def _reset_result(self):
         """Reset the result to unused version.
         """
-        self._description = None
+        self._description = []
         self._rowcount = -1
         if self._result is not None:
             CTaosInterface.freeResult(self._result)
diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py
index 71651afee1..f6fde2619b 100644
--- a/src/connector/python/windows/python2/taos/cursor.py
+++ b/src/connector/python/windows/python2/taos/cursor.py
@@ -27,7 +27,7 @@ class TDengineCursor(object):
     """
 
     def __init__(self, connection=None):
-        self._description = None
+        self._description = []
         self._rowcount = -1
         self._connection = None
         self._result = None
@@ -193,7 +193,7 @@ class TDengineCursor(object):
     def _reset_result(self):
         """Reset the result to unused version.
         """
-        self._description = None
+        self._description = []
         self._rowcount = -1
         if self._result is not None:
             CTaosInterface.freeResult(self._result)
diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py
index b813bba357..db66b99d3b 100644
--- a/src/connector/python/windows/python3/taos/cursor.py
+++ b/src/connector/python/windows/python3/taos/cursor.py
@@ -28,7 +28,7 @@ class TDengineCursor(object):
     """
 
     def __init__(self, connection=None):
-        self._description = None
+        self._description = []
         self._rowcount = -1
         self._connection = None
         self._result = None
@@ -194,7 +194,7 @@ class TDengineCursor(object):
     def _reset_result(self):
         """Reset the result to unused version.
         """
-        self._description = None
+        self._description = []
         self._rowcount = -1
         if self._result is not None:
             CTaosInterface.freeResult(self._result)

From e4dce56692fdec43a09267008eaeb4b7bbbb00c7 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Fri, 4 Dec 2020 11:30:21 +0800
Subject: [PATCH 057/118] [TD-785]Improve the robustness of python connector

---
 src/connector/python/linux/python2/setup.py           | 2 +-
 src/connector/python/linux/python2/taos/__init__.py   | 2 +-
 src/connector/python/linux/python3/setup.py           | 2 +-
 src/connector/python/linux/python3/taos/__init__.py   | 2 +-
 src/connector/python/windows/python2/setup.py         | 2 +-
 src/connector/python/windows/python2/taos/__init__.py | 2 +-
 src/connector/python/windows/python3/setup.py         | 2 +-
 src/connector/python/windows/python3/taos/__init__.py | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py
index b3daa98bdc..92a931b504 100644
--- a/src/connector/python/linux/python2/setup.py
+++ b/src/connector/python/linux/python2/setup.py
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.2",
+    version="2.0.3",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",
diff --git a/src/connector/python/linux/python2/taos/__init__.py b/src/connector/python/linux/python2/taos/__init__.py
index 5d21671eed..62e0536b6f 100644
--- a/src/connector/python/linux/python2/taos/__init__.py
+++ b/src/connector/python/linux/python2/taos/__init__.py
@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0.2'
+apilevel = '2.0.3'
 threadsafety = 0
 paramstyle = 'pyformat'
 
diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py
index f49ebe2b6d..655a12ad13 100644
--- a/src/connector/python/linux/python3/setup.py
+++ b/src/connector/python/linux/python3/setup.py
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.2",
+    version="2.0.3",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",
diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py
index c46131eeaf..8cf095ea68 100644
--- a/src/connector/python/linux/python3/taos/__init__.py
+++ b/src/connector/python/linux/python3/taos/__init__.py
@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0.2'
+apilevel = '2.0.3'
 threadsafety = 0
 paramstyle = 'pyformat'
 
diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py
index 34cb7a04d7..5ddbe83011 100644
--- a/src/connector/python/windows/python2/setup.py
+++ b/src/connector/python/windows/python2/setup.py
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.2",
+    version="2.0.3",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",
diff --git a/src/connector/python/windows/python2/taos/__init__.py b/src/connector/python/windows/python2/taos/__init__.py
index 5d21671eed..62e0536b6f 100644
--- a/src/connector/python/windows/python2/taos/__init__.py
+++ b/src/connector/python/windows/python2/taos/__init__.py
@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0.2'
+apilevel = '2.0.3'
 threadsafety = 0
 paramstyle = 'pyformat'
 
diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py
index c09644d330..ffed304c85 100644
--- a/src/connector/python/windows/python3/setup.py
+++ b/src/connector/python/windows/python3/setup.py
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.2",
+    version="2.0.3",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",
diff --git a/src/connector/python/windows/python3/taos/__init__.py b/src/connector/python/windows/python3/taos/__init__.py
index 0708b9544f..c6dd929a6a 100644
--- a/src/connector/python/windows/python3/taos/__init__.py
+++ b/src/connector/python/windows/python3/taos/__init__.py
@@ -3,7 +3,7 @@ from .connection import TDengineConnection
 from .cursor import TDengineCursor
 
 # Globals
-apilevel = '2.0.2'
+apilevel = '2.0.3'
 threadsafety = 0
 paramstyle = 'pyformat'
 

From 1ae57bbf222c34768e09c9d0c879b5398128a19e Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 4 Dec 2020 03:36:25 +0000
Subject: [PATCH 058/118] fix TD-2322

---
 src/tsdb/src/tsdbMemTable.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c
index 4c51608b5d..999e2deb41 100644
--- a/src/tsdb/src/tsdbMemTable.c
+++ b/src/tsdb/src/tsdbMemTable.c
@@ -786,9 +786,9 @@ static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pT
     if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) {
       tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
                 pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
+      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+      return -1;
     }
-    terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
-    return -1;
   }
 
   return 0;

From b8f38e0568a51814febb2bb3f8ae70b5484a51c7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 4 Dec 2020 12:47:37 +0800
Subject: [PATCH 059/118] [TD-2281]<fix>: filter unsupported mixed up query
 functions.

---
 src/client/src/tscFunctionImpl.c | 8 ++++----
 src/client/src/tscSQLParser.c    | 4 ++++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 56b7f052f7..594a4c39b5 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -4540,11 +4540,11 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) {
  *
  */
 int32_t functionCompatList[] = {
-    // count,       sum,      avg,       min,      max,  stddev,    percentile, apercentile, first,   last
-    1,          1,        1,         1,        1,      1,          1,           1,        1,      1,
-    // last_row,    top,    bottom,     spread,    twa,  leastsqr,     ts,       ts_dummy, tag_dummy, ts_z
+    // count,   sum,      avg,       min,      max,    stddev,    percentile,   apercentile, first,   last
+    1,          1,        1,         1,        1,      1,          1,           1,           1,      1,
+    // last_row,top,      bottom,    spread,   twa,    leastsqr,   ts,          ts_dummy, tag_dummy, ts_z
     4,         -1,       -1,         1,        1,      1,          1,           1,        1,     -1,
-    //  tag,       colprj,  tagprj,   arithmetic, diff, first_dist, last_dist,    interp      rate   irate
+    //  tag,    colprj,   tagprj,    arithmetic, diff, first_dist, last_dist,   interp    rate    irate
     1,          1,        1,         1,       -1,      1,          1,           5,        1,      1,
     // sum_rate, sum_irate, avg_rate, avg_irate
     1,          1,        1,         1,
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index abbe26204d..08b2a7db4e 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -2835,6 +2835,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
 
     if (functionCompatList[functionId] != factor) {
       return false;
+    } else {
+      if (factor == -1) { // two functions with the same -1 flag
+        return false;
+      }
     }
 
     if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) {

From 94756ccd947e62aa26166d899d0ac4620f517f11 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 4 Dec 2020 13:07:23 +0800
Subject: [PATCH 060/118] TD-2331

---
 src/dnode/inc/dnodeMInfos.h |   6 +-
 src/dnode/src/dnodeMInfos.c |  58 +++++++-------
 src/dnode/src/dnodeMgmt.c   |   4 +-
 src/dnode/src/dnodeModule.c |   6 +-
 src/inc/dnode.h             |   2 +-
 src/inc/taosmsg.h           |  14 ++--
 src/mnode/inc/mnodeMnode.h  |   2 +-
 src/mnode/inc/mnodeSdb.h    |   1 +
 src/mnode/src/mnodeMnode.c  | 146 +++++++++++++++++++++---------------
 src/mnode/src/mnodeSdb.c    |  25 ++++--
 src/sync/src/syncMain.c     |   2 +-
 11 files changed, 151 insertions(+), 115 deletions(-)

diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h
index 9c3c85c47e..2c3eef5d5d 100644
--- a/src/dnode/inc/dnodeMInfos.h
+++ b/src/dnode/inc/dnodeMInfos.h
@@ -24,9 +24,9 @@ extern "C" {
 
 int32_t dnodeInitMInfos();
 void    dnodeCleanupMInfos();
-void    dnodeUpdateMInfos(SMnodeInfos *minfos);
-void    dnodeUpdateEpSetForPeer(SRpcEpSet *epSet);
-void    dnodeGetMInfos(SMnodeInfos *minfos);
+void    dnodeUpdateMInfos(SMInfos *pMinfos);
+void    dnodeUpdateEpSetForPeer(SRpcEpSet *pEpSet);
+void    dnodeGetMInfos(SMInfos *pMinfos);
 bool    dnodeIsMasterEp(char *ep);
 
 #ifdef __cplusplus
diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c
index cefe44aebe..162de2243e 100644
--- a/src/dnode/src/dnodeMInfos.c
+++ b/src/dnode/src/dnodeMInfos.c
@@ -22,12 +22,12 @@
 #include "dnodeInt.h"
 #include "dnodeMInfos.h"
 
-static SMnodeInfos tsMInfos;
-static SRpcEpSet   tsMEpSet;
+static SMInfos   tsMInfos;
+static SRpcEpSet tsMEpSet;
 static pthread_mutex_t tsMInfosMutex;
 
-static void    dnodeResetMInfos(SMnodeInfos *minfos);
-static void    dnodePrintMInfos(SMnodeInfos *minfos);
+static void    dnodeResetMInfos(SMInfos *minfos);
+static void    dnodePrintMInfos(SMInfos *minfos);
 static int32_t dnodeReadMInfos();
 static int32_t dnodeWriteMInfos();
 
@@ -44,14 +44,14 @@ int32_t dnodeInitMInfos() {
 
 void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); }
 
-void dnodeUpdateMInfos(SMnodeInfos *minfos) {
-  if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) {
-    dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum);
+void dnodeUpdateMInfos(SMInfos *pMinfos) {
+  if (pMinfos->mnodeNum <= 0 || pMinfos->mnodeNum > 3) {
+    dError("invalid mnode infos, mnodeNum:%d", pMinfos->mnodeNum);
     return;
   }
 
-  for (int32_t i = 0; i < minfos->mnodeNum; ++i) {
-    SMnodeInfo *minfo = &minfos->mnodeInfos[i];
+  for (int32_t i = 0; i < pMinfos->mnodeNum; ++i) {
+    SMInfo *minfo = &pMinfos->mnodeInfos[i];
     minfo->mnodeId = htonl(minfo->mnodeId);
     if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) {
       dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp);
@@ -60,14 +60,14 @@ void dnodeUpdateMInfos(SMnodeInfos *minfos) {
   }
 
   pthread_mutex_lock(&tsMInfosMutex);
-  if (minfos->mnodeNum != tsMInfos.mnodeNum) {
-    dnodeResetMInfos(minfos);
+  if (pMinfos->mnodeNum != tsMInfos.mnodeNum) {
+    dnodeResetMInfos(pMinfos);
     dnodeWriteMInfos();
     sdbUpdateAsync();
   } else {
-    int32_t size = sizeof(SMnodeInfos);
-    if (memcmp(minfos, &tsMInfos, size) != 0) {
-      dnodeResetMInfos(minfos);
+    int32_t size = sizeof(SMInfos);
+    if (memcmp(pMinfos, &tsMInfos, size) != 0) {
+      dnodeResetMInfos(pMinfos);
       dnodeWriteMInfos();
       sdbUpdateAsync();
     }
@@ -99,11 +99,11 @@ bool dnodeIsMasterEp(char *ep) {
   return isMaster;
 }
 
-void dnodeGetMInfos(SMnodeInfos *minfos) {
+void dnodeGetMInfos(SMInfos *pMinfos) {
   pthread_mutex_lock(&tsMInfosMutex);
-  memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos));
+  memcpy(pMinfos, &tsMInfos, sizeof(SMInfos));
   for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) {
-    minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId);
+    pMinfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId);
   }
   pthread_mutex_unlock(&tsMInfosMutex);
 }
@@ -123,15 +123,15 @@ void dnodeGetEpSetForShell(SRpcEpSet *epSet) {
   pthread_mutex_unlock(&tsMInfosMutex);
 }
 
-static void dnodePrintMInfos(SMnodeInfos *minfos) {
-  dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse);
-  for (int32_t i = 0; i < minfos->mnodeNum; i++) {
-    dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp);
+static void dnodePrintMInfos(SMInfos *pMinfos) {
+  dInfo("print minfos, mnodeNum:%d inUse:%d", pMinfos->mnodeNum, pMinfos->inUse);
+  for (int32_t i = 0; i < pMinfos->mnodeNum; i++) {
+    dInfo("mnode index:%d, %s", pMinfos->mnodeInfos[i].mnodeId, pMinfos->mnodeInfos[i].mnodeEp);
   }
 }
 
-static void dnodeResetMInfos(SMnodeInfos *minfos) {
-  if (minfos == NULL) {
+static void dnodeResetMInfos(SMInfos *pMinfos) {
+  if (pMinfos == NULL) {
     tsMEpSet.numOfEps = 1;
     taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]);
 
@@ -142,10 +142,10 @@ static void dnodeResetMInfos(SMnodeInfos *minfos) {
     return;
   }
 
-  if (minfos->mnodeNum == 0) return;
+  if (pMinfos->mnodeNum == 0) return;
 
-  int32_t size = sizeof(SMnodeInfos);
-  memcpy(&tsMInfos, minfos, size);
+  int32_t size = sizeof(SMInfos);
+  memcpy(&tsMInfos, pMinfos, size);
 
   tsMEpSet.inUse = tsMInfos.inUse;
   tsMEpSet.numOfEps = tsMInfos.mnodeNum;
@@ -153,7 +153,7 @@ static void dnodeResetMInfos(SMnodeInfos *minfos) {
     taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]);
   }
 
-  dnodePrintMInfos(minfos);
+  dnodePrintMInfos(pMinfos);
 }
 
 static int32_t dnodeReadMInfos() {
@@ -162,7 +162,7 @@ static int32_t dnodeReadMInfos() {
   char *      content = calloc(1, maxLen + 1);
   cJSON *     root = NULL;
   FILE *      fp = NULL;
-  SMnodeInfos minfos = {0};
+  SMInfos     minfos = {0};
 
   char file[TSDB_FILENAME_LEN + 20] = {0};
   sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
@@ -241,7 +241,7 @@ PARSE_MINFOS_OVER:
   terrno = 0;
 
   for (int32_t i = 0; i < minfos.mnodeNum; ++i) {
-    SMnodeInfo *mInfo = &minfos.mnodeInfos[i];
+    SMInfo *mInfo = &minfos.mnodeInfos[i];
     dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
   }
   dnodeResetMInfos(&minfos);
diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 5c01f64716..15378c77c1 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -472,8 +472,8 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
   }
 
   SStatusRsp *pStatusRsp = pMsg->pCont;
-  SMnodeInfos *minfos = &pStatusRsp->mnodes;
-  dnodeUpdateMInfos(minfos);
+  SMInfos *pMinfos = &pStatusRsp->mnodes;
+  dnodeUpdateMInfos(pMinfos);
 
   SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
   pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c
index 7faa3c8913..f664618f51 100644
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@@ -147,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
   }
 }
 
-bool dnodeStartMnode(SMnodeInfos *minfos) {
-  SMnodeInfos *mnodes = minfos;
+bool dnodeStartMnode(SMInfos *pMinfos) {
+  SMInfos *pMnodes = pMinfos;
 
   if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) {
     dDebug("mnode module is already started, module status:%d", tsModuleStatus);
@@ -159,7 +159,7 @@ bool dnodeStartMnode(SMnodeInfos *minfos) {
   dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus);
   dnodeProcessModuleStatus(moduleStatus);
 
-  sdbUpdateSync(mnodes);
+  sdbUpdateSync(pMnodes);
 
   return true;
 }
diff --git a/src/inc/dnode.h b/src/inc/dnode.h
index eef4490800..1efaa4a24b 100644
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@@ -45,7 +45,7 @@ void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
 int32_t dnodeGetDnodeId();
 void    dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port);
 bool    dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
-bool    dnodeStartMnode(SMnodeInfos *minfos);
+bool    dnodeStartMnode(SMInfos *pMinfos);
 
 void  dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
 void  dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index b4d3bec958..e8e3029244 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -591,13 +591,13 @@ typedef struct {
 typedef struct {
   int32_t mnodeId;
   char    mnodeEp[TSDB_EP_LEN];
-} SMnodeInfo;
+} SMInfo;
 
 typedef struct {
-  int8_t     inUse;
-  int8_t     mnodeNum;
-  SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA];
-} SMnodeInfos;
+  int8_t inUse;
+  int8_t mnodeNum;
+  SMInfo mnodeInfos[TSDB_MAX_REPLICA];
+} SMInfos;
 
 typedef struct {
   int32_t  numOfMnodes;               // tsNumOfMnodes
@@ -632,7 +632,7 @@ typedef struct {
 } SStatusMsg;
 
 typedef struct {
-  SMnodeInfos   mnodes;
+  SMInfos       mnodes;
   SDnodeCfg     dnodeCfg;
   SVgroupAccess vgAccess[];
 } SStatusRsp;
@@ -761,7 +761,7 @@ typedef struct {
 typedef struct {
   int32_t dnodeId;
   char    dnodeEp[TSDB_EP_LEN];  // end point, hostname:port
-  SMnodeInfos mnodes;
+  SMInfos mnodes;
 } SCreateMnodeMsg;
 
 typedef struct {
diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h
index 10cbcebe22..93f2fa11ea 100644
--- a/src/mnode/inc/mnodeMnode.h
+++ b/src/mnode/inc/mnodeMnode.h
@@ -48,7 +48,7 @@ void    mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet);
 char*   mnodeGetMnodeMasterEp();
 
 void    mnodeGetMnodeInfos(void *mnodes);
-void    mnodeUpdateMnodeEpSet();
+void    mnodeUpdateMnodeEpSet(SMInfos *pMnodes);
 
 #ifdef __cplusplus
 }
diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h
index 90c4eac40a..31ea2da640 100644
--- a/src/mnode/inc/mnodeSdb.h
+++ b/src/mnode/inc/mnodeSdb.h
@@ -89,6 +89,7 @@ void*   sdbGetTableByRid(int64_t rid);
 bool    sdbIsMaster();
 bool    sdbIsServing();
 void    sdbUpdateMnodeRoles();
+int32_t sdbGetReplicaNum();
 
 int32_t sdbInsertRow(SSdbRow *pRow);
 int32_t sdbDeleteRow(SSdbRow *pRow);
diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c
index d20d51f82b..ba0b5a1865 100644
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@@ -34,14 +34,14 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-int64_t              tsMnodeRid = -1;
-static void *        tsMnodeSdb = NULL;
-static int32_t       tsMnodeUpdateSize = 0;
-static SRpcEpSet     tsMnodeEpSetForShell;
-static SRpcEpSet     tsMnodeEpSetForPeer;
-static SMnodeInfos   tsMnodeInfos;
-static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+int64_t          tsMnodeRid = -1;
+static void *    tsMnodeSdb = NULL;
+static int32_t   tsMnodeUpdateSize = 0;
+static SRpcEpSet tsMEpForShell;
+static SRpcEpSet tsMEpForPeer;
+static SMInfos   tsMInfos;
+static int32_t   mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t   mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
 #if defined(LINUX)
   static pthread_rwlock_t         tsMnodeLock;
@@ -127,7 +127,7 @@ static int32_t mnodeMnodeActionRestored() {
     mnodeCancelGetNextMnode(pIter);
   }
 
-  mnodeUpdateMnodeEpSet();
+  mnodeUpdateMnodeEpSet(NULL);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -199,93 +199,119 @@ void mnodeCancelGetNextMnode(void *pIter) {
   sdbFreeIter(tsMnodeSdb, pIter);
 }
 
-void mnodeUpdateMnodeEpSet() {
-  mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum());
+void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) {
+  bool    set = false;
+  SMInfos mInfos = {0};
+  mInfo("vgId:1, update mnodes epSet, numOfEps:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos);
+
+  if (pMinfos != NULL) {
+    set = true;
+    mInfos = *pMinfos;
+  }
+  else {
+    int32_t index = 0;
+    void *  pIter = NULL;
+    while (1) {
+      SMnodeObj *pMnode = NULL;
+      pIter = mnodeGetNextMnode(pIter, &pMnode);
+      if (pMnode == NULL) break;
+
+      SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
+      if (pDnode != NULL) {
+        set = true;
+        mInfos.mnodeInfos[index].mnodeId = pMnode->mnodeId;
+        strcpy(mInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp);
+        if (pMnode->role == TAOS_SYNC_ROLE_MASTER) mInfos.inUse = index;
+        index++;
+      } else {
+        set = false;
+      }
+
+      mnodeDecDnodeRef(pDnode);
+      mnodeDecMnodeRef(pMnode);
+    }
+
+    mInfos.mnodeNum = index;
+    if (sdbGetReplicaNum() != mInfos.mnodeNum) {
+      set = false;
+      mDebug("vgId:1, mnodes info not synced, cfg:%d current:%d", sdbGetReplicaNum(), mInfos.mnodeNum);
+    }
+  }
 
   mnodeMnodeWrLock();
 
-  memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet));
-  memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet));
-  memset(&tsMnodeInfos, 0, sizeof(SMnodeInfos));
+  if (set) {
+    memset(&tsMEpForShell, 0, sizeof(SRpcEpSet));
+    memset(&tsMEpForPeer, 0, sizeof(SRpcEpSet));
+    memcpy(&tsMInfos, &mInfos, sizeof(SMInfos));
+    tsMEpForShell.inUse = tsMInfos.inUse;
+    tsMEpForPeer.inUse = tsMInfos.inUse;
+    tsMEpForShell.numOfEps = tsMInfos.mnodeNum;
+    tsMEpForPeer.numOfEps = tsMInfos.mnodeNum;
 
-  int32_t index = 0;
-  void *  pIter = NULL;
-  while (1) {
-    SMnodeObj *pMnode = NULL;
-    pIter = mnodeGetNextMnode(pIter, &pMnode);
-    if (pMnode == NULL) break;
+    mInfo("vgId:1, mnodes epSet is set, num:%d inUse:%d", tsMInfos.mnodeNum, tsMInfos.inUse);
+    for (int index = 0; index < mInfos.mnodeNum; ++index) {
+      SMInfo *pInfo = &tsMInfos.mnodeInfos[index];
+      taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForShell.fqdn[index], &tsMEpForShell.port[index]);
+      taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForPeer.fqdn[index], &tsMEpForPeer.port[index]);
+      tsMEpForPeer.port[index] = tsMEpForPeer.port[index] + TSDB_PORT_DNODEDNODE;
 
-    SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
-    if (pDnode != NULL) {
-      strcpy(tsMnodeEpSetForShell.fqdn[index], pDnode->dnodeFqdn);
-      tsMnodeEpSetForShell.port[index] = htons(pDnode->dnodePort);
-      mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForShell.fqdn[index], htons(tsMnodeEpSetForShell.port[index]));      
+      mInfo("vgId:1, mnode:%d, fqdn:%s shell:%u peer:%u", pInfo->mnodeId, tsMEpForShell.fqdn[index],
+            tsMEpForShell.port[index], tsMEpForPeer.port[index]);
 
-      strcpy(tsMnodeEpSetForPeer.fqdn[index], pDnode->dnodeFqdn);
-      tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE);
-      mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index]));
-
-      tsMnodeInfos.mnodeInfos[index].mnodeId = htonl(pMnode->mnodeId);
-      strcpy(tsMnodeInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp);
-
-      if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
-        tsMnodeEpSetForShell.inUse = index;
-        tsMnodeEpSetForPeer.inUse = index;
-        tsMnodeInfos.inUse = index;
-      }
-
-      mInfo("mnode:%d, ep:%s %s", pDnode->dnodeId, pDnode->dnodeEp, pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
-      index++;
+      tsMEpForShell.port[index] = htons(tsMEpForShell.port[index]);
+      tsMEpForPeer.port[index] = htons(tsMEpForPeer.port[index]);
+      pInfo->mnodeId = htonl(pInfo->mnodeId);
+    }
+  } else {
+    mInfo("vgId:1, mnodes epSet not set, num:%d inUse:%d", tsMInfos.mnodeNum, tsMInfos.inUse);
+    for (int index = 0; index < tsMInfos.mnodeNum; ++index) {
+      mInfo("vgId:1, index:%d, ep:%s:%u", index, tsMEpForShell.fqdn[index], htons(tsMEpForShell.port[index]));
     }
-
-    mnodeDecDnodeRef(pDnode);
-    mnodeDecMnodeRef(pMnode);
   }
 
-  tsMnodeInfos.mnodeNum = index;
-  tsMnodeEpSetForShell.numOfEps = index;
-  tsMnodeEpSetForPeer.numOfEps = index;
-
   mnodeMnodeUnLock();
 }
 
 void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) {
   mnodeMnodeRdLock();
-  *epSet = tsMnodeEpSetForPeer;
+  *epSet = tsMEpForPeer;
   mnodeMnodeUnLock();
 
+  mTrace("vgId:1, mnodes epSet for peer is returned, num:%d inUse:%d", tsMEpForPeer.numOfEps, tsMEpForPeer.inUse);
   for (int32_t i = 0; i < epSet->numOfEps; ++i) {
     if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
       epSet->inUse = (i + 1) % epSet->numOfEps;
-      mTrace("mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
+      mTrace("vgId:1, mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
     } else {
-      mTrace("mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
+      mTrace("vgId:1, mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
     }
   }
 }
 
 void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) {
   mnodeMnodeRdLock();
-  *epSet = tsMnodeEpSetForShell;
+  *epSet = tsMEpForShell;
   mnodeMnodeUnLock();
 
+  mTrace("vgId:1, mnodes epSet for shell is returned, num:%d inUse:%d", tsMEpForShell.numOfEps, tsMEpForShell.inUse);
   for (int32_t i = 0; i < epSet->numOfEps; ++i) {
     if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
       epSet->inUse = (i + 1) % epSet->numOfEps;
-      mTrace("mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
+      mTrace("vgId:1, mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
     } else {
-      mTrace("mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
+      mTrace("vgId:1, mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
     }
   }
 }
 
 char* mnodeGetMnodeMasterEp() {
-  return tsMnodeInfos.mnodeInfos[tsMnodeInfos.inUse].mnodeEp;
+  return tsMInfos.mnodeInfos[tsMInfos.inUse].mnodeEp;
 }
 
-void mnodeGetMnodeInfos(void *mnodeInfos) {
+void mnodeGetMnodeInfos(void *pMinfos) {
   mnodeMnodeRdLock();
-  *(SMnodeInfos *)mnodeInfos = tsMnodeInfos;
+  *(SMInfos *)pMinfos = tsMInfos;
   mnodeMnodeUnLock();
 }
 
@@ -298,7 +324,7 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
   } else {
     pCreate->dnodeId = htonl(dnodeId);
     tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
-    pCreate->mnodes = tsMnodeInfos;
+    pCreate->mnodes = tsMInfos;
     bool found = false;
     for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) {
       if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) {
@@ -336,7 +362,7 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) {
     mError("failed to create mnode, reason:%s", tstrerror(code));
   } else {
     mDebug("mnode is created successfully");
-    mnodeUpdateMnodeEpSet();
+    mnodeUpdateMnodeEpSet(NULL);
     sdbUpdateAsync();
   }
 
@@ -380,7 +406,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId) {
     mnodeDecMnodeRef(pMnode);
   }
 
-  mnodeUpdateMnodeEpSet();
+  mnodeUpdateMnodeEpSet(NULL);
   sdbUpdateAsync();
 }
 
@@ -400,7 +426,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) {
 
   sdbDecRef(tsMnodeSdb, pMnode);
 
-  mnodeUpdateMnodeEpSet();
+  mnodeUpdateMnodeEpSet(NULL);
   sdbUpdateAsync();
 
   return code;
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 2ef758baf1..a8cd595fc8 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -224,11 +224,13 @@ void sdbUpdateMnodeRoles() {
       sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, syncRole[pMnode->role]);
       if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbMgmt.role = pMnode->role;
       mnodeDecMnodeRef(pMnode);
+    } else {
+      sdbDebug("vgId:1, mnode:%d not found", roles.nodeId[i]);
     }
   }
 
   mnodeUpdateClusterId();
-  mnodeUpdateMnodeEpSet();
+  mnodeUpdateMnodeEpSet(NULL);
 }
 
 static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
@@ -308,18 +310,20 @@ void sdbUpdateAsync() {
 }
 
 void sdbUpdateSync(void *pMnodes) {
-  SMnodeInfos *mnodes = pMnodes;
+  SMInfos *pMinfos = pMnodes;
   if (!mnodeIsRunning()) {
     mDebug("vgId:1, mnode not start yet, update sync config later");
     return;
   }
 
-  mDebug("vgId:1, update sync config in sync module, mnodes:%p", pMnodes);
+  mDebug("vgId:1, update sync config, pMnodes:%p", pMnodes);
 
   SSyncCfg syncCfg = {0};
   int32_t  index = 0;
 
-  if (mnodes == NULL) {
+  if (pMinfos == NULL) {
+    mDebug("vgId:1, mInfos not input, use mInfos in sdb, numOfMnodes:%d", syncCfg.replica);
+
     void *pIter = NULL;
     while (1) {
       SMnodeObj *pMnode = NULL;
@@ -339,16 +343,17 @@ void sdbUpdateSync(void *pMnodes) {
       mnodeDecMnodeRef(pMnode);
     }
     syncCfg.replica = index;
-    mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
   } else {
-    for (index = 0; index < mnodes->mnodeNum; ++index) {
-      SMnodeInfo *node = &mnodes->mnodeInfos[index];
+    mDebug("vgId:1, mInfos input, numOfMnodes:%d", syncCfg.replica);
+
+    for (index = 0; index < pMinfos->mnodeNum; ++index) {
+      SMInfo *node = &pMinfos->mnodeInfos[index];
       syncCfg.nodeInfo[index].nodeId = node->mnodeId;
       taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort);
       syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC;
     }
     syncCfg.replica = index;
-    mDebug("vgId:1, mnodes info input, numOfMnodes:%d", syncCfg.replica);
+    mnodeUpdateMnodeEpSet(pMnodes);
   }
 
   syncCfg.quorum = (syncCfg.replica == 1) ? 1 : 2;
@@ -1103,3 +1108,7 @@ static void *sdbWorkerFp(void *pWorker) {
 
   return NULL;
 }
+
+int32_t sdbGetReplicaNum() {
+  return tsSdbMgmt.cfg.replica;
+}
\ No newline at end of file
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
index adac532f2d..7e9e8cc2d0 100644
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@@ -548,7 +548,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
   pPeer->pSyncNode = pNode;
   pPeer->refCount = 1;
 
-  sInfo("%s, it is configured", pPeer->id);
+  sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port);
   int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
   if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
     int32_t checkMs = 100 + (pNode->vgId * 10) % 100;

From 030fc359587bd64ff99adae36ee5d49f1729c528 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Fri, 4 Dec 2020 13:48:14 +0800
Subject: [PATCH 061/118] change version number

---
 cmake/version.inc           | 2 +-
 snap/snapcraft.yaml         | 4 ++--
 src/connector/go            | 2 +-
 src/connector/grafanaplugin | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cmake/version.inc b/cmake/version.inc
index 2f0ec81aea..948c7d2d0b 100644
--- a/cmake/version.inc
+++ b/cmake/version.inc
@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.8.0")
+  SET(TD_VER_NUMBER "2.0.9.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index 059c0650c2..b5d06a4adb 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,6 +1,6 @@
 name: tdengine
 base: core18
-version: '2.0.8.0'
+version: '2.0.9.0'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.0.8.0
+      - usr/lib/libtaos.so.2.0.9.0
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 
diff --git a/src/connector/go b/src/connector/go
index 8c58c512b6..050667e5b4 160000
--- a/src/connector/go
+++ b/src/connector/go
@@ -1 +1 @@
-Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766
+Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f
diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin
index d598db167e..ec77d9049a 160000
--- a/src/connector/grafanaplugin
+++ b/src/connector/grafanaplugin
@@ -1 +1 @@
-Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c
+Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944

From f538fe33bec59029b528d13f45929b46aae6bff3 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 4 Dec 2020 14:10:42 +0800
Subject: [PATCH 062/118] TD-2331

---
 src/mnode/src/mnodeMnode.c | 15 +++++++++------
 src/mnode/src/mnodeSdb.c   |  2 +-
 src/sync/src/syncMain.c    |  2 +-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c
index ba0b5a1865..68acae7dec 100644
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@@ -202,7 +202,7 @@ void mnodeCancelGetNextMnode(void *pIter) {
 void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) {
   bool    set = false;
   SMInfos mInfos = {0};
-  mInfo("vgId:1, update mnodes epSet, numOfEps:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos);
+  mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos);
 
   if (pMinfos != NULL) {
     set = true;
@@ -232,9 +232,9 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) {
     }
 
     mInfos.mnodeNum = index;
-    if (sdbGetReplicaNum() != mInfos.mnodeNum) {
+    if (mInfos.mnodeNum < sdbGetReplicaNum()) {
       set = false;
-      mDebug("vgId:1, mnodes info not synced, cfg:%d current:%d", sdbGetReplicaNum(), mInfos.mnodeNum);
+      mDebug("vgId:1, mnodes info not synced, current:%d syncCfgNum:%d", mInfos.mnodeNum, sdbGetReplicaNum());
     }
   }
 
@@ -316,15 +316,13 @@ void mnodeGetMnodeInfos(void *pMinfos) {
 }
 
 static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
-  mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp);
-
   SCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SCreateMnodeMsg));
   if (pCreate == NULL) {
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   } else {
     pCreate->dnodeId = htonl(dnodeId);
     tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
-    pCreate->mnodes = tsMInfos;
+    mnodeGetMnodeInfos(&pCreate->mnodes);
     bool found = false;
     for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) {
       if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) {
@@ -338,6 +336,11 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
     }
   }
 
+  mDebug("dnode:%d, send create mnode msg to dnode %s, numOfMnodes:%d", dnodeId, dnodeEp, pCreate->mnodes.mnodeNum);
+  for (int32_t i = 0; i < pCreate->mnodes.mnodeNum; ++i) {
+    mDebug("index:%d, mnodeId:%d ep:%s", i, pCreate->mnodes.mnodeInfos[i].mnodeId, pCreate->mnodes.mnodeInfos[i].mnodeEp);
+  }
+
   SRpcMsg rpcMsg = {0};
   rpcMsg.pCont = pCreate;
   rpcMsg.contLen = sizeof(SCreateMnodeMsg);
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index a8cd595fc8..6cc4e09735 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -344,7 +344,7 @@ void sdbUpdateSync(void *pMnodes) {
     }
     syncCfg.replica = index;
   } else {
-    mDebug("vgId:1, mInfos input, numOfMnodes:%d", syncCfg.replica);
+    mDebug("vgId:1, mInfos input, numOfMnodes:%d", pMinfos->mnodeNum);
 
     for (index = 0; index < pMinfos->mnodeNum; ++index) {
       SMInfo *node = &pMinfos->mnodeInfos[index];
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
index 7e9e8cc2d0..b73ca27ce9 100644
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@@ -1134,7 +1134,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
 
   pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL;
   if (pPeer == NULL) {
-    sError("vgId:%d, peer:%s not configured", pNode->vgId, firstPkt.fqdn);
+    sError("vgId:%d, peer:%s:%u not configured", pNode->vgId, firstPkt.fqdn, firstPkt.port);
     taosCloseSocket(connFd);
     // syncSendVpeerCfgMsg(sync);
   } else {

From 1e4c61bb267f6e0fd481d7fc3986a90aa89fdf2b Mon Sep 17 00:00:00 2001
From: zyyang <zyyang@taosdata.com>
Date: Fri, 4 Dec 2020 14:13:40 +0800
Subject: [PATCH 063/118] change jdbc version number

---
 cmake/install.inc                 | 2 +-
 src/connector/jdbc/CMakeLists.txt | 2 +-
 src/connector/jdbc/deploy-pom.xml | 2 +-
 src/connector/jdbc/pom.xml        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmake/install.inc b/cmake/install.inc
index 4b2d4828f8..8418612d4c 100755
--- a/cmake/install.inc
+++ b/cmake/install.inc
@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
   #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
   #INSTALL(TARGETS shell RUNTIME DESTINATION .)
   IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt
index 701a39b209..0eb3eb21ce 100644
--- a/src/connector/jdbc/CMakeLists.txt
+++ b/src/connector/jdbc/CMakeLists.txt
@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
   ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
     POST_BUILD
     COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH}
     COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
     COMMENT "build jdbc driver")
   ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})
diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml
index 1dc2625e62..4564bde81e 100755
--- a/src/connector/jdbc/deploy-pom.xml
+++ b/src/connector/jdbc/deploy-pom.xml
@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.13</version>
+  <version>2.0.14</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>
diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml
index 3d1f402435..7e087ebd9b 100755
--- a/src/connector/jdbc/pom.xml
+++ b/src/connector/jdbc/pom.xml
@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.13</version>
+    <version>2.0.14</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>

From 7d22ca3fb9238bb611f96329f63246268a0286cf Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 4 Dec 2020 14:49:10 +0800
Subject: [PATCH 064/118] [TD-2236]<fix>: fix bugs in groupby normal colunms +
 last query

---
 src/client/src/tscFunctionImpl.c | 55 +++++++++++++++++++++-----------
 1 file changed, 37 insertions(+), 18 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 594a4c39b5..eea4d93de9 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -54,8 +54,8 @@
 
 #define DO_UPDATE_TAG_COLUMNS(ctx, ts)                                           \
   do {                                                                           \
-    for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) {                  \
-      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i];                     \
+    for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) {                  \
+      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i];                     \
       if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) {                             \
         __ctx->tag.i64Key = (ts); \
         __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
@@ -66,8 +66,8 @@
 
 #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx)                   \
   do {                                                          \
-    for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \
-      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i];    \
+    for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
+      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i];    \
       aAggs[TSDB_FUNC_TAG].xFunction(__ctx);                    \
     }                                                           \
   } while (0);
@@ -305,7 +305,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) {
     *type = (int16_t)dataType;
     *bytes = (int16_t)dataBytes;
-    *interBytes = dataBytes;
+    *interBytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo));
   } else if (functionId == TSDB_FUNC_SPREAD) {
     *type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
@@ -1169,8 +1169,8 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp
         if ((*(int32_t *)output < v) ^ isMin) {
           *(int32_t *)output = v;
           
-          for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) {
-            SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i];
+          for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) {
+            SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j];
             aAggs[TSDB_FUNC_TAG].xFunction(__ctx);
           }
           
@@ -1679,16 +1679,35 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
   }
-  
-  SET_VAL(pCtx, 1, 1);
-  memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes);
-  
-  TSKEY ts = pCtx->ptsList[index];
-  DO_UPDATE_TAG_COLUMNS(pCtx, ts);
-  
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  pResInfo->hasResult = DATA_SET_FLAG;
-  pResInfo->complete = true;  // set query completed
+
+  // the scan order is not the required order, ignore it
+  if (pCtx->order != pCtx->param[0].i64Key) {
+    return;
+  }
+
+  if (pCtx->order == TSDB_ORDER_DESC) {
+    SET_VAL(pCtx, 1, 1);
+    memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes);
+
+    TSKEY ts = pCtx->ptsList[index];
+    DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+    pResInfo->complete = true;  // set query completed
+  } else { // in case of ascending order check, all data needs to be checked
+    SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx);
+    TSKEY ts = pCtx->ptsList[index];
+
+    char* buf = GET_ROWCELL_INTERBUF(pResInfo);
+    if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) {
+      pResInfo->hasResult = DATA_SET_FLAG;
+      memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes);
+
+      *(TSKEY*)buf = ts;
+      DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+    }
+  }
 }
 
 static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) {
@@ -1711,7 +1730,7 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind
 
 static void last_dist_function(SQLFunctionCtx *pCtx) {
   /*
-   * 1. for scan data in asc order, no need to check data
+   * 1. for scan data is not the required order
    * 2. for data blocks that are not loaded, no need to check data
    */
   if (pCtx->order != pCtx->param[0].i64Key) {

From f91e989fd3fae4bc29919abf79d51581e3b4789c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 4 Dec 2020 15:12:23 +0800
Subject: [PATCH 065/118] [TD-2199]<fix>: fix bugs in percentile in case of no
 data in table.

---
 src/client/src/tscFunctionImpl.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index eea4d93de9..7921399330 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -2466,7 +2466,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
 static void percentile_function(SQLFunctionCtx *pCtx) {
   int32_t notNullElems = 0;
   
-  SResultRowCellInfo *    pResInfo = GET_RES_INFO(pCtx);
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
 
   // the first stage, only acquire the min/max value
@@ -2567,12 +2567,14 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) {
   double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey;
   
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket;
-  
-  if (pMemBucket->total > 0) {  // check for null
-    *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v);
-  } else {
+  SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo);
+
+  tMemBucket * pMemBucket = ppInfo->pMemBucket;
+  if (pMemBucket == NULL || pMemBucket->total == 0) {  // check for null
+    assert(ppInfo->numOfElems == 0);
     setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
+  } else {
+    *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v);
   }
   
   tMemBucketDestroy(pMemBucket);

From 0fad15ebb21f459f9415bcb1a81146bc2f0216aa Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 4 Dec 2020 15:12:56 +0800
Subject: [PATCH 066/118] [TD-2236] add test cases.

---
 tests/script/general/parser/groupby.sim | 38 +++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim
index 19d9ae84cb..fbe4345a21 100644
--- a/tests/script/general/parser/groupby.sim
+++ b/tests/script/general/parser/groupby.sim
@@ -606,6 +606,44 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00
 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; 
 sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; 
 
+print =================>td-2236
+sql select first(ts),last(ts) from t1 group by c;
+if $rows != 4 then
+  return -1
+endi
+
+if $data00 != @20-03-27 04:11:16.000@ then
+  return -1
+endi
+
+if $data01 != @20-03-27 04:21:16.000@ then
+  return -1
+endi
+
+if $data10 != @20-03-27 04:11:17.000@ then
+  return -1
+endi
+
+if $data11 != @20-03-27 04:31:17.000@ then
+  return -1
+endi
+
+if $data20 != @20-03-27 04:11:18.000@ then
+  return -1
+endi
+
+if $data21 != @20-03-27 04:51:18.000@ then
+  return -1
+endi
+
+if $data30 != @20-03-27 04:11:19.000@ then
+  return -1
+endi
+
+if $data31 != @20-03-27 05:10:19.000@ then
+  return -1
+endi
+
 #sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732'  interval(1m) sliding(15s) group by tbname,t1,t2;
 #if $rows != 40 then
 #  return -1

From 03f088e98beabd90160c5ffee5ae3dca663feac3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 4 Dec 2020 15:52:27 +0800
Subject: [PATCH 067/118] [TD-2298] add test cases.

---
 tests/script/general/parser/groupby.sim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim
index fbe4345a21..44bae6f242 100644
--- a/tests/script/general/parser/groupby.sim
+++ b/tests/script/general/parser/groupby.sim
@@ -606,7 +606,7 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00
 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; 
 sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; 
 
-print =================>td-2236
+print =================>TD-2236
 sql select first(ts),last(ts) from t1 group by c;
 if $rows != 4 then
   return -1

From 4a8f42223dfd5e81386d01779d960f8632267d04 Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Fri, 4 Dec 2020 07:57:11 +0000
Subject: [PATCH 068/118] [TD-2318]<fix>: crash when 5 or more unions are used

---
 src/client/inc/tscUtil.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index cfcecc6582..2c8641da76 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -92,7 +92,7 @@ typedef struct SVgroupTableInfo {
 } SVgroupTableInfo;
 
 static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) {
-  assert(pCmd != NULL && subClauseIndex >= 0 && subClauseIndex < TSDB_MAX_UNION_CLAUSE);
+  assert(pCmd != NULL && subClauseIndex >= 0);
 
   if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) {
     return NULL;

From 7aa282c557108301bf26e563d51cad6147a6651b Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Fri, 4 Dec 2020 16:31:38 +0800
Subject: [PATCH 069/118] [TD-2337]/<fix>: fix *Ref is not there* when unable
 to establish connection

---
 src/client/src/tscSql.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 70b91bd685..fae5b5856f 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -292,16 +292,18 @@ void taos_close(TAOS *taos) {
   pObj->signature = NULL;
   taosTmrStopA(&(pObj->pTimer));
 
-  SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
-  if (pHb != NULL) {
-    if (pHb->rpcRid > 0) {  // wait for rsp from dnode
-      rpcCancelRequest(pHb->rpcRid);
-      pHb->rpcRid = -1;
-    }
+  if (pObj->hbrid > 0) {
+    SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
+    if (pHb != NULL) {
+      if (pHb->rpcRid > 0) {  // wait for rsp from dnode
+        rpcCancelRequest(pHb->rpcRid);
+        pHb->rpcRid = -1;
+      }
 
-    tscDebug("%p HB is freed", pHb);
-    taos_free_result(pHb);
-    taosReleaseRef(tscObjRef, pHb->self);
+      tscDebug("%p HB is freed", pHb);
+      taos_free_result(pHb);
+      taosReleaseRef(tscObjRef, pHb->self);
+    }
   }
 
   int32_t ref = T_REF_DEC(pObj);

From 09e727544dadf5b1425c2857e58821d6588a08e9 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 4 Dec 2020 17:22:19 +0800
Subject: [PATCH 070/118] [TD-225] fix bugs in heartBeatactivitytimer

---
 src/client/src/tscServer.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 5e3acaa0b7..a011ac2f3a 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -191,9 +191,16 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
 void tscProcessActivityTimer(void *handle, void *tmrId) {
   int64_t rid = (int64_t) handle;
   STscObj *pObj = taosAcquireRef(tscRefId, rid);
-  if (pObj == NULL) return; 
+  if (pObj == NULL) {
+    return;
+  }
 
   SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid);
+  if (pHB == NULL) {
+    taosReleaseRef(tscRefId, rid);
+    return;
+  }
+
   assert(pHB->self == pObj->hbrid);
 
   pHB->retry = 0;

From 93413d53f6e826b1c6381f034eab82018ad0094c Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Fri, 4 Dec 2020 17:52:50 +0800
Subject: [PATCH 071/118] [TD-2267]<fix>: remove redundant macro definition

---
 src/inc/taosdef.h | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index ec1e1fc330..ca20293392 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -428,11 +428,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_PORT_SYNC                  10
 #define TSDB_PORT_HTTP                  11
 #define TSDB_PORT_ARBITRATOR            12
-#define TSDB_PORT_DNODESHELL 0
-#define TSDB_PORT_DNODEDNODE 5
-#define TSDB_PORT_SYNC       10
-#define TSDB_PORT_HTTP       11
-#define TSDB_PORT_ARBITRATOR 12
 
 #define TSDB_MAX_WAL_SIZE    (1024*1024)
 

From d82de105c9f987dcb84d33c42b97c412f5a53633 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Fri, 4 Dec 2020 18:40:52 +0800
Subject: [PATCH 072/118] [TD-2185]<test>add test case

---
 tests/pytest/fulltest.sh      |  1 +
 tests/pytest/query/bug2117.py | 27 +++++++++++++++++++++------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index 448a0413e0..8dbc4e3bb0 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -160,6 +160,7 @@ python3 ./test.py -f query/bug1874.py
 python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
 python3 ./test.py -f query/bug2218.py
+python3 ./test.py -f query/bug2117.py
 python3 ./test.py -f query/sliding.py 
 
 #stream
diff --git a/tests/pytest/query/bug2117.py b/tests/pytest/query/bug2117.py
index 1158b78a2a..f637558d79 100644
--- a/tests/pytest/query/bug2117.py
+++ b/tests/pytest/query/bug2117.py
@@ -33,14 +33,29 @@ class TDTestCase:
         for i in range(insertRows):
             ret = tdSql.execute(
                 "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" %
-                (t0+i,i%100,i/2,i%41,i%100,i%100,i*1.0,i%2,'taos'+str(i%100),'涛思'+str(i%100)))
+                (t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i%43),'涛思'+str(i%41)))
         print("==========step2")
         print("test last with group by normal_col ")
-        tdSql.query('select last(c1) from mt0 group by c3')
-        tdSql.checkData(0,0,84)
-        tdSql.checkData(0,1,85)
-        
-
+        tdSql.query('select last(*) from mt0 group by c3')
+        tdSql.checkData(0,1,84)
+        tdSql.checkData(0,9,'涛思0')
+        tdSql.checkData(1,1,85)
+        tdSql.checkData(1,9,'涛思1')
+        tdSql.query('select last(*) from mt0 group by c7')
+        tdSql.checkData(0,1,98)
+        tdSql.checkData(0,9,'涛思14')
+        tdSql.checkData(1,1,99)
+        tdSql.checkData(1,9,'涛思15')
+        tdSql.query('select last(*) from mt0 group by c8')
+        tdSql.checkData(0,3,5)
+        tdSql.checkData(0,4,20)
+        tdSql.checkData(3,1,92)
+        tdSql.checkData(3,9,'涛思8')
+        tdSql.query('select last(*) from mt0 group by c9')
+        tdSql.checkData(0,3,0)
+        tdSql.checkData(0,8,'taos38')
+        tdSql.checkData(40,1,83)
+        tdSql.checkData(40,3,40)
         
     def stop(self):
         tdSql.close()

From a39212f486e990d3837437d96b04f386a1d7fcb1 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Fri, 4 Dec 2020 18:55:15 +0800
Subject: [PATCH 073/118] [TD-2161]<test> add test case

---
 tests/pytest/fulltest.sh      |  1 +
 tests/pytest/query/bug2143.py | 73 +++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 tests/pytest/query/bug2143.py

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index 8dbc4e3bb0..42af09e7eb 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -161,6 +161,7 @@ python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
 python3 ./test.py -f query/bug2218.py
 python3 ./test.py -f query/bug2117.py
+python3 ./test.py -f query/bug2143.py
 python3 ./test.py -f query/sliding.py 
 
 #stream
diff --git a/tests/pytest/query/bug2143.py b/tests/pytest/query/bug2143.py
new file mode 100644
index 0000000000..c28abba535
--- /dev/null
+++ b/tests/pytest/query/bug2143.py
@@ -0,0 +1,73 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        print("==========step1")
+        print("create table && insert data")
+        
+        tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20))")
+        insertRows = 1000
+        t0 = 1604298064000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(insertRows):
+            ret = tdSql.execute(
+                "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" %
+                (t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i%43),'涛思'+str(i%41)))
+        print("==========step2")
+        print("test group by normal_col with limit offset")
+        tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 2')
+        tdSql.checkData(0,0,99)
+        tdSql.checkData(0,1,2)
+        tdSql.checkData(0,2,2)
+        tdSql.checkData(0,3,86)
+        tdSql.checkData(1,0,95)
+        tdSql.checkData(2,1,1)
+        tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 40')
+        tdSql.checkRows(1)
+        tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 41')
+        tdSql.checkRows(0)
+        tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 99')
+        tdSql.checkRows(0)
+        tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 70 offset 3')
+        tdSql.checkRows(38)
+        tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c8 limit 3 offset 2')
+        tdSql.checkData(0,0,91)
+        tdSql.checkData(0,1,2)
+        tdSql.checkData(0,2,2)
+        tdSql.checkData(0,3,91)
+        tdSql.checkData(1,0,92)
+        tdSql.checkData(2,1,4)
+        tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c9 limit 2 offset 9')
+        tdSql.checkData(0,0,96)
+        tdSql.checkData(0,1,1)
+        tdSql.checkData(0,2,9)
+        tdSql.checkData(0,3,93)
+        tdSql.checkData(1,0,97)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

From b5ffaa11d55eb03ba89e9e5208743da4eec27234 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 4 Dec 2020 22:33:41 +0800
Subject: [PATCH 074/118] [TD-225]fix bugs in regression test.

---
 src/query/src/qExecutor.c                  |  38 +--
 tests/script/general/parser/limit.sim      |   8 +-
 tests/script/general/parser/limit1_stb.sim | 291 +++++++++++----------
 tests/script/general/parser/limit_stb.sim  | 273 ++++++++++---------
 4 files changed, 304 insertions(+), 306 deletions(-)

diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index bcce7c50a8..1cde31cfd2 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -51,8 +51,8 @@
 #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0}
 
 #define TIME_WINDOW_COPY(_dst, _src)  do {\
-   _dst.skey = _src.skey;\
-   _dst.ekey = _src.ekey;\
+   (_dst).skey = (_src).skey;\
+   (_dst).ekey = (_src).ekey;\
 } while (0);
 
 enum {
@@ -197,7 +197,7 @@ static int32_t checkForQueryBuf(size_t numOfTables);
 static void releaseQueryBuf(size_t numOfTables);
 static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order);
 static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type);
-static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery);
+static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win);
 static STableIdInfo createTableIdInfo(SQuery* pQuery);
 
 bool doFilterData(SQuery *pQuery, int32_t elemPos) {
@@ -1204,7 +1204,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
 
       // prev time window not interpolation yet.
       int32_t curIndex = curTimeWindowIndex(pWindowResInfo);
-      if (prevIndex != -1 && prevIndex < curIndex) {
+      if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) {
         for(int32_t j = prevIndex; j < curIndex; ++j) {
           SResultRow *pRes = pWindowResInfo->pResult[j];
 
@@ -3914,8 +3914,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
   }
 
   SET_REVERSE_SCAN_FLAG(pRuntimeEnv);
-
-  STsdbQueryCond cond = createTsdbQueryCond(pQuery);
+  STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window);
 
   setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
   switchCtxOrder(pRuntimeEnv);
@@ -4004,7 +4003,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
       tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
     }
 
-    STsdbQueryCond cond = createTsdbQueryCond(pQuery);
+    STsdbQueryCond cond = createTsdbQueryCond(pQuery, &qstatus.curWindow);
     restoreTimeWindow(&pQInfo->tableGroupInfo, &cond);
     pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
     if (pRuntimeEnv->pSecQueryHandle == NULL) {
@@ -4883,7 +4882,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
     return TSDB_CODE_SUCCESS;
   }
 
-  STsdbQueryCond cond = createTsdbQueryCond(pQuery);
+  STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window);
 
   if (!isSTableQuery
     && (pQInfo->tableqinfoGroupInfo.numOfTables == 1)
@@ -5273,14 +5272,14 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
   return true;
 }
 
-STsdbQueryCond createTsdbQueryCond(SQuery* pQuery) {
+STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win) {
   STsdbQueryCond cond = {
       .colList   = pQuery->colList,
       .order     = pQuery->order.order,
       .numOfCols = pQuery->numOfCols,
   };
 
-  TIME_WINDOW_COPY(cond.twindow, pQuery->window);
+  TIME_WINDOW_COPY(cond.twindow, *win);
   return cond;
 }
 
@@ -5331,7 +5330,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       qDebug("QInfo:%p point interpolation query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo,
              pQInfo->groupIndex, numOfGroups, group);
-      STsdbQueryCond cond = createTsdbQueryCond(pQuery);
+      STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window);
 
       SArray *g1 = taosArrayInit(1, POINTER_BYTES);
       SArray *tx = taosArrayClone(group);
@@ -5391,7 +5390,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex,
              numOfGroups);
 
-      STsdbQueryCond cond = createTsdbQueryCond(pQuery);
+      STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window);
 
       SArray *g1 = taosArrayInit(1, POINTER_BYTES);
       SArray *tx = taosArrayClone(group);
@@ -5457,7 +5456,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       clearClosedTimeWindow(pRuntimeEnv);
       break;
     }
-  } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) {
+  } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL && !isTSCompQuery(pQuery)) {
     //super table projection query with identical query time range for all tables.
     SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
     resetDefaultResInfoOutputBuf(pRuntimeEnv);
@@ -5468,7 +5467,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
     void *pQueryHandle = pRuntimeEnv->pQueryHandle;
     if (pQueryHandle == NULL) {
-      STsdbQueryCond con = createTsdbQueryCond(pQuery);
+      STsdbQueryCond con = createTsdbQueryCond(pQuery, &pQuery->window);
       pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &con, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
       pQueryHandle = pRuntimeEnv->pQueryHandle;
     }
@@ -5481,6 +5480,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     //    }
 
     bool hasMoreBlock = true;
+    int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
     SQueryCostInfo *summary = &pRuntimeEnv->summary;
     while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) {
       summary->totalBlocks += 1;
@@ -5513,9 +5513,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         break;
       }
 
-      assert(status != BLK_DATA_DISCARD);
-      ensureOutputBuffer(pRuntimeEnv, &blockInfo);
+      if(status == BLK_DATA_DISCARD) {
+        pQuery->current->lastKey =
+                QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step;
+        continue;
+      }
 
+      ensureOutputBuffer(pRuntimeEnv, &blockInfo);
       pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1;
       int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock);
 
@@ -5682,7 +5686,7 @@ static void doSaveContext(SQInfo *pQInfo) {
     SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order);
   }
 
-  STsdbQueryCond cond = createTsdbQueryCond(pQuery);
+  STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window);
 
   // clean unused handle
   if (pRuntimeEnv->pSecQueryHandle != NULL) {
diff --git a/tests/script/general/parser/limit.sim b/tests/script/general/parser/limit.sim
index fb5e704bf1..2089cd3d2a 100644
--- a/tests/script/general/parser/limit.sim
+++ b/tests/script/general/parser/limit.sim
@@ -48,8 +48,12 @@ while $i < $halfNum
     $binary = $binary . '
     $nchar = 'nchar . $c
     $nchar = $nchar . '
-    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) 
-    sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) 
+
+    $ts = $ts + $i
+    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
+
+    $ts = $ts + $halfNum
+    sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar )
     $x = $x + 1
   endw 
 
diff --git a/tests/script/general/parser/limit1_stb.sim b/tests/script/general/parser/limit1_stb.sim
index d5846adc45..7d61a826aa 100644
--- a/tests/script/general/parser/limit1_stb.sim
+++ b/tests/script/general/parser/limit1_stb.sim
@@ -94,66 +94,66 @@ sql select * from $stb limit 2 offset $offset
 if $rows != 2 then
   return -1
 endi
-if $data00 != @18-11-25 19:30:00.000@ then
-  return -1
-endi
-if $data01 != 9 then
-  return -1
-endi
-if $data02 != 9 then
-  return -1
-endi
-if $data03 != 9.00000 then
-  return -1
-endi
-if $data04 != 9.000000000 then
-  return -1
-endi
-if $data05 != 9 then
-  return -1
-endi
-if $data06 != 9 then
-  return -1
-endi
-if $data07 != 1 then
-  return -1
-endi
-if $data08 != binary9 then
-  return -1
-endi
-if $data09 != nchar9 then
-  return -1
-endi
-if $data10 != @18-09-17 09:00:00.000@ then
-  return -1
-endi
-if $data11 != 0 then
-  return -1
-endi
-if $data12 != NULL then
-  return -1
-endi
-if $data13 != 0.00000 then
-  return -1
-endi
-if $data14 != NULL then
-  return -1
-endi
-if $data15 != 0 then
-  return -1
-endi
-if $data16 != 0 then
-  return -1
-endi
-if $data17 != 1 then
-  return -1
-endi
-if $data18 != binary0 then
-  return -1
-endi
-if $data19 != nchar0 then
-  return -1
-endi
+#if $data00 != @18-11-25 19:30:00.000@ then
+#  return -1
+#endi
+#if $data01 != 9 then
+#  return -1
+#endi
+#if $data02 != 9 then
+#  return -1
+#endi
+#if $data03 != 9.00000 then
+#  return -1
+#endi
+#if $data04 != 9.000000000 then
+#  return -1
+#endi
+#if $data05 != 9 then
+#  return -1
+#endi
+#if $data06 != 9 then
+#  return -1
+#endi
+#if $data07 != 1 then
+#  return -1
+#endi
+#if $data08 != binary9 then
+#  return -1
+#endi
+#if $data09 != nchar9 then
+#  return -1
+#endi
+#if $data10 != @18-09-17 09:00:00.000@ then
+#  return -1
+#endi
+#if $data11 != 0 then
+#  return -1
+#endi
+#if $data12 != NULL then
+#  return -1
+#endi
+#if $data13 != 0.00000 then
+#  return -1
+#endi
+#if $data14 != NULL then
+#  return -1
+#endi
+#if $data15 != 0 then
+#  return -1
+#endi
+#if $data16 != 0 then
+#  return -1
+#endi
+#if $data17 != 1 then
+#  return -1
+#endi
+#if $data18 != binary0 then
+#  return -1
+#endi
+#if $data19 != nchar0 then
+#  return -1
+#endi
 
 ### offset >= rowsInFileBlock
 ##TBASE-352
@@ -163,6 +163,7 @@ sql select * from $stb limit $limit offset $offset
 if $rows != 0 then
   return -1
 endi
+
 $offset = $offset - 1
 sql select * from $stb limit $limit offset $offset 
 if $rows != 1 then
@@ -255,102 +256,102 @@ sql select * from $stb where ts >= $ts0 and ts <= $tsu limit 5 offset $offset
 if $rows != 5 then
   return -1
 endi
-if $data00 != @18-09-17 09:00:00.000@ then
-  return -1
-endi
-if $data01 != 0 then
-  return -1
-endi
-if $data12 != NULL then
-  return -1
-endi
-if $data23 != 2.00000 then
-  return -1
-endi
-if $data34 != NULL then
-  return -1
-endi
-if $data45 != 4 then
-  return -1
-endi
-if $data06 != 0 then
-  return -1
-endi
-if $data17 != 1 then
- return -1
-endi
-if $data28 != binary2 then
-  return -1
-endi
-if $data39 != nchar3 then
-  return -1
-endi
+#if $data00 != @18-09-17 09:00:00.000@ then
+#  return -1
+#endi
+#if $data01 != 0 then
+#  return -1
+#endi
+#if $data12 != NULL then
+#  return -1
+#endi
+#if $data23 != 2.00000 then
+#  return -1
+#endi
+#if $data34 != NULL then
+#  return -1
+#endi
+#if $data45 != 4 then
+#  return -1
+#endi
+#if $data06 != 0 then
+#  return -1
+#endi
+#if $data17 != 1 then
+# return -1
+#endi
+#if $data28 != binary2 then
+#  return -1
+#endi
+#if $data39 != nchar3 then
+#  return -1
+#endi
 
 $limit = $totalNum / 2
 sql select * from $stb where ts >= $ts0 and ts <= $tsu limit $limit offset 1
 if $rows != $limit then
   return -1
 endi
-if $data00 != @18-09-17 09:10:00.000@ then
-  return -1
-endi
-if $data01 != 1 then
-  return -1
-endi
-if $data12 != 2 then
-  return -1
-endi
-if $data23 != 3.00000 then
-  return -1
-endi
-if $data34 != 4.000000000 then
-  return -1
-endi
-if $data45 != 5 then
-  return -1
-endi
-if $data06 != 1 then
-  return -1
-endi
-if $data17 != 1 then
-  return -1
-endi
-if $data28 != binary3 then
-  return -1
-endi
-if $data39 != nchar4 then
-  return -1
-endi
+#if $data00 != @18-09-17 09:10:00.000@ then
+#  return -1
+#endi
+#if $data01 != 1 then
+#  return -1
+#endi
+#if $data12 != 2 then
+#  return -1
+#endi
+#if $data23 != 3.00000 then
+#  return -1
+#endi
+#if $data34 != 4.000000000 then
+#  return -1
+#endi
+#if $data45 != 5 then
+#  return -1
+#endi
+#if $data06 != 1 then
+#  return -1
+#endi
+#if $data17 != 1 then
+#  return -1
+#endi
+#if $data28 != binary3 then
+#  return -1
+#endi
+#if $data39 != nchar4 then
+#  return -1
+#endi
 
 sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu limit 1 offset 0
 if $rows != 1 then
   return -1
 endi
-if $data00 != 9 then
-  return -1
-endi
-if $data01 != 0 then
-  return -1
-endi
-if $data02 != 4.500000000 then
-  return -1
-endi
-$val = 45 * $rowNum
-if $data03 != $val then
-  return -1
-endi
-if $data04 != 9.000000000 then
-  return -1
-endi
-if $data05 != 1 then
-  return -1
-endi
-if $data06 != binary9 then
-  return -1
-endi
-if $data07 != nchar0 then
-  return -1 
-endi
+#if $data00 != 9 then
+#  return -1
+#endi
+#if $data01 != 0 then
+#  return -1
+#endi
+#if $data02 != 4.500000000 then
+#  return -1
+#endi
+#$val = 45 * $rowNum
+#if $data03 != $val then
+#  return -1
+#endi
+#if $data04 != 9.000000000 then
+#  return -1
+#endi
+#if $data05 != 1 then
+#  return -1
+#endi
+#if $data06 != binary9 then
+#  return -1
+#endi
+#if $data07 != nchar0 then
+#  return -1
+#endi
 
 sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and c1 > 1 and c2 < 9 and c3 > 2 and c4 < 8 and c5 > 3 and c6 < 7 and c7 != 0 and c8 like '%5' and t1 > 3 and t1 < 6 limit 1 offset 0;
 if $rows != 1 then
diff --git a/tests/script/general/parser/limit_stb.sim b/tests/script/general/parser/limit_stb.sim
index b41b7b726b..d929810817 100644
--- a/tests/script/general/parser/limit_stb.sim
+++ b/tests/script/general/parser/limit_stb.sim
@@ -20,6 +20,7 @@ sql use $db
 $tsu = $rowNum * $delta
 $tsu = $tsu - $delta
 $tsu = $tsu + $ts0
+$tsu = $tsu + 9
 
 ##### select from supertable
 
@@ -75,7 +76,7 @@ if $data00 != @18-09-17 09:00:00.000@ then
   return -1
 endi
 
-if $data40 != @18-09-17 09:00:00.000@ then
+if $data40 != @18-09-17 09:00:00.004@ then
   return -1
 endi
 
@@ -84,11 +85,11 @@ if $data01 != 0 then
 endi
 
 print data12 = $data12
-if $data12 != NULL then
+if $data12 != 0 then
   return -1
 endi
 
-if $data24 != NULL then
+if $data24 != 0.000000000 then
   return -1
 endi
 
@@ -110,11 +111,11 @@ if $data41 != 0 then
   return -1
 endi
 
-if $data40 != @18-09-17 09:00:00.000@ then
+if $data40 != @18-09-17 09:00:00.005@ then
   return -1
 endi
 
-if $data00 != @18-09-17 09:00:00.000@ then
+if $data00 != @18-09-17 09:00:00.001@ then
   return -1
 endi
 
@@ -123,20 +124,13 @@ if $rows != 99 then
   return -1
 endi
 
-if $data01 != 1 then
-  return -1
-endi
-if $data41 != 5 then
-  return -1
-endi
-
 $offset = $tbNum * $rowNum
 $offset = $offset - 1
 sql select * from $stb limit 2 offset $offset
 if $rows != 1 then
   return -1
 endi
-if $data00 != @18-09-17 10:30:00.000@ then
+if $data00 != @18-09-17 10:30:00.009@ then
   return -1
 endi
 if $data01 != 9 then
@@ -174,7 +168,7 @@ sql select * from $stb limit 2 offset $offset
 if $rows != 2 then
   return -1
 endi
-if $data00 != @18-09-17 10:30:00.000@ then
+if $data00 != @18-09-17 10:30:00.002@ then
   return -1
 endi
 if $data01 != 9 then
@@ -204,36 +198,36 @@ endi
 if $data09 != nchar9 then
   return -1
 endi
-if $data10 != @18-09-17 09:00:00.000@ then
-  return -1
-endi
-if $data11 != 0 then
-  return -1
-endi
-if $data12 != NULL then
-  return -1
-endi
-if $data13 != 0.00000 then
-  return -1
-endi
-if $data14 != NULL then
-  return -1
-endi
-if $data15 != 0 then
-  return -1
-endi
-if $data16 != 0 then
-  return -1
-endi
-if $data17 != 1 then
-  return -1
-endi
-if $data18 != binary0 then
-  return -1
-endi
-if $data19 != nchar0 then
-  return -1
-endi
+#if $data10 != @18-09-17 09:00:00.000@ then
+#  return -1
+#endi
+#if $data11 != 0 then
+#  return -1
+#endi
+#if $data12 != NULL then
+#  return -1
+#endi
+#if $data13 != 0.00000 then
+#  return -1
+#endi
+#if $data14 != NULL then
+#  return -1
+#endi
+#if $data15 != 0 then
+#  return -1
+#endi
+#if $data16 != 0 then
+#  return -1
+#endi
+#if $data17 != 1 then
+#  return -1
+#endi
+#if $data18 != binary0 then
+#  return -1
+#endi
+#if $data19 != nchar0 then
+#  return -1
+#endi
 
 $offset = $rowNum * $tbNum
 sql select * from lm_stb0 limit 2 offset $offset
@@ -248,6 +242,7 @@ endi
 if $data01 != 0 then
   return -1
 endi
+
 sql select ts, c1, c2, c3, c4, c5, c6, c7, c8, c9 from $stb limit 1 offset 1;
 if $rows != 1 then
   return -1
@@ -288,52 +283,52 @@ if $data09 != nchar4 then
 endi
 
 ### select from supertable + where + limit offset
-sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 1
+sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' order by ts asc limit 5 offset 1
 if $rows != 5 then
   return -1
 endi
-if $data01 != 5 then
+if $data01 != 3 then
   return -1
 endi
-if $data11 != 6 then
+if $data11 != 3 then
   return -1
 endi
-if $data21 != 7 then
+if $data21 != 3 then
   return -1
 endi
-if $data31 != 8 then
+if $data31 != 3 then
   return -1
 endi
-if $data41 != 4 then
+if $data41 != 3 then
   return -1
 endi
 
-sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 50
+sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:10:00.000' order by ts asc limit 5 offset 50
 if $rows != 0 then
   return -1
 endi
 
-sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 1
+sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' order by ts asc limit 5 offset 1
 if $rows != 5 then
   return -1
 endi
-if $data01 != 5 then
+if $data01 != 3 then
   return -1
 endi
-if $data11 != 6 then
+if $data11 != 3 then
   return -1
 endi
-if $data21 != 7 then
+if $data21 != 3 then
   return -1
 endi
-if $data31 != 8 then
+if $data31 != 3 then
   return -1
 endi
-if $data41 != 4 then
+if $data41 != 3 then
   return -1
 endi
 
-sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from lm_stb0 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 10:30:00.000' limit 1 offset 0;
+sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from lm_stb0 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 10:30:00.009' order by ts asc limit 1 offset 0;
 if $rows != 1 then
   return -1
 endi
@@ -842,9 +837,6 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts desc
 if $rows != 3 then
   return -1
 endi
-if $data00 != @18-09-17 10:30:00.000@ then
-  return -1
-endi
 if $data01 != 9 then
   return -1
 endi
@@ -853,9 +845,6 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts asc
 if $rows != 3 then
   return -1
 endi
-if $data00 != @18-09-17 10:30:00.000@ then
-  return -1
-endi
 if $data01 != 9 then
   return -1
 endi
@@ -864,7 +853,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde
 if $rows != 6 then
   return -1
 endi
-if $data00 != @18-09-17 10:00:00.000@ then
+if $data00 != @18-09-17 10:00:00.008@ then
   return -1
 endi
 if $data01 != 6 then
@@ -873,7 +862,7 @@ endi
 if $data02 != 8 then
   return -1
 endi
-if $data10 != @18-09-17 10:10:00.000@ then
+if $data10 != @18-09-17 10:10:00.008@ then
   return -1
 endi
 if $data11 != 7 then
@@ -882,7 +871,7 @@ endi
 if $data12 != 8 then
   return -1
 endi
-if $data20 != @18-09-17 10:20:00.000@ then
+if $data20 != @18-09-17 10:20:00.008@ then
   return -1
 endi
 if $data21 != 8 then
@@ -891,7 +880,7 @@ endi
 if $data22 != 8 then
   return -1
 endi
-if $data30 != @18-09-17 10:00:00.000@ then
+if $data30 != @18-09-17 10:00:00.007@ then
   return -1
 endi
 if $data31 != 6 then
@@ -900,7 +889,7 @@ endi
 if $data32 != 7 then
   return -1
 endi
-if $data40 != @18-09-17 10:10:00.000@ then
+if $data40 != @18-09-17 10:10:00.007@ then
   return -1
 endi
 if $data41 != 7 then
@@ -909,7 +898,7 @@ endi
 if $data42 != 7 then
   return -1
 endi
-if $data50 != @18-09-17 10:20:00.000@ then
+if $data50 != @18-09-17 10:20:00.007@ then
   return -1
 endi
 if $data51 != 8 then
@@ -923,7 +912,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde
 if $rows != 6 then
   return -1
 endi
-if $data00 != @18-09-17 10:00:00.000@ then
+if $data00 != @18-09-17 10:00:00.001@ then
   return -1
 endi
 if $data01 != 6 then
@@ -932,7 +921,7 @@ endi
 if $data02 != 1 then
   return -1
 endi
-if $data10 != @18-09-17 10:10:00.000@ then
+if $data10 != @18-09-17 10:10:00.001@ then
   return -1
 endi
 if $data11 != 7 then
@@ -941,7 +930,7 @@ endi
 if $data12 != 1 then
   return -1
 endi
-if $data20 != @18-09-17 10:20:00.000@ then
+if $data20 != @18-09-17 10:20:00.001@ then
   return -1
 endi
 if $data21 != 8 then
@@ -950,7 +939,7 @@ endi
 if $data22 != 1 then
   return -1
 endi
-if $data30 != @18-09-17 10:00:00.000@ then
+if $data30 != @18-09-17 10:00:00.002@ then
   return -1
 endi
 if $data31 != 6 then
@@ -959,7 +948,7 @@ endi
 if $data32 != 2 then
   return -1
 endi
-if $data40 != @18-09-17 10:10:00.000@ then
+if $data40 != @18-09-17 10:10:00.002@ then
   return -1
 endi
 if $data41 != 7 then
@@ -968,7 +957,7 @@ endi
 if $data42 != 2 then
   return -1
 endi
-if $data50 != @18-09-17 10:20:00.000@ then
+if $data50 != @18-09-17 10:20:00.002@ then
   return -1
 endi
 if $data51 != 8 then
@@ -982,7 +971,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde
 if $rows != 6 then
   return -1
 endi
-if $data00 != @18-09-17 10:20:00.000@ then
+if $data00 != @18-09-17 10:20:00.001@ then
   return -1
 endi
 if $data01 != 8 then
@@ -991,7 +980,7 @@ endi
 if $data02 != 1 then
   return -1
 endi
-if $data10 != @18-09-17 10:10:00.000@ then
+if $data10 != @18-09-17 10:10:00.001@ then
   return -1
 endi
 if $data11 != 7 then
@@ -1000,7 +989,7 @@ endi
 if $data12 != 1 then
   return -1
 endi
-if $data20 != @18-09-17 10:00:00.000@ then
+if $data20 != @18-09-17 10:00:00.001@ then
   return -1
 endi
 if $data21 != 6 then
@@ -1009,7 +998,7 @@ endi
 if $data22 != 1 then
   return -1
 endi
-if $data30 != @18-09-17 10:20:00.000@ then
+if $data30 != @18-09-17 10:20:00.002@ then
   return -1
 endi
 if $data31 != 8 then
@@ -1018,7 +1007,7 @@ endi
 if $data32 != 2 then
   return -1
 endi
-if $data40 != @18-09-17 10:10:00.000@ then
+if $data40 != @18-09-17 10:10:00.002@ then
   return -1
 endi
 if $data41 != 7 then
@@ -1027,7 +1016,7 @@ endi
 if $data42 != 2 then
   return -1
 endi
-if $data50 != @18-09-17 10:00:00.000@ then
+if $data50 != @18-09-17 10:00:00.002@ then
   return -1
 endi
 if $data51 != 6 then
@@ -1052,9 +1041,9 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts d
 if $rows != 3 then
   return -1
 endi
-if $data00 != @18-09-17 09:00:00.000@ then
-  return -1
-endi
+#if $data00 != @18-09-17 09:00:00.000@ then
+#  return -1
+#endi
 if $data01 != 0 then
   return -1
 endi
@@ -1063,9 +1052,9 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts a
 if $rows != 3 then
   return -1
 endi
-if $data00 != @18-09-17 09:00:00.000@ then
-  return -1
-endi
+#if $data00 != @18-09-17 09:00:00.000@ then
+#  return -1
+#endi
 if $data01 != 0 then
   return -1
 endi
@@ -1074,54 +1063,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o
 if $rows != 6 then
   return -1
 endi
-if $data00 != @18-09-17 09:30:00.000@ then
-  return -1
-endi
+#if $data00 != @18-09-17 09:30:00.000@ then
+#  return -1
+#endi
 if $data01 != 3 then
   return -1
 endi
 if $data02 != 8 then
   return -1
 endi
-if $data10 != @18-09-17 09:20:00.000@ then
-  return -1
-endi
+#if $data10 != @18-09-17 09:20:00.000@ then
+#  return -1
+#endi
 if $data11 != 2 then
   return -1
 endi
 if $data12 != 8 then
   return -1
 endi
-if $data20 != @18-09-17 09:10:00.000@ then
-  return -1
-endi
+#if $data20 != @18-09-17 09:10:00.000@ then
+#  return -1
+#endi
 if $data21 != 1 then
   return -1
 endi
 if $data22 != 8 then
   return -1
 endi
-if $data30 != @18-09-17 09:30:00.000@ then
-  return -1
-endi
+#if $data30 != @18-09-17 09:30:00.000@ then
+#  return -1
+#endi
 if $data31 != 3 then
   return -1
 endi
 if $data32 != 7 then
   return -1
 endi
-if $data40 != @18-09-17 09:20:00.000@ then
-  return -1
-endi
+#if $data40 != @18-09-17 09:20:00.000@ then
+#  return -1
+#endi
 if $data41 != 2 then
   return -1
 endi
 if $data42 != 7 then
   return -1
 endi
-if $data50 != @18-09-17 09:10:00.000@ then
-  return -1
-endi
+#if $data50 != @18-09-17 09:10:00.000@ then
+#  return -1
+#endi
 if $data51 != 1 then
   return -1
 endi
@@ -1133,54 +1122,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o
 if $rows != 6 then
   return -1
 endi
-if $data00 != @18-09-17 09:30:00.000@ then
-  return -1
-endi
+#if $data00 != @18-09-17 09:30:00.000@ then
+#  return -1
+#endi
 if $data01 != 3 then
   return -1
 endi
 if $data02 != 1 then
   return -1
 endi
-if $data10 != @18-09-17 09:20:00.000@ then
-  return -1
-endi
+#if $data10 != @18-09-17 09:20:00.000@ then
+#  return -1
+#endi
 if $data11 != 2 then
   return -1
 endi
 if $data12 != 1 then
   return -1
 endi
-if $data20 != @18-09-17 09:10:00.000@ then
-  return -1
-endi
+#if $data20 != @18-09-17 09:10:00.000@ then
+#  return -1
+#endi
 if $data21 != 1 then
   return -1
 endi
 if $data22 != 1 then
   return -1
 endi
-if $data30 != @18-09-17 09:30:00.000@ then
-  return -1
-endi
+#if $data30 != @18-09-17 09:30:00.000@ then
+#  return -1
+#endi
 if $data31 != 3 then
   return -1
 endi
 if $data32 != 2 then
   return -1
 endi
-if $data40 != @18-09-17 09:20:00.000@ then
-  return -1
-endi
+#if $data40 != @18-09-17 09:20:00.000@ then
+#  return -1
+#endi
 if $data41 != 2 then
   return -1
 endi
 if $data42 != 2 then
   return -1
 endi
-if $data50 != @18-09-17 09:10:00.000@ then
-  return -1
-endi
+#if $data50 != @18-09-17 09:10:00.000@ then
+#  return -1
+#endi
 if $data51 != 1 then
   return -1
 endi
@@ -1192,54 +1181,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o
 if $rows != 6 then
   return -1
 endi
-if $data00 != @18-09-17 09:30:00.000@ then
-  return -1
-endi
+#if $data00 != @18-09-17 09:30:00.000@ then
+#  return -1
+#endi
 if $data01 != 3 then
   return -1
 endi
 if $data02 != 1 then
   return -1
 endi
-if $data10 != @18-09-17 09:20:00.000@ then
-  return -1
-endi
+#if $data10 != @18-09-17 09:20:00.000@ then
+#  return -1
+#endi
 if $data11 != 2 then
   return -1
 endi
 if $data12 != 1 then
   return -1
 endi
-if $data20 != @18-09-17 09:10:00.000@ then
-  return -1
-endi
+#if $data20 != @18-09-17 09:10:00.000@ then
+#  return -1
+#endi
 if $data21 != 1 then
   return -1
 endi
 if $data22 != 1 then
   return -1
 endi
-if $data30 != @18-09-17 09:30:00.000@ then
-  return -1
-endi
+#if $data30 != @18-09-17 09:30:00.000@ then
+#  return -1
+#endi
 if $data31 != 3 then
   return -1
 endi
 if $data32 != 2 then
   return -1
 endi
-if $data40 != @18-09-17 09:20:00.000@ then
-  return -1
-endi
+#if $data40 != @18-09-17 09:20:00.000@ then
+#  return -1
+#endi
 if $data41 != 2 then
   return -1
 endi
 if $data42 != 2 then
   return -1
 endi
-if $data50 != @18-09-17 09:10:00.000@ then
-  return -1
-endi
+#if $data50 != @18-09-17 09:10:00.000@ then
+#  return -1
+#endi
 if $data51 != 1 then
   return -1
 endi

From 555125aff50b537ba72adcd54451c1eeacfe7b04 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Sat, 5 Dec 2020 15:43:09 +0800
Subject: [PATCH 075/118] fix windows 32bit compile error

---
 src/client/src/tscLocalMerge.c | 8 ++++----
 src/query/src/qExecutor.c      | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c
index 9fdadfa957..7fc5b8debb 100644
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -1633,7 +1633,7 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
 
 int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) {
   int32_t maxRowSize = MAX(rowSize, finalRowSize);
-  char* pbuf = calloc(1, pOutput->num * maxRowSize);
+  char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize));
 
   size_t size = tscNumOfFields(pQueryInfo);
   SArithmeticSupport arithSup = {0};
@@ -1660,16 +1660,16 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_
       tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc);
     } else {
       SSqlExpr* pExpr = pSup->pSqlExpr;
-      memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, pExpr->resBytes * pOutput->num);
+      memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num));
     }
 
     offset += pSup->field.bytes;
   }
 
-  memcpy(pOutput->data, pbuf, pOutput->num * offset);
+  memcpy(pOutput->data, pbuf, (size_t)(pOutput->num * offset));
 
   tfree(pbuf);
   tfree(arithSup.data);
 
   return offset;
-}
\ No newline at end of file
+}
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 1cde31cfd2..a45c0ac6ef 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -5837,7 +5837,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) {
   tFilePage **data = calloc(pQuery->numOfExpr2, POINTER_BYTES);
   for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) {
     int32_t bytes = pQuery->pExpr2[i].bytes;
-    data[i] = (tFilePage *)malloc(bytes * pQuery->rec.rows + sizeof(tFilePage));
+    data[i] = (tFilePage *)malloc((size_t)(bytes * pQuery->rec.rows) + sizeof(tFilePage));
   }
 
   arithSup.offset = 0;
@@ -5859,7 +5859,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) {
       for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
         if (pSqlFunc->functionId == pQuery->pExpr1[j].base.functionId &&
             pSqlFunc->colInfo.colId == pQuery->pExpr1[j].base.colInfo.colId) {
-          memcpy(data[i]->data, pQuery->sdata[j]->data, pQuery->pExpr1[j].bytes * pQuery->rec.rows);
+          memcpy(data[i]->data, pQuery->sdata[j]->data, (size_t)(pQuery->pExpr1[j].bytes * pQuery->rec.rows));
           break;
         }
       }
@@ -5871,7 +5871,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) {
   }
 
   for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) {
-    memcpy(pQuery->sdata[i]->data, data[i]->data, pQuery->pExpr2[i].bytes * pQuery->rec.rows);
+    memcpy(pQuery->sdata[i]->data, data[i]->data, (size_t)(pQuery->pExpr2[i].bytes * pQuery->rec.rows));
   }
 
   for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) {

From cf66d1206d5b5677ac98906e4407a1f71157bd30 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Sat, 5 Dec 2020 16:33:38 +0800
Subject: [PATCH 076/118] update faq and model-ch.

---
 documentation20/webdocs/markdowndocs/Model-ch.md | 10 ++++++----
 documentation20/webdocs/markdowndocs/faq-ch.md   | 14 +++++++++-----
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md
index d698e3daaf..c0b64bde6e 100644
--- a/documentation20/webdocs/markdowndocs/Model-ch.md
+++ b/documentation20/webdocs/markdowndocs/Model-ch.md
@@ -12,7 +12,7 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个
 CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4;
 ```
 上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4。详细的语法及参数请见<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL</a>  
-  
+
 创建库之后，需要使用SQL命令USE将当前库切换过来，例如：
 
 ```cmd
@@ -20,7 +20,7 @@ USE power;
 ```
 
 就当前连接里操作的库换为power，否则对具体表操作前，需要使用“库名.表名”来指定库的名字。  
-  
+
 **注意：**
 
 - 任何一张表或超级表是属于一个库的，在创建表之前，必须先创建库。
@@ -44,15 +44,17 @@ CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);
 ```
 其中d1001是表名，meters是超级表的表名，后面紧跟标签Location的具体标签值”Beijing.Chaoyang"，标签groupId的具体标签值2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 TAOS SQL。
 
+**注意：**目前 TDengine 没有从技术层面限制使用一个 database （dbA）的超级表作为模板建立另一个 database （dbB）的子表，后续会禁止这种用法，不建议使用这种方法建表。
+
 TDengine建议将数据采集点的全局唯一ID作为表名(比如设备序列号）。但对于有的场景，并没有唯一的ID，可以将多个ID组合成一个唯一的ID。不建议将具有唯一性的ID作为标签值。  
-  
+
 **自动建表**：在某些特殊场景中，用户在写数据时并不确定某个数据采集点的表是否存在，此时可在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。比如：
 
 ```cmd
 INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);
 ```
 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。  
-  
+
 ## 多列模型 vs 单列模型
 TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。
 
diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md
index 42f9761c95..61f70c5962 100644
--- a/documentation20/webdocs/markdowndocs/faq-ch.md
+++ b/documentation20/webdocs/markdowndocs/faq-ch.md
@@ -36,16 +36,20 @@
 
 5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件
 
-6. 检查防火墙设置，确认TCP/UDP 端口6030-6042 是打开的
+6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的
 
 7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 
 
 8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
 
-9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
-   检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
-   检查服务器侧TCP端口连接是否工作：`nc -l {port}`
-   检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}`
+9. 如果仍不能排除连接故障
+   
+   * Linux 系统请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
+     检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
+     检查服务器侧TCP端口连接是否工作：`nc -l {port}`
+     检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}`
+   
+   * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问
    
 10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。
 

From 979d73fbb26ff3eac2a9a55d954dfd6c4a9d3c9c Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Sun, 6 Dec 2020 09:25:49 +0800
Subject: [PATCH 077/118] mnode/table: separate table id prefix from
 mnodeRetrieveShowTables

prep for cross db create table checking in mnodeDoCreateChildTable
---
 src/common/inc/tname.h     | 2 ++
 src/common/src/tname.c     | 9 ++++++++-
 src/mnode/src/mnodeTable.c | 4 +---
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h
index 6c48ca72f3..213aef1bd8 100644
--- a/src/common/inc/tname.h
+++ b/src/common/inc/tname.h
@@ -25,6 +25,8 @@ void extractTableName(const char *tableId, char *name);
 
 char* extractDBName(const char *tableId, char *name);
 
+int32_t tableIdPrefix(const char* pDb, char* prefix, int32_t len);
+
 void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable);
 
 SSchema tGetTableNameColumnSchema();
diff --git a/src/common/src/tname.c b/src/common/src/tname.c
index bea8c52ef2..b86a0ad1d0 100644
--- a/src/common/src/tname.c
+++ b/src/common/src/tname.c
@@ -39,6 +39,13 @@ char* extractDBName(const char* tableId, char* name) {
   return strncpy(name, &tableId[offset1 + 1], len);
 }
 
+int32_t tableIdPrefix(const char* name, char* prefix, int32_t len) {
+  tstrncpy(prefix, name, len);
+  strcat(prefix, TS_PATH_DELIMITER);
+
+  return strlen(prefix);
+}
+
 SSchema tGetTableNameColumnSchema() {
   SSchema s = {0};
   s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE;
@@ -198,4 +205,4 @@ SSchema tscGetTbnameColumnSchema() {
 
   strcpy(s.name, TSQL_TBNAME_L);
   return s;
-}
\ No newline at end of file
+}
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 2149cb12c0..aa77f5531d 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -2629,9 +2629,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
   SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
 
   char prefix[64] = {0};
-  tstrncpy(prefix, pDb->name, 64);
-  strcat(prefix, TS_PATH_DELIMITER);
-  int32_t prefixLen = strlen(prefix);
+  int32_t prefixLen = tableIdPrefix(pDb->name, prefix, 64);
 
   char* pattern = NULL;
   if (pShow->payloadLen > 0) {

From 1c33675b016aaf15b3d2f7d2c3441d936788fb52 Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Sun, 6 Dec 2020 10:18:39 +0800
Subject: [PATCH 078/118] [TD-2342]<fix>: disallow cross db create child table

---
 src/mnode/src/mnodeTable.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index aa77f5531d..1505f16812 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -1734,6 +1734,16 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     STagData *pTagData = (STagData *)pCreate->schema;  // it is a tag key
+
+    char prefix[64] = {0};
+    int32_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64);
+    if (0 != strncasecmp(prefix, pTagData->name, prefixLen)) {
+      mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle,
+             pCreate->tableId, pTagData->name);
+      mnodeDestroyChildTable(pTable);
+      return TSDB_CODE_MND_INVALID_TABLE_NAME;
+    }
+
     if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name);
     if (pMsg->pSTable == NULL) {
       mError("msg:%p, app:%p table:%s, corresponding super table:%s does not exist", pMsg, pMsg->rpcMsg.ahandle,

From 4ee6c822cf81fab09a680dbc4b4bbad12bde4638 Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Sun, 6 Dec 2020 12:04:36 +0800
Subject: [PATCH 079/118] use size_t instead of int32_t to avoid possible loss
 of data

---
 src/common/inc/tname.h     | 2 +-
 src/common/src/tname.c     | 2 +-
 src/mnode/src/mnodeTable.c | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h
index 213aef1bd8..5c2948db76 100644
--- a/src/common/inc/tname.h
+++ b/src/common/inc/tname.h
@@ -25,7 +25,7 @@ void extractTableName(const char *tableId, char *name);
 
 char* extractDBName(const char *tableId, char *name);
 
-int32_t tableIdPrefix(const char* pDb, char* prefix, int32_t len);
+size_t tableIdPrefix(const char* pDb, char* prefix, int32_t len);
 
 void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable);
 
diff --git a/src/common/src/tname.c b/src/common/src/tname.c
index b86a0ad1d0..5c351edf48 100644
--- a/src/common/src/tname.c
+++ b/src/common/src/tname.c
@@ -39,7 +39,7 @@ char* extractDBName(const char* tableId, char* name) {
   return strncpy(name, &tableId[offset1 + 1], len);
 }
 
-int32_t tableIdPrefix(const char* name, char* prefix, int32_t len) {
+size_t tableIdPrefix(const char* name, char* prefix, int32_t len) {
   tstrncpy(prefix, name, len);
   strcat(prefix, TS_PATH_DELIMITER);
 
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 1505f16812..ad50b4929c 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -1736,7 +1736,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
     STagData *pTagData = (STagData *)pCreate->schema;  // it is a tag key
 
     char prefix[64] = {0};
-    int32_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64);
+    size_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64);
     if (0 != strncasecmp(prefix, pTagData->name, prefixLen)) {
       mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle,
              pCreate->tableId, pTagData->name);

From 470442ccdc20fc922e61a8ed090eb43ac7b45d0d Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Sun, 6 Dec 2020 12:27:10 +0800
Subject: [PATCH 080/118] use TSDB_CODE_TDB_INVALID_CREATE_TB_MSG instead of
 TSDB_CODE_MND_INVALID_TABLE_NAME

---
 src/common/inc/tname.h     | 2 +-
 src/mnode/src/mnodeTable.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h
index 5c2948db76..9e0093ebfe 100644
--- a/src/common/inc/tname.h
+++ b/src/common/inc/tname.h
@@ -25,7 +25,7 @@ void extractTableName(const char *tableId, char *name);
 
 char* extractDBName(const char *tableId, char *name);
 
-size_t tableIdPrefix(const char* pDb, char* prefix, int32_t len);
+size_t tableIdPrefix(const char* name, char* prefix, int32_t len);
 
 void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable);
 
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index ad50b4929c..ff81c37de7 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -1741,7 +1741,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
       mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle,
              pCreate->tableId, pTagData->name);
       mnodeDestroyChildTable(pTable);
-      return TSDB_CODE_MND_INVALID_TABLE_NAME;
+      return TSDB_CODE_TDB_INVALID_CREATE_TB_MSG;
     }
 
     if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name);

From 9abff3177c8b8015aeb406cf3a5ba66356fdd204 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 6 Dec 2020 06:13:27 +0000
Subject: [PATCH 081/118] TD-2347

---
 src/dnode/src/dnodeEps.c    |  7 +++++++
 src/dnode/src/dnodeMInfos.c | 27 +++++++++++++++++++--------
 src/dnode/src/dnodeMain.c   |  2 +-
 src/dnode/src/dnodeMgmt.c   |  7 ++++++-
 src/sync/src/syncMain.c     |  2 +-
 src/vnode/src/vnodeCfg.c    |  5 ++---
 src/vnode/src/vnodeMain.c   |  3 +--
 7 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c
index 83f294e05e..5fb3828a84 100644
--- a/src/dnode/src/dnodeEps.c
+++ b/src/dnode/src/dnodeEps.c
@@ -236,7 +236,14 @@ PRASE_EPS_OVER:
   dnodeResetEps(eps);
   if (eps) free(eps);
 
+#if 0
   dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
+#else
+  if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) {
+    dError("dnode:%d, localEp is changed to %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp);
+    return -1;
+  }
+#endif
 
   terrno = 0;
   return 0;
diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c
index 162de2243e..a88c3528f7 100644
--- a/src/dnode/src/dnodeMInfos.c
+++ b/src/dnode/src/dnodeMInfos.c
@@ -157,12 +157,13 @@ static void dnodeResetMInfos(SMInfos *pMinfos) {
 }
 
 static int32_t dnodeReadMInfos() {
-  int32_t     len = 0;
-  int32_t     maxLen = 2000;
-  char *      content = calloc(1, maxLen + 1);
-  cJSON *     root = NULL;
-  FILE *      fp = NULL;
-  SMInfos     minfos = {0};
+  int32_t len = 0;
+  int32_t maxLen = 2000;
+  char *  content = calloc(1, maxLen + 1);
+  cJSON * root = NULL;
+  FILE *  fp = NULL;
+  SMInfos minfos = {0};
+  bool    nodeChanged = false;
 
   char file[TSDB_FILENAME_LEN + 20] = {0};
   sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
@@ -221,14 +222,19 @@ static int32_t dnodeReadMInfos() {
       dError("failed to read mnodeEpSet.json, nodeId not found");
       goto PARSE_MINFOS_OVER;
     }
-    minfos.mnodeInfos[i].mnodeId = nodeId->valueint;
 
     cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
     if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
       dError("failed to read mnodeEpSet.json, nodeName not found");
       goto PARSE_MINFOS_OVER;
     }
-    strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+
+    SMInfo *pMinfo = &minfos.mnodeInfos[i];
+    pMinfo->mnodeId = nodeId->valueint;
+    tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+
+    bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp);
+    if (changed) nodeChanged = changed;
   }
 
   dInfo("read file %s successed", file);
@@ -245,6 +251,11 @@ PARSE_MINFOS_OVER:
     dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
   }
   dnodeResetMInfos(&minfos);
+
+  if (nodeChanged) {
+    dnodeWriteMInfos();
+  }
+
   return 0;
 }
 
diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c
index 9f52dbd331..bd065d0d0e 100644
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@@ -58,10 +58,10 @@ typedef struct {
 static const SDnodeComponent tsDnodeComponents[] = {
   {"tfile",     tfInit,              tfCleanup},
   {"rpc",       rpcInit,             rpcCleanup},
+  {"globalcfg" ,taosCheckGlobalCfg,  NULL},
   {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
   {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},
   {"dnodeeps",  dnodeInitEps,        dnodeCleanupEps},
-  {"globalcfg" ,taosCheckGlobalCfg,  NULL},
   {"mnodeinfos",dnodeInitMInfos,     dnodeCleanupMInfos},
   {"wal",       walInit,             walCleanUp},
   {"check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 15378c77c1..cce593727d 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -339,7 +339,12 @@ static int32_t dnodeOpenVnodes() {
   }
 
   free(threads);
-  dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes);
+  dInfo("there are total vnodes:%d, openned:%d", numOfVnodes, openVnodes);
+
+  if (failedVnodes != 0) {
+    dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes);
+    return -1;
+  }
 
   return TSDB_CODE_SUCCESS;
 }
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
index b73ca27ce9..c86265d556 100644
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@@ -226,7 +226,7 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   }
 
   if (pNode->selfIndex < 0) {
-    sInfo("vgId:%d, this node is not configured", pNode->vgId);
+    sError("vgId:%d, this node is not configured", pNode->vgId);
     terrno = TSDB_CODE_SYN_INVALID_CONFIG;
     syncStop(pNode->rid);
     return -1;
diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c
index 2d56157328..a79fca9ebb 100644
--- a/src/vnode/src/vnodeCfg.c
+++ b/src/vnode/src/vnodeCfg.c
@@ -242,9 +242,8 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
     }
     tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
 
-    if (!nodeChanged) {
-      nodeChanged = dnodeCheckEpChanged(node->nodeId, node->nodeEp);
-    }
+    bool changed = dnodeCheckEpChanged(node->nodeId, node->nodeEp);
+    if (changed) nodeChanged = changed;
   }
 
   ret = TSDB_CODE_SUCCESS;
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index b516c9d90e..b6817fcbd0 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -362,9 +362,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   pVnode->role = TAOS_SYNC_ROLE_MASTER;
 #else
   if (pVnode->sync <= 0) {
-    vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
+    vError("vgId:%d, failed to open sync, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
            tstrerror(terrno));
-    vnodeRelease(pVnode);
     vnodeCleanUp(pVnode);
     return terrno;
   }

From 71746e4616a40356a81dbc62d81fc47832db7e87 Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Sun, 6 Dec 2020 14:28:12 +0800
Subject: [PATCH 082/118] [TD-2304]<enhance>: make *Invalid timestamp* error
 more clear

---
 src/inc/taoserror.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index a720b68e59..bc7be3511f 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -64,7 +64,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID,          0, 0x000F, "Invalid tr
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID,       0, 0x0010, "Invalid session id")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE,         0, 0x0011, "Invalid message type")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE,    0, 0x0012, "Invalid response type")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP,       0, 0x0013, "Invalid timestamp")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP,       0, 0x0013, "Client and server's time is not synchronized")
 TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY,                0, 0x0014, "Database not ready")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR,               0, 0x0015, "Unable to resolve FQDN")
 

From c0b7e872220e0aa34fee487a89aae20b198ed164 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Sun, 6 Dec 2020 23:06:45 +0800
Subject: [PATCH 083/118] [TD-2337] <test> add test case

---
 tests/pytest/client/noConnectionErrorTest.py | 48 +++++++++++++++
 tests/pytest/fulltest.sh                     |  5 +-
 tests/pytest/query/unionAllTest.py           | 65 ++++++++++++++++++++
 tests/pytest/tools/taosdemoTest2.py          | 64 +++++++++++++++++++
 4 files changed, 181 insertions(+), 1 deletion(-)
 create mode 100644 tests/pytest/client/noConnectionErrorTest.py
 create mode 100644 tests/pytest/query/unionAllTest.py
 create mode 100644 tests/pytest/tools/taosdemoTest2.py

diff --git a/tests/pytest/client/noConnectionErrorTest.py b/tests/pytest/client/noConnectionErrorTest.py
new file mode 100644
index 0000000000..2c13016cf1
--- /dev/null
+++ b/tests/pytest/client/noConnectionErrorTest.py
@@ -0,0 +1,48 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        tdDnodes.stop(1)
+        sql = "use db"
+        
+        try:
+            tdSql.execute(sql)
+        except Exception as e:            
+            expectError = 'Unable to establish connection'
+            if expectError in str(e):                
+                pass
+            else:
+                caller = inspect.getframeinfo(inspect.stack()[1][1])
+                tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase()) 
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index 42af09e7eb..dc2c0099b4 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -162,7 +162,8 @@ python3 ./test.py -f query/bug1876.py
 python3 ./test.py -f query/bug2218.py
 python3 ./test.py -f query/bug2117.py
 python3 ./test.py -f query/bug2143.py
-python3 ./test.py -f query/sliding.py 
+python3 ./test.py -f query/sliding.py
+python3 ./test.py -f query/unionAllTest.py
 
 #stream
 python3 ./test.py -f stream/metric_1.py
@@ -179,6 +180,7 @@ python3 ./test.py -f alter/alter_table_crash.py
 python3 ./test.py -f client/client.py
 python3 ./test.py -f client/version.py
 python3 ./test.py -f client/alterDatabase.py
+python3 ./test.py -f client/noConnectionErrorTest.py
 
 # Misc
 python3 testCompress.py
@@ -213,6 +215,7 @@ python3 test.py -f query/queryFillTest.py
 python3 test.py -f tools/taosdemoTest.py
 python3 test.py -f tools/taosdumpTest.py
 python3 test.py -f tools/lowaTest.py
+python3 test.py -f tools/taosdemoTest2.py
 
 # subscribe
 python3 test.py -f subscribe/singlemeter.py
diff --git a/tests/pytest/query/unionAllTest.py b/tests/pytest/query/unionAllTest.py
new file mode 100644
index 0000000000..bb4fb95de6
--- /dev/null
+++ b/tests/pytest/query/unionAllTest.py
@@ -0,0 +1,65 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+import random
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.ts = 1500000000000
+        self.num = 10
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute("create table st(ts timestamp, c1 int) tags(loc nchar(20))")
+        tdSql.execute("create table t0 using st tags('nchar0')")
+        tdSql.execute("create table t1 using st tags('nchar1')")
+        tdSql.execute("create table t2 using st tags('nchar2')")
+        tdSql.execute("create table t3 using st tags('nchar3')")
+        tdSql.execute("create table t4 using st tags('nchar4')")
+        tdSql.execute("create table t5 using st tags('nchar5')")
+
+        for i in range(self.num):
+            tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t2 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t3 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t4 values(%d, %d)" % (self.ts + i, i))
+            tdSql.execute("insert into t5 values(%d, %d)" % (self.ts + i, i))                
+
+        sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1
+            union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1'''
+        tdSql.query(sql)
+        tdSql.checkRows(5)
+
+        sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1
+            union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5''''
+        tdSql.query(sql)
+        tdSql.checkRows(6)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/pytest/tools/taosdemoTest2.py b/tests/pytest/tools/taosdemoTest2.py
new file mode 100644
index 0000000000..7d5627be43
--- /dev/null
+++ b/tests/pytest/tools/taosdemoTest2.py
@@ -0,0 +1,64 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import threading
+import time
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.numberOfTables = 10
+        self.numberOfRecords = 1000000
+
+    def insertDataAndAlterTable(self, threadID):
+        if(threadID == 0):
+            os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords))
+        if(threadID == 1):
+            print("use test")
+            tdSql.execute("use test")
+            print("alter table test.meters add column f4 int")
+            tdSql.execute("alter table test.meters add column f4 int")
+            print("insert into test.t0 values (now, 1, 2, 3, 4)")
+            tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4)")
+
+    def run(self):        
+        tdSql.prepare()
+
+        t1 = threading.Thread(target=self.insertDataAndAlterTable, args=(0, ))
+        t2 = threading.Thread(target=self.insertDataAndAlterTable, args=(1, ))
+
+        t1.start()
+        time.sleep(2)
+        t2.start()
+        t1.join()
+        t2.join()
+
+        tdSql.query("select count(*) from test.meters")
+        tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase()) 
\ No newline at end of file

From 485ef89c628c2508fd9d110c78e84672eea9ecdb Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Sun, 6 Dec 2020 23:42:18 +0800
Subject: [PATCH 084/118] delete maxSqlLength.py

---
 tests/pytest/insert/maxSqlLength.py | 93 -----------------------------
 1 file changed, 93 deletions(-)
 delete mode 100644 tests/pytest/insert/maxSqlLength.py

diff --git a/tests/pytest/insert/maxSqlLength.py b/tests/pytest/insert/maxSqlLength.py
deleted file mode 100644
index c114372df2..0000000000
--- a/tests/pytest/insert/maxSqlLength.py
+++ /dev/null
@@ -1,93 +0,0 @@
-###################################################################
-#           Copyright (c) 2016 by TAOS Technologies, Inc.
-#                     All rights reserved.
-#
-#  This file is proprietary and confidential to TAOS Technologies.
-#  No part of this file may be reproduced, stored, transmitted,
-#  disclosed or used in any form or by any means other than as
-#  expressly provided by the written permission from Jianhui Tao
-#
-###################################################################
-
-# -*- coding: utf-8 -*-
-
-import sys
-import datetime
-from util.log import *
-from util.cases import *
-from util.sql import *
-from util.dnodes import *
-import string
-import random
-
-
-class TDTestCase:
-    def init(self, conn, logSql):
-        tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor(), logSql)
-
-        self.ts = 1537146000000
-
-    def get_random_string(self, length):
-        letters = string.ascii_lowercase
-        result_str = ''.join(random.choice(letters) for i in range(length))
-        return result_str
-
-    def run(self):
-        tdSql.prepare()
-
-        tdSql.execute("create table tb(ts timestamp, name1 binary(1000), name2 binary(1000), name3 binary(1000))")
-        
-        sql = "insert into tb values"
-        for i in range(21):
-            value = self.get_random_string(1000)
-            sql += "(%d, '%s', '%s', '%s')" % (self.ts + i, value, value, value)        
-        tdSql.execute(sql)
-
-        self.ts += 21
-        for i in range(22):
-            value = self.get_random_string(1000)
-            sql += "(%d, '%s', '%s', '%s')" % (self.ts + i, value, value, value)        
-        tdSql.error(sql)
-
-        tdSql.query("select * from tb")
-        tdSql.checkRows(21)
-        
-        tdDnodes.stop(1)             
-        tdDnodes.setTestCluster(False)
-        tdDnodes.setValgrind(False)        
-        tdDnodes.deploy(1)                    
-        tdLog.sleep(20)
-        tdDnodes.start(1)
-        tdDnodes.addSimExtraCfg("maxSQLLength", "1048576")
-        
-        
-        tdSql.close()
-        tdSql.prepare()  
-        tdSql.execute("create table tb(ts timestamp, name1 binary(1000), name2 binary(1000), name3 binary(1000))")
-        
-        sql = "insert into tb values"
-        for i in range(22):
-            value = self.get_random_string(1000)
-            sql += "(%d, '%s', '%s', '%s')" % (self.ts + i, value, value, value)        
-        tdSql.execute(sql)
-        
-        tdSql.query("select * from tb")
-        tdSql.checkRows(43)
-            
-        self.ts += 43
-        for i in range(330):
-            value = self.get_random_string(1000)
-            sql += "(%d, '%s', '%s', '%s')" % (self.ts + i, value, value, value)        
-        tdSql.execute(sql)
-
-        tdSql.query("select * from tb")
-        tdSql.checkRows(379)
-                
-    def stop(self):
-        tdSql.close()
-        tdLog.success("%s successfully executed" % __file__)
-
-
-tdCases.addWindows(__file__, TDTestCase())
-tdCases.addLinux(__file__, TDTestCase())

From 5aef8c57acc8a1d719b20da00afda05f5262466c Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 7 Dec 2020 12:57:35 +0800
Subject: [PATCH 085/118] Re implement the TD-1925 code because it cannot merge

---
 CMakeLists.txt                  |   1 -
 cmake/define.inc                |   4 -
 cmake/input.inc                 |   5 -
 src/CMakeLists.txt              |   4 +-
 src/common/inc/tglobal.h        |   1 +
 src/dnode/CMakeLists.txt        |   6 +-
 src/dnode/inc/dnodeCfg.h        |   1 +
 src/dnode/inc/dnodeCheck.h      |   1 +
 src/dnode/inc/dnodeEps.h        |   3 +-
 src/dnode/inc/dnodeInt.h        |   7 +-
 src/dnode/inc/dnodeMInfos.h     |   7 +-
 src/dnode/inc/dnodeMPeer.h      |   1 +
 src/dnode/inc/dnodeMRead.h      |   1 +
 src/dnode/inc/dnodeMWrite.h     |   1 +
 src/dnode/inc/dnodeMain.h       |   1 +
 src/dnode/inc/dnodeModule.h     |   1 +
 src/dnode/inc/dnodePeer.h       |   1 +
 src/dnode/inc/dnodeShell.h      |   1 +
 src/dnode/inc/dnodeStep.h       |  33 ++
 src/dnode/inc/dnodeTelemetry.h  |   1 +
 src/dnode/inc/dnodeVMgmt.h      |  32 ++
 src/dnode/inc/dnodeVRead.h      |   1 +
 src/dnode/inc/dnodeVWrite.h     |   1 +
 src/dnode/inc/dnodeVnodes.h     |  34 ++
 src/dnode/src/dnodeCfg.c        |   3 -
 src/dnode/src/dnodeCheck.c      |   2 -
 src/dnode/src/dnodeEps.c        |   3 -
 src/dnode/src/dnodeMInfos.c     |  25 +-
 src/dnode/src/dnodeMPeer.c      |   9 +-
 src/dnode/src/dnodeMRead.c      |   9 +-
 src/dnode/src/dnodeMWrite.c     |  10 +-
 src/dnode/src/dnodeMain.c       |  87 ++---
 src/dnode/src/dnodeMgmt.c       | 577 --------------------------------
 src/dnode/src/dnodeModule.c     |   5 -
 src/dnode/src/dnodePeer.c       |  24 +-
 src/dnode/src/dnodeShell.c      |  28 +-
 src/dnode/src/dnodeStep.c       |  73 ++++
 src/dnode/src/dnodeSystem.c     |   3 -
 src/dnode/src/dnodeTelemetry.c  |   7 +-
 src/dnode/src/dnodeVMgmt.c      | 239 +++++++++++++
 src/dnode/src/dnodeVRead.c      |   6 +-
 src/dnode/src/dnodeVWrite.c     |   7 +-
 src/dnode/src/dnodeVnodes.c     | 289 ++++++++++++++++
 src/inc/dnode.h                 |  12 +-
 src/inc/taosdef.h               |   7 +
 src/inc/taoserror.h             |  13 +
 src/inc/taosmsg.h               |  13 +-
 src/inc/tsdb.h                  |   2 +-
 src/inc/vnode.h                 |  35 +-
 src/kit/shell/inc/shell.h       |   1 -
 src/kit/shell/src/shellEngine.c |   4 +-
 src/kit/shell/src/shellLinux.c  |  15 +-
 src/kit/shell/src/shellMain.c   |   7 +-
 src/mnode/src/mnodeDb.c         |   7 -
 src/mnode/src/mnodeDnode.c      |  62 ----
 src/mnode/src/mnodeMain.c       |  28 +-
 src/rpc/src/rpcMain.c           |   7 -
 src/tsdb/src/tsdbCommitQueue.c  |   4 +-
 src/util/inc/tnettest.h         |  22 +-
 src/util/src/tnettest.c         | 512 +++++++++++++---------------
 src/util/src/tqueue.c           |   4 +-
 src/vnode/inc/vnodeCfg.h        |   1 +
 src/vnode/inc/vnodeInt.h        |  64 ++--
 src/vnode/inc/vnodeMain.h       |  37 ++
 src/vnode/inc/vnodeMgmt.h       |  42 +++
 src/vnode/inc/vnodeRead.h       |  35 ++
 src/vnode/inc/vnodeStatus.h     |  47 +++
 src/vnode/inc/vnodeSync.h       |  39 +++
 src/vnode/inc/vnodeVersion.h    |   1 +
 src/vnode/inc/vnodeWorker.h     |  33 ++
 src/vnode/inc/vnodeWrite.h      |  35 ++
 src/vnode/src/vnodeCfg.c        |   4 -
 src/vnode/src/vnodeMain.c       | 413 ++++-------------------
 src/vnode/src/vnodeMgmt.c       | 192 +++++++++++
 src/vnode/src/vnodeRead.c       |  18 +-
 src/vnode/src/vnodeStatus.c     | 142 ++++++++
 src/vnode/src/vnodeSync.c       | 151 +++++++++
 src/vnode/src/vnodeVersion.c    |   3 -
 src/vnode/src/vnodeWorker.c     | 208 ++++++++++++
 src/vnode/src/vnodeWrite.c      |  25 +-
 80 files changed, 2202 insertions(+), 1598 deletions(-)
 create mode 100644 src/dnode/inc/dnodeStep.h
 create mode 100644 src/dnode/inc/dnodeVMgmt.h
 create mode 100644 src/dnode/inc/dnodeVnodes.h
 delete mode 100644 src/dnode/src/dnodeMgmt.c
 create mode 100644 src/dnode/src/dnodeStep.c
 create mode 100644 src/dnode/src/dnodeVMgmt.c
 create mode 100644 src/dnode/src/dnodeVnodes.c
 create mode 100644 src/vnode/inc/vnodeMain.h
 create mode 100644 src/vnode/inc/vnodeMgmt.h
 create mode 100644 src/vnode/inc/vnodeRead.h
 create mode 100644 src/vnode/inc/vnodeStatus.h
 create mode 100644 src/vnode/inc/vnodeSync.h
 create mode 100644 src/vnode/inc/vnodeWorker.h
 create mode 100644 src/vnode/inc/vnodeWrite.h
 create mode 100644 src/vnode/src/vnodeMgmt.c
 create mode 100644 src/vnode/src/vnodeStatus.c
 create mode 100644 src/vnode/src/vnodeSync.c
 create mode 100644 src/vnode/src/vnodeWorker.c

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 588526c286..eb2b1cceb4 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -13,7 +13,6 @@ ENDIF ()
 SET(TD_ACCOUNT FALSE)
 SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)
-SET(TD_SYNC TRUE)
 SET(TD_MQTT TRUE)
 SET(TD_TSDB_PLUGINS FALSE)
 
diff --git a/cmake/define.inc b/cmake/define.inc
index 6e64c2709a..782dc625bf 100755
--- a/cmake/define.inc
+++ b/cmake/define.inc
@@ -13,10 +13,6 @@ IF (TD_GRANT)
   ADD_DEFINITIONS(-D_GRANT)
 ENDIF ()
 
-IF (TD_SYNC)
-  ADD_DEFINITIONS(-D_SYNC)
-ENDIF ()
-
 IF (TD_MQTT)
   ADD_DEFINITIONS(-D_MQTT)
 ENDIF ()
diff --git a/cmake/input.inc b/cmake/input.inc
index 1ef2045f57..e8324887a0 100755
--- a/cmake/input.inc
+++ b/cmake/input.inc
@@ -47,11 +47,6 @@ IF (${MQTT} MATCHES "false")
   MESSAGE(STATUS "build without mqtt module")
 ENDIF ()
 
-IF (${SYNC} MATCHES "false")
-  SET(TD_SYNC FALSE)
-  MESSAGE(STATUS "build without sync module")
-ENDIF ()
-
 IF (${RANDOM_FILE_FAIL} MATCHES "true")
   SET(TD_RANDOM_FILE_FAIL TRUE)
   MESSAGE(STATUS "build with random-file-fail enabled")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index a2600785c3..931a0a132e 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -10,9 +10,7 @@ ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)
 ADD_SUBDIRECTORY(kit)
 ADD_SUBDIRECTORY(plugins)
-IF (TD_SYNC)
-  ADD_SUBDIRECTORY(sync)
-ENDIF ()
+ADD_SUBDIRECTORY(sync)
 ADD_SUBDIRECTORY(balance)
 ADD_SUBDIRECTORY(mnode)
 ADD_SUBDIRECTORY(vnode)
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index 851ca57ba9..5b88c9b0d0 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -35,6 +35,7 @@ extern int32_t  tsNumOfMnodes;
 extern int32_t  tsEnableVnodeBak;
 extern int32_t  tsEnableTelemetryReporting;
 extern char     tsEmail[];
+extern char     tsArbitrator[];
 
 // common
 extern int      tsRpcTimer;
diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt
index 5608cfd6d1..ebb25bf858 100644
--- a/src/dnode/CMakeLists.txt
+++ b/src/dnode/CMakeLists.txt
@@ -12,7 +12,7 @@ AUX_SOURCE_DIRECTORY(src SRC)
 
 IF (TD_LINUX)
   ADD_EXECUTABLE(taosd ${SRC})
-  TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4)
+  TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync)
 
   IF (TD_SOMODE_STATIC)
     TARGET_LINK_LIBRARIES(taosd taos_static)
@@ -32,10 +32,6 @@ IF (TD_LINUX)
     TARGET_LINK_LIBRARIES(taosd mqtt)
   ENDIF ()
 
-  IF (TD_SYNC)
-    TARGET_LINK_LIBRARIES(taosd balance sync)
-  ENDIF ()
-
   SET(PREPARE_ENV_CMD "prepare_env_cmd")
   SET(PREPARE_ENV_TARGET "prepare_env_target")
   ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h
index 35d8896460..d74303f325 100644
--- a/src/dnode/inc/dnodeCfg.h
+++ b/src/dnode/inc/dnodeCfg.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitCfg();
 void    dnodeCleanupCfg();
diff --git a/src/dnode/inc/dnodeCheck.h b/src/dnode/inc/dnodeCheck.h
index a4880b3c11..c94b9e9319 100644
--- a/src/dnode/inc/dnodeCheck.h
+++ b/src/dnode/inc/dnodeCheck.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitCheck();
 void    dnodeCleanupCheck();
diff --git a/src/dnode/inc/dnodeEps.h b/src/dnode/inc/dnodeEps.h
index 2a203498c1..a5840997b0 100644
--- a/src/dnode/inc/dnodeEps.h
+++ b/src/dnode/inc/dnodeEps.h
@@ -19,8 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
-#include "taosmsg.h"
+#include "dnodeInt.h"
 
 int32_t dnodeInitEps();
 void    dnodeCleanupEps();
diff --git a/src/dnode/inc/dnodeInt.h b/src/dnode/inc/dnodeInt.h
index f4cbee1d13..7595f5fd02 100644
--- a/src/dnode/inc/dnodeInt.h
+++ b/src/dnode/inc/dnodeInt.h
@@ -19,8 +19,13 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
+#include "taoserror.h"
+#include "taosmsg.h"
 #include "tlog.h"
+#include "trpc.h"
+#include "tglobal.h"
+#include "dnode.h"
+#include "vnode.h"
 
 extern int32_t dDebugFlag;
 
diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h
index 2c3eef5d5d..f05e2b6f7b 100644
--- a/src/dnode/inc/dnodeMInfos.h
+++ b/src/dnode/inc/dnodeMInfos.h
@@ -19,8 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
-#include "taosmsg.h"
+#include "dnodeInt.h"
 
 int32_t dnodeInitMInfos();
 void    dnodeCleanupMInfos();
@@ -29,6 +28,10 @@ void    dnodeUpdateEpSetForPeer(SRpcEpSet *pEpSet);
 void    dnodeGetMInfos(SMInfos *pMinfos);
 bool    dnodeIsMasterEp(char *ep);
 
+void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/dnode/inc/dnodeMPeer.h b/src/dnode/inc/dnodeMPeer.h
index 00221baa22..b7e566d7e4 100644
--- a/src/dnode/inc/dnodeMPeer.h
+++ b/src/dnode/inc/dnodeMPeer.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMPeer();
 void    dnodeCleanupMPeer();
diff --git a/src/dnode/inc/dnodeMRead.h b/src/dnode/inc/dnodeMRead.h
index 8a8e71227d..279098d30e 100644
--- a/src/dnode/inc/dnodeMRead.h
+++ b/src/dnode/inc/dnodeMRead.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMRead();
 void    dnodeCleanupMRead();
diff --git a/src/dnode/inc/dnodeMWrite.h b/src/dnode/inc/dnodeMWrite.h
index 6a3d41bc81..8d4fcce3be 100644
--- a/src/dnode/inc/dnodeMWrite.h
+++ b/src/dnode/inc/dnodeMWrite.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMWrite();
 void    dnodeCleanupMWrite();
diff --git a/src/dnode/inc/dnodeMain.h b/src/dnode/inc/dnodeMain.h
index c1480407bd..ca79d53afd 100644
--- a/src/dnode/inc/dnodeMain.h
+++ b/src/dnode/inc/dnodeMain.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitSystem();
 void    dnodeCleanUpSystem();
diff --git a/src/dnode/inc/dnodeModule.h b/src/dnode/inc/dnodeModule.h
index 8618de3244..edcefbdd0c 100644
--- a/src/dnode/inc/dnodeModule.h
+++ b/src/dnode/inc/dnodeModule.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitModules();
 void    dnodeStartModules();
diff --git a/src/dnode/inc/dnodePeer.h b/src/dnode/inc/dnodePeer.h
index 0dcf48f232..6d337ef6dc 100644
--- a/src/dnode/inc/dnodePeer.h
+++ b/src/dnode/inc/dnodePeer.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitServer();
 void    dnodeCleanupServer();
diff --git a/src/dnode/inc/dnodeShell.h b/src/dnode/inc/dnodeShell.h
index 300c86c599..3fa66d6a3b 100644
--- a/src/dnode/inc/dnodeShell.h
+++ b/src/dnode/inc/dnodeShell.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitShell();
 void    dnodeCleanupShell();
diff --git a/src/dnode/inc/dnodeStep.h b/src/dnode/inc/dnodeStep.h
new file mode 100644
index 0000000000..e181e19c46
--- /dev/null
+++ b/src/dnode/inc/dnodeStep.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_STEP_H
+#define TDENGINE_DNODE_STEP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "dnodeInt.h"
+
+int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize);
+void    dnodeStepCleanup(SStep *pSteps, int32_t stepSize);
+void    dnodeReportStep(char *name, char *desc, int8_t finished);
+void    dnodeSendStartupStep(SRpcMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/dnode/inc/dnodeTelemetry.h b/src/dnode/inc/dnodeTelemetry.h
index 6fb62556ae..e4fd5a0376 100644
--- a/src/dnode/inc/dnodeTelemetry.h
+++ b/src/dnode/inc/dnodeTelemetry.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitTelemetry();
 void    dnodeCleanupTelemetry();
diff --git a/src/dnode/inc/dnodeVMgmt.h b/src/dnode/inc/dnodeVMgmt.h
new file mode 100644
index 0000000000..821196defc
--- /dev/null
+++ b/src/dnode/inc/dnodeVMgmt.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_VMGMT_H
+#define TDENGINE_DNODE_VMGMT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "dnodeInt.h"
+
+int32_t dnodeInitVMgmt();
+void    dnodeCleanupVMgmt();
+void    dnodeDispatchToVMgmtQueue(SRpcMsg *rpcMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/dnode/inc/dnodeVRead.h b/src/dnode/inc/dnodeVRead.h
index 5b17693146..30dfb1b3a4 100644
--- a/src/dnode/inc/dnodeVRead.h
+++ b/src/dnode/inc/dnodeVRead.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitVRead();
 void    dnodeCleanupVRead();
diff --git a/src/dnode/inc/dnodeVWrite.h b/src/dnode/inc/dnodeVWrite.h
index 759e9ca8a5..2ddff210f8 100644
--- a/src/dnode/inc/dnodeVWrite.h
+++ b/src/dnode/inc/dnodeVWrite.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitVWrite();
 void    dnodeCleanupVWrite();
diff --git a/src/dnode/inc/dnodeVnodes.h b/src/dnode/inc/dnodeVnodes.h
new file mode 100644
index 0000000000..1785ed3d06
--- /dev/null
+++ b/src/dnode/inc/dnodeVnodes.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_VNODES_H
+#define TDENGINE_DNODE_VNODES_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "dnodeInt.h"
+
+int32_t dnodeInitVnodes();
+void    dnodeCleanupVnodes();
+int32_t dnodeInitTimer();
+void    dnodeCleanupTimer();
+void    dnodeSendStatusMsgToMnode();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c
index 16d109a13a..89249d773b 100644
--- a/src/dnode/src/dnodeCfg.c
+++ b/src/dnode/src/dnodeCfg.c
@@ -16,9 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeCfg.h"
 
 static SDnodeCfg tsCfg = {0};
diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c
index a9ee4ac649..be26bb967b 100644
--- a/src/dnode/src/dnodeCheck.c
+++ b/src/dnode/src/dnodeCheck.c
@@ -15,8 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tglobal.h"
-#include "dnodeInt.h"
 #include "dnodeCheck.h"
 
 typedef struct {
diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c
index 5fb3828a84..09151533e2 100644
--- a/src/dnode/src/dnodeEps.c
+++ b/src/dnode/src/dnodeEps.c
@@ -16,10 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
 #include "hash.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeEps.h"
 
 static SDnodeEps *tsEps = NULL;
diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c
index a88c3528f7..7c385a889d 100644
--- a/src/dnode/src/dnodeMInfos.c
+++ b/src/dnode/src/dnodeMInfos.c
@@ -16,10 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeMInfos.h"
 
 static SMInfos   tsMInfos;
@@ -297,3 +294,25 @@ static int32_t dnodeWriteMInfos() {
   dInfo("successed to write %s", file);
   return 0;
 }
+
+void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
+  SRpcConnInfo connInfo = {0};
+  rpcGetConnInfo(rpcMsg->handle, &connInfo);
+
+  SRpcEpSet epSet = {0};
+  if (forShell) {
+    dnodeGetEpSetForShell(&epSet);
+  } else {
+    dnodeGetEpSetForPeer(&epSet);
+  }
+
+  dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
+         taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
+
+  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+    dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
+    epSet.port[i] = htons(epSet.port[i]);
+  }
+
+  rpcSendRedirectRsp(rpcMsg->handle, &epSet);
+}
\ No newline at end of file
diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c
index ee6dc5212e..0863666f76 100644
--- a/src/dnode/src/dnodeMPeer.c
+++ b/src/dnode/src/dnodeMPeer.c
@@ -15,16 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "tqueue.h"
 #include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "dnodeVMgmt.h"
+#include "dnodeMInfos.h"
 #include "dnodeMWrite.h"
 
 typedef struct {
diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c
index 65f3af7b3b..0fc6400d99 100644
--- a/src/dnode/src/dnodeMRead.c
+++ b/src/dnode/src/dnodeMRead.c
@@ -15,16 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "tqueue.h"
 #include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "dnodeVMgmt.h"
+#include "dnodeMInfos.h"
 #include "dnodeMRead.h"
 
 typedef struct {
diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c
index ef2d49ef42..bc387e2171 100644
--- a/src/dnode/src/dnodeMWrite.c
+++ b/src/dnode/src/dnodeMWrite.c
@@ -15,17 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "ttimer.h"
 #include "tqueue.h"
-#include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "dnodeVMgmt.h"
+#include "dnodeMInfos.h"
 #include "dnodeMWrite.h"
 
 typedef struct {
diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c
index bd065d0d0e..502c410ce3 100644
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@@ -16,15 +16,12 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taos.h"
-#include "tutil.h"
 #include "tconfig.h"
-#include "tglobal.h"
 #include "tfile.h"
 #include "twal.h"
-#include "trpc.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+// #include "tfs.h"
+#include "tsync.h"
+#include "dnodeStep.h"
 #include "dnodePeer.h"
 #include "dnodeModule.h"
 #include "dnodeEps.h"
@@ -33,6 +30,8 @@
 #include "dnodeCheck.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
+#include "dnodeVMgmt.h"
+#include "dnodeVnodes.h"
 #include "dnodeMRead.h"
 #include "dnodeMWrite.h"
 #include "dnodeMPeer.h"
@@ -45,38 +44,32 @@ static int32_t dnodeInitStorage();
 static void    dnodeCleanupStorage();
 static void    dnodeSetRunStatus(SRunStatus status);
 static void    dnodeCheckDataDirOpenned(char *dir);
-static int32_t dnodeInitComponents();
-static void    dnodeCleanupComponents(int32_t stepId);
 static int     dnodeCreateDir(const char *dir);
 
-typedef struct {
-  const char *const name;
-  int32_t (*init)();
-  void (*cleanup)();
-} SDnodeComponent;
-
-static const SDnodeComponent tsDnodeComponents[] = {
-  {"tfile",     tfInit,              tfCleanup},
-  {"rpc",       rpcInit,             rpcCleanup},
-  {"globalcfg" ,taosCheckGlobalCfg,  NULL},
-  {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
-  {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},
-  {"dnodeeps",  dnodeInitEps,        dnodeCleanupEps},
-  {"mnodeinfos",dnodeInitMInfos,     dnodeCleanupMInfos},
-  {"wal",       walInit,             walCleanUp},
-  {"check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
-  {"vread",     dnodeInitVRead,      dnodeCleanupVRead},
-  {"vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
-  {"mread",     dnodeInitMRead,      dnodeCleanupMRead},
-  {"mwrite",    dnodeInitMWrite,     dnodeCleanupMWrite},
-  {"mpeer",     dnodeInitMPeer,      dnodeCleanupMPeer},  
-  {"client",    dnodeInitClient,     dnodeCleanupClient},
-  {"server",    dnodeInitServer,     dnodeCleanupServer},
-  {"mgmt",      dnodeInitMgmt,       dnodeCleanupMgmt},
-  {"modules",   dnodeInitModules,    dnodeCleanupModules},
-  {"mgmt-tmr",  dnodeInitMgmtTimer,  dnodeCleanupMgmtTimer},
-  {"shell",     dnodeInitShell,      dnodeCleanupShell},
-  {"telemetry", dnodeInitTelemetry,  dnodeCleanupTelemetry},
+static SStep tsDnodeSteps[] = {
+  {"dnode-tfile",     tfInit,              tfCleanup},
+  {"dnode-rpc",       rpcInit,             rpcCleanup},
+  {"dnode-globalcfg", taosCheckGlobalCfg,  NULL},
+  {"dnode-storage",   dnodeInitStorage,    dnodeCleanupStorage},
+  {"dnode-cfg",       dnodeInitCfg,        dnodeCleanupCfg},
+  {"dnode-eps",       dnodeInitEps,        dnodeCleanupEps},
+  {"dnode-minfos",    dnodeInitMInfos,     dnodeCleanupMInfos},
+  {"dnode-wal",       walInit,             walCleanUp},
+  {"dnode-sync",      syncInit,            syncCleanUp},
+  {"dnode-check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
+  {"dnode-vread",     dnodeInitVRead,      dnodeCleanupVRead},
+  {"dnode-vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
+  {"dnode-vmgmt",     dnodeInitVMgmt,      dnodeCleanupVMgmt},
+  {"dnode-mread",     dnodeInitMRead,      dnodeCleanupMRead},
+  {"dnode-mwrite",    dnodeInitMWrite,     dnodeCleanupMWrite},
+  {"dnode-mpeer",     dnodeInitMPeer,      dnodeCleanupMPeer},  
+  {"dnode-client",    dnodeInitClient,     dnodeCleanupClient},
+  {"dnode-server",    dnodeInitServer,     dnodeCleanupServer},
+  {"dnode-vnodes",    dnodeInitVnodes,     dnodeCleanupVnodes},
+  {"dnode-modules",   dnodeInitModules,    dnodeCleanupModules},
+  {"dnode-tmr",       dnodeInitTimer,      dnodeCleanupTimer},
+  {"dnode-shell",     dnodeInitShell,      dnodeCleanupShell},
+  {"dnode-telemetry", dnodeInitTelemetry,  dnodeCleanupTelemetry},
 };
 
 static int dnodeCreateDir(const char *dir) {
@@ -87,24 +80,14 @@ static int dnodeCreateDir(const char *dir) {
   return 0;
 }
 
-static void dnodeCleanupComponents(int32_t stepId) {
-  for (int32_t i = stepId; i >= 0; i--) {
-    if (tsDnodeComponents[i].cleanup) {
-      (*tsDnodeComponents[i].cleanup)();
-    }
-  }
+static void dnodeCleanupComponents() {
+  int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep);
+  dnodeStepCleanup(tsDnodeSteps, stepSize);
 }
 
 static int32_t dnodeInitComponents() {
-  int32_t code = 0;
-  for (int32_t i = 0; i < sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]); i++) {
-    if (tsDnodeComponents[i].init() != 0) {
-      dnodeCleanupComponents(i);
-      code = -1;
-      break;
-    }
-  }
-  return code;
+  int32_t stepSize = sizeof(tsDnodeSteps) / sizeof(SStep);
+  return dnodeStepInit(tsDnodeSteps, stepSize);
 }
 
 int32_t dnodeInitSystem() {
@@ -151,7 +134,7 @@ int32_t dnodeInitSystem() {
 void dnodeCleanUpSystem() {
   if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
     dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
-    dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1);
+    dnodeCleanupComponents();
     taos_cleanup();
     taosCloseLog();
   }
diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
deleted file mode 100644
index cce593727d..0000000000
--- a/src/dnode/src/dnodeMgmt.c
+++ /dev/null
@@ -1,577 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "cJSON.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "ttimer.h"
-#include "tsdb.h"
-#include "twal.h"
-#include "tqueue.h"
-#include "tsync.h"
-#include "ttimer.h"
-#include "tbn.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "vnode.h"
-#include "mnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
-#include "dnodeEps.h"
-#include "dnodeCfg.h"
-#include "dnodeMInfos.h"
-#include "dnodeVRead.h"
-#include "dnodeVWrite.h"
-#include "dnodeModule.h"
-
-typedef struct {
-  pthread_t thread;
-  int32_t   threadIndex;
-  int32_t   failed;
-  int32_t   opened;
-  int32_t   vnodeNum;
-  int32_t * vnodeList;
-} SOpenVnodeThread;
-
-typedef struct {
-  SRpcMsg rpcMsg;
-  char    pCont[];
-} SMgmtMsg;
-
-void *            tsDnodeTmr = NULL;
-static void *     tsStatusTimer = NULL;
-static uint32_t   tsRebootTime;
-static taos_qset  tsMgmtQset = NULL;
-static taos_queue tsMgmtQueue = NULL;
-static pthread_t  tsQthread;
-
-static void   dnodeProcessStatusRsp(SRpcMsg *pMsg);
-static void   dnodeSendStatusMsg(void *handle, void *tmrId);
-static void  *dnodeProcessMgmtQueue(void *param);
-
-static int32_t  dnodeOpenVnodes();
-static void     dnodeCloseVnodes();
-static int32_t  dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
-static int32_t  dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
-static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
-
-int32_t dnodeInitMgmt() {
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeProcessAlterVnodeMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeProcessDropVnodeMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
-  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
-
-  dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP,  dnodeProcessStatusRsp);
-  tsRebootTime = taosGetTimestampSec();
-
-  int32_t code = vnodeInitResources();
-  if (code != TSDB_CODE_SUCCESS) {
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  // create the queue and thread to handle the message 
-  tsMgmtQset = taosOpenQset();
-  if (tsMgmtQset == NULL) {
-    dError("failed to create the mgmt queue set");
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  tsMgmtQueue = taosOpenQueue();
-  if (tsMgmtQueue == NULL) {
-    dError("failed to create the mgmt queue");
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  taosAddIntoQset(tsMgmtQset, tsMgmtQueue, NULL);
-
-  pthread_attr_t thAttr;
-  pthread_attr_init(&thAttr);
-  pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
-
-  code = pthread_create(&tsQthread, &thAttr, dnodeProcessMgmtQueue, NULL);
-  pthread_attr_destroy(&thAttr);
-  if (code != 0) {
-    dError("failed to create thread to process mgmt queue, reason:%s", strerror(errno));
-    dnodeCleanupMgmt();
-    return -1; 
-  }
-
-  code = dnodeOpenVnodes();
-  if (code != TSDB_CODE_SUCCESS) {
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  dInfo("dnode mgmt is initialized");
- 
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t dnodeInitMgmtTimer() {
-  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
-  if (tsDnodeTmr == NULL) {
-    dError("failed to init dnode timer");
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
-  dInfo("dnode mgmt timer is initialized");
-  return TSDB_CODE_SUCCESS;
-}
-
-void dnodeSendStatusMsgToMnode() {
-  if (tsDnodeTmr != NULL && tsStatusTimer != NULL) {
-    dInfo("force send status msg to mnode");
-    taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer);
-  }
-}
-
-void dnodeCleanupMgmtTimer() {
-  if (tsStatusTimer != NULL) {
-    taosTmrStopA(&tsStatusTimer);
-    tsStatusTimer = NULL;
-  }
-
-  if (tsDnodeTmr != NULL) {
-    taosTmrCleanUp(tsDnodeTmr);
-    tsDnodeTmr = NULL;
-  }
-}
-
-void dnodeCleanupMgmt() {
-  dnodeCleanupMgmtTimer();
-  dnodeCloseVnodes();
-
-  if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset);
-  if (tsQthread) pthread_join(tsQthread, NULL);
-
-  if (tsMgmtQueue) taosCloseQueue(tsMgmtQueue);
-  if (tsMgmtQset) taosCloseQset(tsMgmtQset);
-  tsMgmtQset = NULL;
-  tsMgmtQueue = NULL;
-
-  vnodeCleanupResources();
-}
-
-static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
-  int32_t   size = sizeof(SMgmtMsg) + pMsg->contLen;
-  SMgmtMsg *pMgmt = taosAllocateQitem(size);
-  if (pMgmt == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-
-  pMgmt->rpcMsg = *pMsg;
-  pMgmt->rpcMsg.pCont = pMgmt->pCont;
-  memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
-  taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
-  int32_t code = dnodeWriteToMgmtQueue(pMsg);
-  if (code != TSDB_CODE_SUCCESS) {
-    SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
-    rpcSendResponse(&rsp);
-  }
-
-  rpcFreeCont(pMsg->pCont);
-}
-
-static void *dnodeProcessMgmtQueue(void *param) {
-  SMgmtMsg *pMgmt;
-  SRpcMsg * pMsg;
-  SRpcMsg   rsp = {0};
-  int32_t   qtype;
-  void *    handle;
-
-  while (1) {
-    if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
-      dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
-      break;
-    }
-
-    pMsg = &pMgmt->rpcMsg;
-    dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]);
-    if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
-      rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
-    } else {
-      rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-    }
-
-    rsp.handle = pMsg->handle;
-    rsp.pCont = NULL;
-    rpcSendResponse(&rsp);
-
-    taosFreeQitem(pMsg);
-  }
-
-  return NULL;
-}
-
-static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
-  DIR *dir = opendir(tsVnodeDir);
-  if (dir == NULL) {
-    return TSDB_CODE_DND_NO_WRITE_ACCESS;
-  }
-
-  *numOfVnodes = 0;
-  struct dirent *de = NULL;
-  while ((de = readdir(dir)) != NULL) {
-    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-    if (de->d_type & DT_DIR) {
-      if (strncmp("vnode", de->d_name, 5) != 0) continue;
-      int32_t vnode = atoi(de->d_name + 5);
-      if (vnode == 0) continue;
-
-      (*numOfVnodes)++;
-
-      if (*numOfVnodes >= TSDB_MAX_VNODES) {
-        dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
-        continue;
-      } else {
-        vnodeList[*numOfVnodes - 1] = vnode;
-      }
-    }
-  }
-  closedir(dir);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void *dnodeOpenVnode(void *param) {
-  SOpenVnodeThread *pThread = param;
-  char vnodeDir[TSDB_FILENAME_LEN * 3];
-
-  dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
-
-  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
-    int32_t vgId = pThread->vnodeList[v];
-    snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vgId);
-    if (vnodeOpen(vgId, vnodeDir) < 0) {
-      dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
-      pThread->failed++;
-    } else {
-      dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex);
-      pThread->opened++;
-    }
-  }
-
-  dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
-         pThread->failed);
-  return NULL;
-}
-
-static int32_t dnodeOpenVnodes() {
-  int32_t vnodeList[TSDB_MAX_VNODES] = {0};
-  int32_t numOfVnodes = 0;
-  int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
-
-  if (status != TSDB_CODE_SUCCESS) {
-    dInfo("get dnode list failed");
-    return status;
-  }
-
-  int32_t threadNum = tsNumOfCores;
-  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
-  SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread));
-  for (int32_t t = 0; t < threadNum; ++t) {
-    threads[t].threadIndex = t;
-    threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t));
-  }
-
-  for (int32_t v = 0; v < numOfVnodes; ++v) {
-    int32_t t = v % threadNum;
-    SOpenVnodeThread *pThread = &threads[t];
-    pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
-  }
-
-  dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
-
-  for (int32_t t = 0; t < threadNum; ++t) {
-    SOpenVnodeThread *pThread = &threads[t];
-    if (pThread->vnodeNum == 0) continue;
-
-    pthread_attr_t thAttr;
-    pthread_attr_init(&thAttr);
-    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
-    if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) {
-      dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
-    }
-
-    pthread_attr_destroy(&thAttr);
-  }
-
-  int32_t openVnodes = 0;
-  int32_t failedVnodes = 0;
-  for (int32_t t = 0; t < threadNum; ++t) {
-    SOpenVnodeThread *pThread = &threads[t];
-    if (pThread->vnodeNum > 0 && pThread->thread) {
-      pthread_join(pThread->thread, NULL);
-    }
-    openVnodes += pThread->opened;
-    failedVnodes += pThread->failed;
-    free(pThread->vnodeList);
-  }
-
-  free(threads);
-  dInfo("there are total vnodes:%d, openned:%d", numOfVnodes, openVnodes);
-
-  if (failedVnodes != 0) {
-    dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes);
-    return -1;
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void dnodeCloseVnodes() {
-  int32_t vnodeList[TSDB_MAX_VNODES]= {0};
-  int32_t numOfVnodes = 0;
-  int32_t status;
-
-  status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
-
-  if (status != TSDB_CODE_SUCCESS) {
-    dInfo("get dnode list failed");
-    return;
-  }
-
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    vnodeClose(vnodeList[i]);
-  }
-
-  dInfo("total vnodes:%d are all closed", numOfVnodes);
-}
-
-static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
-  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
-  pCreate->cfg.vgId                = htonl(pCreate->cfg.vgId);
-  pCreate->cfg.cfgVersion          = htonl(pCreate->cfg.cfgVersion);
-  pCreate->cfg.maxTables           = htonl(pCreate->cfg.maxTables);
-  pCreate->cfg.cacheBlockSize      = htonl(pCreate->cfg.cacheBlockSize);
-  pCreate->cfg.totalBlocks         = htonl(pCreate->cfg.totalBlocks);
-  pCreate->cfg.daysPerFile         = htonl(pCreate->cfg.daysPerFile);
-  pCreate->cfg.daysToKeep1         = htonl(pCreate->cfg.daysToKeep1);
-  pCreate->cfg.daysToKeep2         = htonl(pCreate->cfg.daysToKeep2);
-  pCreate->cfg.daysToKeep          = htonl(pCreate->cfg.daysToKeep);
-  pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
-  pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
-  pCreate->cfg.fsyncPeriod         = htonl(pCreate->cfg.fsyncPeriod);
-  pCreate->cfg.commitTime          = htonl(pCreate->cfg.commitTime);
-
-  for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
-    pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
-  }
-
-  return pCreate;
-}
-
-static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
-  SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
-
-  void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
-  if (pVnode != NULL) {
-    dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId);
-    vnodeRelease(pVnode);
-    return TSDB_CODE_SUCCESS;
-  } else {
-    dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId);
-    return vnodeCreate(pCreate);
-  }
-}
-
-static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
-  SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
-
-  void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
-  if (pVnode != NULL) {
-    dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId);
-    int32_t code = vnodeAlter(pVnode, pAlter);
-    vnodeRelease(pVnode);
-    return code;
-  } else {
-    dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
-    return TSDB_CODE_VND_INVALID_VGROUP_ID;
-  }
-}
-
-static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
-  SDropVnodeMsg *pDrop = rpcMsg->pCont;
-  pDrop->vgId = htonl(pDrop->vgId);
-
-  return vnodeDrop(pDrop->vgId);
-}
-
-static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
-//  SAlterStreamMsg *pStream = pCont;
-//  pStream->uid    = htobe64(pStream->uid);
-//  pStream->stime  = htobe64(pStream->stime);
-//  pStream->vnode  = htonl(pStream->vnode);
-//  pStream->sid    = htonl(pStream->sid);
-//  pStream->status = htonl(pStream->status);
-//
-//  int32_t code = dnodeCreateStream(pStream);
-
-  return 0;
-}
-
-static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
-  SCfgDnodeMsg *pCfg = pMsg->pCont;
-  return taosCfgDynamicOptions(pCfg->config);
-}
-
-static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
-  SCreateMnodeMsg *pCfg = pMsg->pCont;
-  pCfg->dnodeId = htonl(pCfg->dnodeId);
-  if (pCfg->dnodeId != dnodeGetDnodeId()) {
-    dDebug("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
-    return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
-  }
-
-  if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) {
-    dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
-    return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
-  }
-
-  dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
-  for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
-    pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
-    dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
-  }
-
-  dnodeStartMnode(&pCfg->mnodes);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
-  if (pMsg->code != TSDB_CODE_SUCCESS) {
-    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    return;
-  }
-
-  SStatusRsp *pStatusRsp = pMsg->pCont;
-  SMInfos *pMinfos = &pStatusRsp->mnodes;
-  dnodeUpdateMInfos(pMinfos);
-
-  SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
-  pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
-  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
-  pCfg->dnodeId = htonl(pCfg->dnodeId);
-  dnodeUpdateCfg(pCfg);
-
-  vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
-
-  SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
-  dnodeUpdateEps(pEps);
-
-  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-}
-
-static void dnodeSendStatusMsg(void *handle, void *tmrId) {
-  if (tsDnodeTmr == NULL) {
-    dError("dnode timer is already released");
-    return;
-  }
-
-  if (tsStatusTimer == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    dError("failed to start status timer");
-    return;
-  }
-
-  int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-  SStatusMsg *pStatus = rpcMallocCont(contLen);
-  if (pStatus == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    dError("failed to malloc status message");
-    return;
-  }
-
-  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
-  pStatus->dnodeId          = htonl(dnodeGetDnodeId());
-  pStatus->version          = htonl(tsVersion);
-  pStatus->lastReboot       = htonl(tsRebootTime);
-  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
-  pStatus->diskAvailable    = tsAvailDataDirGB;
-  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
-  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
-
-  // fill cluster cfg parameters
-  pStatus->clusterCfg.numOfMnodes        = htonl(tsNumOfMnodes);
-  pStatus->clusterCfg.enableBalance      = htonl(tsEnableBalance);
-  pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
-  pStatus->clusterCfg.offlineThreshold   = htonl(tsOfflineThreshold);
-  pStatus->clusterCfg.statusInterval     = htonl(tsStatusInterval);
-  pStatus->clusterCfg.maxtablesPerVnode  = htonl(tsMaxTablePerVnode);
-  pStatus->clusterCfg.maxVgroupsPerDb    = htonl(tsMaxVgroupsPerDb);
-  tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN);
-  tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64);
-  pStatus->clusterCfg.checkTime = 0;
-  char timestr[32] = "1970-01-01 00:00:00.00";
-  (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
-  tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
-  tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);  
-  
-  vnodeBuildStatusMsg(pStatus);
-  contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
-  pStatus->openVnodes = htons(pStatus->openVnodes);
-  
-  SRpcMsg rpcMsg = {
-    .pCont   = pStatus,
-    .contLen = contLen,
-    .msgType = TSDB_MSG_TYPE_DM_STATUS
-  };
-
-  SRpcEpSet epSet;
-  dnodeGetEpSetForPeer(&epSet);
-  dnodeSendMsgToDnode(&epSet, &rpcMsg);
-}
-
-void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
-  SRpcConnInfo connInfo = {0};
-  rpcGetConnInfo(rpcMsg->handle, &connInfo);
-
-  SRpcEpSet epSet = {0};
-  if (forShell) {
-    dnodeGetEpSetForShell(&epSet);
-  } else {
-    dnodeGetEpSetForPeer(&epSet);
-  }
-  
-  dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
-         taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
-
-  for (int i = 0; i < epSet.numOfEps; ++i) {
-    dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
-    epSet.port[i] = htons(epSet.port[i]);
-  }
-
-  rpcSendRedirectRsp(rpcMsg->handle, &epSet);
-}
diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c
index f664618f51..9eb52cbf5a 100644
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@@ -15,15 +15,10 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosdef.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "mnode.h"
 #include "http.h"
 #include "tmqtt.h"
 #include "monitor.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeModule.h"
 
 typedef struct {
diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c
index 6b5b28622b..bf74e14963 100644
--- a/src/dnode/src/dnodePeer.c
+++ b/src/dnode/src/dnodePeer.c
@@ -21,15 +21,12 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "dnodeVMgmt.h"
 #include "dnodeVWrite.h"
 #include "dnodeMPeer.h"
 #include "dnodeMInfos.h"
+#include "dnodeStep.h"
 
 static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
@@ -44,19 +41,19 @@ int32_t dnodeInitServer() {
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeDispatchToVWriteQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeDispatchToVWriteQueue;
 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeDispatchToMgmtQueue; 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeDispatchToMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToVMgmtQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeDispatchToVMgmtQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeDispatchToVMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToVMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToVMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToVMgmtQueue;
 
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = dnodeDispatchToMPeerQueue;
-  
+
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort    = tsDnodeDnodePort;
@@ -91,8 +88,9 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     .pCont   = NULL,
     .contLen = 0
   };
-  
+
   if (pMsg->pCont == NULL) return;
+  if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) return dnodeSendStartupStep(pMsg);
 
   if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
     rspMsg.code = TSDB_CODE_APP_NOT_READY;
diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c
index 89f657f789..d76af4e3dc 100644
--- a/src/dnode/src/dnodeShell.c
+++ b/src/dnode/src/dnodeShell.c
@@ -15,20 +15,14 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosdef.h"
-#include "taosmsg.h"
-#include "tglobal.h"
-#include "tutil.h"
 #include "http.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
 #include "dnodeMRead.h"
 #include "dnodeMWrite.h"
 #include "dnodeShell.h"
+#include "dnodeStep.h"
 
 static void  (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void    dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
@@ -74,6 +68,8 @@ int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = dnodeDispatchToMReadQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = dnodeDispatchToMReadQueue;
 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST]   = dnodeSendStartupStep;
+
   int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
   numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
   if (numOfThreads < 1) {
@@ -142,7 +138,23 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   }
 }
 
+static int32_t dnodeAuthNettestUser(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  if (strcmp(user, "nettestinternal") == 0) {
+    char pass[32] = {0};
+    taosEncryptPass((uint8_t *)user, strlen(user), pass);
+    *spi = 0;
+    *encrypt = 0;
+    *ckey = 0;
+    memcpy(secret, pass, TSDB_KEY_LEN);
+    dTrace("nettest user is authorized");
+    return 0;
+  }
+
+  return -1;
+}
+
 static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0;  
   int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
   if (code != TSDB_CODE_APP_NOT_READY) return code;
 
@@ -220,4 +232,4 @@ SStatisInfo dnodeGetStatisInfo() {
   }
 
   return info;
-}
+}
\ No newline at end of file
diff --git a/src/dnode/src/dnodeStep.c b/src/dnode/src/dnodeStep.c
new file mode 100644
index 0000000000..0f535b9470
--- /dev/null
+++ b/src/dnode/src/dnodeStep.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "dnodeStep.h"
+
+static SStartupStep tsStartupStep;
+
+void dnodeReportStep(char *name, char *desc, int8_t finished) {
+  tstrncpy(tsStartupStep.name, name, sizeof(tsStartupStep.name));
+  tstrncpy(tsStartupStep.desc, desc, sizeof(tsStartupStep.desc));
+  tsStartupStep.finished = finished;
+}
+
+void dnodeSendStartupStep(SRpcMsg *pMsg) {
+  dInfo("nettest msg is received, cont:%s", (char *)pMsg->pCont);
+
+  SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep));
+  memcpy(pStep, &tsStartupStep, sizeof(SStartupStep));
+
+  dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished);
+
+  SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStep, .contLen = sizeof(SStartupStep)};
+  rpcSendResponse(&rpcRsp);
+  rpcFreeCont(pMsg->pCont);
+}
+
+void taosStepCleanupImp(SStep *pSteps, int32_t stepId) {
+  for (int32_t step = stepId; step >= 0; step--) {
+    SStep *pStep = pSteps + step;
+    dDebug("step:%s will cleanup", pStep->name);
+    if (pStep->cleanupFp != NULL) {
+      (*pStep->cleanupFp)();
+    }
+  }
+}
+
+int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) {
+  for (int32_t step = 0; step < stepSize; step++) {
+    SStep *pStep = pSteps + step;
+    if (pStep->initFp == NULL) continue;
+
+    dnodeReportStep(pStep->name, "Start initialization", 0);
+
+    int32_t code = (*pStep->initFp)();
+    if (code != 0) {
+      dDebug("step:%s will init", pStep->name);
+      taosStepCleanupImp(pSteps, step);
+      return code;
+    }
+
+    dnodeReportStep(pStep->name, "Initialization complete", step + 1 >= stepSize);
+  }
+
+  return 0;
+}
+
+void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { 
+  return taosStepCleanupImp(pSteps, stepSize - 1);
+}
\ No newline at end of file
diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c
index 56316e9619..a135cda055 100644
--- a/src/dnode/src/dnodeSystem.c
+++ b/src/dnode/src/dnodeSystem.c
@@ -16,9 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tgrant.h"
-#include "tutil.h"
-#include "tglobal.h"
-#include "dnodeInt.h"
 #include "dnodeMain.h"
 
 static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c
index e973f9901f..85f0137d89 100644
--- a/src/dnode/src/dnodeTelemetry.c
+++ b/src/dnode/src/dnodeTelemetry.c
@@ -15,9 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "tglobal.h"
-#include "tutil.h"
 #include "osTime.h"
 #include "tsocket.h"
 #include "tbuffer.h"
@@ -32,8 +29,6 @@
 #include "mnodeTable.h"
 #include "mnodeSdb.h"
 #include "mnodeAcct.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeTelemetry.h"
 
 static tsem_t tsExitSem;
@@ -313,4 +308,4 @@ void dnodeCleanupTelemetry() {
     pthread_join(tsTelemetryThread, NULL);
     tsem_destroy(&tsExitSem);
   }
-}
+}
\ No newline at end of file
diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c
new file mode 100644
index 0000000000..87302026ec
--- /dev/null
+++ b/src/dnode/src/dnodeVMgmt.c
@@ -0,0 +1,239 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tqueue.h"
+#include "dnodeVMgmt.h"
+
+typedef struct {
+  SRpcMsg rpcMsg;
+  char    pCont[];
+} SMgmtMsg;
+
+static taos_qset  tsMgmtQset = NULL;
+static taos_queue tsMgmtQueue = NULL;
+static pthread_t  tsQthread;
+
+static void *  dnodeProcessMgmtQueue(void *param);
+static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
+static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
+static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
+static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
+static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
+static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
+static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
+
+int32_t dnodeInitVMgmt() {
+  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
+  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeProcessAlterVnodeMsg;
+  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeProcessDropVnodeMsg;
+  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg;
+  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
+  dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
+
+  int32_t code = vnodeInitMgmt();
+  if (code != TSDB_CODE_SUCCESS) return -1;
+
+  tsMgmtQset = taosOpenQset();
+  if (tsMgmtQset == NULL) {
+    dError("failed to create the vmgmt queue set");
+    return -1;
+  }
+
+  tsMgmtQueue = taosOpenQueue();
+  if (tsMgmtQueue == NULL) {
+    dError("failed to create the vmgmt queue");
+    return -1;
+  }
+
+  taosAddIntoQset(tsMgmtQset, tsMgmtQueue, NULL);
+
+  pthread_attr_t thAttr;
+  pthread_attr_init(&thAttr);
+  pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+  code = pthread_create(&tsQthread, &thAttr, dnodeProcessMgmtQueue, NULL);
+  pthread_attr_destroy(&thAttr);
+  if (code != 0) {
+    dError("failed to create thread to process vmgmt queue, reason:%s", strerror(errno));
+    return -1;
+  }
+
+  dInfo("dnode vmgmt is initialized");
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeCleanupVMgmt() {
+  if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset);
+  if (tsQthread) pthread_join(tsQthread, NULL);
+
+  if (tsMgmtQueue) taosCloseQueue(tsMgmtQueue);
+  if (tsMgmtQset) taosCloseQset(tsMgmtQset);
+
+  tsMgmtQset = NULL;
+  tsMgmtQueue = NULL;
+
+  vnodeCleanupMgmt();
+}
+
+static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
+  int32_t   size = sizeof(SMgmtMsg) + pMsg->contLen;
+  SMgmtMsg *pMgmt = taosAllocateQitem(size);
+  if (pMgmt == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
+
+  pMgmt->rpcMsg = *pMsg;
+  pMgmt->rpcMsg.pCont = pMgmt->pCont;
+  memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
+  taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeDispatchToVMgmtQueue(SRpcMsg *pMsg) {
+  int32_t code = dnodeWriteToMgmtQueue(pMsg);
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
+    rpcSendResponse(&rsp);
+  }
+
+  rpcFreeCont(pMsg->pCont);
+}
+
+static void *dnodeProcessMgmtQueue(void *param) {
+  SMgmtMsg *pMgmt;
+  SRpcMsg * pMsg;
+  SRpcMsg   rsp = {0};
+  int32_t   qtype;
+  void *    handle;
+
+  while (1) {
+    if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
+      dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
+      break;
+    }
+
+    pMsg = &pMgmt->rpcMsg;
+    dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]);
+    if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
+      rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
+    } else {
+      rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
+    }
+
+    dDebug("msg:%p, is processed, code:0x%x", pMgmt, rsp.code);
+    if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) {
+      rsp.handle = pMsg->handle;
+      rsp.pCont = NULL;
+      rpcSendResponse(&rsp);
+    }
+
+    taosFreeQitem(pMsg);
+  }
+
+  return NULL;
+}
+
+static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
+  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
+  pCreate->cfg.vgId                = htonl(pCreate->cfg.vgId);
+  pCreate->cfg.cfgVersion          = htonl(pCreate->cfg.cfgVersion);
+  pCreate->cfg.maxTables           = htonl(pCreate->cfg.maxTables);
+  pCreate->cfg.cacheBlockSize      = htonl(pCreate->cfg.cacheBlockSize);
+  pCreate->cfg.totalBlocks         = htonl(pCreate->cfg.totalBlocks);
+  pCreate->cfg.daysPerFile         = htonl(pCreate->cfg.daysPerFile);
+  pCreate->cfg.daysToKeep1         = htonl(pCreate->cfg.daysToKeep1);
+  pCreate->cfg.daysToKeep2         = htonl(pCreate->cfg.daysToKeep2);
+  pCreate->cfg.daysToKeep          = htonl(pCreate->cfg.daysToKeep);
+  pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
+  pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
+  pCreate->cfg.fsyncPeriod         = htonl(pCreate->cfg.fsyncPeriod);
+  pCreate->cfg.commitTime          = htonl(pCreate->cfg.commitTime);
+
+  for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
+    pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
+  }
+
+  return pCreate;
+}
+
+static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
+  SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
+
+  void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
+  if (pVnode != NULL) {
+    dDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId);
+    vnodeRelease(pVnode);
+    return TSDB_CODE_SUCCESS;
+  } else {
+    dDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId);
+    return vnodeCreate(pCreate);
+  }
+}
+
+static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
+  SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
+
+  void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
+  if (pVnode != NULL) {
+    dDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId);
+    int32_t code = vnodeAlter(pVnode, pAlter);
+    vnodeRelease(pVnode);
+    return code;
+  } else {
+    dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
+    return TSDB_CODE_VND_INVALID_VGROUP_ID;
+  }
+}
+
+static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
+  SDropVnodeMsg *pDrop = rpcMsg->pCont;
+  pDrop->vgId = htonl(pDrop->vgId);
+
+  return vnodeDrop(pDrop->vgId);
+}
+
+static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
+  return 0;
+}
+
+static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
+  SCfgDnodeMsg *pCfg = pMsg->pCont;
+  return taosCfgDynamicOptions(pCfg->config);
+}
+
+static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
+  SCreateMnodeMsg *pCfg = pMsg->pCont;
+  pCfg->dnodeId = htonl(pCfg->dnodeId);
+  if (pCfg->dnodeId != dnodeGetDnodeId()) {
+    dDebug("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
+    return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
+  }
+
+  if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) {
+    dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
+    return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
+  }
+
+  dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
+  for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
+    pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
+    dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
+  }
+
+  dnodeStartMnode(&pCfg->mnodes);
+
+  return TSDB_CODE_SUCCESS;
+}
\ No newline at end of file
diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c
index b42a627a3a..07496b142a 100644
--- a/src/dnode/src/dnodeVRead.c
+++ b/src/dnode/src/dnodeVRead.c
@@ -15,12 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "tqueue.h"
-#include "vnode.h"
-#include "dnodeInt.h"
+#include "dnodeVRead.h"
 
 typedef struct {
   pthread_t thread;    // thread
diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c
index 6d4b50ee54..a5ae8ac830 100644
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@@ -15,13 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "tqueue.h"
-#include "twal.h"
-#include "vnode.h"
-#include "dnodeInt.h"
+#include "dnodeVWrite.h"
 
 typedef struct {
   taos_qall qall;
diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c
new file mode 100644
index 0000000000..ba7f7625fa
--- /dev/null
+++ b/src/dnode/src/dnodeVnodes.c
@@ -0,0 +1,289 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "ttimer.h"
+#include "dnodeEps.h"
+#include "dnodeCfg.h"
+#include "dnodeMInfos.h"
+#include "dnodeVnodes.h"
+
+typedef struct {
+  pthread_t thread;
+  int32_t   threadIndex;
+  int32_t   failed;
+  int32_t   opened;
+  int32_t   vnodeNum;
+  int32_t * vnodeList;
+} SOpenVnodeThread;
+
+void *          tsDnodeTmr = NULL;
+static void *   tsStatusTimer = NULL;
+static uint32_t tsRebootTime = 0;
+
+static void dnodeSendStatusMsg(void *handle, void *tmrId);
+static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
+
+int32_t dnodeInitTimer() {
+  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
+  if (tsDnodeTmr == NULL) {
+    dError("failed to init dnode timer");
+    return -1;
+  }
+
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp);
+
+  tsRebootTime = taosGetTimestampSec();
+  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
+
+  dInfo("dnode timer is initialized");
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeCleanupTimer() {
+  if (tsStatusTimer != NULL) {
+    taosTmrStopA(&tsStatusTimer);
+    tsStatusTimer = NULL;
+  }
+
+  if (tsDnodeTmr != NULL) {
+    taosTmrCleanUp(tsDnodeTmr);
+    tsDnodeTmr = NULL;
+  }
+}
+
+static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
+  DIR *dir = opendir(tsVnodeDir);
+  if (dir == NULL) return TSDB_CODE_DND_NO_WRITE_ACCESS;
+
+  *numOfVnodes = 0;
+  struct dirent *de = NULL;
+  while ((de = readdir(dir)) != NULL) {
+    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+    if (de->d_type & DT_DIR) {
+      if (strncmp("vnode", de->d_name, 5) != 0) continue;
+      int32_t vnode = atoi(de->d_name + 5);
+      if (vnode == 0) continue;
+
+      (*numOfVnodes)++;
+
+      if (*numOfVnodes >= TSDB_MAX_VNODES) {
+        dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
+        continue;
+      } else {
+        vnodeList[*numOfVnodes - 1] = vnode;
+      }
+    }
+  }
+  closedir(dir);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void *dnodeOpenVnode(void *param) {
+  SOpenVnodeThread *pThread = param;
+
+  dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
+
+  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
+    int32_t vgId = pThread->vnodeList[v];
+    if (vnodeOpen(vgId) < 0) {
+      dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
+      pThread->failed++;
+    } else {
+      dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex);
+      pThread->opened++;
+    }
+  }
+
+  dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
+         pThread->failed);
+  return NULL;
+}
+
+int32_t dnodeInitVnodes() {
+  int32_t vnodeList[TSDB_MAX_VNODES] = {0};
+  int32_t numOfVnodes = 0;
+  int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
+
+  if (status != TSDB_CODE_SUCCESS) {
+    dInfo("get dnode list failed");
+    return status;
+  }
+
+  int32_t threadNum = tsNumOfCores;
+  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
+  SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread));
+  for (int32_t t = 0; t < threadNum; ++t) {
+    threads[t].threadIndex = t;
+    threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t));
+  }
+
+  for (int32_t v = 0; v < numOfVnodes; ++v) {
+    int32_t t = v % threadNum;
+    SOpenVnodeThread *pThread = &threads[t];
+    pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
+  }
+
+  dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
+
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SOpenVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum == 0) continue;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+    if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) {
+      dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
+    }
+
+    pthread_attr_destroy(&thAttr);
+  }
+
+  int32_t openVnodes = 0;
+  int32_t failedVnodes = 0;
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SOpenVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum > 0 && pThread->thread) {
+      pthread_join(pThread->thread, NULL);
+    }
+    openVnodes += pThread->opened;
+    failedVnodes += pThread->failed;
+    free(pThread->vnodeList);
+  }
+
+  free(threads);
+  dInfo("there are total vnodes:%d, openned:%d", numOfVnodes, openVnodes);
+
+  if (failedVnodes != 0) {
+    dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes);
+    return -1;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeCleanupVnodes() {
+  int32_t vnodeList[TSDB_MAX_VNODES]= {0};
+  int32_t numOfVnodes = 0;
+  int32_t status;
+
+  status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
+
+  if (status != TSDB_CODE_SUCCESS) {
+    dInfo("get dnode list failed");
+    return;
+  }
+
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    vnodeClose(vnodeList[i]);
+  }
+
+  dInfo("total vnodes:%d are all closed", numOfVnodes);
+}
+
+static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
+  if (pMsg->code != TSDB_CODE_SUCCESS) {
+    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    return;
+  }
+
+  SStatusRsp *pStatusRsp = pMsg->pCont;
+  SMInfos *minfos = &pStatusRsp->mnodes;
+  dnodeUpdateMInfos(minfos);
+
+  SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
+  pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
+  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
+  pCfg->dnodeId = htonl(pCfg->dnodeId);
+  dnodeUpdateCfg(pCfg);
+
+  vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
+
+  SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
+  dnodeUpdateEps(pEps);
+
+  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+}
+
+static void dnodeSendStatusMsg(void *handle, void *tmrId) {
+  if (tsDnodeTmr == NULL) {
+    dError("dnode timer is already released");
+    return;
+  }
+
+  if (tsStatusTimer == NULL) {
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    dError("failed to start status timer");
+    return;
+  }
+
+  int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
+  SStatusMsg *pStatus = rpcMallocCont(contLen);
+  if (pStatus == NULL) {
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    dError("failed to malloc status message");
+    return;
+  }
+
+  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
+  pStatus->dnodeId          = htonl(dnodeGetDnodeId());
+  pStatus->version          = htonl(tsVersion);
+  pStatus->lastReboot       = htonl(tsRebootTime);
+  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
+  pStatus->diskAvailable    = tsAvailDataDirGB;
+  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
+  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
+
+  // fill cluster cfg parameters
+  pStatus->clusterCfg.numOfMnodes        = htonl(tsNumOfMnodes);
+  pStatus->clusterCfg.enableBalance      = htonl(tsEnableBalance);
+  pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
+  pStatus->clusterCfg.offlineThreshold   = htonl(tsOfflineThreshold);
+  pStatus->clusterCfg.statusInterval     = htonl(tsStatusInterval);
+  pStatus->clusterCfg.maxtablesPerVnode  = htonl(tsMaxTablePerVnode);
+  pStatus->clusterCfg.maxVgroupsPerDb    = htonl(tsMaxVgroupsPerDb);
+  tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN);
+  tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64);
+  pStatus->clusterCfg.checkTime = 0;
+  char timestr[32] = "1970-01-01 00:00:00.00";
+  (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
+  tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
+  tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);  
+
+  vnodeBuildStatusMsg(pStatus);
+  contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
+  pStatus->openVnodes = htons(pStatus->openVnodes);
+
+  SRpcMsg rpcMsg = {
+    .pCont   = pStatus,
+    .contLen = contLen,
+    .msgType = TSDB_MSG_TYPE_DM_STATUS
+  };
+
+  SRpcEpSet epSet;
+  dnodeGetEpSetForPeer(&epSet);
+  dnodeSendMsgToDnode(&epSet, &rpcMsg);
+}
+
+void dnodeSendStatusMsgToMnode() {
+  if (tsDnodeTmr != NULL && tsStatusTimer != NULL) {
+    dInfo("force send status msg to mnode");
+    taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer);
+  }
+}
\ No newline at end of file
diff --git a/src/inc/dnode.h b/src/inc/dnode.h
index 1efaa4a24b..9dd95e32d7 100644
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@@ -71,8 +71,18 @@ void    dnodeDelayReprocessMWriteMsg(void *pMsg);
 
 void    dnodeSendStatusMsgToMnode();
 
+typedef struct {
+  char *name;
+  int32_t (*initFp)();
+  void (*cleanupFp)();
+} SStep;
+
+int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize);
+void    dnodeStepCleanup(SStep *pSteps, int32_t stepSize);
+void    dnodeReportStep(char *name, char *desc, int8_t finished);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif
\ No newline at end of file
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index ca20293392..0cc06be1db 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -286,6 +286,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_SHOW_SQL_LEN         512
 #define TSDB_SLOW_QUERY_SQL_LEN   512
 
+#define TSDB_STEP_NAME_LEN        32
+#define TSDB_STEP_DESC_LEN        128
+
 #define TSDB_MQTT_HOSTNAME_LEN    64
 #define TSDB_MQTT_PORT_LEN        8
 #define TSDB_MQTT_USER_LEN        24
@@ -439,6 +442,10 @@ typedef enum {
   TAOS_QTYPE_QUERY = 4
 } EQType;
 
+#define TSDB_MAX_TIERS           3
+#define TSDB_MAX_DISKS_PER_TIER 16
+#define TSDB_MAX_DISKS           (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER)
+
 typedef enum {
   TSDB_SUPER_TABLE  = 0,   // super table
   TSDB_CHILD_TABLE  = 1,   // table created from super table
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index bc7be3511f..e0d7e01843 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -192,6 +192,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        0, 0x0400, "Message no
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY,            0, 0x0401, "Dnode out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS,          0, 0x0402, "No permission for disk files in dnode")
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN,          0, 0x0403, "Invalid message length")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS,       0, 0x0404, "Action in progress")
 
 // vnode 
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS,       0, 0x0500, "Action in progress")
@@ -394,6 +395,18 @@ TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ,        0, 0x2113, "src bad se
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE,     0, 0x2114, "src incomplete")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL,        0, 0x2115, "src general")
 
+// tfs
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY,             0, 0x2200, "tfs out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG,                 0, 0x2201, "tfs invalid mount config")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT,            0, 0x2202, "tfs too many mount")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY,               0, 0x2203, "tfs duplicate primary mount")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_PRIMARY_DISK,           0, 0x2204, "tfs no primary mount")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_MOUNT_AT_TIER,          0, 0x2205, "tfs no mount at tier")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS,       0, 0x2206, "tfs file already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL,               0, 0x2207, "tfs invalid level")
+TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK,             0, 0x2208, "tfs no valid disk")
+
+
 #ifdef TAOS_ERROR_C
 };
 #endif
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index e8e3029244..ea9d608d92 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -105,10 +105,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_AUTH, "auth" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
-
-  
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" )
-
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" )
 
 #ifndef TAOS_MESSAGE_C
   TSDB_MSG_TYPE_MAX  // 105
@@ -840,6 +837,14 @@ typedef struct {
   char ckey[TSDB_KEY_LEN];
 } SAuthMsg, SAuthRsp;
 
+typedef struct {
+  int8_t  finished;
+  int8_t  reserved1[7];
+  char    name[TSDB_STEP_NAME_LEN];
+  char    desc[TSDB_STEP_DESC_LEN];
+  char    reserved2[64];
+} SStartupStep;
+
 #pragma pack(pop)
 
 #ifdef __cplusplus
diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h
index 58859f42bc..1769bd6566 100644
--- a/src/inc/tsdb.h
+++ b/src/inc/tsdb.h
@@ -321,7 +321,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
  */
 void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
 
-int  tsdbInitCommitQueue(int nthreads);
+int  tsdbInitCommitQueue();
 void tsdbDestroyCommitQueue();
 int  tsdbSyncCommit(TSDB_REPO_T *repo);
 void tsdbIncCommitRef(int vgId);
diff --git a/src/inc/vnode.h b/src/inc/vnode.h
index 5f643295d6..33bb2c278c 100644
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@@ -19,18 +19,9 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
 #include "trpc.h"
 #include "twal.h"
 
-typedef enum _VN_STATUS {
-  TAOS_VN_STATUS_INIT = 0,
-  TAOS_VN_STATUS_READY = 1,
-  TAOS_VN_STATUS_CLOSING = 2,
-  TAOS_VN_STATUS_UPDATING = 3,
-  TAOS_VN_STATUS_RESET = 4,
-} EVnodeStatus;
-
 typedef struct {
   int32_t len;
   void *  rsp;
@@ -60,29 +51,35 @@ typedef struct {
   SWalHead pHead[];
 } SVWriteMsg;
 
+// vnodeStatus
 extern char *vnodeStatus[];
 
+// vnodeMain
 int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeDrop(int32_t vgId);
-int32_t vnodeOpen(int32_t vgId, char *rootDir);
+int32_t vnodeOpen(int32_t vgId);
 int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeClose(int32_t vgId);
 
-void*   vnodeAcquire(int32_t vgId);        // add refcount
-void    vnodeRelease(void *pVnode);        // dec refCount
+// vnodeMgmt
+int32_t vnodeInitMgmt();
+void    vnodeCleanupMgmt();
+void*   vnodeAcquire(int32_t vgId);
+void    vnodeRelease(void *pVnode);
 void*   vnodeGetWal(void *pVnode);
+int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
+void    vnodeBuildStatusMsg(void *pStatus);
+void    vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
 
+// vnodeWrite
 int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg);
 void    vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite);
 int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet);
-int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
-void    vnodeBuildStatusMsg(void *pStatus);
+
+// vnodeSync
 void    vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code);
-void    vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
-
-int32_t vnodeInitResources();
-void    vnodeCleanupResources();
 
+// vnodeRead
 int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
 void    vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead);
 int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
@@ -91,4 +88,4 @@ int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
 }
 #endif
 
-#endif
+#endif
\ No newline at end of file
diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h
index d65c943e28..2c6e4a308c 100644
--- a/src/kit/shell/inc/shell.h
+++ b/src/kit/shell/inc/shell.h
@@ -51,7 +51,6 @@ typedef struct SShellArguments {
   char* commands;
   int   abort;
   int   port;
-  int   endPort;
   int   pktLen;
   char* netTestRole;
 } SShellArguments;
diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index 22f01ac142..af8beb7987 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -32,14 +32,14 @@
 /**************** Global variables ****************/
 #ifdef _TD_POWER_
 char      CLIENT_VERSION[] = "Welcome to the PowerDB shell from %s, Client Version:%s\n"
-                             "Copyright (c) 2017 by PowerDB, Inc. All rights reserved.\n\n";
+                             "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n";
 char      PROMPT_HEADER[] = "power> ";
 
 char      CONTINUE_PROMPT[] = "    -> ";
 int       prompt_size = 7;
 #else
 char      CLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
-                             "Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.\n\n";
+                             "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
 char      PROMPT_HEADER[] = "taos> ";
 
 char      CONTINUE_PROMPT[] = "   -> ";
diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c
index 3226ad830a..f896253fb4 100644
--- a/src/kit/shell/src/shellLinux.c
+++ b/src/kit/shell/src/shellLinux.c
@@ -46,8 +46,7 @@ static struct argp_option options[] = {
   {"thread",     'T', "THREADNUM",  0,                   "Number of threads when using multi-thread to import data."},
   {"database",   'd', "DATABASE",   0,                   "Database to use when connecting to the server."},
   {"timezone",   't', "TIMEZONE",   0,                   "Time zone of the shell, default is local."},
-  {"netrole",    'n', "NETROLE",    0,                   "Net role when network connectivity test, default is NULL, options: client|clients|server."},
-  {"endport",    'e', "ENDPORT",    0,                   "Net test end port, default is 6042."},
+  {"netrole",    'n', "NETROLE",    0,                   "Net role when network connectivity test, default is NULL, options: client|server|rpc|startup."},
   {"pktlen",     'l', "PKTLEN",     0,                   "Packet length used for net test, default is 1000 bytes."},
   {0}};
 
@@ -130,20 +129,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
     case 'd':
       arguments->database = arg;
       break;
-
     case 'n':
       arguments->netTestRole = arg;
       break;
-
-    case 'e':
-      if (arg) {
-        arguments->endPort = atoi(arg);
-      } else {
-        fprintf(stderr, "Invalid end port\n");
-        return -1;
-      }
-      break;
-
     case 'l':
       if (arg) {
         arguments->pktLen = atoi(arg);
@@ -152,7 +140,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         return -1;
       }
       break;
-      
     case OPT_ABORT:
       arguments->abort = 1;
       break;
diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c
index 2083ad3e9b..a2ce78d36f 100644
--- a/src/kit/shell/src/shellMain.c
+++ b/src/kit/shell/src/shellMain.c
@@ -61,8 +61,7 @@ SShellArguments args = {
   .file = "\0",
   .dir = "\0",
   .threadNum = 5,
-  .commands = NULL,  
-  .endPort = 6042,
+  .commands = NULL,
   .pktLen = 1000,
   .netTestRole = NULL
 };
@@ -81,9 +80,7 @@ int main(int argc, char* argv[]) {
 
   if (args.netTestRole && args.netTestRole[0] != 0) {
     taos_init();
-    CmdArguments cmdArgs;
-    memcpy(&cmdArgs, &args, sizeof(SShellArguments));
-    taosNetTest(&cmdArgs);
+    taosNetTest(args.netTestRole, args.host, args.port, args.pktLen);
     exit(0);
   }
 
diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index c971a945aa..25dbb10536 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -317,13 +317,6 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
 
-#ifndef _SYNC
-  if (pCfg->replications != 1) {
-    mError("invalid db option replications:%d can only be 1 in this version", pCfg->replications);
-    return TSDB_CODE_MND_INVALID_DB_OPTION;
-  }
-#endif
-
   if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
     mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE);
     return TSDB_CODE_MND_INVALID_DB_OPTION;
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 0ff50b2307..f297dd51dd 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -111,9 +111,6 @@ static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) {
 static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) {
   SDnodeObj *pDnode = pRow->pObj;
  
-#ifndef _SYNC 
-  mnodeDropAllDnodeVgroups(pDnode);
-#endif  
   mnodeDropMnodeLocal(pDnode->dnodeId);
   bnNotify();
   mnodeUpdateDnodeEps();
@@ -705,11 +702,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
 
   mInfo("dnode:%d, start to drop it", pDnode->dnodeId);
 
-#ifndef _SYNC
-  int32_t code = mnodeDropDnode(pDnode, pMsg);
-#else
   int32_t code = bnDropDnode(pDnode);
-#endif  
   mnodeDecDnodeRef(pDnode);
   return code;
 }
@@ -1179,58 +1172,3 @@ static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) {
     default:return "any";
   }
 }
-
-#ifndef _SYNC
-
-int32_t bnInit() { return TSDB_CODE_SUCCESS; }
-void    bnCleanUp() {}
-void    bnNotify() {}
-void    bnCheckModules() {}
-void    bnReset() {}
-int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
-
-char* syncRole[] = {
-  "offline",
-  "unsynced",
-  "syncing",
-  "slave",
-  "master"
-};
-
-int32_t bnAllocVnodes(SVgObj *pVgroup) {
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  SDnodeObj *pSelDnode = NULL;
-  float      vnodeUsage = 1000.0;
-
-  while (1) {
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    if (pDnode->numOfCores > 0 && pDnode->openVnodes < TSDB_MAX_VNODES) {
-      float openVnodes = pDnode->openVnodes;
-      if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum;
-
-      float usage = openVnodes / pDnode->numOfCores;
-      if (usage <= vnodeUsage) {
-        pSelDnode = pDnode;
-        vnodeUsage = usage;
-      }
-    }
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  if (pSelDnode == NULL) {
-    mError("failed to alloc vnode to vgroup");
-    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
-  }
-
-  pVgroup->vnodeGid[0].dnodeId = pSelDnode->dnodeId;
-  pVgroup->vnodeGid[0].pDnode = pSelDnode;
-
-  mDebug("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes);
-  return TSDB_CODE_SUCCESS;
-}
-
-#endif 
-
diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c
index 86f2c821f9..7b520c6022 100644
--- a/src/mnode/src/mnodeMain.c
+++ b/src/mnode/src/mnodeMain.c
@@ -37,16 +37,10 @@
 #include "mnodeShow.h"
 #include "mnodeProfile.h"
 
-typedef struct {
-  const char *const name;
-  int               (*init)();
-  void              (*cleanup)();
-} SMnodeComponent;
-
 void *tsMnodeTmr = NULL;
 static bool tsMgmtIsRunning = false;
 
-static const SMnodeComponent tsMnodeComponents[] = {
+static SStep tsMnodeSteps[] = {
   {"sdbref",  sdbInitRef,       sdbCleanUpRef},
   {"profile", mnodeInitProfile, mnodeCleanupProfile},
   {"cluster", mnodeInitCluster, mnodeCleanupCluster},
@@ -67,22 +61,14 @@ static void mnodeInitTimer();
 static void mnodeCleanupTimer();
 static bool mnodeNeedStart() ;
 
-static void mnodeCleanupComponents(int32_t stepId) {
-  for (int32_t i = stepId; i >= 0; i--) {
-    tsMnodeComponents[i].cleanup();
-  }
+static void mnodeCleanupComponents() {
+  int32_t stepSize = sizeof(tsMnodeSteps) / sizeof(SStep);
+  dnodeStepCleanup(tsMnodeSteps, stepSize);
 }
 
 static int32_t mnodeInitComponents() {
-  int32_t code = 0;
-  for (int32_t i = 0; i < sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]); i++) {
-    if (tsMnodeComponents[i].init() != 0) {
-      mnodeCleanupComponents(i);
-      code = -1;
-      break;
-    }
-  }
-  return code;
+  int32_t stepSize = sizeof(tsMnodeSteps) / sizeof(SStep);
+  return dnodeStepInit(tsMnodeSteps, stepSize);
 }
 
 int32_t mnodeStartSystem() {
@@ -132,7 +118,7 @@ void mnodeCleanupSystem() {
     dnodeFreeMReadQueue();
     dnodeFreeMPeerQueue();
     mnodeCleanupTimer();
-    mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
+    mnodeCleanupComponents();
 
     mInfo("mnode is cleaned up");
   }
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 00a97d7bc2..587c079fe6 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -1086,13 +1086,6 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
         tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
       } 
     } else { // msg is passed to app only parsing is ok 
-
-      if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
-        rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS);
-        rpcFreeMsg(pRecv->msg); 
-        return pConn;
-      }
-      
       rpcProcessIncomingMsg(pConn, pHead, pContext);
     }
   }
diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c
index c86b8f32b7..75a2cbcb8d 100644
--- a/src/tsdb/src/tsdbCommitQueue.c
+++ b/src/tsdb/src/tsdbCommitQueue.c
@@ -14,6 +14,7 @@
  */
 
 #include "os.h"
+#include "tglobal.h"
 #include "tlist.h"
 #include "tref.h"
 #include "tsdbMain.h"
@@ -36,7 +37,8 @@ static void *tsdbLoopCommit(void *arg);
 
 SCommitQueue tsCommitQueue = {0};
 
-int tsdbInitCommitQueue(int nthreads) {
+int tsdbInitCommitQueue() {
+  int nthreads = tsNumOfCommitThreads;
   SCommitQueue *pQueue = &tsCommitQueue;
 
   if (nthreads < 1) nthreads = 1;
diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h
index 426df5cbb2..b7585bd715 100644
--- a/src/util/inc/tnettest.h
+++ b/src/util/inc/tnettest.h
@@ -20,27 +20,7 @@
 extern "C" {
 #endif
 
-typedef struct CmdArguments {
-  char* host;
-  char* password;
-  char* user;
-  char* auth;
-  char* database;
-  char* timezone;
-  bool  is_raw_time;
-  bool  is_use_passwd;
-  char  file[TSDB_FILENAME_LEN];
-  char  dir[TSDB_FILENAME_LEN];
-  int   threadNum;
-  char* commands;
-  int   abort;
-  int   port;
-  int   endPort;
-  int   pktLen;
-  char* netTestRole;
-} CmdArguments;
-
-void taosNetTest(CmdArguments* args);
+void taosNetTest(char *role, char *host, int port, int pkgLen);
 
 #ifdef __cplusplus
 }
diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c
index 6fd5265983..c269d9a1ff 100644
--- a/src/util/src/tnettest.c
+++ b/src/util/src/tnettest.c
@@ -13,50 +13,42 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "taoserror.h"
 #include "tulog.h"
-#include "tconfig.h"
 #include "tglobal.h"
 #include "tsocket.h"
 #include "trpc.h"
 #include "rpcHead.h"
-#include "tutil.h"
-#include "tnettest.h"
 
-#define    MAX_PKG_LEN          (64*1000)
-#define    BUFFER_SIZE          (MAX_PKG_LEN + 1024)
+#define MAX_PKG_LEN (64 * 1000)
+#define BUFFER_SIZE (MAX_PKG_LEN + 1024)
+
+extern int32_t tsRpcMaxUdpSize;
 
 typedef struct {
+  char *   hostFqdn;
   uint32_t hostIp;
-  uint16_t port;
-  uint16_t pktLen;
-} info_s;
+  int32_t  port;
+  int32_t  pktLen;
+} STestInfo;
 
-extern int tsRpcMaxUdpSize;
-
-static char g_user[TSDB_USER_LEN+1]     = {0};
-static char g_pass[TSDB_PASSWORD_LEN+1] = {0};
-static char g_serverFqdn[TSDB_FQDN_LEN] = {0};
-static uint16_t g_startPort = 0;
-static uint16_t g_endPort   = 6042;
-static uint32_t g_pktLen    = 0;
-
-
-static void *bindUdpPort(void *sarg) {
-  info_s *pinfo = (info_s *)sarg;
-  int     port = pinfo->port;
-  SOCKET  serverSocket;
+static void *taosNetBindUdpPort(void *sarg) {
+  STestInfo *pinfo = (STestInfo *)sarg;
+  int32_t    port = pinfo->port;
+  SOCKET     serverSocket;
+  char       buffer[BUFFER_SIZE];
+  int32_t    iDataNum;
+  socklen_t  sin_size;
 
   struct sockaddr_in server_addr;
   struct sockaddr_in clientAddr;
-  char               buffer[BUFFER_SIZE];
-  int                iDataNum;
-    
+
   if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) {
-    perror("socket");
+    uError("failed to create udp socket since %s", strerror(errno));
     return NULL;
   }
 
@@ -66,28 +58,25 @@ static void *bindUdpPort(void *sarg) {
   server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
 
   if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
-    perror("connect");
+    uError("failed to bind udp port:%d since %s", port, strerror(errno));
     return NULL;
   }
 
-  socklen_t sin_size;
+  uInfo("udp server at port:%d is listening", port);
 
   while (1) {
     memset(buffer, 0, BUFFER_SIZE);
-
     sin_size = sizeof(*(struct sockaddr *)&server_addr);
-
     iDataNum = recvfrom(serverSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&clientAddr, &sin_size);
 
     if (iDataNum < 0) {
-      perror("recvfrom null");
+      uDebug("failed to perform recvfrom func at %d since %s", port, strerror(errno));
       continue;
     }
-    if (iDataNum > 0) {
-      printf("recv Client: %s pkg from UDP port: %d, pkg len: %d\n", taosInetNtoa(clientAddr.sin_addr), port, iDataNum);
-      //printf("Read msg from udp:%s ... %s\n", buffer, buffer+iDataNum-16);
 
-      sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int)sin_size);
+    if (iDataNum > 0) {
+      uInfo("UDP: recv:%d bytes from %s:%d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
+      sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size);
     }
   }
 
@@ -95,20 +84,20 @@ static void *bindUdpPort(void *sarg) {
   return NULL;
 }
 
-static void *bindTcpPort(void *sarg) {
-  info_s *pinfo = (info_s *)sarg;
-  int     port = pinfo->port;
-  SOCKET  serverSocket;
-
+static void *taosNetBindTcpPort(void *sarg) {
   struct sockaddr_in server_addr;
   struct sockaddr_in clientAddr;
-  int                addr_len = sizeof(clientAddr);
-  SOCKET             client;
-  char               buffer[BUFFER_SIZE];
-  int                iDataNum = 0;
+
+  STestInfo *pinfo = sarg;
+  int32_t    port = pinfo->port;
+  SOCKET     serverSocket;
+  int32_t    addr_len = sizeof(clientAddr);
+  SOCKET     client;
+  char       buffer[BUFFER_SIZE];
+  int32_t    iDataNum = 0;
 
   if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) {
-    printf("socket() fail: %s", strerror(errno));
+    uError("failed to create tcp socket since %s", strerror(errno));
     return NULL;
   }
 
@@ -118,28 +107,30 @@ static void *bindTcpPort(void *sarg) {
   server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
 
   if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
-    printf("port:%d bind() fail: %s", port, strerror(errno));
+    uError("failed to bind tcp port:%d since %s", port, strerror(errno));
     return NULL;
   }
 
   if (listen(serverSocket, 5) < 0) {
-    printf("listen() fail: %s", strerror(errno));
+    uError("failed to listen tcp port:%d since %s", port, strerror(errno));
     return NULL;
   }
 
-  //printf("Bind port: %d success\n", port);
+  uInfo("tcp server at port:%d is listening", port);
+
   while (1) {
     client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len);
     if (client < 0) {
-      printf("accept() fail: %s", strerror(errno));
+      uDebug("failed to accept from tcp port:%d since %s", port, strerror(errno));
       continue;
     }
 
     iDataNum = 0;
     memset(buffer, 0, BUFFER_SIZE);
-    int   nleft, nread;
-    char *ptr = buffer;
+    int32_t nleft, nread;
+    char *  ptr = buffer;
     nleft = pinfo->pktLen;
+
     while (nleft > 0) {
       nread = recv(client, ptr, BUFFER_SIZE, 0);
 
@@ -149,7 +140,7 @@ static void *bindTcpPort(void *sarg) {
         if (errno == EINTR) {
           continue;
         } else {
-          printf("recv Client: %s pkg from TCP port: %d fail:%s.\n", taosInetNtoa(clientAddr.sin_addr), port, strerror(errno));
+          uError("failed to perform recv func at %d since %s", port, strerror(errno));
           taosCloseSocket(serverSocket);
           return NULL;
         }
@@ -157,11 +148,11 @@ static void *bindTcpPort(void *sarg) {
         nleft -= nread;
         ptr += nread;
         iDataNum += nread;
-      }      
+      }
     }
-    
-    printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", taosInetNtoa(clientAddr.sin_addr), port, iDataNum);
+
     if (iDataNum > 0) {
+      uInfo("TCP: recv:%d bytes from %s:%d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
       send(client, buffer, iDataNum, 0);
     }
   }
@@ -170,39 +161,38 @@ static void *bindTcpPort(void *sarg) {
   return NULL;
 }
 
-static int checkTcpPort(info_s *info) {
+static int32_t taosNetCheckTcpPort(STestInfo *info) {
+  SOCKET  clientSocket;
+  char    sendbuf[BUFFER_SIZE];
+  char    recvbuf[BUFFER_SIZE];
+  int32_t iDataNum = 0;
+
   struct sockaddr_in serverAddr;
-  SOCKET             clientSocket;
-  char               sendbuf[BUFFER_SIZE];
-  char               recvbuf[BUFFER_SIZE];
-  int                iDataNum = 0;
   if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
-    printf("socket() fail: %s\n", strerror(errno));
+    uError("failed to create tcp client socket since %s", strerror(errno));
     return -1;
   }
 
   // set send and recv overtime
-  struct timeval timeout;  
-  timeout.tv_sec = 2;  //s
-  timeout.tv_usec = 0; //us  
-  if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
-    perror("setsockopt send timer failed:");
+  struct timeval timeout;
+  timeout.tv_sec = 2;   // s
+  timeout.tv_usec = 0;  // us
+  if (setsockopt(clientSocket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
+    uError("failed to setsockopt send timer since %s", strerror(errno));
   }
-  if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
-    perror("setsockopt recv timer failed:");
+  if (setsockopt(clientSocket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
+    uError("failed to setsockopt recv timer since %s", strerror(errno));
   }
 
   serverAddr.sin_family = AF_INET;
   serverAddr.sin_port = htons(info->port);
-
   serverAddr.sin_addr.s_addr = info->hostIp;
 
-  //printf("=================================\n");
   if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) {
-    printf("connect() fail: %s\t", strerror(errno));
+    uError("failed to connect port:%d since %s", info->port, strerror(errno));
     return -1;
   }
-  //printf("Connect to: %s:%d...success\n", host, port);
+
   memset(sendbuf, 0, BUFFER_SIZE);
   memset(recvbuf, 0, BUFFER_SIZE);
 
@@ -214,9 +204,10 @@ static int checkTcpPort(info_s *info) {
   send(clientSocket, sendbuf, info->pktLen, 0);
 
   memset(recvbuf, 0, BUFFER_SIZE);
-  int   nleft, nread;
-  char *ptr = recvbuf;
+  int32_t nleft, nread;
+  char *  ptr = recvbuf;
   nleft = info->pktLen;
+
   while (nleft > 0) {
     nread = recv(clientSocket, ptr, BUFFER_SIZE, 0);;
   
@@ -226,7 +217,7 @@ static int checkTcpPort(info_s *info) {
       if (errno == EINTR) {
         continue;
       } else {
-        printf("recv ack pkg from TCP port: %d fail:%s.\n", info->port, strerror(errno));
+        uError("faild to recv pkg from TCP port:%d since %s", info->port, strerror(errno));
         taosCloseSocket(clientSocket);
         return -1;
       }
@@ -234,45 +225,46 @@ static int checkTcpPort(info_s *info) {
       nleft -= nread;
       ptr += nread;
       iDataNum += nread;
-    }      
+    }
   }
 
   if (iDataNum < info->pktLen) {
-    printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, info->port);
+    uError("TCP: received ack:%d bytes, less than send:%d bytes from port:%d", iDataNum, info->pktLen, info->port);
     return -1;
   }
-  //printf("Read ack pkg len:%d from tcp port: %d, buffer: %s  %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8);
 
   taosCloseSocket(clientSocket);
   return 0;
 }
 
-static int checkUdpPort(info_s *info) {
+static int32_t taosNetCheckUdpPort(STestInfo *info) {
+  SOCKET  clientSocket;
+  char    sendbuf[BUFFER_SIZE];
+  char    recvbuf[BUFFER_SIZE];
+  int32_t iDataNum = 0;
+
   struct sockaddr_in serverAddr;
-  SOCKET             clientSocket;
-  char               sendbuf[BUFFER_SIZE];
-  char               recvbuf[BUFFER_SIZE];
-  int                iDataNum = 0;
+
   if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) {
-    perror("socket");
+    uError("failed to create udp client socket since %s", strerror(errno));
     return -1;
   }
 
-  // set overtime 
+  // set overtime
   struct timeval timeout;
-  timeout.tv_sec = 2;  //s
-  timeout.tv_usec = 0; //us
-  if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
-    perror("setsockopt send timer failed:");
+  timeout.tv_sec = 2;   // s
+  timeout.tv_usec = 0;  // us
+  if (setsockopt(clientSocket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
+    uError("failed to setsockopt send timer since %s", strerror(errno));
   }
-  if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
-    perror("setsockopt recv timer failed:");
+  if (setsockopt(clientSocket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
+    uError("failed to setsockopt recv timer since %s", strerror(errno));
   }
-  
+
   serverAddr.sin_family = AF_INET;
   serverAddr.sin_port = htons(info->port);
   serverAddr.sin_addr.s_addr = info->hostIp;
-  
+
   memset(sendbuf, 0, BUFFER_SIZE);
   memset(recvbuf, 0, BUFFER_SIZE);
 
@@ -283,69 +275,66 @@ static int checkUdpPort(info_s *info) {
 
   socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr);
 
-  int code = sendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int)sin_size);
+  int32_t code = sendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size);
   if (code < 0) {
-    perror("sendto");
+    uError("failed to perform sendto func since %s", strerror(errno));
     return -1;
   }
 
   iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size);
 
   if (iDataNum < info->pktLen) {
-    printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\t\t", iDataNum, info->pktLen, info->port);
+    uError("UDP: received ack:%d bytes, less than send:%d bytes from port:%d", iDataNum, info->pktLen, info->port);
     return -1;
   }
-  
-  //printf("Read ack pkg len:%d from udp port: %d, buffer: %s  %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8);
+
   taosCloseSocket(clientSocket);
   return 0;
 }
 
-static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uint16_t pktLen) {
-  int ret;
-  info_s info;
-  memset(&info, 0, sizeof(info_s));
-  info.hostIp   = hostIp;
-  info.pktLen   = pktLen;
+static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort, int32_t pktLen) {
+  int32_t   ret;
+  STestInfo info;
 
-  for (uint16_t port = startPort; port <= maxPort; port++) {
-    //printf("test: %s:%d\n", info.host, port);
-    printf("\n");
+  memset(&info, 0, sizeof(STestInfo));
+  info.hostIp = hostIp;
+  info.pktLen = pktLen;
 
+  for (int32_t port = startPort; port <= endPort; port++) {
     info.port = port;
-    ret = checkTcpPort(&info);
+    ret = taosNetCheckTcpPort(&info);
     if (ret != 0) {
-      printf("tcp port:%d test fail.\t\n", port);
+      uError("failed to test tcp port:%d", port);
     } else {
-      printf("tcp port:%d test ok.\t\t", port);
+      uInfo("successed to test tcp port:%d", port);
     }
-    
-    ret = checkUdpPort(&info);
+
+    ret = taosNetCheckUdpPort(&info);
     if (ret != 0) {
-      printf("udp port:%d test fail.\t\n", port);
+      uError("failed to test udp port:%d", port);
     } else {
-      printf("udp port:%d test ok.\t\t", port);
+      uInfo("successed to test udp port:%d", port);
     }
   }
-  
-  printf("\n");
-  return ;
+  return;
 }
 
-void* tnetInitRpc(char* secretEncrypt, char spi) {
+void *taosNetInitRpc(char *secretEncrypt, char spi) {
   SRpcInit rpcInit;
-  void* pRpcConn = NULL;
+  void *   pRpcConn = NULL;
+
+  char user[] = "nettestinternal";
+  char pass[] = "nettestinternal";
+  taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt);
 
-  taosEncryptPass((uint8_t *)g_pass, strlen(g_pass), secretEncrypt);
-      
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort = 0;
-  rpcInit.label = "NET-TEST";
+  rpcInit.label = "NT";
   rpcInit.numOfThreads = 1;  // every DB connection has only one thread
   rpcInit.cfp = NULL;
   rpcInit.sessions = 16;
   rpcInit.connType = TAOS_CONN_CLIENT;
-  rpcInit.user = g_user;
+  rpcInit.user = user;
   rpcInit.idleTime = 2000;
   rpcInit.ckey = "key";
   rpcInit.spi = spi;
@@ -355,17 +344,18 @@ void* tnetInitRpc(char* secretEncrypt, char spi) {
   return pRpcConn;
 }
 
-static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi) {  
+static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupStep *pStep) {
   SRpcEpSet epSet;
   SRpcMsg   reqMsg;
   SRpcMsg   rspMsg;
-  void*     pRpcConn;
+  void *    pRpcConn;
 
   char secretEncrypt[32] = {0};
 
-  pRpcConn = tnetInitRpc(secretEncrypt, spi);
+  pRpcConn = taosNetInitRpc(secretEncrypt, spi);
   if (NULL == pRpcConn) {
-    return -1;
+    uError("failed to init client rpc");
+    return TSDB_CODE_RPC_NETWORK_UNAVAIL;
   }
 
   memset(&epSet, 0, sizeof(SRpcEpSet));
@@ -373,205 +363,171 @@ static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktL
   epSet.numOfEps = 1;
   epSet.port[0] = port;
   strcpy(epSet.fqdn[0], serverFqdn);
-  
+
   reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST;
   reqMsg.pCont = rpcMallocCont(pktLen);
   reqMsg.contLen = pktLen;
   reqMsg.code = 0;
   reqMsg.handle = NULL;   // rpc handle returned to app
-  reqMsg.ahandle = NULL;   // app handle set by client
+  reqMsg.ahandle = NULL;  // app handle set by client
+  strcpy(reqMsg.pCont, "nettest");
 
   rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg);
 
-  // handle response
   if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) {
-    //printf("code:%d[%s]\n", rspMsg.code, tstrerror(rspMsg.code));
-    return -1;
+    uDebug("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code));
+    return rspMsg.code;
   }
-  
+
+  int32_t code = 0;
+  if (pStep != NULL && rspMsg.pCont != NULL && rspMsg.contLen > 0 && rspMsg.contLen <= sizeof(SStartupStep)) {
+    memcpy(pStep, rspMsg.pCont, rspMsg.contLen);
+    code = 1;
+  }
+
   rpcFreeCont(rspMsg.pCont);
-
   rpcClose(pRpcConn);
-
-  return 0;
+  return code;
 }
 
-static void rpcCheckPort(uint32_t hostIp) {
-  int ret;
-  char spi;
+static int32_t taosNetParseStartup(SStartupStep *pCont) {
+  SStartupStep *pStep = pCont;
+  uInfo("step:%s desc:%s", pStep->name, pStep->desc);
 
-  for (uint16_t port = g_startPort; port <= g_endPort; port++) {
-    //printf("test: %s:%d\n", info.host, port);
-    printf("\n");
+  if (pStep->finished) {
+    uInfo("check startup finished");
+  }
 
-    //================ check tcp port ================ 
-    int32_t pktLen;
-    if (g_pktLen <= tsRpcMaxUdpSize) {
-      pktLen = tsRpcMaxUdpSize + 1000;
-    } else {
-      pktLen = g_pktLen;
+  return pStep->finished ? 0 : 1;
+}
+
+static void taosNetTestStartup(char *host, int32_t port) {
+  uInfo("check startup, host:%s port:%d\n", host, port);
+
+  SStartupStep *pStep = malloc(sizeof(SStartupStep));
+  while (1) {
+    int32_t code = taosNetCheckRpc(host, port + TSDB_PORT_DNODEDNODE, 20, 0, pStep);
+    if (code > 0) {
+      code = taosNetParseStartup(pStep);
     }
 
-    spi = 1;
-    ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
-    if (ret != 0) {
-      spi = 0;
-      ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
-      if (ret != 0) {        
-        printf("TCP port:%d test fail.\t\t", port);
-      } else {
-        //printf("tcp port:%d test ok.\t\t", port);
-        printf("TCP port:\033[32m%d test OK\033[0m\t\t", port);
-      }      
+    if (code > 0) {
+      uDebug("continue check startup step");
     } else {
-      //printf("tcp port:%d test ok.\t\t", port);
-      printf("TCP port:\033[32m%d test OK\033[0m\t\t", port);
-    }
-
-    //================ check udp port ================ 
-    if (g_pktLen >= tsRpcMaxUdpSize) {
-      pktLen = tsRpcMaxUdpSize - 1000;
-    } else {
-      pktLen = g_pktLen;
-    }
-    
-    spi = 0;
-    ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
-    if (ret != 0) {
-      spi = 1;
-      ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
-      if (ret != 0) {        
-        printf("udp port:%d test fail.\t\n", port);
-      } else {
-        //printf("udp port:%d test ok.\t\n", port);
-        printf("UDP port:\033[32m%d test OK\033[0m\t\n", port);
-      }      
-    } else {
-      //printf("udp port:%d test ok.\t\n", port);
-      printf("UDP port:\033[32m%d test OK\033[0m\t\n", port);
+      if (code < 0) {
+        uError("failed to check startup step, code:0x%x %s", code, tstrerror(code));
+      }
+      break;
     }
   }
-  
-  printf("\n");
-  return ;
+
+  free(pStep);
 }
 
-static void taosNetTestClient(int flag) {
-  uint32_t serverIp = taosGetIpFromFqdn(g_serverFqdn);
+static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) {
+  int32_t endPort = startPort + 9;
+  char    spi = 0;
+
+  uInfo("check rpc, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen);
+    
+  for (uint16_t port = startPort; port <= endPort; port++) {
+    int32_t sendpkgLen;
+    if (pkgLen <= tsRpcMaxUdpSize) {
+      sendpkgLen = tsRpcMaxUdpSize + 1000;
+    } else {
+      sendpkgLen = pkgLen;
+    }
+
+    int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL);
+    if (ret < 0) {
+      uError("failed to test tcp port:%d", port);
+    } else {
+      uInfo("successed to test tcp port:%d", port);
+    }
+
+    if (pkgLen >= tsRpcMaxUdpSize) {
+      sendpkgLen = tsRpcMaxUdpSize - 1000;
+    } else {
+      sendpkgLen = pkgLen;
+    }
+
+    ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL);
+    if (ret < 0) {
+      uError("failed to test udp port:%d", port);
+    } else {
+      uInfo("successed to test udp port:%d", port);
+    }
+  }
+}
+
+static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) {
+  int32_t endPort = startPort + 11;
+  uInfo("work as client, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen);
+
+  uint32_t serverIp = taosGetIpFromFqdn(host);
   if (serverIp == 0xFFFFFFFF) {
-    printf("Failed to resolve FQDN:%s", g_serverFqdn); 
+    uError("failed to resolve fqdn:%s", host);
     exit(-1);
   }
 
-  if (0 == flag) {
-    checkPort(serverIp, g_startPort, g_endPort, g_pktLen);
-  } else {
-    rpcCheckPort(serverIp);
-  }
-  
-  return;
+  uInfo("server ip:%s is resolved from host:%s", taosIpStr(serverIp), host);
+  taosNetCheckPort(serverIp, startPort, endPort, pkgLen);
 }
 
-static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) {
+static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) {
+  int32_t endPort = startPort + 11;
+  uInfo("work as server, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen);
 
-  int port = startPort;
-  int num = endPort - startPort + 1;
+  int32_t port = startPort;
+  int32_t num = endPort - startPort + 1;
+  if (num < 0) num = 1;
 
-  if (num < 0) {
-    num = 1;
-  }
-  
   pthread_t *pids = malloc(2 * num * sizeof(pthread_t));
-  info_s *     tinfos = malloc(num * sizeof(info_s));
-  info_s *     uinfos = malloc(num * sizeof(info_s));
+  STestInfo *tinfos = malloc(num * sizeof(STestInfo));
+  STestInfo *uinfos = malloc(num * sizeof(STestInfo));
 
-  for (size_t i = 0; i < num; i++) {
-    info_s *tcpInfo = tinfos + i;
-    tcpInfo->port = (uint16_t)(port + i);
-    tcpInfo->pktLen = pktLen;
+  for (int32_t i = 0; i < num; i++) {
+    STestInfo *tcpInfo = tinfos + i;
+    tcpInfo->port = port + i;
+    tcpInfo->pktLen = pkgLen;
 
-    if (pthread_create(pids + i, NULL, bindTcpPort, tcpInfo) != 0) 
-    {
-      printf("create thread fail, port:%d.\n", port);
+    if (pthread_create(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) {
+      uInfo("failed to create tcp test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port);
       exit(-1);
     }
 
-    info_s *udpInfo = uinfos + i;
+    STestInfo *udpInfo = uinfos + i;
     udpInfo->port = (uint16_t)(port + i);
-    if (pthread_create(pids + num + i, NULL, bindUdpPort, udpInfo) != 0)
-    {                          
-      printf("create thread fail, port:%d.\n", port);
+    if (pthread_create(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) {
+      uInfo("failed to create udp test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port);
       exit(-1);
     }
   }
-  
-  for (int i = 0; i < num; i++) {
+
+  for (int32_t i = 0; i < num; i++) {
     pthread_join(pids[i], NULL);
     pthread_join(pids[(num + i)], NULL);
   }
 }
 
+void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen) {
+  tscEmbedded = 1;
+  if (host == NULL) host = tsLocalFqdn;
+  if (port == 0) port = tsServerPort;
+  if (pkgLen <= 10) pkgLen = 1000;
+  if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN;
 
-void taosNetTest(CmdArguments *args) {
-  if (0 == args->pktLen) {
-    g_pktLen = 1000;
+  if (0 == strcmp("client", role)) {
+    taosNetTestClient(host, port, pkgLen);
+  } else if (0 == strcmp("server", role)) {
+    taosNetTestServer(host, port, pkgLen);
+  } else if (0 == strcmp("rpc", role)) {
+    taosNetTestRpc(host, port, pkgLen);
+  } else if (0 == strcmp("startup", role)) {
+    taosNetTestStartup(host, port);
   } else {
-    g_pktLen = args->pktLen;
-  }
-  
-  if (args->port && args->endPort) {
-    if (args->port > args->endPort) {
-      printf("endPort[%d] must not lesss port[%d]\n", args->endPort, args->port);
-      exit(-1);
-    }
-  }  
- 
-  if (args->host && args->host[0] != 0) {
-    if (strlen(args->host) >= TSDB_EP_LEN) {
-      printf("host invalid: %s\n", args->host);
-      exit(-1);
-    }
-
-    taosGetFqdnPortFromEp(args->host, g_serverFqdn, &g_startPort);
-  } else {
-    tstrncpy(g_serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN);    
-    g_startPort = tsServerPort;
-  }
-  
-  if (args->port) {
-    g_startPort = args->port;
-  }
-  
-  if (args->endPort) {
-    g_endPort = args->endPort;
-  }
-  
-  if (g_startPort > g_endPort) {
-    printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort);
-    exit(-1);
+    taosNetTestStartup(host, port);
   }
 
-  
-  if (args->is_use_passwd) {
-    if (args->password == NULL) args->password = getpass("Enter password: ");
-  } else {
-    args->password = TSDB_DEFAULT_PASS;
-  }
-  tstrncpy(g_pass, args->password, TSDB_PASSWORD_LEN);
-
-  if (args->user == NULL) {
-    args->user = TSDB_DEFAULT_USER;
-  }
-  tstrncpy(g_user, args->user, TSDB_USER_LEN);
-  
-  if (0 == strcmp("client", args->netTestRole)) {
-    printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen);
-    taosNetTestClient(0);
-  } else if (0 == strcmp("clients", args->netTestRole)) {
-    printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen);
-    taosNetTestClient(1);
-  } else if (0 == strcmp("server", args->netTestRole)) {
-    taosNetTestServer(g_startPort, g_endPort, g_pktLen);
-  }
+  tscEmbedded = 0;
 }
-
diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c
index 143ce7c474..c15ae729ed 100644
--- a/src/util/src/tqueue.c
+++ b/src/util/src/tqueue.c
@@ -358,8 +358,8 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand
     if (queue->head) {
         pNode = queue->head;
         *pitem = pNode->item;
-        *type = pNode->type;
-        *phandle = queue->ahandle;
+        if (type) *type = pNode->type;
+        if (phandle) *phandle = queue->ahandle;
         queue->head = pNode->next;
         if (queue->head == NULL) 
           queue->tail = NULL;
diff --git a/src/vnode/inc/vnodeCfg.h b/src/vnode/inc/vnodeCfg.h
index c5887fef5d..ba148c07c1 100644
--- a/src/vnode/inc/vnodeCfg.h
+++ b/src/vnode/inc/vnodeCfg.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "vnodeInt.h"
 
 int32_t vnodeReadCfg(SVnodeObj *pVnode);
 int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg);
diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h
index e468c2e83e..401c217b9a 100644
--- a/src/vnode/inc/vnodeInt.h
+++ b/src/vnode/inc/vnodeInt.h
@@ -19,11 +19,11 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
 #include "tlog.h"
 #include "tsync.h"
-#include "twal.h"
 #include "tcq.h"
+#include "tsdb.h"
+#include "vnode.h"
 
 extern int32_t vDebugFlag;
 
@@ -35,39 +35,37 @@ extern int32_t vDebugFlag;
 #define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
 
 typedef struct {
-  int32_t      vgId;      // global vnode group ID
-  int32_t      refCount;  // reference count
-  int32_t      queuedWMsg;
-  int32_t      queuedRMsg;
-  int32_t      flowctrlLevel;
-  int8_t       status; 
-  int8_t       role;   
-  int8_t       accessState;
-  int8_t       isFull;
-  int8_t       isCommiting;
-  uint64_t     version;   // current version 
-  uint64_t     fversion;  // version on saved data file
-  void        *wqueue;
-  void        *rqueue;
-  void        *wal;
-  void        *tsdb;
-  int64_t      sync;
-  void        *events;
-  void        *cq;  // continuous query
-  int32_t      cfgVersion;
-  STsdbCfg     tsdbCfg;
-  SSyncCfg     syncCfg;
-  SWalCfg      walCfg;
-  void        *qMgmt;
-  char        *rootDir;
-  tsem_t       sem;
-  int8_t       dropped;
-  char         db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  int32_t  vgId;      // global vnode group ID
+  int32_t  refCount;  // reference count
+  int32_t  queuedWMsg;
+  int32_t  queuedRMsg;
+  int32_t  flowctrlLevel;
+  int8_t   status;
+  int8_t   role;
+  int8_t   accessState;
+  int8_t   isFull;
+  int8_t   isCommiting;
+  uint64_t version;   // current version
+  uint64_t fversion;  // version on saved data file
+  void *   wqueue;
+  void *   rqueue;
+  void *   wal;
+  void *   tsdb;
+  int64_t  sync;
+  void *   events;
+  void *   cq;  // continuous query
+  int32_t  cfgVersion;
+  STsdbCfg tsdbCfg;
+  SSyncCfg syncCfg;
+  SWalCfg  walCfg;
+  void *   qMgmt;
+  char *   rootDir;
+  tsem_t   sem;
+  int8_t   dropped;
+  char     db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  pthread_mutex_t statusMutex;
 } SVnodeObj;
 
-void vnodeInitWriteFp(void);
-void vnodeInitReadFp(void);
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/vnode/inc/vnodeMain.h b/src/vnode/inc/vnodeMain.h
new file mode 100644
index 0000000000..058b6bd090
--- /dev/null
+++ b/src/vnode/inc/vnodeMain.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_VNODE_MAIN_H
+#define TDENGINE_VNODE_MAIN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeDrop(int32_t vgId);
+int32_t vnodeOpen(int32_t vgId);
+int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeClose(int32_t vgId);
+
+int32_t vnodeReset(SVnodeObj *pVnode);
+void    vnodeDestroy(SVnodeObj *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/vnode/inc/vnodeMgmt.h b/src/vnode/inc/vnodeMgmt.h
new file mode 100644
index 0000000000..5a7e745619
--- /dev/null
+++ b/src/vnode/inc/vnodeMgmt.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_VNODE_MGMT_H
+#define TDENGINE_VNODE_MGMT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t vnodeInitMgmt();
+void    vnodeCleanupMgmt();
+
+void*   vnodeAcquire(int32_t vgId);
+void    vnodeRelease(void *pVnode);
+void*   vnodeGetWal(void *pVnode);
+
+int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
+void    vnodeBuildStatusMsg(void *pStatus);
+void    vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
+
+void    vnodeAddIntoHash(SVnodeObj* pVnode);
+void    vnodeRemoveFromHash(SVnodeObj * pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/vnode/inc/vnodeRead.h b/src/vnode/inc/vnodeRead.h
new file mode 100644
index 0000000000..f2953d79f4
--- /dev/null
+++ b/src/vnode/inc/vnodeRead.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_VNODE_READ_H
+#define TDENGINE_VNODE_READ_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t vnodeInitRead(void);
+void    vnodeCleanupRead(void);
+
+int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
+void    vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead);
+int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/vnode/inc/vnodeStatus.h b/src/vnode/inc/vnodeStatus.h
new file mode 100644
index 0000000000..00ac47df65
--- /dev/null
+++ b/src/vnode/inc/vnodeStatus.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_VNODE_STATUS_H
+#define TDENGINE_VNODE_STATUS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+typedef enum _VN_STATUS {
+  TAOS_VN_STATUS_INIT = 0,
+  TAOS_VN_STATUS_READY = 1,
+  TAOS_VN_STATUS_CLOSING = 2,
+  TAOS_VN_STATUS_UPDATING = 3,
+  TAOS_VN_STATUS_RESET = 4,
+} EVnodeStatus;
+
+bool vnodeSetInitStatus(SVnodeObj* pVnode);
+bool vnodeSetReadyStatus(SVnodeObj* pVnode);
+bool vnodeSetClosingStatus(SVnodeObj* pVnode);
+bool vnodeSetUpdatingStatus(SVnodeObj* pVnode);
+bool vnodeSetResetStatus(SVnodeObj* pVnode);
+
+bool vnodeInInitStatus(SVnodeObj* pVnode);
+bool vnodeInReadyStatus(SVnodeObj* pVnode);
+bool vnodeInClosingStatus(SVnodeObj* pVnode);
+bool vnodeInResetStatus(SVnodeObj* pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/vnode/inc/vnodeSync.h b/src/vnode/inc/vnodeSync.h
new file mode 100644
index 0000000000..ae02ca17cb
--- /dev/null
+++ b/src/vnode/inc/vnodeSync.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_VNODE_SYNC_H
+#define TDENGINE_VNODE_SYNC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver);
+int32_t  vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId);
+void     vnodeNotifyRole(int32_t vgId, int8_t role);
+void     vnodeCtrlFlow(int32_t vgId, int32_t level);
+int32_t  vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion);
+void     vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code);
+int32_t  vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam);
+int32_t  vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver);
+
+void     vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/vnode/inc/vnodeVersion.h b/src/vnode/inc/vnodeVersion.h
index 1d086cb21f..913e3915ab 100644
--- a/src/vnode/inc/vnodeVersion.h
+++ b/src/vnode/inc/vnodeVersion.h
@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "vnodeInt.h"
 
 int32_t vnodeReadVersion(SVnodeObj *pVnode);
 int32_t vnodeSaveVersion(SVnodeObj *pVnode);
diff --git a/src/vnode/inc/vnodeWorker.h b/src/vnode/inc/vnodeWorker.h
new file mode 100644
index 0000000000..abb0aa80ab
--- /dev/null
+++ b/src/vnode/inc/vnodeWorker.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_VNODE_WORKER_H
+#define TDENGINE_VNODE_WORKER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t vnodeInitMWorker();
+void    vnodeCleanupMWorker();
+int32_t vnodeOpenInMWorker(int32_t vgId, void *rpcHandle);
+int32_t vnodeCleanupInMWorker(int32_t vgId, void *rpcHandle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/vnode/inc/vnodeWrite.h b/src/vnode/inc/vnodeWrite.h
new file mode 100644
index 0000000000..8b3f0fdb58
--- /dev/null
+++ b/src/vnode/inc/vnodeWrite.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_VNODE_WRITE_H
+#define TDENGINE_VNODE_WRITE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t vnodeInitWrite(void);
+void    vnodeCleanupWrite(void);
+
+int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg);
+void    vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite);
+int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c
index a79fca9ebb..e0881db000 100644
--- a/src/vnode/src/vnodeCfg.c
+++ b/src/vnode/src/vnodeCfg.c
@@ -15,13 +15,9 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosmsg.h"
-#include "taoserror.h"
 #include "cJSON.h"
 #include "tglobal.h"
-#include "tsdb.h"
 #include "dnode.h"
-#include "vnodeInt.h"
 #include "vnodeCfg.h"
 
 static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index b6817fcbd0..b37a0b568e 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -18,77 +18,17 @@
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "tglobal.h"
-#include "trpc.h"
-#include "tsdb.h"
-#include "tutil.h"
-#include "vnode.h"
-#include "vnodeInt.h"
+// #include "tfs.h"
 #include "query.h"
 #include "dnode.h"
 #include "vnodeCfg.h"
+#include "vnodeStatus.h"
+#include "vnodeSync.h"
 #include "vnodeVersion.h"
+#include "vnodeMgmt.h"
 
-static SHashObj*tsVnodesHash;
-static void     vnodeCleanUp(SVnodeObj *pVnode);
-static int32_t  vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno);
-static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
-static int32_t  vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId);
-static void     vnodeNotifyRole(int32_t vgId, int8_t role);
-static void     vnodeCtrlFlow(int32_t vgId, int32_t level);
-static int32_t  vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion);
-static void     vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code);
-static int32_t  vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam);
-static int32_t  vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver);
-
-#ifndef _SYNC
-int64_t syncStart(const SSyncInfo *info) { return NULL; }
-int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype) { return 0; }
-void    syncStop(int64_t rid) {}
-int32_t syncReconfig(int64_t rid, const SSyncCfg *cfg) { return 0; }
-int32_t syncGetNodesRole(int64_t rid, SNodesRole *cfg) { return 0; }
-void    syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {}
-#endif
-
-char* vnodeStatus[] = {
-  "init",
-  "ready",
-  "closing",
-  "updating",
-  "reset"
-};
-
-int32_t vnodeInitResources() {
-  int32_t code = syncInit();
-  if (code != 0) return code;
-
-  vnodeInitWriteFp();
-  vnodeInitReadFp();
-
-  tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-  if (tsVnodesHash == NULL) {
-    vError("failed to init vnode list");
-    return TSDB_CODE_VND_OUT_OF_MEMORY;
-  }
-
-  if (tsdbInitCommitQueue(tsNumOfCommitThreads) < 0) {
-    vError("failed to init vnode commit queue");
-    return terrno;
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-void vnodeCleanupResources() {
-  tsdbDestroyCommitQueue();
-
-  if (tsVnodesHash != NULL) {
-    vDebug("vnode list is cleanup");
-    taosHashCleanup(tsVnodesHash);
-    tsVnodesHash = NULL;
-  }
-
-  syncCleanUp();
-}
+static void    vnodeCleanUp(SVnodeObj *pVnode);
+static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno);
 
 int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
   int32_t code;
@@ -155,7 +95,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
 
   vInfo("vgId:%d, vnode dir is created, walLevel:%d fsyncPeriod:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel,
         pVnodeCfg->cfg.fsyncPeriod);
-  code = vnodeOpen(pVnodeCfg->cfg.vgId, rootDir);
+  code = vnodeOpen(pVnodeCfg->cfg.vgId);
 
   return code;
 }
@@ -176,73 +116,81 @@ int32_t vnodeDrop(int32_t vgId) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) {
-  SVnodeObj *pVnode = vparam;
-
-  // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS
-  // cfgVersion can be corrected by status msg
-  if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_UPDATING) != TAOS_VN_STATUS_READY) {
-    vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId);
-    return TSDB_CODE_SUCCESS;
-  }
-
+static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) {
   int32_t code = vnodeWriteCfg(pVnodeCfg);
   if (code != TSDB_CODE_SUCCESS) {
-    pVnode->status = TAOS_VN_STATUS_READY;
     return code; 
   }
 
   code = vnodeReadCfg(pVnode);
   if (code != TSDB_CODE_SUCCESS) {
-    pVnode->status = TAOS_VN_STATUS_READY;
     return code; 
   }
 
   code = walAlter(pVnode->wal, &pVnode->walCfg);
   if (code != TSDB_CODE_SUCCESS) {
-    pVnode->status = TAOS_VN_STATUS_READY;
     return code;
   }
 
   code = syncReconfig(pVnode->sync, &pVnode->syncCfg);
   if (code != TSDB_CODE_SUCCESS) {
-    pVnode->status = TAOS_VN_STATUS_READY;
     return code; 
-  } 
+  }
 
   if (pVnode->tsdb) {
     code = tsdbConfigRepo(pVnode->tsdb, &pVnode->tsdbCfg);
     if (code != TSDB_CODE_SUCCESS) {
-      pVnode->status = TAOS_VN_STATUS_READY;
-      return code; 
+      return code;
     }
   }
 
-  pVnode->status = TAOS_VN_STATUS_READY;
-  vDebug("vgId:%d, vnode is altered", pVnode->vgId);
-
-  return TSDB_CODE_SUCCESS;
+  return 0;
 }
 
-int32_t vnodeOpen(int32_t vnode, char *rootDir) {
-  char temp[TSDB_FILENAME_LEN];
+int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) {
+  SVnodeObj *pVnode = vparam;
+
+  // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS
+  // cfgVersion can be corrected by status msg
+  if (!vnodeSetUpdatingStatus(pVnode)) {
+    vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t code = vnodeAlterImp(pVnode, pVnodeCfg);
+  vnodeSetReadyStatus(pVnode);
+
+  if (code != 0) {
+    vError("vgId:%d, failed to alter vnode, code:0x%x", pVnode->vgId, code);
+  } else {
+    vDebug("vgId:%d, vnode is altered", pVnode->vgId);
+  }
+
+  return code;
+}
+
+int32_t vnodeOpen(int32_t vgId) {
+  char temp[TSDB_FILENAME_LEN * 3];
+  char rootDir[TSDB_FILENAME_LEN * 2];
+  snprintf(rootDir, TSDB_FILENAME_LEN * 2, "%s/vnode%d", tsVnodeDir, vgId);
 
   SVnodeObj *pVnode = calloc(sizeof(SVnodeObj), 1);
   if (pVnode == NULL) {
-    vError("vgId:%d, failed to open vnode since no enough memory", vnode);
+    vError("vgId:%d, failed to open vnode since no enough memory", vgId);
     return TAOS_SYSTEM_ERROR(errno);
   }
 
   atomic_add_fetch_32(&pVnode->refCount, 1);
 
-  pVnode->vgId     = vnode;
-  pVnode->status   = TAOS_VN_STATUS_INIT;
+  pVnode->vgId     = vgId;
   pVnode->fversion = 0;
   pVnode->version  = 0;  
   pVnode->tsdbCfg.tsdbId = pVnode->vgId;
   pVnode->rootDir = strdup(rootDir);
   pVnode->accessState = TSDB_VN_ALL_ACCCESS;
   tsem_init(&pVnode->sem, 0, 0);
+  pthread_mutex_init(&pVnode->statusMutex, NULL);
+  vnodeSetInitStatus(pVnode);
 
   int32_t code = vnodeReadCfg(pVnode);
   if (code != TSDB_CODE_SUCCESS) {
@@ -272,7 +220,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
     sprintf(cqCfg.user, "_root");
     strcpy(cqCfg.pass, tsInternalPass);
     strcpy(cqCfg.db, pVnode->db);
-    cqCfg.vgId = vnode;
+    cqCfg.vgId = vgId;
     cqCfg.cqWrite = vnodeWriteToCache;
     pVnode->cq = cqOpen(pVnode, &cqCfg);
     if (pVnode->cq == NULL) {
@@ -341,13 +289,13 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
   tsdbIncCommitRef(pVnode->vgId);
 
-  taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
+  vnodeAddIntoHash(pVnode);
 
   SSyncInfo syncInfo;
   syncInfo.vgId = pVnode->vgId;
   syncInfo.version = pVnode->version;
   syncInfo.syncCfg = pVnode->syncCfg;
-  sprintf(syncInfo.path, "%s", rootDir);
+  tstrncpy(syncInfo.path, rootDir, TSDB_FILENAME_LEN);
   syncInfo.getWalInfo = vnodeGetWalInfo;
   syncInfo.getFileInfo = vnodeGetFileInfo;
   syncInfo.writeToCache = vnodeWriteToCache;
@@ -358,18 +306,14 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   syncInfo.getVersion = vnodeGetVersion;
   pVnode->sync = syncStart(&syncInfo);
 
-#ifndef _SYNC
-  pVnode->role = TAOS_SYNC_ROLE_MASTER;
-#else
   if (pVnode->sync <= 0) {
     vError("vgId:%d, failed to open sync, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
            tstrerror(terrno));
     vnodeCleanUp(pVnode);
     return terrno;
   }
-#endif
 
-  pVnode->status = TAOS_VN_STATUS_READY;
+  vnodeSetReadyStatus(pVnode);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -384,24 +328,9 @@ int32_t vnodeClose(int32_t vgId) {
   return 0;
 }
 
-void vnodeRelease(void *vparam) {
-  if (vparam == NULL) return;
-  SVnodeObj *pVnode = vparam;
-  int32_t    code = 0;
-  int32_t    vgId = pVnode->vgId;
-
-  int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
-  vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", vgId, refCount, pVnode);
-  assert(refCount >= 0);
-
-  if (refCount > 0) {
-    if (pVnode->status == TAOS_VN_STATUS_RESET && refCount <= 3) {
-      tsem_post(&pVnode->sem);
-    }
-    return;
-  }
-
-  vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", vgId, refCount, pVnode);
+void vnodeDestroy(SVnodeObj *pVnode) {
+  int32_t code = 0;
+  int32_t vgId = pVnode->vgId;
 
   if (pVnode->qMgmt) {
     qCleanupQueryMgmt(pVnode->qMgmt);
@@ -464,121 +393,20 @@ void vnodeRelease(void *vparam) {
   }
 
   tsem_destroy(&pVnode->sem);
+  pthread_mutex_destroy(&pVnode->statusMutex);
   free(pVnode);
   tsdbDecCommitRef(vgId);
-
-  int32_t count = taosHashGetSize(tsVnodesHash);
-  vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count);
 }
 
-static void vnodeIncRef(void *ptNode) {
-  assert(ptNode != NULL);
-
-  SVnodeObj **ppVnode = (SVnodeObj **)ptNode;
-  assert(ppVnode);
-  assert(*ppVnode);
-
-  SVnodeObj *pVnode = *ppVnode;
-  atomic_add_fetch_32(&pVnode->refCount, 1);
-  vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
-}
-
-void *vnodeAcquire(int32_t vgId) {
-  SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
-
-  if (ppVnode == NULL || *ppVnode == NULL) {
-    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
-    vDebug("vgId:%d, not exist", vgId);
-    return NULL;
-  }
-
-  return *ppVnode;
-}
-
-void *vnodeGetWal(void *pVnode) {
-  return ((SVnodeObj *)pVnode)->wal;
-}
-
-static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
-  int64_t totalStorage = 0;
-  int64_t compStorage = 0;
-  int64_t pointsWritten = 0;
-
-  if (pVnode->status != TAOS_VN_STATUS_READY) return;
-  if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
-
-  if (pVnode->tsdb) {
-    tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
-  }
-
-  SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
-  pLoad->vgId = htonl(pVnode->vgId);
-  pLoad->cfgVersion = htonl(pVnode->cfgVersion);
-  pLoad->totalStorage = htobe64(totalStorage);
-  pLoad->compStorage = htobe64(compStorage);
-  pLoad->pointsWritten = htobe64(pointsWritten);
-  pLoad->status = pVnode->status;
-  pLoad->role = pVnode->role;
-  pLoad->replica = pVnode->syncCfg.replica;  
-}
-
-int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
-  void *pIter = taosHashIterate(tsVnodesHash, NULL);
-  while (pIter) {
-    SVnodeObj **pVnode = pIter;
-    if (*pVnode) {
-
-    (*numOfVnodes)++;
-    if (*numOfVnodes >= TSDB_MAX_VNODES) {
-      vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES);
-      continue;
-    } else {
-      vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId;
-    }
-
-    }
-
-    pIter = taosHashIterate(tsVnodesHash, pIter);    
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-void vnodeBuildStatusMsg(void *param) {
-  SStatusMsg *pStatus = param;
-
-  void *pIter = taosHashIterate(tsVnodesHash, NULL);
-  while (pIter) {
-    SVnodeObj **pVnode = pIter;
-    if (*pVnode) {
-      vnodeBuildVloadMsg(*pVnode, pStatus);
-    }
-    pIter = taosHashIterate(tsVnodesHash, pIter);
-  }
-}
-
-void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    pAccess[i].vgId = htonl(pAccess[i].vgId);
-    SVnodeObj *pVnode = vnodeAcquire(pAccess[i].vgId);
-    if (pVnode != NULL) {
-      pVnode->accessState = pAccess[i].accessState;
-      if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) {
-        vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState);
-      }
-      vnodeRelease(pVnode);
-    }
-  }
-}
 
 static void vnodeCleanUp(SVnodeObj *pVnode) {
   // remove from hash, so new messages wont be consumed
-  taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t));
+  vnodeRemoveFromHash(pVnode);
 
-  if (pVnode->status != TAOS_VN_STATUS_INIT) {
+  if (!vnodeInInitStatus(pVnode)) {
     // it may be in updateing or reset state, then it shall wait
     int32_t i = 0;
-    while (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_CLOSING) !=
-           TAOS_VN_STATUS_READY) {
+    while (!vnodeSetClosingStatus(pVnode)) {
       if (++i % 1000 == 0) {
         sched_yield();
       }
@@ -614,7 +442,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
     pVnode->isCommiting = 1;
     pVnode->fversion = pVnode->version;
     vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
-    if (pVnode->status != TAOS_VN_STATUS_INIT) {
+    if (!vnodeInInitStatus(pVnode)) {
       return walRenew(pVnode->wal);
     }
     return 0;
@@ -624,7 +452,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
     vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
     pVnode->isCommiting = 0;
     pVnode->isFull = 0;
-    if (pVnode->status != TAOS_VN_STATUS_INIT) {
+    if (!vnodeInInitStatus(pVnode)) {
       walRemoveOneOldFile(pVnode->wal);
     }
     return vnodeSaveVersion(pVnode);
@@ -633,75 +461,12 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
   return 0;
 }
 
-static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size,
-                                 uint64_t *fversion) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while get file info", vgId);
-    return 0;
-  }
-
-  *fversion = pVnode->fversion;
-  uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size);
-
-  vnodeRelease(pVnode);
-  return ret;
-}
-
-static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while get wal info", vgId);
-    return -1;
-  }
-
-  int32_t code = walGetWalFile(pVnode->wal, fileName, fileId);
-
-  vnodeRelease(pVnode);
-  return code;
-}
-
-static void vnodeNotifyRole(int32_t vgId, int8_t role) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vTrace("vgId:%d, vnode not found while notify role", vgId);
-    return;
-  }
-
-  vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]);
-  pVnode->role = role;
-  dnodeSendStatusMsgToMnode();
-
-  if (pVnode->role == TAOS_SYNC_ROLE_MASTER) {
-    cqStart(pVnode->cq);
-  } else {
-    cqStop(pVnode->cq);
-  }
-
-  vnodeRelease(pVnode);
-}
-
-static void vnodeCtrlFlow(int32_t vgId, int32_t level) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vTrace("vgId:%d, vnode not found while flow ctrl", vgId);
-    return;
-  }
-
-  if (pVnode->flowctrlLevel != level) {
-    vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level);
-    pVnode->flowctrlLevel = level;
-  }
-
-  vnodeRelease(pVnode);
-}
-
-static int32_t vnodeResetTsdb(SVnodeObj *pVnode) {
+int32_t vnodeReset(SVnodeObj *pVnode) {
   char rootDir[128] = "\0";
   sprintf(rootDir, "%s/tsdb", pVnode->rootDir);
 
-  if (pVnode->status != TAOS_VN_STATUS_CLOSING && pVnode->status != TAOS_VN_STATUS_INIT) {
-    pVnode->status = TAOS_VN_STATUS_RESET;
+  if (!vnodeSetResetStatus(pVnode)) {
+    return -1;
   }
 
   void *tsdb = pVnode->tsdb;
@@ -724,70 +489,8 @@ static int32_t vnodeResetTsdb(SVnodeObj *pVnode) {
   appH.cqDropFunc = cqDrop;
   pVnode->tsdb = tsdbOpenRepo(rootDir, &appH);
 
-  pVnode->status = TAOS_VN_STATUS_READY;
+  vnodeSetReadyStatus(pVnode);
   vnodeRelease(pVnode);
 
   return 0;
 }
-
-static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while notify file synced", vgId);
-    return 0;
-  }
-
-  pVnode->fversion = fversion;
-  pVnode->version = fversion;
-  vnodeSaveVersion(pVnode);
-
-  vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
-  int32_t code = vnodeResetTsdb(pVnode);
-
-  vnodeRelease(pVnode);
-  return code;
-}
-
-void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) {
-  void *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while confirm forward", vgId);
-    return;
-  }
-
-  dnodeSendRpcVWriteRsp(pVnode, wparam, code);
-  vnodeRelease(pVnode);
-}
-
-static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while write to cache", vgId);
-    return TSDB_CODE_VND_INVALID_VGROUP_ID;
-  }
-
-  int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam);
-
-  vnodeRelease(pVnode);
-  return code;
-}
-
-static int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while write to cache", vgId);
-    return -1;
-  }
-
-  int32_t code = 0;
-  if (pVnode->isCommiting) {
-    vDebug("vgId:%d, vnode is commiting while get version", vgId);
-    code = -1;
-  } else {
-    *fver = pVnode->fversion;
-    *wver = pVnode->version;
-  }
-
-  vnodeRelease(pVnode);
-  return code;
-}
diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c
new file mode 100644
index 0000000000..631ec13ee8
--- /dev/null
+++ b/src/vnode/src/vnodeMgmt.c
@@ -0,0 +1,192 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "dnode.h"
+#include "vnodeStatus.h"
+#include "vnodeWorker.h"
+#include "vnodeRead.h"
+#include "vnodeWrite.h"
+#include "vnodeMain.h"
+
+static SHashObj *tsVnodesHash = NULL;
+
+static int32_t vnodeInitHash(void);
+static void    vnodeCleanupHash(void);
+static void    vnodeIncRef(void *ptNode);
+
+static SStep tsVnodeSteps[] = {
+  {"vnode-worker", vnodeInitMWorker,    vnodeCleanupMWorker},
+  {"vnode-write",  vnodeInitWrite,      vnodeCleanupWrite},
+  {"vnode-read",   vnodeInitRead,       vnodeCleanupRead},
+  {"vnode-hash",   vnodeInitHash,       vnodeCleanupHash},
+  {"tsdb-queue",   tsdbInitCommitQueue, tsdbDestroyCommitQueue}
+};
+
+int32_t vnodeInitMgmt() {
+  int32_t stepSize = sizeof(tsVnodeSteps) / sizeof(SStep);
+  return dnodeStepInit(tsVnodeSteps, stepSize);
+}
+
+void vnodeCleanupMgmt() {
+  int32_t stepSize = sizeof(tsVnodeSteps) / sizeof(SStep);
+  dnodeStepCleanup(tsVnodeSteps, stepSize);
+}
+
+static int32_t vnodeInitHash() {
+  tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (tsVnodesHash == NULL) {
+    vError("failed to init vnode mgmt");
+    return -1;
+  }
+
+  return 0;
+}
+
+static void vnodeCleanupHash() {
+  if (tsVnodesHash != NULL) {
+    vDebug("vnode mgmt is cleanup");
+    taosHashCleanup(tsVnodesHash);
+    tsVnodesHash = NULL;
+  }
+}
+
+void *vnodeGetWal(void *pVnode) {
+  return ((SVnodeObj *)pVnode)->wal;
+}
+
+void vnodeAddIntoHash(SVnodeObj *pVnode) {
+  taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
+}
+
+void vnodeRemoveFromHash(SVnodeObj *pVnode) { 
+  taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t));
+}
+
+static void vnodeIncRef(void *ptNode) {
+  assert(ptNode != NULL);
+
+  SVnodeObj **ppVnode = (SVnodeObj **)ptNode;
+  assert(ppVnode);
+  assert(*ppVnode);
+
+  SVnodeObj *pVnode = *ppVnode;
+  atomic_add_fetch_32(&pVnode->refCount, 1);
+  vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+}
+
+void *vnodeAcquire(int32_t vgId) {
+  SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
+
+  if (ppVnode == NULL || *ppVnode == NULL) {
+    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
+    vDebug("vgId:%d, not exist", vgId);
+    return NULL;
+  }
+
+  return *ppVnode;
+}
+
+void vnodeRelease(void *vparam) {
+  SVnodeObj *pVnode = vparam;
+  if (vparam == NULL) return;
+
+  int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
+  vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode);
+  assert(refCount >= 0);
+
+  if (refCount > 0) {
+    if (vnodeInResetStatus(pVnode) && refCount <= 3) {
+      tsem_post(&pVnode->sem);
+    }
+  } else {
+    vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode);
+    vnodeDestroy(pVnode);
+    int32_t count = taosHashGetSize(tsVnodesHash);
+    vDebug("vgId:%d, vnode is destroyed, vnodes:%d", pVnode->vgId, count);
+  }
+}
+
+static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
+  int64_t totalStorage = 0;
+  int64_t compStorage = 0;
+  int64_t pointsWritten = 0;
+
+  if (!vnodeInReadyStatus(pVnode)) return;
+  if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
+
+  if (pVnode->tsdb) {
+    tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
+  }
+
+  SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
+  pLoad->vgId = htonl(pVnode->vgId);
+  pLoad->cfgVersion = htonl(pVnode->cfgVersion);
+  pLoad->totalStorage = htobe64(totalStorage);
+  pLoad->compStorage = htobe64(compStorage);
+  pLoad->pointsWritten = htobe64(pointsWritten);
+  pLoad->status = pVnode->status;
+  pLoad->role = pVnode->role;
+  pLoad->replica = pVnode->syncCfg.replica;  
+}
+
+int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
+  void *pIter = taosHashIterate(tsVnodesHash, NULL);
+  while (pIter) {
+    SVnodeObj **pVnode = pIter;
+    if (*pVnode) {
+
+    (*numOfVnodes)++;
+    if (*numOfVnodes >= TSDB_MAX_VNODES) {
+      vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES);
+      continue;
+    } else {
+      vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId;
+    }
+
+    }
+
+    pIter = taosHashIterate(tsVnodesHash, pIter);    
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+void vnodeBuildStatusMsg(void *param) {
+  SStatusMsg *pStatus = param;
+
+  void *pIter = taosHashIterate(tsVnodesHash, NULL);
+  while (pIter) {
+    SVnodeObj **pVnode = pIter;
+    if (*pVnode) {
+      vnodeBuildVloadMsg(*pVnode, pStatus);
+    }
+    pIter = taosHashIterate(tsVnodesHash, pIter);
+  }
+}
+
+void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    pAccess[i].vgId = htonl(pAccess[i].vgId);
+    SVnodeObj *pVnode = vnodeAcquire(pAccess[i].vgId);
+    if (pVnode != NULL) {
+      pVnode->accessState = pAccess[i].accessState;
+      if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) {
+        vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState);
+      }
+      vnodeRelease(pVnode);
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index ed6d29505f..34921a93b3 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -16,26 +16,26 @@
 #define _DEFAULT_SOURCE
 #define _NON_BLOCKING_RETRIEVE  0
 #include "os.h"
-#include "tglobal.h"
-#include "taoserror.h"
 #include "taosmsg.h"
-#include "query.h"
-#include "trpc.h"
-#include "tsdb.h"
-#include "vnode.h"
-#include "vnodeInt.h"
 #include "tqueue.h"
+#include "tglobal.h"
+#include "query.h"
+#include "vnodeStatus.h"
 
 static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead);
 static int32_t  vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
 static int32_t  vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
 static int32_t  vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId);
 
-void vnodeInitReadFp(void) {
+int32_t vnodeInitRead(void) {
   vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg;
   vnodeProcessReadMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg;
+
+  return 0;
 }
 
+void vnodeCleanupRead() {}
+
 //
 // After the fetch request enters the vnode queue, if the vnode cannot provide services, the process function are
 // still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the
@@ -54,7 +54,7 @@ int32_t vnodeProcessRead(void *vparam, SVReadMsg *pRead) {
 }
 
 static int32_t vnodeCheckRead(SVnodeObj *pVnode) {
-  if (pVnode->status != TAOS_VN_STATUS_READY) {
+  if (!vnodeInReadyStatus(pVnode)) {
     vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
            pVnode->refCount, pVnode);
     return TSDB_CODE_APP_NOT_READY;
diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c
new file mode 100644
index 0000000000..d09a6a8663
--- /dev/null
+++ b/src/vnode/src/vnodeStatus.c
@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "vnodeStatus.h"
+
+char* vnodeStatus[] = {
+  "init",
+  "ready",
+  "closing",
+  "updating",
+  "reset"
+};
+
+bool vnodeSetInitStatus(SVnodeObj* pVnode) {
+  pthread_mutex_lock(&pVnode->statusMutex);
+  pVnode->status = TAOS_VN_STATUS_INIT;
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return true;
+}
+
+bool vnodeSetReadyStatus(SVnodeObj* pVnode) {
+  bool set = false;
+  pthread_mutex_lock(&pVnode->statusMutex);
+
+  if (pVnode->status == TAOS_VN_STATUS_INIT || pVnode->status == TAOS_VN_STATUS_READY ||
+      pVnode->status == TAOS_VN_STATUS_UPDATING || pVnode->status == TAOS_VN_STATUS_RESET) {
+    pVnode->status = TAOS_VN_STATUS_READY;
+    set = true;
+  } else {
+    vDebug("vgId:%d, cannot set status:ready, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]);
+  }
+
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return set;
+}
+
+bool vnodeSetClosingStatus(SVnodeObj* pVnode) {
+  bool set = false;
+  pthread_mutex_lock(&pVnode->statusMutex);
+
+  if (pVnode->status == TAOS_VN_STATUS_READY) {
+    pVnode->status = TAOS_VN_STATUS_CLOSING;
+    set = true;
+  } else {
+    vTrace("vgId:%d, cannot set status:closing, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]);
+  }
+
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return set;
+}
+
+bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) {
+  bool set = false;
+  pthread_mutex_lock(&pVnode->statusMutex);
+
+  if (pVnode->status == TAOS_VN_STATUS_READY) {
+    pVnode->status = TAOS_VN_STATUS_UPDATING;
+    set = true;
+  } else {
+    vDebug("vgId:%d, cannot set status:updating, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]);
+  }
+
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return set;
+}
+
+bool vnodeSetResetStatus(SVnodeObj* pVnode) {
+  bool set = false;
+  pthread_mutex_lock(&pVnode->statusMutex);
+
+  if (pVnode->status != TAOS_VN_STATUS_CLOSING && pVnode->status != TAOS_VN_STATUS_INIT) {
+    pVnode->status = TAOS_VN_STATUS_RESET;
+    set = true;
+  } else {
+    vDebug("vgId:%d, cannot set status:reset, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]);
+  }
+
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return set;
+}
+
+bool vnodeInInitStatus(SVnodeObj* pVnode) {
+  bool in = false;
+  pthread_mutex_lock(&pVnode->statusMutex);
+
+  if (pVnode->status == TAOS_VN_STATUS_INIT) {
+    in = true;
+  }
+
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return in;
+}
+
+bool vnodeInReadyStatus(SVnodeObj* pVnode) {
+  bool in = false;
+  pthread_mutex_lock(&pVnode->statusMutex);
+
+  if (pVnode->status == TAOS_VN_STATUS_READY) {
+    in = true;
+  }
+
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return in;
+}
+
+bool vnodeInClosingStatus(SVnodeObj* pVnode) {
+  bool in = false;
+  pthread_mutex_lock(&pVnode->statusMutex);
+
+  if (pVnode->status == TAOS_VN_STATUS_CLOSING) {
+    in = true;
+  }
+
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return in;
+}
+
+bool vnodeInResetStatus(SVnodeObj* pVnode) {
+  bool in = false;
+  pthread_mutex_lock(&pVnode->statusMutex);
+
+  if (pVnode->status == TAOS_VN_STATUS_RESET) {
+    in = true;
+  }
+
+  pthread_mutex_unlock(&pVnode->statusMutex);
+  return in;
+}
diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c
new file mode 100644
index 0000000000..c67132c41f
--- /dev/null
+++ b/src/vnode/src/vnodeSync.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "query.h"
+#include "dnode.h"
+#include "vnodeVersion.h"
+#include "vnodeMain.h"
+
+uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while get file info", vgId);
+    return 0;
+  }
+
+  *fver = pVnode->fversion;
+  uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size);
+
+  vnodeRelease(pVnode);
+  return ret;
+}
+
+int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while get wal info", vgId);
+    return -1;
+  }
+
+  int32_t code = walGetWalFile(pVnode->wal, fileName, fileId);
+
+  vnodeRelease(pVnode);
+  return code;
+}
+
+void vnodeNotifyRole(int32_t vgId, int8_t role) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vTrace("vgId:%d, vnode not found while notify role", vgId);
+    return;
+  }
+
+  vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]);
+  pVnode->role = role;
+  dnodeSendStatusMsgToMnode();
+
+  if (pVnode->role == TAOS_SYNC_ROLE_MASTER) {
+    cqStart(pVnode->cq);
+  } else {
+    cqStop(pVnode->cq);
+  }
+
+  vnodeRelease(pVnode);
+}
+
+void vnodeCtrlFlow(int32_t vgId, int32_t level) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vTrace("vgId:%d, vnode not found while flow ctrl", vgId);
+    return;
+  }
+
+  if (pVnode->flowctrlLevel != level) {
+    vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level);
+    pVnode->flowctrlLevel = level;
+  }
+
+  vnodeRelease(pVnode);
+}
+
+int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while notify file synced", vgId);
+    return 0;
+  }
+
+  pVnode->fversion = fversion;
+  pVnode->version = fversion;
+  vnodeSaveVersion(pVnode);
+
+  vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
+  int32_t code = vnodeReset(pVnode);
+
+  vnodeRelease(pVnode);
+  return code;
+}
+
+void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) {
+  void *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while confirm forward", vgId);
+    return;
+  }
+
+  dnodeSendRpcVWriteRsp(pVnode, wparam, code);
+  vnodeRelease(pVnode);
+}
+
+int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while write to cache", vgId);
+    return TSDB_CODE_VND_INVALID_VGROUP_ID;
+  }
+
+  int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam);
+
+  vnodeRelease(pVnode);
+  return code;
+}
+
+int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while write to cache", vgId);
+    return -1;
+  }
+
+  int32_t code = 0;
+  if (pVnode->isCommiting) {
+    vDebug("vgId:%d, vnode is commiting while get version", vgId);
+    code = -1;
+  } else {
+    *fver = pVnode->fversion;
+    *wver = pVnode->version;
+  }
+
+  vnodeRelease(pVnode);
+  return code;
+}
+
+void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code) {
+  SVnodeObj *pVnode = vparam;
+  syncConfirmForward(pVnode->sync, version, code);
+}
diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c
index 8f6360b4f9..fb3b3ebd9e 100644
--- a/src/vnode/src/vnodeVersion.c
+++ b/src/vnode/src/vnodeVersion.c
@@ -15,11 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
 #include "cJSON.h"
 #include "tglobal.h"
-#include "tsdb.h"
-#include "vnodeInt.h"
 #include "vnodeVersion.h"
 
 int32_t vnodeReadVersion(SVnodeObj *pVnode) {
diff --git a/src/vnode/src/vnodeWorker.c b/src/vnode/src/vnodeWorker.c
new file mode 100644
index 0000000000..4608d5e126
--- /dev/null
+++ b/src/vnode/src/vnodeWorker.c
@@ -0,0 +1,208 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "taosmsg.h"
+#include "tutil.h"
+#include "tqueue.h"
+#include "tglobal.h"
+#include "vnodeWorker.h"
+
+typedef enum {
+  VNODE_WORKER_ACTION_CREATE,
+  VNODE_WORKER_ACTION_DELETE
+} EVMWorkerAction;
+
+typedef struct {
+  int32_t vgId;
+  int32_t code;
+  void *  rpcHandle;
+  SVnodeObj *pVnode;
+  EVMWorkerAction action;
+} SVMWorkerMsg;
+
+typedef struct {
+  pthread_t thread;
+  int32_t   workerId;
+} SVMWorker;
+
+typedef struct {
+  int32_t    curNum;
+  int32_t    maxNum;
+  SVMWorker *worker;
+} SVMWorkerPool;
+
+static SVMWorkerPool tsVMWorkerPool;
+static taos_qset     tsVMWorkerQset;
+static taos_queue    tsVMWorkerQueue;
+
+static void *vnodeMWorkerFunc(void *param);
+
+static int32_t vnodeStartMWorker() {
+  tsVMWorkerQueue = taosOpenQueue();
+  if (tsVMWorkerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
+
+  taosAddIntoQset(tsVMWorkerQset, tsVMWorkerQueue, NULL);
+
+  for (int32_t i = tsVMWorkerPool.curNum; i < tsVMWorkerPool.maxNum; ++i) {
+    SVMWorker *pWorker = tsVMWorkerPool.worker + i;
+    pWorker->workerId = i;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&pWorker->thread, &thAttr, vnodeMWorkerFunc, pWorker) != 0) {
+      vError("failed to create thread to process vmworker queue, reason:%s", strerror(errno));
+    }
+
+    pthread_attr_destroy(&thAttr);
+
+    tsVMWorkerPool.curNum = i + 1;
+    vDebug("vmworker:%d is launched, total:%d", pWorker->workerId, tsVMWorkerPool.maxNum);
+  }
+
+  vDebug("vmworker queue:%p is allocated", tsVMWorkerQueue);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t vnodeInitMWorker() {
+  tsVMWorkerQset = taosOpenQset();
+
+  tsVMWorkerPool.maxNum = 1;
+  tsVMWorkerPool.curNum = 0;
+  tsVMWorkerPool.worker = calloc(sizeof(SVMWorker), tsVMWorkerPool.maxNum);
+
+  if (tsVMWorkerPool.worker == NULL) return -1;
+  for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) {
+    SVMWorker *pWorker = tsVMWorkerPool.worker + i;
+    pWorker->workerId = i;
+    vDebug("vmworker:%d is created", i);
+  }
+
+  vDebug("vmworker is initialized, num:%d qset:%p", tsVMWorkerPool.maxNum, tsVMWorkerQset);
+
+  return vnodeStartMWorker();
+}
+
+static void vnodeStopMWorker() {
+  vDebug("vmworker queue:%p is freed", tsVMWorkerQueue);
+  taosCloseQueue(tsVMWorkerQueue);
+  tsVMWorkerQueue = NULL;
+}
+
+void vnodeCleanupMWorker() {
+  for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) {
+    SVMWorker *pWorker = tsVMWorkerPool.worker + i;
+    if (pWorker->thread) {
+      taosQsetThreadResume(tsVMWorkerQset);
+    }
+    vDebug("vmworker:%d is closed", i);
+  }
+
+  for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) {
+    SVMWorker *pWorker = tsVMWorkerPool.worker + i;
+    vDebug("vmworker:%d start to join", i);
+    if (pWorker->thread) {
+      pthread_join(pWorker->thread, NULL);
+    }
+    vDebug("vmworker:%d join success", i);
+  }
+
+  vDebug("vmworker is closed, qset:%p", tsVMWorkerQset);
+
+  taosCloseQset(tsVMWorkerQset);
+  tsVMWorkerQset = NULL;
+  tfree(tsVMWorkerPool.worker);
+
+  vnodeStopMWorker();
+}
+
+static int32_t vnodeWriteIntoMWorker(int32_t vgId, EVMWorkerAction action,void *rpcHandle) {
+  SVMWorkerMsg *pMsg = taosAllocateQitem(sizeof(SVMWorkerMsg));
+  if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY;
+
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) return TSDB_CODE_VND_INVALID_VGROUP_ID;
+
+  pMsg->vgId = vgId;
+  pMsg->pVnode = pVnode;
+  pMsg->rpcHandle = rpcHandle;
+  pMsg->action = action;
+
+  int32_t code = taosWriteQitem(tsVMWorkerQueue, TAOS_QTYPE_RPC, pMsg);
+  if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS;
+
+  return code;
+}
+
+int32_t vnodeOpenInMWorker(int32_t vgId, void *rpcHandle) {
+  vTrace("vgId:%d, will open in vmworker", vgId);
+  return vnodeWriteIntoMWorker(vgId, VNODE_WORKER_ACTION_CREATE, rpcHandle);
+}
+
+int32_t vnodeCleanupInMWorker(int32_t vgId, void *rpcHandle) {
+  vTrace("vgId:%d, will cleanup in vmworker", vgId);
+  return vnodeWriteIntoMWorker(vgId, VNODE_WORKER_ACTION_DELETE, rpcHandle);
+}
+
+static void vnodeFreeMWorkerMsg(SVMWorkerMsg *pMsg) {
+  vTrace("vgId:%d, disposed in vmworker", pMsg->vgId);
+  vnodeRelease(pMsg->pVnode);
+  taosFreeQitem(pMsg);
+}
+
+static void vnodeSendVMWorkerRpcRsp(SVMWorkerMsg *pMsg) {
+  SRpcMsg rpcRsp = {
+    .handle = pMsg->rpcHandle,
+    .code = pMsg->code,
+  };
+
+  rpcSendResponse(&rpcRsp);
+  vnodeFreeMWorkerMsg(pMsg);
+}
+
+static void vnodeProcessMWorkerMsg(SVMWorkerMsg *pMsg) {
+  pMsg->code = 0;
+
+  switch (pMsg->action) {
+    case VNODE_WORKER_ACTION_CREATE:
+      pMsg->code = vnodeOpen(pMsg->vgId);
+      break;
+    case VNODE_WORKER_ACTION_DELETE:
+      pMsg->code = vnodeDrop(pMsg->vgId);
+      break;
+    default:
+      break;
+  }
+}
+
+static void *vnodeMWorkerFunc(void *param) {
+  while (1) {
+    SVMWorkerMsg *pMsg = NULL;
+    if (taosReadQitemFromQset(tsVMWorkerQset, NULL, (void **)&pMsg, NULL) == 0) {
+      vDebug("qset:%p, vmworker got no message from qset, exiting", tsVMWorkerQset);
+      break;
+    }
+
+    vTrace("vgId:%d, action:%d will be processed in vmworker queue", pMsg->vgId, pMsg->action);
+    vnodeProcessMWorkerMsg(pMsg);
+    vnodeSendVMWorkerRpcRsp(pMsg);
+  }
+
+  return NULL;
+}
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index 268d1fb53b..a826a4903f 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -19,17 +19,9 @@
 #include "taoserror.h"
 #include "tglobal.h"
 #include "tqueue.h"
-#include "trpc.h"
-#include "tsdb.h"
-#include "twal.h"
-#include "tsync.h"
 #include "ttimer.h"
-#include "tdataformat.h"
-#include "vnode.h"
-#include "vnodeInt.h"
-#include "syncInt.h"
-#include "tcq.h"
 #include "dnode.h"
+#include "vnodeStatus.h"
 
 #define MAX_QUEUED_MSG_NUM 10000
 
@@ -43,15 +35,19 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
 static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
 static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite);
 
-void vnodeInitWriteFp(void) {
+int32_t vnodeInitWrite(void) {
   vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT]          = vnodeProcessSubmitMsg;
   vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessCreateTableMsg;
   vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]   = vnodeProcessDropTableMsg;
   vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = vnodeProcessAlterTableMsg;
   vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = vnodeProcessDropStableMsg;
   vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL]  = vnodeProcessUpdateTagValMsg;
+
+  return 0;
 }
 
+void vnodeCleanupWrite() {}
+
 int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) {
   int32_t    code = 0;
   SVnodeObj *pVnode = vparam;
@@ -68,7 +64,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
          taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version, pVnode->version);
 
   if (pHead->version == 0) {  // from client or CQ
-    if (pVnode->status != TAOS_VN_STATUS_READY) {
+    if (!vnodeInReadyStatus(pVnode)) {
       vDebug("vgId:%d, msg:%s not processed since vstatus:%d, qtype:%s hver:%" PRIu64, pVnode->vgId,
              taosMsg[pHead->msgType], pVnode->status, qtypeStr[qtype], pHead->version);
       return TSDB_CODE_APP_NOT_READY;  // it may be in deleting or closing state
@@ -118,7 +114,7 @@ static int32_t vnodeCheckWrite(void *vparam) {
     return TSDB_CODE_APP_NOT_READY;
   }
 
-  if (pVnode->status == TAOS_VN_STATUS_CLOSING) {
+  if (vnodeInClosingStatus(pVnode)) {
     vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
            pVnode->refCount, pVnode);
     return TSDB_CODE_APP_NOT_READY;
@@ -132,11 +128,6 @@ static int32_t vnodeCheckWrite(void *vparam) {
   return TSDB_CODE_SUCCESS;
 }
 
-void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code) {
-  SVnodeObj *pVnode = vparam;
-  syncConfirmForward(pVnode->sync, version, code);
-}
-
 static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
   int32_t code = TSDB_CODE_SUCCESS;
 

From 9083dc986db3896f3b9e005f71eb13c65dbaa9db Mon Sep 17 00:00:00 2001
From: zyyang <zyyang@taosdata.com>
Date: Mon, 7 Dec 2020 13:13:53 +0800
Subject: [PATCH 086/118] [TD-2364]<fix>: mybatis-plus pagenation with druid
 datasource

---
 tests/examples/JDBC/mybatisplus-demo/pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/examples/JDBC/mybatisplus-demo/pom.xml b/tests/examples/JDBC/mybatisplus-demo/pom.xml
index 8535f3b797..b827fa5f6f 100644
--- a/tests/examples/JDBC/mybatisplus-demo/pom.xml
+++ b/tests/examples/JDBC/mybatisplus-demo/pom.xml
@@ -38,6 +38,11 @@
             <artifactId>h2</artifactId>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>druid</artifactId>
+            <version>1.1.17</version>
+        </dependency>
 
         <dependency>
             <groupId>com.taosdata.jdbc</groupId>

From b1d11b84efc5763fd8e41c2e582b3d37870b0a87 Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Mon, 7 Dec 2020 06:16:58 +0000
Subject: [PATCH 087/118] [TD-2366]<fix>: is (not) null for binary/nchar

---
 src/client/src/tscSQLParser.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 91b00e0109..dd336bcf05 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -3282,7 +3282,12 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC
       ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0);
 
   if (pColFilter->filterstr) {
-    if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) {
+    if (pExpr->nSQLOptr != TK_EQ
+      && pExpr->nSQLOptr != TK_NE
+      && pExpr->nSQLOptr != TK_ISNULL
+      && pExpr->nSQLOptr != TK_NOTNULL
+      && pExpr->nSQLOptr != TK_LIKE
+      ) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
     }
   } else {

From 5b28a59a97ae6a7bebed91367af551d6c4b15b01 Mon Sep 17 00:00:00 2001
From: zyyang <zyyang@taosdata.com>
Date: Mon, 7 Dec 2020 15:10:40 +0800
Subject: [PATCH 088/118] change

---
 tests/examples/JDBC/mybatisplus-demo/pom.xml   |  2 +-
 .../src/main/resources/application.yml         | 18 ++++++------------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/tests/examples/JDBC/mybatisplus-demo/pom.xml b/tests/examples/JDBC/mybatisplus-demo/pom.xml
index b827fa5f6f..e59b915d2c 100644
--- a/tests/examples/JDBC/mybatisplus-demo/pom.xml
+++ b/tests/examples/JDBC/mybatisplus-demo/pom.xml
@@ -47,7 +47,7 @@
         <dependency>
             <groupId>com.taosdata.jdbc</groupId>
             <artifactId>taos-jdbcdriver</artifactId>
-            <version>2.0.11</version>
+            <version>2.0.14</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml
index 96667f28b8..71e518602e 100644
--- a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml
+++ b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml
@@ -1,17 +1,5 @@
 spring:
   datasource:
-    #    driver-class-name: org.h2.Driver
-    #    schema: classpath:db/schema-mysql.sql
-    #    data: classpath:db/data-mysql.sql
-    #    url: jdbc:h2:mem:test
-    #    username: root
-    #    password: test
-
-    #    driver-class-name: com.mysql.jdbc.Driver
-    #    url: jdbc:mysql://master:3306/test?useSSL=false
-    #    username: root
-    #    password: 123456
-
     driver-class-name: com.taosdata.jdbc.TSDBDriver
     url: jdbc:TAOS://localhost:6030/mp_test
     user: root
@@ -20,6 +8,12 @@ spring:
     locale: en_US.UTF-8
     timezone: UTC-8
 
+    druid:
+      initial-size: 5
+      min-idle: 5
+      max-active: 5
+
+
 mybatis-plus:
   configuration:
     map-underscore-to-camel-case: false

From a5ab6bffb4bab156d80f4dcdf29817617172ec36 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 7 Dec 2020 07:34:51 +0000
Subject: [PATCH 089/118] scripts

---
 tests/script/general/stable/values.sim | 33 ++++++++++++++++----------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/tests/script/general/stable/values.sim b/tests/script/general/stable/values.sim
index d0c1783851..51488aabef 100644
--- a/tests/script/general/stable/values.sim
+++ b/tests/script/general/stable/values.sim
@@ -18,16 +18,25 @@ sql create table vdb0.vtb00 using vdb0.mt tags( 0 )
 sql create table vdb0.vtb01 using vdb0.mt tags( 0 )
 
 sql create database vdb1
-sql create table vdb1.vtb10 using vdb0.mt tags( 1 )
-sql create table vdb1.vtb11 using vdb0.mt tags( 1 )
+sql create table vdb1.mt (ts timestamp, tbcol int) TAGS(tgcol int)
+sql_error create table vdb1.vtb10 using vdb0.mt tags( 1 )
+sql_error create table vdb1.vtb11 using vdb0.mt tags( 1 )
+sql create table vdb1.vtb10 using vdb1.mt tags( 1 )
+sql create table vdb1.vtb11 using vdb1.mt tags( 1 )
 
 sql create database vdb2
-sql create table vdb2.vtb20 using vdb0.mt tags( 2 )
-sql create table vdb2.vtb21 using vdb0.mt tags( 2 )
+sql create table vdb2.mt (ts timestamp, tbcol int) TAGS(tgcol int)
+sql_error create table vdb2.vtb20 using vdb0.mt tags( 2 )
+sql_error create table vdb2.vtb21 using vdb0.mt tags( 2 )
+sql create table vdb2.vtb20 using vdb2.mt tags( 2 )
+sql create table vdb2.vtb21 using vdb2.mt tags( 2 )
 
 sql create database vdb3
-sql create table vdb3.vtb30 using vdb0.mt tags( 3 )
-sql create table vdb3.vtb31 using vdb0.mt tags( 3 )
+sql create table vdb3.mt (ts timestamp, tbcol int) TAGS(tgcol int)
+sql_error create table vdb3.vtb20 using vdb0.mt tags( 2 )
+sql_error create table vdb3.vtb21 using vdb0.mt tags( 2 )
+sql create table vdb3.vtb30 using vdb3.mt tags( 3 )
+sql create table vdb3.vtb31 using vdb3.mt tags( 3 )
 
 print =============== step2
 sql insert into vdb0.vtb00 values (1519833600000 , 10) (1519833600001, 20) (1519833600002, 30) 
@@ -40,7 +49,7 @@ sql insert into vdb3.vtb30 values (1519833600000 , 13) (1519833600001, 23) (1519
 sql insert into vdb3.vtb31 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) 
 sql select * from vdb0.mt
 
-if $rows != 24 then 
+if $rows != 6 then 
   return -1
 endi
 
@@ -55,7 +64,7 @@ sql insert into vdb3.vtb30 values (1519833600003 , 43) (1519833600005, 53) (1519
 sql insert into vdb3.vtb31 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) 
 sql select * from vdb0.mt
 
-if $rows != 48 then 
+if $rows != 12 then 
   return -1
 endi
 
@@ -66,7 +75,7 @@ sql insert into vdb2.vtb20 values(1519833600006, 62) (1519833600007, 72) vdb2.vt
 sql insert into vdb3.vtb30 values(1519833600006, 63) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 63) (1519833600007, 73) 
 sql select * from vdb0.mt
 
-if $rows != 64 then 
+if $rows != 16 then 
   return -1
 endi
 
@@ -77,7 +86,7 @@ sql insert into vdb2.vtb20 values(1519833600008, 82) (1519833600007, 72) vdb2.vt
 sql insert into vdb3.vtb30 values(1519833600008, 83) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 83) (1519833600007, 73) 
 sql select * from vdb0.mt
 
-if $rows != 68 then 
+if $rows != 17 then 
   return -1
 endi
 
@@ -87,7 +96,7 @@ sql insert into vdb0.vtb01 values(1519833600009, 90) (1519833600010, 100) vdb1.v
 
 sql select * from vdb0.mt
 
-if $rows != 84 then 
+if $rows != 21 then 
   return -1
 endi
 
@@ -97,7 +106,7 @@ sql insert into vdb0.vtb01 values(1519833600012, 120) (1519833600011, 110) vdb1.
 
 sql select * from vdb0.mt
 
-if $rows != 100 then 
+if $rows != 25 then 
   return -1
 endi
 

From d6356319d011f1a031edc614e119356a046d5751 Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Mon, 7 Dec 2020 08:18:11 +0000
Subject: [PATCH 090/118] [TD-2310]<feature>: add dest table into show streams

---
 src/client/inc/tsclient.h    |  3 +++
 src/client/src/tscProfile.c  |  5 +++++
 src/client/src/tscStream.c   |  4 ++++
 src/cq/src/cqMain.c          | 13 +++++++++----
 src/cq/test/cqtest.c         |  2 +-
 src/inc/taosmsg.h            |  1 +
 src/inc/tcq.h                |  2 +-
 src/inc/tsdb.h               |  2 +-
 src/mnode/src/mnodeProfile.c | 10 ++++++++++
 src/tsdb/src/tsdbMain.c      |  2 +-
 src/tsdb/src/tsdbMeta.c      |  2 +-
 11 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index a1b6174de0..748a9b2996 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -382,6 +382,7 @@ typedef struct SSqlObj {
 
 typedef struct SSqlStream {
   SSqlObj *pSql;
+  const char* dstTable;
   uint32_t streamId;
   char     listed;
   bool     isProject;
@@ -408,6 +409,8 @@ typedef struct SSqlStream {
   struct SSqlStream *prev, *next;
 } SSqlStream;
 
+void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable);
+
 int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn);
 void    tscInitMsgsFp();
 
diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c
index 18fc79c474..f813ff85d9 100644
--- a/src/client/src/tscProfile.c
+++ b/src/client/src/tscProfile.c
@@ -262,6 +262,11 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
   SSqlStream *pStream = pObj->streamList;
   while (pStream) {
     tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql));
+    if (pStream->dstTable == NULL) {
+      pSdesc->dstTable[0] = 0;
+    } else {
+      tstrncpy(pSdesc->dstTable, pStream->dstTable, sizeof(pSdesc->dstTable));
+    }
     pSdesc->streamId = htonl(pStream->streamId);
     pSdesc->num = htobe64(pStream->num);
 
diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c
index 68c3bcae16..74b8e4d958 100644
--- a/src/client/src/tscStream.c
+++ b/src/client/src/tscStream.c
@@ -535,6 +535,10 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) {
            pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr);
 }
 
+void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) {
+  pStream->dstTable = dstTable;
+}
+
 TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
                               int64_t stime, void *param, void (*callback)(void *)) {
   STscObj *pObj = (STscObj *)taos;
diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c
index efb8795962..3968d5b8c9 100644
--- a/src/cq/src/cqMain.c
+++ b/src/cq/src/cqMain.c
@@ -57,6 +57,7 @@ typedef struct SCqObj {
   uint64_t       uid;
   int32_t        tid;      // table ID
   int32_t        rowSize;  // bytes of a row
+  char *         dstTable;
   char *         sqlStr;   // SQL string
   STSchema *     pSchema;  // pointer to schema array
   void *         pStream;
@@ -185,7 +186,7 @@ void cqStop(void *handle) {
   pthread_mutex_unlock(&pContext->mutex);
 }
 
-void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
+void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema) {
   if (tsEnableStream == 0) {
     return NULL;
   }
@@ -195,9 +196,11 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *
   if (pObj == NULL) return NULL;
 
   pObj->uid = uid;
-  pObj->tid = tid;
-  pObj->sqlStr = malloc(strlen(sqlStr)+1);
-  strcpy(pObj->sqlStr, sqlStr);
+  pObj->tid = sid;
+  if (dstTable != NULL) {
+    pObj->dstTable = strdup(dstTable);
+  }
+  pObj->sqlStr = strdup(sqlStr);
 
   pObj->pSchema = tdDupSchema(pSchema);
   pObj->rowSize = schemaTLen(pSchema);
@@ -247,6 +250,7 @@ void cqDrop(void *handle) {
 
   cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); 
   tdFreeSchema(pObj->pSchema);
+  free(pObj->dstTable);
   free(pObj->sqlStr);
   free(pObj);
 
@@ -292,6 +296,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
   if (pObj->pStream == NULL) {
     pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
     if (pObj->pStream) {
+      tscSetStreamDestTable(pObj->pStream, pObj->dstTable);
       pContext->num++;
       cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
     } else {
diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c
index 41380f0d86..f378835f0a 100644
--- a/src/cq/test/cqtest.c
+++ b/src/cq/test/cqtest.c
@@ -70,7 +70,7 @@ int main(int argc, char *argv[]) {
   tdDestroyTSchemaBuilder(&schemaBuilder);
 
   for (int sid =1; sid<10; ++sid) {
-    cqCreate(pCq, sid, sid, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema);
+    cqCreate(pCq, sid, sid, NULL, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema);
   }
 
   tdFreeSchema(pSchema);
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index e8e3029244..6404f12034 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -790,6 +790,7 @@ typedef struct {
 
 typedef struct {
   char     sql[TSDB_SHOW_SQL_LEN];
+  char     dstTable[TSDB_TABLE_NAME_LEN];
   uint32_t streamId;
   int64_t  num;  // number of computing/cycles
   int64_t  useconds;
diff --git a/src/inc/tcq.h b/src/inc/tcq.h
index afa744a9c4..ad123d4080 100644
--- a/src/inc/tcq.h
+++ b/src/inc/tcq.h
@@ -42,7 +42,7 @@ void  cqStart(void *handle);
 void  cqStop(void *handle);
 
 // cqCreate is called by TSDB to start an instance of CQ 
-void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema);
+void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema);
 
 // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate
 void  cqDrop(void *handle);
diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h
index 58859f42bc..42100438fd 100644
--- a/src/inc/tsdb.h
+++ b/src/inc/tsdb.h
@@ -48,7 +48,7 @@ typedef struct {
   void *cqH;
   int (*notifyStatus)(void *, int status, int eno);
   int (*eventCallBack)(void *);
-  void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
+  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema);
   void (*cqDropFunc)(void *handle);
 } STsdbAppH;
 
diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c
index 36b6ff7a59..3256d5cd59 100644
--- a/src/mnode/src/mnodeProfile.c
+++ b/src/mnode/src/mnodeProfile.c
@@ -450,6 +450,12 @@ static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
+  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "dest table");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
   pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "ip:port");
@@ -524,6 +530,10 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v
       STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]);
       cols++;
 
+      pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->dstTable, pShow->bytes[cols]);
+      cols++;
+
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port);
       STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]);
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index 3990c0c516..9d65325001 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -872,7 +872,7 @@ static void tsdbStartStream(STsdbRepo *pRepo) {
   for (int i = 0; i < pMeta->maxTables; i++) {
     STable *pTable = pMeta->tables[i];
     if (pTable && pTable->type == TSDB_STREAM_TABLE) {
-      pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql,
+      pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql,
                                                      tsdbGetTableSchemaImpl(pTable, false, false, -1));
     }
   }
diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c
index 25c815b74e..9dfa147c8f 100644
--- a/src/tsdb/src/tsdbMeta.c
+++ b/src/tsdb/src/tsdbMeta.c
@@ -828,7 +828,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo
 
   if (lock && tsdbUnlockRepoMeta(pRepo) < 0) return -1;
   if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE && addIdx) {
-    pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql,
+    pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql,
                                                    tsdbGetTableSchemaImpl(pTable, false, false, -1));
   }
 

From 2db5f1e873d4446defdbe181c5696b8a184409f3 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 7 Dec 2020 17:11:04 +0800
Subject: [PATCH 091/118] TD-2370

---
 src/balance/src/bnMain.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c
index 383f981913..d80488fe9f 100644
--- a/src/balance/src/bnMain.c
+++ b/src/balance/src/bnMain.c
@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "tref.h"
 #include "tsync.h"
 #include "tglobal.h"
 #include "dnode.h"
@@ -28,7 +29,9 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-static SBnMgmt tsBnMgmt;;
+extern int64_t tsDnodeRid;
+extern int64_t tsSdbRid;
+static SBnMgmt tsBnMgmt;
 static void  bnMonitorDnodeModule();
 
 static void bnLock() {
@@ -529,6 +532,9 @@ void bnCheckStatus() {
   void *     pIter = NULL;
   SDnodeObj *pDnode = NULL;
 
+  void *dnodeSdb = taosAcquireRef(tsSdbRid, tsDnodeRid);
+  if (dnodeSdb == NULL) return;
+
   while (1) {
     pIter = mnodeGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
@@ -543,6 +549,8 @@ void bnCheckStatus() {
     }
     mnodeDecDnodeRef(pDnode);
   }
+
+  taosReleaseRef(tsSdbRid, tsDnodeRid);
 }
 
 void bnCheckModules() {

From b39ceed07eeec3772cdafb9d759654c17c4d51f6 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 7 Dec 2020 17:42:52 +0800
Subject: [PATCH 092/118] TD-2324

---
 src/vnode/inc/vnodeMain.h   |  1 +
 src/vnode/inc/vnodeWorker.h |  4 ++--
 src/vnode/src/vnodeMain.c   | 13 ++++++------
 src/vnode/src/vnodeWorker.c | 42 +++++++++++++++++--------------------
 4 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/src/vnode/inc/vnodeMain.h b/src/vnode/inc/vnodeMain.h
index 058b6bd090..e1ddcdc36a 100644
--- a/src/vnode/inc/vnodeMain.h
+++ b/src/vnode/inc/vnodeMain.h
@@ -28,6 +28,7 @@ int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeClose(int32_t vgId);
 
 int32_t vnodeReset(SVnodeObj *pVnode);
+void    vnodeCleanUp(SVnodeObj *pVnode);
 void    vnodeDestroy(SVnodeObj *pVnode);
 
 #ifdef __cplusplus
diff --git a/src/vnode/inc/vnodeWorker.h b/src/vnode/inc/vnodeWorker.h
index abb0aa80ab..01d9d42900 100644
--- a/src/vnode/inc/vnodeWorker.h
+++ b/src/vnode/inc/vnodeWorker.h
@@ -23,8 +23,8 @@ extern "C" {
 
 int32_t vnodeInitMWorker();
 void    vnodeCleanupMWorker();
-int32_t vnodeOpenInMWorker(int32_t vgId, void *rpcHandle);
-int32_t vnodeCleanupInMWorker(int32_t vgId, void *rpcHandle);
+int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode);
+int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode);
 
 #ifdef __cplusplus
 }
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index b37a0b568e..e95387b62c 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -26,8 +26,9 @@
 #include "vnodeSync.h"
 #include "vnodeVersion.h"
 #include "vnodeMgmt.h"
+#include "vnodeWorker.h"
+#include "vnodeMain.h"
 
-static void    vnodeCleanUp(SVnodeObj *pVnode);
 static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno);
 
 int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
@@ -110,6 +111,8 @@ int32_t vnodeDrop(int32_t vgId) {
   vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
   pVnode->dropped = 1;
 
+  // remove from hash, so new messages wont be consumed
+  vnodeRemoveFromHash(pVnode);
   vnodeRelease(pVnode);
   vnodeCleanUp(pVnode);
 
@@ -309,6 +312,7 @@ int32_t vnodeOpen(int32_t vgId) {
   if (pVnode->sync <= 0) {
     vError("vgId:%d, failed to open sync, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
            tstrerror(terrno));
+    vnodeRemoveFromHash(pVnode);
     vnodeCleanUp(pVnode);
     return terrno;
   }
@@ -322,6 +326,7 @@ int32_t vnodeClose(int32_t vgId) {
   if (pVnode == NULL) return 0;
 
   vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode);
+  vnodeRemoveFromHash(pVnode);
   vnodeRelease(pVnode);
   vnodeCleanUp(pVnode);
 
@@ -398,11 +403,7 @@ void vnodeDestroy(SVnodeObj *pVnode) {
   tsdbDecCommitRef(vgId);
 }
 
-
-static void vnodeCleanUp(SVnodeObj *pVnode) {
-  // remove from hash, so new messages wont be consumed
-  vnodeRemoveFromHash(pVnode);
-
+void vnodeCleanUp(SVnodeObj *pVnode) {
   if (!vnodeInInitStatus(pVnode)) {
     // it may be in updateing or reset state, then it shall wait
     int32_t i = 0;
diff --git a/src/vnode/src/vnodeWorker.c b/src/vnode/src/vnodeWorker.c
index 4608d5e126..d6053cf18e 100644
--- a/src/vnode/src/vnodeWorker.c
+++ b/src/vnode/src/vnodeWorker.c
@@ -21,10 +21,11 @@
 #include "tqueue.h"
 #include "tglobal.h"
 #include "vnodeWorker.h"
+#include "vnodeMain.h"
 
 typedef enum {
-  VNODE_WORKER_ACTION_CREATE,
-  VNODE_WORKER_ACTION_DELETE
+  VNODE_WORKER_ACTION_CLEANUP,
+  VNODE_WORKER_ACTION_DESTROUY
 } EVMWorkerAction;
 
 typedef struct {
@@ -132,14 +133,11 @@ void vnodeCleanupMWorker() {
   vnodeStopMWorker();
 }
 
-static int32_t vnodeWriteIntoMWorker(int32_t vgId, EVMWorkerAction action,void *rpcHandle) {
+static int32_t vnodeWriteIntoMWorker(SVnodeObj *pVnode, EVMWorkerAction action, void *rpcHandle) {
   SVMWorkerMsg *pMsg = taosAllocateQitem(sizeof(SVMWorkerMsg));
   if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY;
 
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) return TSDB_CODE_VND_INVALID_VGROUP_ID;
-
-  pMsg->vgId = vgId;
+  pMsg->vgId = pVnode->vgId;
   pMsg->pVnode = pVnode;
   pMsg->rpcHandle = rpcHandle;
   pMsg->action = action;
@@ -150,29 +148,27 @@ static int32_t vnodeWriteIntoMWorker(int32_t vgId, EVMWorkerAction action,void *
   return code;
 }
 
-int32_t vnodeOpenInMWorker(int32_t vgId, void *rpcHandle) {
-  vTrace("vgId:%d, will open in vmworker", vgId);
-  return vnodeWriteIntoMWorker(vgId, VNODE_WORKER_ACTION_CREATE, rpcHandle);
+int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode) {
+  vTrace("vgId:%d, will cleanup in vmworker", pVnode->vgId);
+  return vnodeWriteIntoMWorker(pVnode, VNODE_WORKER_ACTION_CLEANUP, NULL);
 }
 
-int32_t vnodeCleanupInMWorker(int32_t vgId, void *rpcHandle) {
-  vTrace("vgId:%d, will cleanup in vmworker", vgId);
-  return vnodeWriteIntoMWorker(vgId, VNODE_WORKER_ACTION_DELETE, rpcHandle);
+int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode) {
+  vTrace("vgId:%d, will destroy in vmworker", pVnode->vgId);
+  return vnodeWriteIntoMWorker(pVnode, VNODE_WORKER_ACTION_DESTROUY, NULL);
 }
 
 static void vnodeFreeMWorkerMsg(SVMWorkerMsg *pMsg) {
   vTrace("vgId:%d, disposed in vmworker", pMsg->vgId);
-  vnodeRelease(pMsg->pVnode);
   taosFreeQitem(pMsg);
 }
 
 static void vnodeSendVMWorkerRpcRsp(SVMWorkerMsg *pMsg) {
-  SRpcMsg rpcRsp = {
-    .handle = pMsg->rpcHandle,
-    .code = pMsg->code,
-  };
+  if (pMsg->rpcHandle != NULL) {
+    SRpcMsg rpcRsp = {.handle = pMsg->rpcHandle, .code = pMsg->code};
+    rpcSendResponse(&rpcRsp);
+  }
 
-  rpcSendResponse(&rpcRsp);
   vnodeFreeMWorkerMsg(pMsg);
 }
 
@@ -180,11 +176,11 @@ static void vnodeProcessMWorkerMsg(SVMWorkerMsg *pMsg) {
   pMsg->code = 0;
 
   switch (pMsg->action) {
-    case VNODE_WORKER_ACTION_CREATE:
-      pMsg->code = vnodeOpen(pMsg->vgId);
+    case VNODE_WORKER_ACTION_CLEANUP:
+      vnodeCleanUp(pMsg->pVnode);
       break;
-    case VNODE_WORKER_ACTION_DELETE:
-      pMsg->code = vnodeDrop(pMsg->vgId);
+    case VNODE_WORKER_ACTION_DESTROUY:
+      vnodeDestroy(pMsg->pVnode);
       break;
     default:
       break;

From 1fc3380019b0be8912d257cba189fe0510973cd2 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 7 Dec 2020 17:54:13 +0800
Subject: [PATCH 093/118] TD-2370

---
 src/util/src/tqueue.c     | 2 +-
 src/vnode/src/vnodeMain.c | 2 +-
 src/vnode/src/vnodeMgmt.c | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c
index c15ae729ed..d72bc5f412 100644
--- a/src/util/src/tqueue.c
+++ b/src/util/src/tqueue.c
@@ -366,7 +366,7 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand
         queue->numOfItems--;
         atomic_sub_fetch_32(&qset->numOfItems, 1);
         code = 1;
-        uTrace("item:%p is read out from queue:%p, type:%d items:%d", *pitem, queue, *type, queue->numOfItems);
+        uTrace("item:%p is read out from queue:%p, type:%d items:%d", *pitem, queue, pNode->type, queue->numOfItems);
     } 
 
     pthread_mutex_unlock(&queue->mutex);
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index e95387b62c..e286a972dc 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -114,7 +114,7 @@ int32_t vnodeDrop(int32_t vgId) {
   // remove from hash, so new messages wont be consumed
   vnodeRemoveFromHash(pVnode);
   vnodeRelease(pVnode);
-  vnodeCleanUp(pVnode);
+  vnodeCleanupInMWorker(pVnode);
 
   return TSDB_CODE_SUCCESS;
 }
diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c
index 631ec13ee8..cf42690d7d 100644
--- a/src/vnode/src/vnodeMgmt.c
+++ b/src/vnode/src/vnodeMgmt.c
@@ -114,7 +114,7 @@ void vnodeRelease(void *vparam) {
     }
   } else {
     vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode);
-    vnodeDestroy(pVnode);
+    vnodeDestroyInMWorker(pVnode);
     int32_t count = taosHashGetSize(tsVnodesHash);
     vDebug("vgId:%d, vnode is destroyed, vnodes:%d", pVnode->vgId, count);
   }

From aa1a5700fba114815a42aa2b486b7ff18db6b170 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 7 Dec 2020 23:52:07 +0800
Subject: [PATCH 094/118] TD-2371

---
 src/client/src/tscAsync.c         |   2 +-
 src/client/src/tscSql.c           |   2 +-
 src/client/src/tscSystem.c        |   7 -
 src/dnode/src/dnodeMain.c         |   2 +
 src/kit/shell/src/shellLinux.c    |   2 +-
 src/plugins/http/src/httpSql.c    |   4 +-
 src/plugins/http/src/httpSystem.c |   4 -
 src/util/inc/tnote.h              |  53 ++--
 src/util/src/tnote.c              | 395 ++++++++++++++----------------
 9 files changed, 219 insertions(+), 252 deletions(-)

diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index 3ff8a68d8f..6b2722b43f 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -96,7 +96,7 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa
     return;
   }
   
-  taosNotePrintTsc(sqlstr);
+  nPrintTsc(sqlstr);
   
   SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
   if (pSql == NULL) {
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index fae5b5856f..8e165241f0 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -344,7 +344,7 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES
     return NULL;
   }
 
-  taosNotePrintTsc(sqlstr);
+  nPrintTsc(sqlstr);
 
   SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
   if (pSql == NULL) {
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 03b6ac8404..132f8158fc 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -41,7 +41,6 @@ int     tscRefId = -1;
 int tscNumOfThreads;
 
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
-void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
 //void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet);
 
 void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
@@ -78,7 +77,6 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon
   return 0;
 }
 
-
 void taos_init_imp(void) {
   char temp[128]  = {0};
   
@@ -111,11 +109,6 @@ void taos_init_imp(void) {
   }
 
   taosSetCoreDump();
-
-  if (tsTscEnableRecordSql != 0) {
-    taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
-  }
-
   tscInitMsgsFp();
   int queueSize = tsMaxConnections*2;
 
diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c
index 502c410ce3..79efe5aa0e 100644
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taos.h"
+#include "tnote.h"
 #include "tconfig.h"
 #include "tfile.h"
 #include "twal.h"
@@ -98,6 +99,7 @@ int32_t dnodeInitSystem() {
   taosInitGlobalCfg();
   taosReadGlobalLogCfg();
   taosSetCoreDump();
+  taosInitNotes();
   signal(SIGPIPE, SIG_IGN);
 
   if (dnodeCreateDir(tsLogDir) < 0) {
diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c
index f896253fb4..6f4ee3fc50 100644
--- a/src/kit/shell/src/shellLinux.c
+++ b/src/kit/shell/src/shellLinux.c
@@ -46,7 +46,7 @@ static struct argp_option options[] = {
   {"thread",     'T', "THREADNUM",  0,                   "Number of threads when using multi-thread to import data."},
   {"database",   'd', "DATABASE",   0,                   "Database to use when connecting to the server."},
   {"timezone",   't', "TIMEZONE",   0,                   "Time zone of the shell, default is local."},
-  {"netrole",    'n', "NETROLE",    0,                   "Net role when network connectivity test, default is NULL, options: client|server|rpc|startup."},
+  {"netrole",    'n', "NETROLE",    0,                   "Net role when network connectivity test, default is startup, options: client|server|rpc|startup."},
   {"pktlen",     'l', "PKTLEN",     0,                   "Packet length used for net test, default is 1000 bytes."},
   {0}};
 
diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c
index 564f555c40..3e517c6fa6 100644
--- a/src/plugins/http/src/httpSql.c
+++ b/src/plugins/http/src/httpSql.c
@@ -181,7 +181,7 @@ void httpProcessMultiSql(HttpContext *pContext) {
   char *sql = httpGetCmdsString(pContext, cmd->sql);
   httpTraceL("context:%p, fd:%d, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, pContext->user,
              multiCmds->pos, sql);
-  taosNotePrintHttp(sql);
+  nPrintHttp(sql);
   taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext);
 }
 
@@ -329,7 +329,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) {
   }
 
   httpTraceL("context:%p, fd:%d, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->user, sql);
-  taosNotePrintHttp(sql);
+  nPrintHttp(sql);
   taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext);
 }
 
diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c
index 3b8858b62e..34a70a658b 100644
--- a/src/plugins/http/src/httpSystem.c
+++ b/src/plugins/http/src/httpSystem.c
@@ -37,7 +37,6 @@ void opInitHandle(HttpServer* pServer) {}
 #endif
 
 HttpServer tsHttpServer;
-void taosInitNote(int32_t numOfNoteLines, int32_t maxNotes, char* lable);
 
 int32_t httpInitSystem() {
   strcpy(tsHttpServer.label, "rest");
@@ -48,9 +47,6 @@ int32_t httpInitSystem() {
 
   pthread_mutex_init(&tsHttpServer.serverMutex, NULL);
 
-  if (tsHttpEnableRecordSql != 0) {
-    taosInitNote(tsNumOfLogLines / 10, 1, (char*)"http_note");
-  }
   restInitHandle(&tsHttpServer);
   adminInitHandle(&tsHttpServer);
   gcInitHandle(&tsHttpServer);
diff --git a/src/util/inc/tnote.h b/src/util/inc/tnote.h
index 552224abf0..b28a53fc69 100644
--- a/src/util/inc/tnote.h
+++ b/src/util/inc/tnote.h
@@ -20,41 +20,38 @@
 extern "C" {
 #endif
 
-#include "os.h"
-#include "tutil.h"
-#include "tglobal.h"
-
 #define MAX_NOTE_LINE_SIZE 66000
 #define NOTE_FILE_NAME_LEN 300
-  
-typedef struct _taosNoteInfo {
-  int  taosNoteFileNum ;
-  int  taosNoteMaxLines;
-  int  taosNoteLines;
-  char taosNoteName[NOTE_FILE_NAME_LEN];
-  int  taosNoteFlag;
-  int  taosNoteFd;
-  int  taosNoteOpenInProgress;
-  pthread_mutex_t taosNoteMutex;
-}taosNoteInfo;
-  
-void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...);
 
-extern taosNoteInfo  m_HttpNote;
-extern taosNoteInfo  m_TscNote;
+typedef struct {
+  int32_t fileNum;
+  int32_t maxLines;
+  int32_t lines;
+  int32_t flag;
+  int32_t fd;
+  int32_t openInProgress;
+  char    name[NOTE_FILE_NAME_LEN];
+  pthread_mutex_t mutex;
+} SNoteObj;
 
-extern int   tsHttpEnableRecordSql;
-extern int   tsTscEnableRecordSql;
+extern SNoteObj tsHttpNote;
+extern SNoteObj tsTscNote;
+extern SNoteObj tsErrorNote;
 
-#define taosNotePrintHttp(...)               \
+void taosInitNotes();
+void taosNotePrint(SNoteObj* pNote, const char* const format, ...);
+
+#define nPrintHttp(...)                      \
   if (tsHttpEnableRecordSql) {               \
-    taosNotePrint(&m_HttpNote, __VA_ARGS__); \
+    taosNotePrint(&tsHttpNote, __VA_ARGS__); \
   }
-    
-#define taosNotePrintTsc(...)                 \
-    if (tsTscEnableRecordSql) {               \
-      taosNotePrint(&m_TscNote, __VA_ARGS__); \
-    }
+
+#define nPrintTsc(...)                      \
+  if (tsTscEnableRecordSql) {               \
+    taosNotePrint(&tsTscNote, __VA_ARGS__); \
+  }
+
+#define nError(...) taosNotePrint(&tsErrorNote, __VA_ARGS__);
 
 #ifdef __cplusplus
 }
diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 9536f6fb70..04dd58afce 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -13,277 +13,256 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
+#include "tutil.h"
+#include "tglobal.h"
 #include "tnote.h"
 
-taosNoteInfo  m_HttpNote;
-taosNoteInfo  m_TscNote;
+SNoteObj tsHttpNote;
+SNoteObj tsTscNote;
+SNoteObj tsErrorNote;
 
-int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote);
+static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj * pNote);
+static void taosCloseNoteByFd(int32_t oldFd, SNoteObj * pNote);
 
-void taosInitNote(int numOfNoteLines, int maxNotes, char* lable)
-{
-    taosNoteInfo * pNote = NULL;
-    char temp[128] = { 0 };
+void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, char *name) {
+  memset(pNote, 0, sizeof(SNoteObj));
+  pNote->fileNum = 1;
+  pNote->fd = -1;
 
-    if (strcasecmp(lable, "http_note") == 0) {
-        pNote = &m_HttpNote;
-        sprintf(temp, "%s/httpnote", tsLogDir);
-    } else if (strcasecmp(lable, "tsc_note") == 0) {
-        pNote = &m_TscNote;        
-        sprintf(temp, "%s/tscnote-%d", tsLogDir, getpid());
-    } else {
-        return;
-    }
+  if (taosOpenNoteWithMaxLines(name, numOfLines, maxNotes, pNote) < 0) {
+    fprintf(stderr, "failed to init note file\n");
+  }
 
-    memset(pNote, 0, sizeof(taosNoteInfo));
-    pNote->taosNoteFileNum        = 1;
-    //pNote->taosNoteMaxLines       = 0;
-    //pNote->taosNoteLines          = 0;
-    //pNote->taosNoteFlag           = 0;
-    pNote->taosNoteFd             = -1;
-    //pNote->taosNoteOpenInProgress = 0;
-
-    if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes, pNote) < 0)
-        fprintf(stderr, "failed to init note file\n");
-
-    taosNotePrint(pNote, "==================================================");
-    taosNotePrint(pNote, "===================  new note  ===================");
-    taosNotePrint(pNote, "==================================================");
+  taosNotePrint(pNote, "==================================================");
+  taosNotePrint(pNote, "===================  new note  ===================");
+  taosNotePrint(pNote, "==================================================");
 }
 
-void taosCloseNoteByFd(int oldFd, taosNoteInfo * pNote);
-bool taosLockNote(int fd, taosNoteInfo * pNote)
-{
-    if (fd < 0) return false;
+void taosInitNotes() {
+  char name[TSDB_FILENAME_LEN * 2] = {0};
 
-    if (pNote->taosNoteFileNum > 1) {
-        int ret = (int)(flock(fd, LOCK_EX | LOCK_NB));
-        if (ret == 0) {
-            return true;
-        }
-    }
+  if (tsTscEnableRecordSql) {
+    snprintf(name, TSDB_FILENAME_LEN * 2, "%s/tscsql-%d", tsLogDir, taosGetPId());
+    taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name);
+  }
 
-    return false;
+  if (tsHttpEnableRecordSql) {
+    snprintf(name, TSDB_FILENAME_LEN * 2, "%s/httpsql", tsLogDir);
+    taosInitNote(tsNumOfLogLines, 1, &tsHttpNote, name);
+  }
+
+  if (tscEmbedded == 0) {
+    snprintf(name, TSDB_FILENAME_LEN * 2, "%s/note", tsLogDir);
+    taosInitNote(tsNumOfLogLines, 1, &tsErrorNote, name);
+  }
 }
 
-void taosUnLockNote(int fd, taosNoteInfo * pNote)
-{
-    if (fd < 0) return;
+bool taosLockNote(int32_t fd, SNoteObj *pNote) {
+  if (fd < 0) return false;
 
-    if (pNote->taosNoteFileNum > 1) {
-        flock(fd, LOCK_UN | LOCK_NB);
+  if (pNote->fileNum > 1) {
+    int32_t ret = (int32_t)(flock(fd, LOCK_EX | LOCK_NB));
+    if (ret == 0) {
+      return true;
     }
+  }
+
+  return false;
 }
 
-void *taosThreadToOpenNewNote(void *param)
-{
-    char name[NOTE_FILE_NAME_LEN * 2];
-    taosNoteInfo * pNote = (taosNoteInfo *)param;
+void taosUnLockNote(int32_t fd, SNoteObj *pNote) {
+  if (fd < 0) return;
 
-    pNote->taosNoteFlag ^= 1;
-    pNote->taosNoteLines = 0;
-    sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
+  if (pNote->fileNum > 1) {
+    flock(fd, LOCK_UN | LOCK_NB);
+  }
+}
 
-    umask(0);
+void *taosThreadToOpenNewNote(void *param) {
+  char      name[NOTE_FILE_NAME_LEN * 2];
+  SNoteObj *pNote = (SNoteObj *)param;
 
-    int fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
-    if (fd < 0) {
-      return NULL;
-    }
+  pNote->flag ^= 1;
+  pNote->lines = 0;
+  sprintf(name, "%s.%d", pNote->name, pNote->flag);
 
-    taosLockNote(fd, pNote);
-    (void)lseek(fd, 0, SEEK_SET);
+  umask(0);
 
-    int oldFd = pNote->taosNoteFd;
-    pNote->taosNoteFd = fd;
-    pNote->taosNoteLines = 0;
-    pNote->taosNoteOpenInProgress = 0;
-    taosNotePrint(pNote, "===============  new note is opened  =============");
-
-    taosCloseNoteByFd(oldFd, pNote);
+  int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+  if (fd < 0) {
     return NULL;
+  }
+
+  taosLockNote(fd, pNote);
+  (void)lseek(fd, 0, SEEK_SET);
+
+  int32_t oldFd = pNote->fd;
+  pNote->fd = fd;
+  pNote->lines = 0;
+  pNote->openInProgress = 0;
+  taosNotePrint(pNote, "===============  new note is opened  =============");
+
+  taosCloseNoteByFd(oldFd, pNote);
+  return NULL;
 }
 
-int taosOpenNewNote(taosNoteInfo * pNote)
-{
-    pthread_mutex_lock(&pNote->taosNoteMutex);
+int32_t taosOpenNewNote(SNoteObj *pNote) {
+  pthread_mutex_lock(&pNote->mutex);
 
-    if (pNote->taosNoteLines > pNote->taosNoteMaxLines && pNote->taosNoteOpenInProgress == 0) {
-        pNote->taosNoteOpenInProgress = 1;
+  if (pNote->lines > pNote->maxLines && pNote->openInProgress == 0) {
+    pNote->openInProgress = 1;
 
-        taosNotePrint(pNote, "===============  open new note  ==================");
-        pthread_t pattern;
-        pthread_attr_t attr;
-        pthread_attr_init(&attr);
-        pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+    taosNotePrint(pNote, "===============  open new note  ==================");
+    pthread_t      pattern;
+    pthread_attr_t attr;
+    pthread_attr_init(&attr);
+    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 
-        pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void*)pNote);
-        pthread_attr_destroy(&attr);
-    }
+    pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void *)pNote);
+    pthread_attr_destroy(&attr);
+  }
 
-    pthread_mutex_unlock(&pNote->taosNoteMutex);
+  pthread_mutex_unlock(&pNote->mutex);
 
-    return pNote->taosNoteFd;
+  return pNote->fd;
 }
 
-bool taosCheckNoteIsOpen(char *noteName, taosNoteInfo * pNote)
-{
-    /*
-    int exist = access(noteName, F_OK);
-    if (exist != 0) {
-        return false;
-    }
-    */
+bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) {
+  int32_t fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+  if (fd < 0) {
+    fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno));
+    return true;
+  }
 
-    int fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
-    if (fd < 0) {
-        fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno));
-        return true;
-    }
-
-    if (taosLockNote(fd, pNote)) {
-        taosUnLockNote(fd, pNote);
-        close(fd);
-        return false;
-    }
-    else {
-        close(fd);
-        return true;
-    }
+  if (taosLockNote(fd, pNote)) {
+    taosUnLockNote(fd, pNote);
+    close(fd);
+    return false;
+  } else {
+    close(fd);
+    return true;
+  }
 }
 
-void taosGetNoteName(char *fn, taosNoteInfo * pNote)
-{
-    if (pNote->taosNoteFileNum > 1) {
-        for (int i = 0; i < pNote->taosNoteFileNum; i++) {
-            char fileName[NOTE_FILE_NAME_LEN];
+void taosGetNoteName(char *fn, SNoteObj *pNote) {
+  if (pNote->fileNum > 1) {
+    for (int32_t i = 0; i < pNote->fileNum; i++) {
+      char fileName[NOTE_FILE_NAME_LEN];
 
-            sprintf(fileName, "%s%d.0", fn, i);
-            bool file1open = taosCheckNoteIsOpen(fileName, pNote);
+      sprintf(fileName, "%s%d.0", fn, i);
+      bool file1open = taosCheckNoteIsOpen(fileName, pNote);
 
-            sprintf(fileName, "%s%d.1", fn, i);
-            bool file2open = taosCheckNoteIsOpen(fileName, pNote);
+      sprintf(fileName, "%s%d.1", fn, i);
+      bool file2open = taosCheckNoteIsOpen(fileName, pNote);
 
-            if (!file1open && !file2open) {
-                sprintf(pNote->taosNoteName, "%s%d", fn, i);
-                return;
-            }
-        }
+      if (!file1open && !file2open) {
+        sprintf(pNote->name, "%s%d", fn, i);
+        return;
+      }
     }
+  }
 
-    if (strlen(fn) < NOTE_FILE_NAME_LEN) {
-      strcpy(pNote->taosNoteName, fn);
-    }
+  if (strlen(fn) < NOTE_FILE_NAME_LEN) {
+    strcpy(pNote->name, fn);
+  }
 }
 
-int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote)
-{
-    char name[NOTE_FILE_NAME_LEN * 2] = "\0";
-    struct stat  notestat0, notestat1;
-    int size;
+int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) {
+  char    name[NOTE_FILE_NAME_LEN * 2] = "\0";
+  int32_t size;
+  struct stat notestat0, notestat1;
 
-    pNote->taosNoteMaxLines = maxLines;
-    pNote->taosNoteFileNum = maxNoteNum;
-    taosGetNoteName(fn, pNote);
+  pNote->maxLines = maxLines;
+  pNote->fileNum = maxNoteNum;
+  taosGetNoteName(fn, pNote);
 
-    if (strlen(fn) > NOTE_FILE_NAME_LEN * 2 - 2) {
-      fprintf(stderr, "the len of file name overflow:%s\n", fn);
-      return -1;
-    } 
+  if (strlen(fn) > NOTE_FILE_NAME_LEN * 2 - 2) {
+    fprintf(stderr, "the len of file name overflow:%s\n", fn);
+    return -1;
+  }
 
+  strcpy(name, fn);
+  strcat(name, ".0");
+
+  // if none of the note files exist, open 0, if both exists, open the old one
+  if (stat(name, &notestat0) < 0) {
+    pNote->flag = 0;
+  } else {
     strcpy(name, fn);
-    strcat(name, ".0");
-
-    // if none of the note files exist, open 0, if both exists, open the old one
-    if (stat(name, &notestat0) < 0) {
-        pNote->taosNoteFlag = 0;
+    strcat(name, ".1");
+    if (stat(name, &notestat1) < 0) {
+      pNote->flag = 1;
     } else {
-        strcpy(name, fn);
-        strcat(name, ".1");
-        if (stat(name, &notestat1) < 0) {
-            pNote->taosNoteFlag = 1;
-        }
-        else {
-            pNote->taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1;
-        }
+      pNote->flag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1;
     }
+  }
 
-    char noteName[NOTE_FILE_NAME_LEN * 2] = "\0";
-    sprintf(noteName, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
-    pthread_mutex_init(&pNote->taosNoteMutex, NULL);
+  char noteName[NOTE_FILE_NAME_LEN * 2] = "\0";
+  sprintf(noteName, "%s.%d", pNote->name, pNote->flag);
+  pthread_mutex_init(&pNote->mutex, NULL);
 
-    umask(0);
-    pNote->taosNoteFd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+  umask(0);
+  pNote->fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
 
-    if (pNote->taosNoteFd < 0) {
-        fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno));
-        return -1;
-    }
-    taosLockNote(pNote->taosNoteFd, pNote);
+  if (pNote->fd < 0) {
+    fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno));
+    return -1;
+  }
+  taosLockNote(pNote->fd, pNote);
 
-    // only an estimate for number of lines
-    struct stat filestat;
-    if (fstat(pNote->taosNoteFd, &filestat) < 0) {
-      fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno));
-      return -1;
-    }    
-    size = (int)filestat.st_size;
-    pNote->taosNoteLines = size / 60;
+  // only an estimate for number of lines
+  struct stat filestat;
+  if (fstat(pNote->fd, &filestat) < 0) {
+    fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno));
+    return -1;
+  }
+  size = (int32_t)filestat.st_size;
+  pNote->lines = size / 60;
 
-    lseek(pNote->taosNoteFd, 0, SEEK_END);
+  lseek(pNote->fd, 0, SEEK_END);
 
-    return 0;
+  return 0;
 }
 
-void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
-{
-    va_list argpointer;
-    char    buffer[MAX_NOTE_LINE_SIZE+2];
-    int     len;
-    struct  tm      Tm, *ptm;
-    struct  timeval timeSecs;
-    time_t  curTime;
+void taosNotePrint(SNoteObj *pNote, const char *const format, ...) {
+  va_list        argpointer;
+  char           buffer[MAX_NOTE_LINE_SIZE + 2];
+  int32_t        len;
+  struct tm      Tm, *ptm;
+  struct timeval timeSecs;
+  time_t         curTime;
 
-    gettimeofday(&timeSecs, NULL);
-    curTime = timeSecs.tv_sec;
-    ptm = localtime_r(&curTime, &Tm);
-#ifndef LINUX
-  len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%lld ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,
-                ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec, taosGetPthreadId());
-#else
-  len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %lx ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min,
-                ptm->tm_sec, (int)timeSecs.tv_usec, (unsigned long int)pthread_self());
-#endif
-    va_start(argpointer, format);
-    len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer);
-    va_end(argpointer);
+  gettimeofday(&timeSecs, NULL);
+  curTime = timeSecs.tv_sec;
+  ptm = localtime_r(&curTime, &Tm);
+  len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour,
+                ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId());
+  va_start(argpointer, format);
+  len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer);
+  va_end(argpointer);
 
-    if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2;
+  if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2;
 
-    buffer[len++] = '\n';
-    buffer[len] = 0;
+  buffer[len++] = '\n';
+  buffer[len] = 0;
 
-    if (pNote->taosNoteFd >= 0)  {
-        taosWrite(pNote->taosNoteFd, buffer, (unsigned int)len);
+  if (pNote->fd >= 0) {
+    taosWrite(pNote->fd, buffer, len);
 
-        if (pNote->taosNoteMaxLines > 0) {
-            pNote->taosNoteLines++;
-            if ((pNote->taosNoteLines > pNote->taosNoteMaxLines) && (pNote->taosNoteOpenInProgress == 0))
-                taosOpenNewNote(pNote);
-        }
+    if (pNote->maxLines > 0) {
+      pNote->lines++;
+      if ((pNote->lines > pNote->maxLines) && (pNote->openInProgress == 0)) taosOpenNewNote(pNote);
     }
+  }
 }
 
-void taosCloseNote(taosNoteInfo * pNote)
-{
-    taosCloseNoteByFd(pNote->taosNoteFd, pNote);
-}
+void taosCloseNote(SNoteObj *pNote) { taosCloseNoteByFd(pNote->fd, pNote); }
 
-void taosCloseNoteByFd(int fd, taosNoteInfo * pNote)
-{
-    if (fd >= 0) {
-        taosUnLockNote(fd, pNote);
-        close(fd);
-    }
+void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) {
+  if (fd >= 0) {
+    taosUnLockNote(fd, pNote);
+    close(fd);
+  }
 }

From aa3df5f7fac7976f1adf267977bce3d8062944d9 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 10:28:13 +0800
Subject: [PATCH 095/118] TD-2371

---
 src/util/src/tlog.c  |  1 -
 src/util/src/tnote.c | 39 ++++++++++++++++++++-------------------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c
index ad3a922304..0ba5e49c6e 100644
--- a/src/util/src/tlog.c
+++ b/src/util/src/tlog.c
@@ -287,7 +287,6 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   tsLogObj.fileNum = maxFileNum;
   taosGetLogFileName(fn);
 
-
   if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
     strcpy(name, fn);
     strcat(name, ".0");
diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 04dd58afce..3338d8f301 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -53,7 +53,7 @@ void taosInitNotes() {
     taosInitNote(tsNumOfLogLines, 1, &tsHttpNote, name);
   }
 
-  if (tscEmbedded == 0) {
+  if (tscEmbedded == 1) {
     snprintf(name, TSDB_FILENAME_LEN * 2, "%s/note", tsLogDir);
     taosInitNote(tsNumOfLogLines, 1, &tsErrorNote, name);
   }
@@ -170,36 +170,37 @@ void taosGetNoteName(char *fn, SNoteObj *pNote) {
 }
 
 int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) {
-  char    name[NOTE_FILE_NAME_LEN * 2] = "\0";
+  char    name[NOTE_FILE_NAME_LEN * 2] = {0};
   int32_t size;
-  struct stat notestat0, notestat1;
+  struct stat logstat0, logstat1;
 
   pNote->maxLines = maxLines;
   pNote->fileNum = maxNoteNum;
   taosGetNoteName(fn, pNote);
 
-  if (strlen(fn) > NOTE_FILE_NAME_LEN * 2 - 2) {
-    fprintf(stderr, "the len of file name overflow:%s\n", fn);
-    return -1;
+  if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) {
+    strcpy(name, fn);
+    strcat(name, ".0");
   }
+  bool log0Exist = stat(name, &logstat0) >= 0;
 
-  strcpy(name, fn);
-  strcat(name, ".0");
-
-  // if none of the note files exist, open 0, if both exists, open the old one
-  if (stat(name, &notestat0) < 0) {
-    pNote->flag = 0;
-  } else {
+  if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) {
     strcpy(name, fn);
     strcat(name, ".1");
-    if (stat(name, &notestat1) < 0) {
-      pNote->flag = 1;
-    } else {
-      pNote->flag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1;
-    }
+  }
+  bool log1Exist = stat(name, &logstat1) >= 0;
+
+  if (!log0Exist && !log1Exist) {
+    pNote->flag = 0;
+  } else if (!log1Exist) {
+    pNote->flag = 0;
+  } else if (!log0Exist) {
+    pNote->flag = 1;
+  } else {
+    pNote->flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1;
   }
 
-  char noteName[NOTE_FILE_NAME_LEN * 2] = "\0";
+  char noteName[NOTE_FILE_NAME_LEN * 2] = {0};
   sprintf(noteName, "%s.%d", pNote->name, pNote->flag);
   pthread_mutex_init(&pNote->mutex, NULL);
 

From c3a7a9576daf9e2e1421d3054c743cdba58ead87 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 10:56:50 +0800
Subject: [PATCH 096/118] TD-2371

---
 src/util/inc/tnote.h |  5 +++--
 src/util/src/tlog.c  |  2 ++
 src/util/src/tnote.c | 49 +++++++++++++++++++++++---------------------
 3 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/src/util/inc/tnote.h b/src/util/inc/tnote.h
index b28a53fc69..b4c0c7501b 100644
--- a/src/util/inc/tnote.h
+++ b/src/util/inc/tnote.h
@@ -36,10 +36,11 @@ typedef struct {
 
 extern SNoteObj tsHttpNote;
 extern SNoteObj tsTscNote;
-extern SNoteObj tsErrorNote;
+extern SNoteObj tsInfoNote;
 
 void taosInitNotes();
 void taosNotePrint(SNoteObj* pNote, const char* const format, ...);
+void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len);
 
 #define nPrintHttp(...)                      \
   if (tsHttpEnableRecordSql) {               \
@@ -51,7 +52,7 @@ void taosNotePrint(SNoteObj* pNote, const char* const format, ...);
     taosNotePrint(&tsTscNote, __VA_ARGS__); \
   }
 
-#define nError(...) taosNotePrint(&tsErrorNote, __VA_ARGS__);
+#define nInfo(buffer, len) taosNotePrintBuffer(&tsInfoNote, buffer, len);
 
 #ifdef __cplusplus
 }
diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c
index 0ba5e49c6e..c0f89e8465 100644
--- a/src/util/src/tlog.c
+++ b/src/util/src/tlog.c
@@ -17,6 +17,7 @@
 #include "os.h"
 #include "tulog.h"
 #include "tlog.h"
+#include "tnote.h"
 #include "tutil.h"
 
 #define MAX_LOGLINE_SIZE              (1000)
@@ -400,6 +401,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
   }
 
   if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len);
+  if (dflag == 255) nInfo(buffer, len);
 }
 
 void taosDumpData(unsigned char *msg, int32_t len) {
diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 3338d8f301..82ce2c314d 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -21,12 +21,12 @@
 
 SNoteObj tsHttpNote;
 SNoteObj tsTscNote;
-SNoteObj tsErrorNote;
+SNoteObj tsInfoNote;
 
-static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj * pNote);
-static void taosCloseNoteByFd(int32_t oldFd, SNoteObj * pNote);
+static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote);
+static void    taosCloseNoteByFd(int32_t oldFd, SNoteObj *pNote);
 
-void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, char *name) {
+static void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, char *name) {
   memset(pNote, 0, sizeof(SNoteObj));
   pNote->fileNum = 1;
   pNote->fd = -1;
@@ -54,12 +54,12 @@ void taosInitNotes() {
   }
 
   if (tscEmbedded == 1) {
-    snprintf(name, TSDB_FILENAME_LEN * 2, "%s/note", tsLogDir);
-    taosInitNote(tsNumOfLogLines, 1, &tsErrorNote, name);
+    snprintf(name, TSDB_FILENAME_LEN * 2, "%s/taosinfo", tsLogDir);
+    taosInitNote(tsNumOfLogLines, 1, &tsInfoNote, name);
   }
 }
 
-bool taosLockNote(int32_t fd, SNoteObj *pNote) {
+static bool taosLockNote(int32_t fd, SNoteObj *pNote) {
   if (fd < 0) return false;
 
   if (pNote->fileNum > 1) {
@@ -72,7 +72,7 @@ bool taosLockNote(int32_t fd, SNoteObj *pNote) {
   return false;
 }
 
-void taosUnLockNote(int32_t fd, SNoteObj *pNote) {
+static void taosUnLockNote(int32_t fd, SNoteObj *pNote) {
   if (fd < 0) return;
 
   if (pNote->fileNum > 1) {
@@ -80,7 +80,7 @@ void taosUnLockNote(int32_t fd, SNoteObj *pNote) {
   }
 }
 
-void *taosThreadToOpenNewNote(void *param) {
+static void *taosThreadToOpenNewNote(void *param) {
   char      name[NOTE_FILE_NAME_LEN * 2];
   SNoteObj *pNote = (SNoteObj *)param;
 
@@ -108,7 +108,7 @@ void *taosThreadToOpenNewNote(void *param) {
   return NULL;
 }
 
-int32_t taosOpenNewNote(SNoteObj *pNote) {
+static int32_t taosOpenNewNote(SNoteObj *pNote) {
   pthread_mutex_lock(&pNote->mutex);
 
   if (pNote->lines > pNote->maxLines && pNote->openInProgress == 0) {
@@ -129,7 +129,7 @@ int32_t taosOpenNewNote(SNoteObj *pNote) {
   return pNote->fd;
 }
 
-bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) {
+static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) {
   int32_t fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
   if (fd < 0) {
     fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno));
@@ -146,7 +146,7 @@ bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) {
   }
 }
 
-void taosGetNoteName(char *fn, SNoteObj *pNote) {
+static void taosGetNoteName(char *fn, SNoteObj *pNote) {
   if (pNote->fileNum > 1) {
     for (int32_t i = 0; i < pNote->fileNum; i++) {
       char fileName[NOTE_FILE_NAME_LEN];
@@ -169,7 +169,7 @@ void taosGetNoteName(char *fn, SNoteObj *pNote) {
   }
 }
 
-int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) {
+static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) {
   char    name[NOTE_FILE_NAME_LEN * 2] = {0};
   int32_t size;
   struct stat logstat0, logstat1;
@@ -227,6 +227,16 @@ int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum,
   return 0;
 }
 
+void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) {
+  if (pNote->fd < 0) return;
+  taosWrite(pNote->fd, buffer, len);
+
+  if (pNote->maxLines > 0) {
+    pNote->lines++;
+    if ((pNote->lines > pNote->maxLines) && (pNote->openInProgress == 0)) taosOpenNewNote(pNote);
+  }
+}
+
 void taosNotePrint(SNoteObj *pNote, const char *const format, ...) {
   va_list        argpointer;
   char           buffer[MAX_NOTE_LINE_SIZE + 2];
@@ -249,19 +259,12 @@ void taosNotePrint(SNoteObj *pNote, const char *const format, ...) {
   buffer[len++] = '\n';
   buffer[len] = 0;
 
-  if (pNote->fd >= 0) {
-    taosWrite(pNote->fd, buffer, len);
-
-    if (pNote->maxLines > 0) {
-      pNote->lines++;
-      if ((pNote->lines > pNote->maxLines) && (pNote->openInProgress == 0)) taosOpenNewNote(pNote);
-    }
-  }
+  taosNotePrintBuffer(pNote, buffer, len);
 }
 
-void taosCloseNote(SNoteObj *pNote) { taosCloseNoteByFd(pNote->fd, pNote); }
+// static void taosCloseNote(SNoteObj *pNote) { taosCloseNoteByFd(pNote->fd, pNote); }
 
-void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) {
+static void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) {
   if (fd >= 0) {
     taosUnLockNote(fd, pNote);
     close(fd);

From 9b5594334d10d67248592b9ca2fdb092e1ef8f96 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 8 Dec 2020 10:57:50 +0800
Subject: [PATCH 097/118] [TD-2356]<test>add new Jenkinsfile

---
 tests/Jenkinsfile | 130 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)
 create mode 100644 tests/Jenkinsfile

diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile
new file mode 100644
index 0000000000..49e25a2f5e
--- /dev/null
+++ b/tests/Jenkinsfile
@@ -0,0 +1,130 @@
+properties([pipelineTriggers([githubPush()])])
+node {
+    git url: 'https://github.com/liuyq-617/TDengine'
+}
+
+def pre_test(){
+    catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                sudo rmtaos
+                '''
+    }
+    sh '''
+    cd ${WKC}
+    rm -rf *
+    cd ${WK}
+    git reset --hard
+    git checkout develop
+    git pull
+    cd ${WKC}
+    rm -rf *
+    mv ${WORKSPACE}/* .
+    cd ${WK}
+    export TZ=Asia/Harbin
+    date
+    rm -rf ${WK}/debug
+    mkdir debug
+    cd debug
+    cmake .. > /dev/null
+    make > /dev/null
+    make install > /dev/null
+    cd ${WKC}/tests
+    '''
+    return 1
+}
+pipeline {
+  agent none
+  environment{
+      WK = '/var/lib/jenkins/workspace/TDinternal'
+      WKC= '/var/lib/jenkins/workspace/TDinternal/community'
+  }
+
+  stages {
+      stage('Parallel test stage') {
+      parallel {
+        stage('python p1') {
+          agent{label 'p1'}
+          steps {
+            pre_test()
+            sh '''
+            cd ${WKC}/tests
+            ./test-all.sh p1
+            date'''
+          }
+        }
+        stage('test_b1') {
+          agent{label 'master'}
+          steps {
+            pre_test()
+            sh '''
+            cd ${WKC}/tests
+            ./test-all.sh b1
+            date'''
+          }
+        }
+
+        stage('test_crash_gen') {
+          agent{label "b2"}
+          steps {
+            pre_test()
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./crash_gen.sh -a -p -t 4 -s 2000
+                '''
+            }
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./handle_crash_gen_val_log.sh
+                '''
+            }
+            sh '''
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b2
+            date
+            '''
+          }
+        }
+
+        stage('test_valgrind') {
+          agent{label "b3"}
+
+          steps {
+            pre_test()
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                ./valgrind-test.sh 2>&1 > mem-error-out.log
+                ./handle_val_log.sh
+                '''
+            }           
+            sh '''
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b3
+            date'''
+          }
+        }
+       stage('python p2'){
+         agent{label "p2"}
+         steps{
+            pre_test()         
+            sh '''
+            date
+            cd ${WKC}/tests
+            ./test-all.sh p2
+            date
+            '''
+          
+         }
+       }
+       
+          
+      }
+    }
+
+  }
+  
+}

From 97b64d5654cd10aa402cc2d7b8b76f0f853968a7 Mon Sep 17 00:00:00 2001
From: zyyang <zyyang@taosdata.com>
Date: Tue, 8 Dec 2020 11:07:11 +0800
Subject: [PATCH 098/118] [TD-2364]<fix>: fix mybatis-plus pagenation
 exceptions

---
 cmake/install.inc                                         | 2 +-
 src/connector/jdbc/CMakeLists.txt                         | 2 +-
 src/connector/jdbc/deploy-pom.xml                         | 3 +--
 src/connector/jdbc/pom.xml                                | 2 +-
 .../src/main/java/com/taosdata/jdbc/TSDBStatement.java    | 8 +++++---
 5 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/cmake/install.inc b/cmake/install.inc
index 8418612d4c..55b3fa188f 100755
--- a/cmake/install.inc
+++ b/cmake/install.inc
@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
   #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
   #INSTALL(TARGETS shell RUNTIME DESTINATION .)
   IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.15-dist.jar DESTINATION connector/jdbc)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt
index 0eb3eb21ce..e289f1ae1b 100644
--- a/src/connector/jdbc/CMakeLists.txt
+++ b/src/connector/jdbc/CMakeLists.txt
@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
   ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
     POST_BUILD
     COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.15-dist.jar ${LIBRARY_OUTPUT_PATH}
     COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
     COMMENT "build jdbc driver")
   ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})
diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml
index 4564bde81e..1a86bc57dc 100755
--- a/src/connector/jdbc/deploy-pom.xml
+++ b/src/connector/jdbc/deploy-pom.xml
@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.14</version>
+  <version>2.0.15</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>
@@ -36,7 +36,6 @@
     </developer>
   </developers>
 
-
   <dependencies>
     <dependency>
       <groupId>commons-logging</groupId>
diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml
index 7e087ebd9b..25a36e3a48 100755
--- a/src/connector/jdbc/pom.xml
+++ b/src/connector/jdbc/pom.xml
@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.14</version>
+    <version>2.0.15</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java
index cdd88b825e..cd2a768a38 100644
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java
@@ -81,7 +81,7 @@ public class TSDBStatement implements Statement {
         }
 
         if (!this.connector.isUpdateQuery(pSql)) {
-        	TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer);
+            TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer);
             res.setBatchFetch(this.connection.getBatchFetch());
             return res;
         } else {
@@ -125,7 +125,8 @@ public class TSDBStatement implements Statement {
     }
 
     public int getMaxFieldSize() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        return 0;
+//        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
     }
 
     public void setMaxFieldSize(int max) throws SQLException {
@@ -218,7 +219,8 @@ public class TSDBStatement implements Statement {
     }
 
     public int getFetchDirection() throws SQLException {
-        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+        return ResultSet.FETCH_FORWARD;
+//        throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
     }
 
     /*

From b21f78d3206fdd72746ff255802adabc8c0bb26e Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 11:10:04 +0800
Subject: [PATCH 099/118] TD-2371

---
 src/client/src/tscSystem.c | 2 ++
 src/util/inc/tnote.h       | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 132f8158fc..1eddeacc65 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -17,6 +17,7 @@
 #include "taosmsg.h"
 #include "tref.h"
 #include "trpc.h"
+#include "tnote.h"
 #include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
@@ -102,6 +103,7 @@ void taos_init_imp(void) {
 
     taosReadGlobalCfg();
     taosCheckGlobalCfg();
+    taosInitNotes();
 
     rpcInit();
     tscDebug("starting to initialize TAOS client ...");
diff --git a/src/util/inc/tnote.h b/src/util/inc/tnote.h
index b4c0c7501b..7511b61f41 100644
--- a/src/util/inc/tnote.h
+++ b/src/util/inc/tnote.h
@@ -52,7 +52,10 @@ void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len);
     taosNotePrint(&tsTscNote, __VA_ARGS__); \
   }
 
-#define nInfo(buffer, len) taosNotePrintBuffer(&tsInfoNote, buffer, len);
+#define nInfo(buffer, len)                         \
+  if (tscEmbedded == 1) {                          \
+    taosNotePrintBuffer(&tsInfoNote, buffer, len); \
+  }
 
 #ifdef __cplusplus
 }

From be5940fe7ec789b257b98991e83d4ede1d98fcb7 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 13:06:22 +0800
Subject: [PATCH 100/118] TD-2367

---
 src/mnode/src/mnodeTable.c         | 38 ++++++++++++++++++++----------
 src/plugins/http/src/httpContext.c |  2 +-
 src/plugins/http/src/httpSession.c |  2 +-
 src/util/inc/hash.h                |  2 +-
 src/util/src/hash.c                |  2 +-
 5 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index ff81c37de7..09db5fefd7 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -397,13 +397,14 @@ static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) {
 
   if (pStable->vgHash == NULL) {
     pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+    mDebug("table:%s, create hash:%p", pStable->info.tableId, pStable->vgHash);
   }
 
   if (pStable->vgHash != NULL) {
     if (taosHashGet(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)) == NULL) {
       taosHashPut(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId));
-      mDebug("table:%s, vgId:%d is put into stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId,
-             (int32_t)taosHashGetSize(pStable->vgHash));
+      mDebug("table:%s, vgId:%d is put into stable hash:%p, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId,
+             pStable->vgHash, taosHashGetSize(pStable->vgHash));
     }
   }
 }
@@ -416,13 +417,14 @@ static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable)
   SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId);
   if (pVgroup == NULL) {
     taosHashRemove(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId));
-    mDebug("table:%s, vgId:%d is remove from stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId,
-           (int32_t)taosHashGetSize(pStable->vgHash));
+    mDebug("table:%s, vgId:%d is remove from stable hash:%p sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId,
+           pStable->vgHash, taosHashGetSize(pStable->vgHash));
   }
   mnodeDecVgroupRef(pVgroup);
 }
 
 static void mnodeDestroySuperTable(SSTableObj *pStable) {
+  mDebug("table:%s, is destroyed, stable hash:%p", pStable->info.tableId, pStable->vgHash);
   if (pStable->vgHash != NULL) {
     taosHashCleanup(pStable->vgHash);
     pStable->vgHash = NULL;
@@ -464,6 +466,9 @@ static int32_t mnodeSuperTableActionUpdate(SSdbRow *pRow) {
   SSTableObj *pNew = pRow->pObj;
   SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId);
   if (pTable != NULL && pTable != pNew) {
+    mDebug("table:%s, will be updated, hash:%p sizeOfVgList:%d, new hash:%p sizeOfVgList:%d", pTable->info.tableId,
+           pTable->vgHash, taosHashGetSize(pTable->vgHash), pNew->vgHash, taosHashGetSize(pNew->vgHash));
+
     void *oldTableId = pTable->info.tableId;
     void *oldSchema = pTable->schema;
     void *oldVgHash = pTable->vgHash;
@@ -479,6 +484,9 @@ static int32_t mnodeSuperTableActionUpdate(SSdbRow *pRow) {
     free(pNew);
     free(oldTableId);
     free(oldSchema);
+
+    mDebug("table:%s, update finished, hash:%p sizeOfVgList:%d", pTable->info.tableId, pTable->vgHash,
+           taosHashGetSize(pTable->vgHash));
   }
 
   mnodeDecTableRef(pTable);
@@ -783,8 +791,8 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
 
   if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
     SSTableObj *pSTable = (SSTableObj *)pMsg->pTable;
-    mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d",
-          pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash));
+    mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", pMsg,
+          pMsg->rpcMsg.ahandle, pDrop->tableId, pSTable->uid, pSTable->numOfTables, taosHashGetSize(pSTable->vgHash));
     return mnodeProcessDropSuperTableMsg(pMsg);
   } else {
     SCTableObj *pCTable = (SCTableObj *)pMsg->pTable;
@@ -925,7 +933,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
   if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
 
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
-   if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
+  mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle,
+        pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash));
+
+  if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
     int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL);
     while (pVgId) {
       SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
@@ -938,8 +949,9 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
       pDrop->uid = htobe64(pStable->uid);
       mnodeExtractTableName(pStable->info.tableId, pDrop->tableId);
 
-      mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d", pMsg, pMsg->rpcMsg.ahandle,
-            pStable->info.tableId, pVgroup->vgId);
+      mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d, hash:%p sizeOfVgList:%d", pMsg,
+            pMsg->rpcMsg.ahandle, pStable->info.tableId, pVgroup->vgId, pStable->vgHash,
+            taosHashGetSize(pStable->vgHash));
       SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup);
       SRpcMsg   rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
       dnodeSendMsgToDnode(&epSet, &rpcMsg);
@@ -1482,8 +1494,8 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
 
   pMsg->rpcRsp.rsp = pMeta;
 
-  mDebug("msg:%p, app:%p stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg, pMsg->rpcMsg.ahandle,
-         pTable->info.tableId, pTable->uid);
+  mDebug("msg:%p, app:%p stable:%s, uid:%" PRIu64 " table meta is retrieved, sizeOfVgList:%d numOfTables:%d", pMsg,
+         pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->uid, taosHashGetSize(pTable->vgHash), pTable->numOfTables);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1512,7 +1524,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
   char *msg = (char *)pRsp + sizeof(SSTableVgroupRspMsg);
 
   for (int32_t i = 0; i < numOfTable; ++i) {
-    char *          stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
+    char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
     SSTableObj *pTable = mnodeGetSuperTable(stableName);
     if (pTable == NULL) {
       mError("msg:%p, app:%p stable:%s, not exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, stableName);
@@ -1533,6 +1545,8 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
       msg += sizeof(SVgroupsMsg);
     } else {
       SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg;
+      mDebug("msg:%p, app:%p stable:%s, hash:%p sizeOfVgList:%d will be returned", pMsg, pMsg->rpcMsg.ahandle,
+             pTable->info.tableId, pTable->vgHash, taosHashGetSize(pTable->vgHash));
 
       int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL);
       int32_t  vgSize = 0;
diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c
index 22f464924e..266228e7ac 100644
--- a/src/plugins/http/src/httpContext.c
+++ b/src/plugins/http/src/httpContext.c
@@ -79,7 +79,7 @@ bool httpInitContexts() {
 void httpCleanupContexts() {
   if (tsHttpServer.contextCache != NULL) {
     SCacheObj *cache = tsHttpServer.contextCache;
-    httpInfo("context cache is cleanuping, size:%" PRIzu "", taosHashGetSize(cache->pHashTable));
+    httpInfo("context cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable));
     taosCacheCleanup(tsHttpServer.contextCache);
     tsHttpServer.contextCache = NULL;
   }
diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c
index a96e4433b0..35ce0160b2 100644
--- a/src/plugins/http/src/httpSession.c
+++ b/src/plugins/http/src/httpSession.c
@@ -107,7 +107,7 @@ static void httpDestroySession(void *data) {
 void httpCleanUpSessions() {
   if (tsHttpServer.sessionCache != NULL) {
     SCacheObj *cache = tsHttpServer.sessionCache;
-    httpInfo("session cache is cleanuping, size:%" PRIzu "", taosHashGetSize(cache->pHashTable));
+    httpInfo("session cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable));
     taosCacheCleanup(tsHttpServer.sessionCache);
     tsHttpServer.sessionCache = NULL;
   }
diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h
index b0319d3e13..5bada93d1c 100644
--- a/src/util/inc/hash.h
+++ b/src/util/inc/hash.h
@@ -82,7 +82,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
  * @param pHashObj
  * @return
  */
-size_t taosHashGetSize(const SHashObj *pHashObj);
+int32_t taosHashGetSize(const SHashObj *pHashObj);
 
 /**
  * put element into hash table, if the element with the same key exists, update it
diff --git a/src/util/src/hash.c b/src/util/src/hash.c
index 0e3e0d3e24..7a835e87e7 100644
--- a/src/util/src/hash.c
+++ b/src/util/src/hash.c
@@ -189,7 +189,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
   return pHashObj;
 }
 
-size_t taosHashGetSize(const SHashObj *pHashObj) { return (pHashObj == NULL) ? 0 : pHashObj->size; }
+int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)((pHashObj == NULL) ? 0 : pHashObj->size); }
 
 int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
   uint32_t   hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);

From 20436cd3b26882d42353fee44f65ce6c9d3df56a Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 14:13:53 +0800
Subject: [PATCH 101/118] TD-2363

---
 src/mnode/src/mnodeTable.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 09db5fefd7..6297bb21d0 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -396,7 +396,7 @@ static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) {
   atomic_add_fetch_32(&pStable->numOfTables, 1);
 
   if (pStable->vgHash == NULL) {
-    pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+    pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
     mDebug("table:%s, create hash:%p", pStable->info.tableId, pStable->vgHash);
   }
 

From f8fc442b63a47b8a2e7273d322032c0d35d00552 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 14:37:30 +0800
Subject: [PATCH 102/118] test

---
 tests/test/c/hashPerformance.c | 60 +++++++++++++++++++++++++++-------
 1 file changed, 48 insertions(+), 12 deletions(-)

diff --git a/tests/test/c/hashPerformance.c b/tests/test/c/hashPerformance.c
index db3be0e950..111ea25a09 100644
--- a/tests/test/c/hashPerformance.c
+++ b/tests/test/c/hashPerformance.c
@@ -24,15 +24,17 @@
 #define GREEN "\033[1;32m"
 #define NC "\033[0m"
 
-int32_t capacity = 100000;
-int32_t q1Times = 1;
-int32_t q2Times = 1;
+int32_t capacity = 128;
+int32_t q1Times = 10;
+int32_t q2Times = 10;
 int32_t keyNum = 100000;
-int32_t printInterval = 10000;
+int32_t printInterval = 1000;
+void *  hashHandle;
+pthread_t thread;
 
 typedef struct HashTestRow {
-  int32_t size;
-  void *  ptr;
+  int32_t keySize;
+  char    key[100];
 } HashTestRow;
 
 void shellParseArgument(int argc, char *argv[]);
@@ -40,7 +42,7 @@ void shellParseArgument(int argc, char *argv[]);
 void testHashPerformance() {
   int64_t    initialMs = taosGetTimestampMs();
   _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
-  void *     hashHandle = taosHashInit(capacity, hashFp, true);
+  hashHandle = taosHashInit(128, hashFp, true, HASH_NO_LOCK);
 
   int64_t startMs = taosGetTimestampMs();
   float   seconds = (startMs - initialMs) / 1000.0;
@@ -48,17 +50,25 @@ void testHashPerformance() {
 
   for (int32_t t = 1; t <= keyNum; ++t) {
     HashTestRow row = {0};
-    char key[100] = {0};
-    int32_t keySize = sprintf(key, "0.db.st%d", t);
+    row.keySize = sprintf(row.key, "0.db.st%d", t);
 
     for (int32_t q = 0; q < q1Times; q++) {
-      taosHashGet(hashHandle, &key, keySize);
+      taosHashGet(hashHandle, row.key, row.keySize);
     }
 
-    taosHashPut(hashHandle, key, keySize, &row, sizeof(HashTestRow));
+    taosHashPut(hashHandle, row.key, row.keySize, &row, sizeof(HashTestRow));
 
     for (int32_t q = 0; q < q2Times; q++) {
-      taosHashGet(hashHandle, &key, keySize);
+      taosHashGet(hashHandle, row.key, row.keySize);
+    }
+
+    // test iterator
+    {
+      HashTestRow *row = taosHashIterate(hashHandle, NULL);
+      while (row) {
+        taosHashGet(hashHandle, row->key, row->keySize);
+        row = taosHashIterate(hashHandle, row);
+      }
     }
 
     if (t % printInterval == 0) {
@@ -80,9 +90,35 @@ void testHashPerformance() {
   taosHashCleanup(hashHandle);
 }
 
+void *multiThreadFunc(void *param) {
+  for (int i = 0; i < 100; ++i) {
+    taosMsleep(1000);
+    HashTestRow *row = taosHashIterate(hashHandle, NULL);
+    while (row) {
+      taosHashGet(hashHandle, row->key, row->keySize);
+      row = taosHashIterate(hashHandle, row);
+    }
+    int64_t hashSize = taosHashGetSize(hashHandle);
+    pPrint("i:%d hashSize:%ld", i, hashSize);
+  }
+
+  return NULL;
+}
+
+void multiThreadTest() {
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  
+  // Start threads to write
+  pthread_create(&thread, &thattr, multiThreadFunc, NULL);
+}
+
 int main(int argc, char *argv[]) {
   shellParseArgument(argc, argv);
+  multiThreadTest();
   testHashPerformance();
+  pthread_join(thread, NULL);
 }
 
 void printHelp() {

From ce8e8ab6c7aa9e71635f4762f7a49a5e85be2efb Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Tue, 8 Dec 2020 15:37:11 +0800
Subject: [PATCH 103/118] [TD-2274]<fix>: make ep inconsistent log message more
 clear

---
 src/dnode/src/dnodeEps.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c
index 09151533e2..103710bf6f 100644
--- a/src/dnode/src/dnodeEps.c
+++ b/src/dnode/src/dnodeEps.c
@@ -237,7 +237,7 @@ PRASE_EPS_OVER:
   dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
 #else
   if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) {
-    dError("dnode:%d, localEp is changed to %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp);
+    dError("dnode:%d, localEp is different from %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp);
     return -1;
   }
 #endif

From 4cc10970bc32c9576ac43ff1094f65fcb9a82af3 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 8 Dec 2020 15:59:51 +0800
Subject: [PATCH 104/118] [TD-2328]add test case for MaxSQLLen

---
 tests/pytest/bug2265.py  | 85 ++++++++++++++++++++++++++++++++++++++++
 tests/pytest/fulltest.sh |  1 +
 tests/pytest/pytest_1.sh |  2 +-
 3 files changed, 87 insertions(+), 1 deletion(-)
 create mode 100644 tests/pytest/bug2265.py

diff --git a/tests/pytest/bug2265.py b/tests/pytest/bug2265.py
new file mode 100644
index 0000000000..e78233928f
--- /dev/null
+++ b/tests/pytest/bug2265.py
@@ -0,0 +1,85 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import taos
+if __name__ == "__main__":
+
+    logSql = True
+    deployPath = ""
+    testCluster = False
+    valgrind = 0
+
+    print("start to execute %s" % __file__)
+    tdDnodes.init(deployPath)
+    tdDnodes.setTestCluster(testCluster)
+    tdDnodes.setValgrind(valgrind)
+
+    tdDnodes.stopAll()
+    tdDnodes.addSimExtraCfg("maxSQLLength", "1048576")
+    tdDnodes.deploy(1)
+    tdDnodes.start(1)
+    host = '127.0.0.1'
+
+    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+
+    tdCases.logSql(logSql)
+    print('1')
+    conn = taos.connect(
+        host,
+        config=tdDnodes.getSimCfgPath())
+
+    tdSql.init(conn.cursor(), True)
+        
+    print("==========step1")
+    print("create table ")
+    tdSql.execute("create database db")
+    tdSql.execute("use db")
+    tdSql.execute("create table t1 (ts timestamp, c1 int,c2 int ,c3 int)")
+
+    print("==========step2")
+    print("insert maxSQLLength data ")
+    data = 'insert into t1 values'
+    ts = 1604298064000
+    i = 0
+    while ((len(data)<(1024*1024)) & (i < 32767 - 1) ):
+        data += '(%s,%d,%d,%d)'%(ts+i,i%1000,i%1000,i%1000)
+        i+=1
+    tdSql.execute(data)
+    
+    print("==========step4")
+    print("insert data batch larger than 32767 ")
+    i = 0
+    while ((len(data)<(1024*1024)) & (i < 32767) ):
+        data += '(%s,%d,%d,%d)'%(ts+i,i%1000,i%1000,i%1000)
+        i+=1
+    tdSql.error(data)
+
+    print("==========step4")
+    print("insert data larger than maxSQLLength ")
+    tdSql.execute("create table t2 (ts timestamp, c1 binary(50))")
+    data = 'insert into t2 values'
+    i = 0
+    while ((len(data)<(1024*1024)) & (i < 32767 - 1 ) ):
+        data += '(%s,%s)'%(ts+i,'a'*50)
+        i+=1
+    tdSql.error(data)      
+    tdSql.close()
+    tdLog.success("%s successfully executed" % __file__)
+        
+   
+
diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index dc2c0099b4..87e792e073 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -20,6 +20,7 @@ python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
 python3 ./test.py -f insert/before_1970.py
+python3 bug2265.py
 
 python3 ./test.py -f table/alter_wal0.py
 python3 ./test.py -f table/column_name.py
diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh
index 52f5a30f4e..6eab25130a 100755
--- a/tests/pytest/pytest_1.sh
+++ b/tests/pytest/pytest_1.sh
@@ -159,7 +159,7 @@ python3 ./test.py -f query/bug1874.py
 python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
 python3 ./test.py -f query/bug2218.py 
-
+python3 bug2265.py
 #stream
 python3 ./test.py -f stream/metric_1.py
 python3 ./test.py -f stream/new.py

From ffa51bb99089a32c1168c70f12281b90ac4acb8b Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Tue, 8 Dec 2020 15:51:00 +0800
Subject: [PATCH 105/118] [TD-2114]<test>: Imporve performance test script

---
 tests/perftest-scripts/perftest-query.sh      |  13 +-
 .../pytest/insert/insertFromCSVPerformance.py | 131 ++++++++++++
 tests/pytest/query/queryPerformance.py        | 189 ++++++++++--------
 tests/pytest/query/unionAllTest.py            |   2 +-
 tests/pytest/tools/taosdemoPerformance.py     |  93 +++++++++
 5 files changed, 343 insertions(+), 85 deletions(-)
 create mode 100644 tests/pytest/insert/insertFromCSVPerformance.py
 create mode 100644 tests/pytest/tools/taosdemoPerformance.py

diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh
index 51bb9b36c3..b96daa5464 100755
--- a/tests/perftest-scripts/perftest-query.sh
+++ b/tests/perftest-scripts/perftest-query.sh
@@ -65,7 +65,18 @@ function runQueryPerfTest {
 	echoInfo "Run Performance Test"
 	cd $WORK_DIR/TDengine/tests/pytest
 	
-	python3 query/queryPerformance.py 0 | tee -a $PERFORMANCE_TEST_REPORT
+	python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+
+	python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+
+	yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -t 1000 -n 1000 > taosdemoperf.txt
+
+	CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'`
+	INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'`
+	REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'`
+	
+	python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND | tee -a $PERFORMANCE_TEST_REPORT
+	[ -f taosdemoperf.txt ] && rm taosdemoperf.txt
 }
 
 
diff --git a/tests/pytest/insert/insertFromCSVPerformance.py b/tests/pytest/insert/insertFromCSVPerformance.py
new file mode 100644
index 0000000000..84fd1d7cca
--- /dev/null
+++ b/tests/pytest/insert/insertFromCSVPerformance.py
@@ -0,0 +1,131 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+import datetime
+import csv
+import random
+import pandas as pd
+import argparse
+import os.path
+
+class insertFromCSVPerformace:
+    def __init__(self, commitID, dbName, stbName, branchName):
+        self.commitID = commitID
+        self.dbName = dbName
+        self.stbName = stbName
+        self.branchName = branchName
+        self.ts = 1500074556514
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taosperf"
+        self.conn = taos.connect(
+            self.host,
+            self.user,
+            self.password,
+            self.config)
+        
+    def writeCSV(self):
+        with open('test3.csv','w', encoding='utf-8', newline='') as csvFile:
+            writer = csv.writer(csvFile, dialect='excel')
+            for i in range(1000000):
+                newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000)
+                d = datetime.datetime.fromtimestamp(newTimestamp / 1000)
+                dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f"))
+                writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)])
+    
+    def removCSVHeader(self):
+        data = pd.read_csv("ordered.csv")
+        data = data.drop([0])
+        data.to_csv("ordered.csv", header = False, index = False)
+    
+    def createTables(self):
+        cursor = self.conn.cursor()
+                    
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists %s(ts timestamp, in_order_time float, out_of_order_time float, commit_id binary(50)) tags(branch binary(50))" % self.stbName)
+        cursor.execute("create table if not exists %s using %s tags('%s')" % (self.branchName, self.stbName, self.branchName))
+
+        cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+        cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+
+        cursor.close()
+    
+    def run(self):
+        cursor = self.conn.cursor()
+        cursor.execute("use %s" % self.dbName)
+        print("==================== CSV insert performance ====================")
+        
+        totalTime = 0
+        for i in range(10):
+            cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+            startTime = time.time()
+            cursor.execute("insert into t1 file 'outoforder.csv'")
+            totalTime += time.time() - startTime
+            cursor.execute("drop table if exists t1")            
+        out_of_order_time = (float) (totalTime / 10)
+        print("Out of Order - Insert time: %f" % out_of_order_time)                      
+        
+        totalTime = 0
+        for i in range(10):
+            cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+            startTime = time.time()
+            cursor.execute("insert into t2 file 'ordered.csv'")
+            totalTime += time.time() - startTime
+            cursor.execute("drop table if exists t2")
+
+        in_order_time = (float) (totalTime / 10)
+        print("In order - Insert time: %f" % in_order_time)
+        cursor.execute("insert into %s values(now, %f, %f, '%s')" % (self.branchName, in_order_time, out_of_order_time, self.commitID))
+        
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()    
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',
+        default='null',
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--stable-name',
+        action='store',
+        default='csv_insert',
+        type=str,
+        help='Database name to be created (default: csv_insert)')
+    parser.add_argument(
+        '-b',
+        '--branch-name',
+        action='store',
+        default='develop',
+        type=str,
+        help='branch name (default: develop)')
+    
+    args = parser.parse_args()
+    perftest = insertFromCSVPerformace(args.commit_id, args.database_name, args.stable_name, args.branch_name)
+
+    perftest.createTables()
+    perftest.run()
\ No newline at end of file
diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py
index 72af38450c..720ae745cb 100644
--- a/tests/pytest/query/queryPerformance.py
+++ b/tests/pytest/query/queryPerformance.py
@@ -16,10 +16,16 @@ import sys
 import os
 import taos
 import time
+import argparse
 
 
 class taosdemoQueryPerformace:
-    def initConnection(self):
+    def __init__(self, clearCache, commitID, dbName, stbName, tbPerfix):
+        self.clearCache = clearCache
+        self.commitID = commitID
+        self.dbName = dbName
+        self.stbName = stbName
+        self.tbPerfix = tbPerfix
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
@@ -30,92 +36,109 @@ class taosdemoQueryPerformace:
             self.password,
             self.config)
 
+    def createPerfTables(self):
+        cursor = self.conn.cursor()
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50)) tags(query_id int, query_sql binary(300))" % self.stbName)
+
+        sql = "select count(*) from test.meters"
+        tableid = 1
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.meters"
+        tableid = 2
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select count(*) from test.meters where loc='beijing'"
+        tableid = 3
+        cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.meters where areaid=10"
+        tableid = 4
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.t10 interval(10s)"
+        tableid = 5
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select last_row(*) from meters"
+        tableid = 6
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select * from meters"
+        tableid = 7
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'"
+        tableid = 8
+        cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+
+        cursor.close()
 
     def query(self): 
-        cursor = self.conn.cursor()                  
-        cursor.execute("use test")
+        cursor = self.conn.cursor() 
+        print("==================== query performance ====================")
 
-        totalTime = 0
-        for i in range(100):       
-            if(sys.argv[1] == '1'):
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")  
-            startTime = time.time()            
-            cursor.execute("select count(*) from test.meters")
-            totalTime += time.time() - startTime
-        print("query time for: select count(*) from test.meters %f seconds" % (totalTime / 100))
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("select tbname, query_id, query_sql from %s" % self.stbName)      
 
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):  
-                # root permission is required              
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.meters")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.meters %f seconds" % (totalTime / 100))
+        for data in cursor:
+            table_name = data[0]
+            query_id = data[1]
+            sql = data[2]            
+            
+            totalTime = 0            
+            cursor2 = self.conn.cursor()
+            cursor2.execute("use test")
+            for i in range(100):       
+                if(self.clearCache == True):
+                    # root permission is required
+                    os.system("echo 3 > /proc/sys/vm/drop_caches")                
+                
+                startTime = time.time()         
+                cursor2.execute(sql)
+                totalTime += time.time() - startTime
+            cursor2.close()    
+            print("query time for: %s %f seconds" % (sql, totalTime / 100))
+                        
+            cursor3 = self.conn.cursor()
+            cursor3.execute("insert into %s.%s values(now, %f, '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID))
 
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select count(*) from test.meters where loc='beijing'")
-            totalTime += time.time() - startTime
-        print("query time for: select count(*) from test.meters where loc='beijing' %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required              
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.meters where areaid=10")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.meters where areaid=10 %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):                
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.t10 interval(10s)")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.t10 interval(10s) %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required           
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select last_row(*) from meters")
-            totalTime += time.time() - startTime
-        print("query time for: select last_row(*) from meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required           
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select * from meters")
-            totalTime += time.time() - startTime
-        print("query time for: select * from meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required        
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000' %f seconds" % (totalTime / 100))
+        cursor3.close()
+        cursor.close()
 
 if __name__ == '__main__':
-    perftest = taosdemoQueryPerformace()
-    perftest.initConnection()
-    perftest.query()
\ No newline at end of file
+    parser = argparse.ArgumentParser()    
+    parser.add_argument(
+        '-r',
+        '--remove-cache',                
+        action='store_true',
+        default=False,
+        help='clear cache before query (default: False)')
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',
+        default='null',
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--stable-name',
+        action='store',
+        default='query_tb',
+        type=str,
+        help='table name to be created (default: query_tb)')
+    parser.add_argument(
+        '-p',
+        '--table-perfix',
+        action='store',
+        default='q',
+        type=str,
+        help='table name perfix (default: q)')
+    
+    args = parser.parse_args()
+    perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix)
+    perftest.createPerfTables()
+    perftest.query()
diff --git a/tests/pytest/query/unionAllTest.py b/tests/pytest/query/unionAllTest.py
index bb4fb95de6..1b69c8ac4d 100644
--- a/tests/pytest/query/unionAllTest.py
+++ b/tests/pytest/query/unionAllTest.py
@@ -52,7 +52,7 @@ class TDTestCase:
         tdSql.checkRows(5)
 
         sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1
-            union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5''''
+            union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5' limit 1'''
         tdSql.query(sql)
         tdSql.checkRows(6)
 
diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py
new file mode 100644
index 0000000000..6b6296e61a
--- /dev/null
+++ b/tests/pytest/tools/taosdemoPerformance.py
@@ -0,0 +1,93 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+import datetime
+import csv
+import random
+import pandas as pd
+import argparse
+import os.path
+
+class taosdemoPerformace:
+    def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond):
+        self.commitID = commitID
+        self.dbName = dbName
+        self.createTableTime = createTableTime
+        self.insertRecordsTime = insertRecordsTime
+        self.recordsPerSecond = recordsPerSecond        
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taosperf"
+        self.conn = taos.connect(
+            self.host,
+            self.user,
+            self.password,
+            self.config)    
+    
+    def createTablesAndStoreData(self):
+        cursor = self.conn.cursor()
+                    
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50))")
+        print("==================== taosdemo performance ====================")
+        print("create tables time: %f" % self.createTableTime)
+        print("insert records time: %f" % self.insertRecordsTime)
+        print("records per second: %f" % self.recordsPerSecond)
+        cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s')" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID))
+        cursor.execute("drop database if exists taosdemo_insert_test")
+
+        cursor.close()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',        
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--create-table',
+        action='store',        
+        type=float,
+        help='create table time')
+    parser.add_argument(
+        '-i',
+        '--insert-records',
+        action='store',        
+        type=float,
+        help='insert records time')
+    parser.add_argument(
+        '-r',
+        '---records-per-second',
+        action='store',        
+        type=float,
+        help='records per request')
+    
+    args = parser.parse_args()
+
+    perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second)    
+    perftest.createTablesAndStoreData()
\ No newline at end of file

From 5aeafa80acf452543fd5970b6ddb40ecc8e214c7 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 8 Dec 2020 16:25:03 +0800
Subject: [PATCH 106/118] [TD-2333]<test>test max table len

---
 tests/pytest/fulltest.sh               |  1 +
 tests/pytest/pytest_1.sh               |  1 +
 tests/pytest/table/max_table_length.py | 55 ++++++++++++++++++++++++++
 3 files changed, 57 insertions(+)
 create mode 100644 tests/pytest/table/max_table_length.py

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index 87e792e073..b4a8bd635f 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -28,6 +28,7 @@ python3 ./test.py -f table/column_num.py
 python3 ./test.py -f table/db_table.py
 python3 ./test.py -f table/create_sensitive.py
 #python3 ./test.py -f table/tablename-boundary.py
+python3 ./test.py  -f table/max_table_length.py
 
 # tag
 python3 ./test.py -f tag_lite/filter.py
diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh
index 6eab25130a..3102396178 100755
--- a/tests/pytest/pytest_1.sh
+++ b/tests/pytest/pytest_1.sh
@@ -26,6 +26,7 @@ python3 ./test.py -f table/column_num.py
 python3 ./test.py -f table/db_table.py
 python3 ./test.py -f table/create_sensitive.py
 #python3 ./test.py -f table/tablename-boundary.py
+python3 ./test.py  -f table/max_table_length.py
 
 # tag
 python3 ./test.py -f tag_lite/filter.py
diff --git a/tests/pytest/table/max_table_length.py b/tests/pytest/table/max_table_length.py
new file mode 100644
index 0000000000..ec34f3008f
--- /dev/null
+++ b/tests/pytest/table/max_table_length.py
@@ -0,0 +1,55 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, db_test.stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+
+        tdLog.info("check nchar")
+        tdSql.error("create database anal (ts timestamp ,i nchar(4094))")
+        tdSql.execute(
+            "create table anal (ts timestamp ,i nchar(4093))")
+
+        print("==============step2")
+        tdLog.info("check binary")
+        tdSql.error("create database anal (ts timestamp ,i binary(16375))")
+        tdSql.execute(
+            "create table anal1 (ts timestamp ,i binary(16374))")
+        
+        print("==============step3")
+        tdLog.info("check int & binary")
+        tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)")
+        tdSql.execute("create table anal2 (ts timestamp ,i binary(16370),j int)")
+        tdSql.execute("create table anal3 (ts timestamp ,i binary(16366), j int, k int)")
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

From b38fbb8199dff8f46795f6cf868661e65fae844b Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 8 Dec 2020 16:36:01 +0800
Subject: [PATCH 107/118] [TD-2332]<test>test diff&&top

---
 tests/pytest/fulltest.sh      |  1 +
 tests/pytest/pytest_1.sh      |  1 +
 tests/pytest/query/bug2281.py | 47 +++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+)
 create mode 100644 tests/pytest/query/bug2281.py

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index b4a8bd635f..8d386aa066 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -29,6 +29,7 @@ python3 ./test.py -f table/db_table.py
 python3 ./test.py -f table/create_sensitive.py
 #python3 ./test.py -f table/tablename-boundary.py
 python3 ./test.py  -f table/max_table_length.py
+python3 ./test.py -f query/bug2281.py
 
 # tag
 python3 ./test.py -f tag_lite/filter.py
diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh
index 3102396178..e756ea02fa 100755
--- a/tests/pytest/pytest_1.sh
+++ b/tests/pytest/pytest_1.sh
@@ -160,6 +160,7 @@ python3 ./test.py -f query/bug1874.py
 python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
 python3 ./test.py -f query/bug2218.py 
+python3 ./test.py -f query/bug2281.py
 python3 bug2265.py
 #stream
 python3 ./test.py -f stream/metric_1.py
diff --git a/tests/pytest/query/bug2281.py b/tests/pytest/query/bug2281.py
new file mode 100644
index 0000000000..b8eb17f5cd
--- /dev/null
+++ b/tests/pytest/query/bug2281.py
@@ -0,0 +1,47 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        print("==========step1")
+        print("create table && insert data")
+        
+        tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float)")
+        insertRows = 10
+        t0 = 1604298064000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(insertRows):
+            ret = tdSql.execute(
+                "insert into t1 values (%d , %d,%d)" %
+                (t0+i,i%100,i/2.0))
+
+        print("==========step2")
+        print("query diff && top")
+        tdSql.error('select diff(c1),top(c2) from t1')
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

From 6b1e3ba72a5b323d170fa7180409b77f56163aad Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Tue, 8 Dec 2020 16:41:03 +0800
Subject: [PATCH 108/118] [TD-2375]<test>test percentile from blank table

---
 tests/pytest/fulltest.sh      |  5 +++--
 tests/pytest/pytest_1.sh      |  1 +
 tests/pytest/query/bug2119.py | 41 +++++++++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 tests/pytest/query/bug2119.py

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index 8d386aa066..b58100ef0a 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -29,7 +29,7 @@ python3 ./test.py -f table/db_table.py
 python3 ./test.py -f table/create_sensitive.py
 #python3 ./test.py -f table/tablename-boundary.py
 python3 ./test.py  -f table/max_table_length.py
-python3 ./test.py -f query/bug2281.py
+
 
 # tag
 python3 ./test.py -f tag_lite/filter.py
@@ -167,7 +167,8 @@ python3 ./test.py -f query/bug2117.py
 python3 ./test.py -f query/bug2143.py
 python3 ./test.py -f query/sliding.py
 python3 ./test.py -f query/unionAllTest.py
-
+python3 ./test.py -f query/bug2281.py
+python3 ./test.py -f query/bug2119.py
 #stream
 python3 ./test.py -f stream/metric_1.py
 python3 ./test.py -f stream/new.py
diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh
index e756ea02fa..28afbfcdf0 100755
--- a/tests/pytest/pytest_1.sh
+++ b/tests/pytest/pytest_1.sh
@@ -161,6 +161,7 @@ python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
 python3 ./test.py -f query/bug2218.py 
 python3 ./test.py -f query/bug2281.py
+python3 ./test.py -f query/bug2119.py
 python3 bug2265.py
 #stream
 python3 ./test.py -f stream/metric_1.py
diff --git a/tests/pytest/query/bug2119.py b/tests/pytest/query/bug2119.py
new file mode 100644
index 0000000000..4224e55596
--- /dev/null
+++ b/tests/pytest/query/bug2119.py
@@ -0,0 +1,41 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        print("==========step1")
+        print("create table && insert data")
+        
+        tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float)")
+       
+
+        print("==========step2")
+        print("query percentile from blank table")
+        tdSql.query('select percentile(c1,1) from t1')
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

From 4a53c0a9c2ef36623d44c8ffcb3d687104333498 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Tue, 8 Dec 2020 18:34:32 +0800
Subject: [PATCH 109/118] [TD-2342]<test>: add test case

---
 tests/pytest/fulltest.sh                      | 10 +++--
 tests/pytest/pytest_1.sh                      | 10 +++--
 .../pytest/table/createTableFromAnotherDb.py  | 41 +++++++++++++++++++
 3 files changed, 55 insertions(+), 6 deletions(-)
 create mode 100644 tests/pytest/table/createTableFromAnotherDb.py

diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index b58100ef0a..983f437297 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -22,6 +22,7 @@ python3 ./test.py -f insert/insertIntoTwoTables.py
 python3 ./test.py -f insert/before_1970.py
 python3 bug2265.py
 
+#table
 python3 ./test.py -f table/alter_wal0.py
 python3 ./test.py -f table/column_name.py
 python3 ./test.py -f table/column_num.py
@@ -29,6 +30,12 @@ python3 ./test.py -f table/db_table.py
 python3 ./test.py -f table/create_sensitive.py
 #python3 ./test.py -f table/tablename-boundary.py
 python3 ./test.py  -f table/max_table_length.py
+python3 ./test.py -f table/alter_column.py
+python3 ./test.py -f table/boundary.py
+python3 ./test.py -f table/create-a-lot.py
+python3 ./test.py -f table/create.py
+python3 ./test.py -f table/del_stable.py
+python3 ./test.py -f table/queryWithTaosdKilled.py
 
 
 # tag
@@ -138,9 +145,6 @@ python3 ./test.py -f user/pass_len.py
 # stable
 python3 ./test.py -f stable/query_after_reset.py
 
-# table
-python3 ./test.py -f table/del_stable.py
-
 #query
 python3 ./test.py -f query/filter.py
 python3 ./test.py -f query/filterCombo.py
diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh
index 28afbfcdf0..6e0c8ef8b2 100755
--- a/tests/pytest/pytest_1.sh
+++ b/tests/pytest/pytest_1.sh
@@ -20,6 +20,7 @@ python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
 
+#table
 python3 ./test.py -f table/alter_wal0.py
 python3 ./test.py -f table/column_name.py
 python3 ./test.py -f table/column_num.py
@@ -27,6 +28,12 @@ python3 ./test.py -f table/db_table.py
 python3 ./test.py -f table/create_sensitive.py
 #python3 ./test.py -f table/tablename-boundary.py
 python3 ./test.py  -f table/max_table_length.py
+python3 ./test.py -f table/alter_column.py
+python3 ./test.py -f table/boundary.py
+python3 ./test.py -f table/create-a-lot.py
+python3 ./test.py -f table/create.py
+python3 ./test.py -f table/del_stable.py
+python3 ./test.py -f table/queryWithTaosdKilled.py
 
 # tag
 python3 ./test.py -f tag_lite/filter.py
@@ -135,9 +142,6 @@ python3 ./test.py -f user/pass_len.py
 # stable
 python3 ./test.py -f stable/query_after_reset.py
 
-# table
-python3 ./test.py -f table/del_stable.py
-
 #query
 python3 ./test.py -f query/filter.py
 python3 ./test.py -f query/filterCombo.py
diff --git a/tests/pytest/table/createTableFromAnotherDb.py b/tests/pytest/table/createTableFromAnotherDb.py
new file mode 100644
index 0000000000..b40e72404c
--- /dev/null
+++ b/tests/pytest/table/createTableFromAnotherDb.py
@@ -0,0 +1,41 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")        
+        tdSql.execute("create table db.cars(ts timestamp, c int) tags(id int);")
+        tdSql.execute("create database db2")
+        tdSql.error("create table db2.car1 using db.cars tags(1)")
+        tdSql.error("insert into db2.car1 using db1.cars tags(1) values(now, 1);")
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

From 51aaef543065111258027ce76c838fd8e3c182c2 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 19:27:06 +0800
Subject: [PATCH 110/118] nettest role

---
 src/kit/shell/src/shellWindows.c |  22 ++++
 src/util/src/tnettest.c          | 204 +++++++++++++++----------------
 2 files changed, 123 insertions(+), 103 deletions(-)

diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c
index ce98681391..a92831de25 100644
--- a/src/kit/shell/src/shellWindows.c
+++ b/src/kit/shell/src/shellWindows.c
@@ -45,6 +45,10 @@ void printHelp() {
   printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server.");
   printf("%s%s\n", indent, "-t");
   printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
+  printf("%s%s\n", indent, "-n");
+  printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup.");
+  printf("%s%s\n", indent, "-l");
+  printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes.");
 
   exit(EXIT_SUCCESS);
 }
@@ -137,6 +141,24 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
         exit(EXIT_FAILURE);
       }
     }
+    // For time zone
+    else if (strcmp(argv[i], "-n") == 0) {
+      if (i < argc - 1) {
+        arguments->netTestRole = argv[++i];
+      } else {
+        fprintf(stderr, "option -n requires an argument\n");
+        exit(EXIT_FAILURE);
+      }
+    }
+    // For time zone
+    else if (strcmp(argv[i], "-l") == 0) {
+      if (i < argc - 1) {
+        arguments->pktLen = atoi(argv[++i]);
+      } else {
+        fprintf(stderr, "option -l requires an argument\n");
+        exit(EXIT_FAILURE);
+      }
+    }
     // For temperory command TODO
     else if (strcmp(argv[i], "--help") == 0) {
       printHelp();
diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c
index c269d9a1ff..4619271599 100644
--- a/src/util/src/tnettest.c
+++ b/src/util/src/tnettest.c
@@ -43,12 +43,13 @@ static void *taosNetBindUdpPort(void *sarg) {
   char       buffer[BUFFER_SIZE];
   int32_t    iDataNum;
   socklen_t  sin_size;
+  int32_t bufSize = 1024000;
 
   struct sockaddr_in server_addr;
   struct sockaddr_in clientAddr;
 
   if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) {
-    uError("failed to create udp socket since %s", strerror(errno));
+    uError("failed to create UDP socket since %s", strerror(errno));
     return NULL;
   }
 
@@ -58,11 +59,23 @@ static void *taosNetBindUdpPort(void *sarg) {
   server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
 
   if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
-    uError("failed to bind udp port:%d since %s", port, strerror(errno));
+    uError("failed to bind UDP port:%d since %s", port, strerror(errno));
     return NULL;
   }
 
-  uInfo("udp server at port:%d is listening", port);
+  if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
+    uError("failed to set the send buffer size for UDP socket\n");
+    taosCloseSocket(serverSocket);
+    return NULL;
+  }
+
+  if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
+    uError("failed to set the receive buffer size for UDP socket\n");
+    taosCloseSocket(serverSocket);
+    return NULL;
+  }
+
+  uInfo("UDP server at port:%d is listening", port);
 
   while (1) {
     memset(buffer, 0, BUFFER_SIZE);
@@ -76,7 +89,7 @@ static void *taosNetBindUdpPort(void *sarg) {
 
     if (iDataNum > 0) {
       uInfo("UDP: recv:%d bytes from %s:%d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
-      sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size);
+      taosSendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size);
     }
   }
 
@@ -94,10 +107,9 @@ static void *taosNetBindTcpPort(void *sarg) {
   int32_t    addr_len = sizeof(clientAddr);
   SOCKET     client;
   char       buffer[BUFFER_SIZE];
-  int32_t    iDataNum = 0;
 
   if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) {
-    uError("failed to create tcp socket since %s", strerror(errno));
+    uError("failed to create TCP socket since %s", strerror(errno));
     return NULL;
   }
 
@@ -106,130 +118,103 @@ static void *taosNetBindTcpPort(void *sarg) {
   server_addr.sin_port = htons(port);
   server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
 
+  int32_t reuse = 1;
+  if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
+    uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
+    taosCloseSocket(serverSocket);
+    return NULL;
+  }
+
   if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
-    uError("failed to bind tcp port:%d since %s", port, strerror(errno));
+    uError("failed to bind TCP port:%d since %s", port, strerror(errno));
     return NULL;
   }
 
-  if (listen(serverSocket, 5) < 0) {
-    uError("failed to listen tcp port:%d since %s", port, strerror(errno));
+  
+  if (taosKeepTcpAlive(serverSocket) < 0) {
+    uError("failed to set tcp server keep-alive option since %s", strerror(errno));
+    taosCloseSocket(serverSocket);
     return NULL;
   }
 
-  uInfo("tcp server at port:%d is listening", port);
+  if (listen(serverSocket, 10) < 0) {
+    uError("failed to listen TCP port:%d since %s", port, strerror(errno));
+    return NULL;
+  }
+
+  uInfo("TCP server at port:%d is listening", port);
 
   while (1) {
     client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len);
     if (client < 0) {
-      uDebug("failed to accept from tcp port:%d since %s", port, strerror(errno));
+      uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno));
       continue;
     }
 
-    iDataNum = 0;
-    memset(buffer, 0, BUFFER_SIZE);
-    int32_t nleft, nread;
-    char *  ptr = buffer;
-    nleft = pinfo->pktLen;
-
-    while (nleft > 0) {
-      nread = recv(client, ptr, BUFFER_SIZE, 0);
-
-      if (nread == 0) {
-        break;
-      } else if (nread < 0) {
-        if (errno == EINTR) {
-          continue;
-        } else {
-          uError("failed to perform recv func at %d since %s", port, strerror(errno));
-          taosCloseSocket(serverSocket);
-          return NULL;
-        }
-      } else {
-        nleft -= nread;
-        ptr += nread;
-        iDataNum += nread;
-      }
+    int32_t ret = taosReadMsg(client, buffer, pinfo->pktLen);
+    if (ret < 0 || ret != pinfo->pktLen) {
+      uError("TCP: failed to read %d bytes at port:%d since %s", port, strerror(errno));
+      taosCloseSocket(serverSocket);
+      return NULL;
     }
 
-    if (iDataNum > 0) {
-      uInfo("TCP: recv:%d bytes from %s:%d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
-      send(client, buffer, iDataNum, 0);
+    uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port);
+
+    ret = taosWriteMsg(client, buffer, pinfo->pktLen);
+    if (ret < 0) {
+      uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, strerror(errno), port);
+      taosCloseSocket(serverSocket);
+      return NULL;
     }
+
+    uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port);
   }
-  
+
   taosCloseSocket(serverSocket);
   return NULL;
 }
 
 static int32_t taosNetCheckTcpPort(STestInfo *info) {
-  SOCKET  clientSocket;
-  char    sendbuf[BUFFER_SIZE];
-  char    recvbuf[BUFFER_SIZE];
-  int32_t iDataNum = 0;
+  SOCKET clientSocket;
+  char   buffer[BUFFER_SIZE] = {0};
 
-  struct sockaddr_in serverAddr;
   if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
-    uError("failed to create tcp client socket since %s", strerror(errno));
+    uError("failed to create TCP client socket since %s", strerror(errno));
     return -1;
   }
 
-  // set send and recv overtime
-  struct timeval timeout;
-  timeout.tv_sec = 2;   // s
-  timeout.tv_usec = 0;  // us
-  if (setsockopt(clientSocket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
-    uError("failed to setsockopt send timer since %s", strerror(errno));
-  }
-  if (setsockopt(clientSocket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
-    uError("failed to setsockopt recv timer since %s", strerror(errno));
+  int32_t reuse = 1;
+  if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
+    uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
+    taosCloseSocket(clientSocket);
+    return -1;
   }
 
+  struct sockaddr_in serverAddr;
+  memset((char *)&serverAddr, 0, sizeof(serverAddr));
   serverAddr.sin_family = AF_INET;
-  serverAddr.sin_port = htons(info->port);
+  serverAddr.sin_port = (uint16_t)htons((uint16_t)info->port);
   serverAddr.sin_addr.s_addr = info->hostIp;
 
   if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) {
-    uError("failed to connect port:%d since %s", info->port, strerror(errno));
+    uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno));
     return -1;
   }
 
-  memset(sendbuf, 0, BUFFER_SIZE);
-  memset(recvbuf, 0, BUFFER_SIZE);
+  taosKeepTcpAlive(clientSocket);
 
-  struct in_addr ipStr;
-  memcpy(&ipStr, &info->hostIp, 4);
-  sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port);
-  sprintf(sendbuf + info->pktLen - 16, "1122334455667788");
+  sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port);
+  sprintf(buffer + info->pktLen - 16, "1122334455667788");
 
-  send(clientSocket, sendbuf, info->pktLen, 0);
-
-  memset(recvbuf, 0, BUFFER_SIZE);
-  int32_t nleft, nread;
-  char *  ptr = recvbuf;
-  nleft = info->pktLen;
-
-  while (nleft > 0) {
-    nread = recv(clientSocket, ptr, BUFFER_SIZE, 0);;
-  
-    if (nread == 0) {
-      break;
-    } else if (nread < 0) {
-      if (errno == EINTR) {
-        continue;
-      } else {
-        uError("faild to recv pkg from TCP port:%d since %s", info->port, strerror(errno));
-        taosCloseSocket(clientSocket);
-        return -1;
-      }
-    } else {
-      nleft -= nread;
-      ptr += nread;
-      iDataNum += nread;
-    }
+  int32_t ret = taosWriteMsg(clientSocket, buffer, info->pktLen);
+  if (ret < 0) {
+    uError("TCP: failed to write msg to %s:%d since %s", info->port, taosIpStr(info->hostIp), strerror(errno));
+    return -1;
   }
 
-  if (iDataNum < info->pktLen) {
-    uError("TCP: received ack:%d bytes, less than send:%d bytes from port:%d", iDataNum, info->pktLen, info->port);
+  ret = taosReadMsg(clientSocket, buffer, info->pktLen);
+  if (ret < 0) {
+    uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno));
     return -1;
   }
 
@@ -237,11 +222,13 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) {
   return 0;
 }
 
+
 static int32_t taosNetCheckUdpPort(STestInfo *info) {
   SOCKET  clientSocket;
   char    sendbuf[BUFFER_SIZE];
   char    recvbuf[BUFFER_SIZE];
   int32_t iDataNum = 0;
+  int32_t bufSize = 1024000;
 
   struct sockaddr_in serverAddr;
 
@@ -254,13 +241,23 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) {
   struct timeval timeout;
   timeout.tv_sec = 2;   // s
   timeout.tv_usec = 0;  // us
-  if (setsockopt(clientSocket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
+  if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
     uError("failed to setsockopt send timer since %s", strerror(errno));
   }
-  if (setsockopt(clientSocket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
+  if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
     uError("failed to setsockopt recv timer since %s", strerror(errno));
   }
 
+  if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
+    uError("failed to set the send buffer size for UDP socket\n");
+    return -1;
+  }
+
+  if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
+    uError("failed to set the receive buffer size for UDP socket\n");
+    return -1;
+  }
+
   serverAddr.sin_family = AF_INET;
   serverAddr.sin_port = htons(info->port);
   serverAddr.sin_addr.s_addr = info->hostIp;
@@ -275,7 +272,7 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) {
 
   socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr);
 
-  int32_t code = sendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size);
+  int32_t code = taosSendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size);
   if (code < 0) {
     uError("failed to perform sendto func since %s", strerror(errno));
     return -1;
@@ -304,16 +301,16 @@ static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort
     info.port = port;
     ret = taosNetCheckTcpPort(&info);
     if (ret != 0) {
-      uError("failed to test tcp port:%d", port);
+      uError("failed to test TCP port:%d", port);
     } else {
-      uInfo("successed to test tcp port:%d", port);
+      uInfo("successed to test TCP port:%d", port);
     }
 
     ret = taosNetCheckUdpPort(&info);
     if (ret != 0) {
-      uError("failed to test udp port:%d", port);
+      uError("failed to test UDP port:%d", port);
     } else {
-      uInfo("successed to test udp port:%d", port);
+      uInfo("successed to test UDP port:%d", port);
     }
   }
   return;
@@ -440,9 +437,9 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) {
 
     int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL);
     if (ret < 0) {
-      uError("failed to test tcp port:%d", port);
+      uError("failed to test TCP port:%d", port);
     } else {
-      uInfo("successed to test tcp port:%d", port);
+      uInfo("successed to test TCP port:%d", port);
     }
 
     if (pkgLen >= tsRpcMaxUdpSize) {
@@ -453,9 +450,9 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) {
 
     ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL);
     if (ret < 0) {
-      uError("failed to test udp port:%d", port);
+      uError("failed to test UDP port:%d", port);
     } else {
-      uInfo("successed to test udp port:%d", port);
+      uInfo("successed to test UDP port:%d", port);
     }
   }
 }
@@ -492,14 +489,15 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) {
     tcpInfo->pktLen = pkgLen;
 
     if (pthread_create(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) {
-      uInfo("failed to create tcp test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port);
+      uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port);
       exit(-1);
     }
 
     STestInfo *udpInfo = uinfos + i;
-    udpInfo->port = (uint16_t)(port + i);
+    udpInfo->port = port + i;
+    tcpInfo->pktLen = pkgLen;
     if (pthread_create(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) {
-      uInfo("failed to create udp test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port);
+      uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port);
       exit(-1);
     }
   }

From 553e66bd36f2c3dca31ee2c39a159f98f4f1b2ea Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 19:32:22 +0800
Subject: [PATCH 111/118] compile error

---
 src/util/src/tnettest.c   | 6 +++---
 src/vnode/src/vnodeMain.c | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c
index 4619271599..c76b90e7be 100644
--- a/src/util/src/tnettest.c
+++ b/src/util/src/tnettest.c
@@ -153,7 +153,7 @@ static void *taosNetBindTcpPort(void *sarg) {
 
     int32_t ret = taosReadMsg(client, buffer, pinfo->pktLen);
     if (ret < 0 || ret != pinfo->pktLen) {
-      uError("TCP: failed to read %d bytes at port:%d since %s", port, strerror(errno));
+      uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno));
       taosCloseSocket(serverSocket);
       return NULL;
     }
@@ -162,7 +162,7 @@ static void *taosNetBindTcpPort(void *sarg) {
 
     ret = taosWriteMsg(client, buffer, pinfo->pktLen);
     if (ret < 0) {
-      uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, strerror(errno), port);
+      uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno));
       taosCloseSocket(serverSocket);
       return NULL;
     }
@@ -208,7 +208,7 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) {
 
   int32_t ret = taosWriteMsg(clientSocket, buffer, info->pktLen);
   if (ret < 0) {
-    uError("TCP: failed to write msg to %s:%d since %s", info->port, taosIpStr(info->hostIp), strerror(errno));
+    uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno));
     return -1;
   }
 
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index e286a972dc..36983c1cf0 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -441,7 +441,6 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
 
   if (status == TSDB_STATUS_COMMIT_START) {
     pVnode->isCommiting = 1;
-    pVnode->fversion = pVnode->version;
     vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
     if (!vnodeInInitStatus(pVnode)) {
       return walRenew(pVnode->wal);
@@ -450,9 +449,10 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
   }
 
   if (status == TSDB_STATUS_COMMIT_OVER) {
-    vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
     pVnode->isCommiting = 0;
     pVnode->isFull = 0;
+    pVnode->fversion = pVnode->version;
+    vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
     if (!vnodeInInitStatus(pVnode)) {
       walRemoveOneOldFile(pVnode->wal);
     }

From 77e853e4c3f5345c1b54e2e8a11908fdd1563272 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 20:14:55 +0800
Subject: [PATCH 112/118] minor changes

---
 src/util/src/tnettest.c | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c
index 4619271599..0481fe61d3 100644
--- a/src/util/src/tnettest.c
+++ b/src/util/src/tnettest.c
@@ -87,10 +87,13 @@ static void *taosNetBindUdpPort(void *sarg) {
       continue;
     }
 
+    uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
+
     if (iDataNum > 0) {
-      uInfo("UDP: recv:%d bytes from %s:%d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
-      taosSendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size);
+      iDataNum = taosSendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size);
     }
+
+    uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
   }
 
   taosCloseSocket(serverSocket);
@@ -222,11 +225,9 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) {
   return 0;
 }
 
-
 static int32_t taosNetCheckUdpPort(STestInfo *info) {
   SOCKET  clientSocket;
-  char    sendbuf[BUFFER_SIZE];
-  char    recvbuf[BUFFER_SIZE];
+  char    buffer[BUFFER_SIZE] = {0};
   int32_t iDataNum = 0;
   int32_t bufSize = 1024000;
 
@@ -262,26 +263,25 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) {
   serverAddr.sin_port = htons(info->port);
   serverAddr.sin_addr.s_addr = info->hostIp;
 
-  memset(sendbuf, 0, BUFFER_SIZE);
-  memset(recvbuf, 0, BUFFER_SIZE);
-
   struct in_addr ipStr;
   memcpy(&ipStr, &info->hostIp, 4);
-  sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port);
-  sprintf(sendbuf + info->pktLen - 16, "1122334455667788");
+  sprintf(buffer, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port);
+  sprintf(buffer + info->pktLen - 16, "1122334455667788");
 
   socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr);
 
-  int32_t code = taosSendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size);
-  if (code < 0) {
-    uError("failed to perform sendto func since %s", strerror(errno));
+  iDataNum = taosSendto(clientSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size);
+  if (iDataNum < 0 || iDataNum != info->pktLen) {
+    uError("UDP: failed to perform sendto func since %s", strerror(errno));
     return -1;
   }
 
-  iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size);
+  memset(buffer, 0, BUFFER_SIZE);
+  sin_size = sizeof(*(struct sockaddr *)&serverAddr);
+  iDataNum = recvfrom(clientSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size);
 
-  if (iDataNum < info->pktLen) {
-    uError("UDP: received ack:%d bytes, less than send:%d bytes from port:%d", iDataNum, info->pktLen, info->port);
+  if (iDataNum < 0 || iDataNum != info->pktLen) {
+    uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno));
     return -1;
   }
 
@@ -313,7 +313,6 @@ static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort
       uInfo("successed to test UDP port:%d", port);
     }
   }
-  return;
 }
 
 void *taosNetInitRpc(char *secretEncrypt, char spi) {

From 6691dccb8f41865b91394d8fac54a71dbc6b5f1d Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 20:28:46 +0800
Subject: [PATCH 113/118] minor changes

---
 src/util/src/tnettest.c | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c
index 40e5e198a8..89601147a5 100644
--- a/src/util/src/tnettest.c
+++ b/src/util/src/tnettest.c
@@ -238,17 +238,6 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) {
     return -1;
   }
 
-  // set overtime
-  struct timeval timeout;
-  timeout.tv_sec = 2;   // s
-  timeout.tv_usec = 0;  // us
-  if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
-    uError("failed to setsockopt send timer since %s", strerror(errno));
-  }
-  if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) {
-    uError("failed to setsockopt recv timer since %s", strerror(errno));
-  }
-
   if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
     uError("failed to set the send buffer size for UDP socket\n");
     return -1;

From 1c1dac72c1205bbc128d09dc20890242e1d21ac8 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Dec 2020 22:17:02 +0800
Subject: [PATCH 114/118] TD-2371

---
 src/util/src/tnote.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 82ce2c314d..f2db0b3316 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -228,7 +228,7 @@ static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxN
 }
 
 void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) {
-  if (pNote->fd < 0) return;
+  if (pNote->fd <= 0) return;
   taosWrite(pNote->fd, buffer, len);
 
   if (pNote->maxLines > 0) {

From adc4cfd92c5d5920b5cb87b668e0fa0fc2d4d91a Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 9 Dec 2020 10:34:51 +0800
Subject: [PATCH 115/118] TD-2323

---
 src/dnode/inc/dnodeInt.h    |  8 ++++++++
 src/dnode/inc/dnodeModule.h |  2 +-
 src/dnode/inc/dnodeVnodes.h |  4 ++--
 src/dnode/src/dnodeMain.c   | 24 ++++++++++++++++++++++--
 src/dnode/src/dnodeModule.c | 26 +++++++++++++++-----------
 src/dnode/src/dnodeVnodes.c | 19 ++++---------------
 src/inc/dnode.h             |  7 -------
 7 files changed, 52 insertions(+), 38 deletions(-)

diff --git a/src/dnode/inc/dnodeInt.h b/src/dnode/inc/dnodeInt.h
index 7595f5fd02..1327cd4433 100644
--- a/src/dnode/inc/dnodeInt.h
+++ b/src/dnode/inc/dnodeInt.h
@@ -36,6 +36,14 @@ extern int32_t dDebugFlag;
 #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
 #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
 
+typedef enum {
+  TSDB_RUN_STATUS_INITIALIZE,
+  TSDB_RUN_STATUS_RUNING,
+  TSDB_RUN_STATUS_STOPPED
+} SRunStatus;
+
+SRunStatus dnodeGetRunStatus();
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/dnode/inc/dnodeModule.h b/src/dnode/inc/dnodeModule.h
index edcefbdd0c..e645784c8f 100644
--- a/src/dnode/inc/dnodeModule.h
+++ b/src/dnode/inc/dnodeModule.h
@@ -22,8 +22,8 @@ extern "C" {
 #include "dnodeInt.h"
 
 int32_t dnodeInitModules();
-void    dnodeStartModules();
 void    dnodeCleanupModules();
+bool    dnodeStartMnode(SMInfos *pMinfos);
 void    dnodeProcessModuleStatus(uint32_t moduleStatus);
 
 #ifdef __cplusplus
diff --git a/src/dnode/inc/dnodeVnodes.h b/src/dnode/inc/dnodeVnodes.h
index 1785ed3d06..e60dd290ce 100644
--- a/src/dnode/inc/dnodeVnodes.h
+++ b/src/dnode/inc/dnodeVnodes.h
@@ -23,8 +23,8 @@ extern "C" {
 
 int32_t dnodeInitVnodes();
 void    dnodeCleanupVnodes();
-int32_t dnodeInitTimer();
-void    dnodeCleanupTimer();
+int32_t dnodeInitStatusTimer();
+void    dnodeCleanupStatusTimer();
 void    dnodeSendStatusMsgToMnode();
 
 #ifdef __cplusplus
diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c
index 79efe5aa0e..730dcf3681 100644
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@@ -17,6 +17,7 @@
 #include "os.h"
 #include "taos.h"
 #include "tnote.h"
+#include "ttimer.h"
 #include "tconfig.h"
 #include "tfile.h"
 #include "twal.h"
@@ -39,6 +40,7 @@
 #include "dnodeShell.h"
 #include "dnodeTelemetry.h"
 
+void *tsDnodeTmr = NULL;
 static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED;
 
 static int32_t dnodeInitStorage();
@@ -68,8 +70,8 @@ static SStep tsDnodeSteps[] = {
   {"dnode-server",    dnodeInitServer,     dnodeCleanupServer},
   {"dnode-vnodes",    dnodeInitVnodes,     dnodeCleanupVnodes},
   {"dnode-modules",   dnodeInitModules,    dnodeCleanupModules},
-  {"dnode-tmr",       dnodeInitTimer,      dnodeCleanupTimer},
   {"dnode-shell",     dnodeInitShell,      dnodeCleanupShell},
+  {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer},
   {"dnode-telemetry", dnodeInitTelemetry,  dnodeCleanupTelemetry},
 };
 
@@ -91,6 +93,23 @@ static int32_t dnodeInitComponents() {
   return dnodeStepInit(tsDnodeSteps, stepSize);
 }
 
+static int32_t dnodeInitTmr() {
+  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
+  if (tsDnodeTmr == NULL) {
+    dError("failed to init dnode timer");
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dnodeCleanupTmr() {
+  if (tsDnodeTmr != NULL) {
+    taosTmrCleanUp(tsDnodeTmr);
+    tsDnodeTmr = NULL;
+  }
+}
+
 int32_t dnodeInitSystem() {
   dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
   tscEmbedded  = 1;
@@ -100,6 +119,7 @@ int32_t dnodeInitSystem() {
   taosReadGlobalLogCfg();
   taosSetCoreDump();
   taosInitNotes();
+  dnodeInitTmr();
   signal(SIGPIPE, SIG_IGN);
 
   if (dnodeCreateDir(tsLogDir) < 0) {
@@ -125,7 +145,6 @@ int32_t dnodeInitSystem() {
     return -1;
   }
 
-  dnodeStartModules();
   dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
 
   dInfo("TDengine is initialized successfully");
@@ -136,6 +155,7 @@ int32_t dnodeInitSystem() {
 void dnodeCleanUpSystem() {
   if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
     dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
+    dnodeCleanupTmr();
     dnodeCleanupComponents();
     taos_cleanup();
     taosCloseLog();
diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c
index 9eb52cbf5a..62de85445c 100644
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@@ -97,6 +97,20 @@ void dnodeCleanupModules() {
   }
 }
 
+static int32_t dnodeStartModules() {
+  for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) {
+    if (tsModule[module].enable && tsModule[module].startFp) {
+      int32_t code = (*tsModule[module].startFp)();
+      if (code != 0) {
+        dError("failed to start module:%s, code:%d", tsModule[module].name, code);
+        return code;
+      }
+    }
+  }
+
+  return 0;
+}
+
 int32_t dnodeInitModules() {
   dnodeAllocModules();
 
@@ -110,17 +124,7 @@ int32_t dnodeInitModules() {
   }
 
   dInfo("dnode modules is initialized");
-  return 0;
-}
-
-void dnodeStartModules() {
-  for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) {
-    if (tsModule[module].enable && tsModule[module].startFp) {
-      if ((*tsModule[module].startFp)() != 0) {
-        dError("failed to start module:%s", tsModule[module].name);
-      }
-    }
-  }
+  return dnodeStartModules();
 }
 
 void dnodeProcessModuleStatus(uint32_t moduleStatus) {
diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c
index ba7f7625fa..85b997d94c 100644
--- a/src/dnode/src/dnodeVnodes.c
+++ b/src/dnode/src/dnodeVnodes.c
@@ -30,39 +30,28 @@ typedef struct {
   int32_t * vnodeList;
 } SOpenVnodeThread;
 
-void *          tsDnodeTmr = NULL;
+extern void *   tsDnodeTmr;
 static void *   tsStatusTimer = NULL;
 static uint32_t tsRebootTime = 0;
 
 static void dnodeSendStatusMsg(void *handle, void *tmrId);
 static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
 
-int32_t dnodeInitTimer() {
-  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
-  if (tsDnodeTmr == NULL) {
-    dError("failed to init dnode timer");
-    return -1;
-  }
-
+int32_t dnodeInitStatusTimer() {
   dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp);
 
   tsRebootTime = taosGetTimestampSec();
   taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
 
-  dInfo("dnode timer is initialized");
+  dInfo("dnode status timer is initialized");
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeCleanupTimer() {
+void dnodeCleanupStatusTimer() {
   if (tsStatusTimer != NULL) {
     taosTmrStopA(&tsStatusTimer);
     tsStatusTimer = NULL;
   }
-
-  if (tsDnodeTmr != NULL) {
-    taosTmrCleanUp(tsDnodeTmr);
-    tsDnodeTmr = NULL;
-  }
 }
 
 static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
diff --git a/src/inc/dnode.h b/src/inc/dnode.h
index 9dd95e32d7..cb7e9f0b0d 100644
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@@ -29,13 +29,6 @@ typedef struct {
   int32_t httpReqNum;
 } SStatisInfo;
 
-typedef enum {
-  TSDB_RUN_STATUS_INITIALIZE,
-  TSDB_RUN_STATUS_RUNING,
-  TSDB_RUN_STATUS_STOPPED
-} SRunStatus;
-
-SRunStatus  dnodeGetRunStatus();
 SStatisInfo dnodeGetStatisInfo();
 
 bool    dnodeIsFirstDeploy();

From 77acab22975afcb6999a3324466003eb479b760a Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Wed, 9 Dec 2020 14:24:29 +0800
Subject: [PATCH 116/118] [TD-2292]<fix> multi exec taos_query when fail

---
 src/kit/taosdump/taosdump.c | 717 +++++++++++++++---------------------
 1 file changed, 307 insertions(+), 410 deletions(-)

diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c
index a7258c9724..bdfea26294 100644
--- a/src/kit/taosdump/taosdump.c
+++ b/src/kit/taosdump/taosdump.c
@@ -64,7 +64,10 @@ enum _show_tables_index {
   TSDB_SHOW_TABLES_NAME_INDEX,
   TSDB_SHOW_TABLES_CREATED_TIME_INDEX,
   TSDB_SHOW_TABLES_COLUMNS_INDEX,
-  TSDB_SHOW_TABLES_METRIC_INDEX,
+  TSDB_SHOW_TABLES_METRIC_INDEX,  
+  TSDB_SHOW_TABLES_UID_INDEX,  
+  TSDB_SHOW_TABLES_TID_INDEX,
+  TSDB_SHOW_TABLES_VGID_INDEX,  
   TSDB_MAX_SHOW_TABLES
 };
 
@@ -92,24 +95,27 @@ typedef struct {
 extern char version[];
 
 typedef struct {
-  char name[TSDB_DB_NAME_LEN + 1];
-  int32_t  tables;
+  char     name[TSDB_DB_NAME_LEN + 1];
+  char     create_time[32];
+  int32_t  ntables;
   int32_t  vgroups;  
-  int16_t  replications;
+  int16_t  replica;
   int16_t  quorum;
-  int16_t  daysPerFile;  
-  int16_t  daysToKeep;
-  int16_t  daysToKeep1;
-  int16_t  daysToKeep2;
-  int32_t  cacheBlockSize; //MB
-  int32_t  totalBlocks;
-  int32_t  minRowsPerFileBlock;
-  int32_t  maxRowsPerFileBlock;
-  int8_t   walLevel;
-  int32_t  fsyncPeriod;
-  int8_t   compression;
-  int8_t   precision;   // time resolution
+  int16_t  days;  
+  char     keeplist[32];
+  //int16_t  daysToKeep;
+  //int16_t  daysToKeep1;
+  //int16_t  daysToKeep2;
+  int32_t  cache; //MB
+  int32_t  blocks;
+  int32_t  minrows;
+  int32_t  maxrows;
+  int8_t   wallevel;
+  int32_t  fsync;
+  int8_t   comp;
+  char     precision[8];   // time resolution
   int8_t   update;
+  char     status[16];
 } SDbInfo;
 
 typedef struct {
@@ -128,8 +134,17 @@ typedef struct {
   int32_t   totalThreads;
   char      dbName[TSDB_TABLE_NAME_LEN + 1];
   void     *taosCon;
+  int64_t   rowsOfDumpOut;
+  int64_t   tablesOfDumpOut;
 } SThreadParaObj;
 
+typedef struct {
+  int64_t   totalRowsOfDumpOut;
+  int64_t   totalChildTblsOfDumpOut;
+  int32_t   totalSuperTblsOfDumpOut;
+  int32_t   totalDatabasesOfDumpOut;
+} resultStatistics;
+
 static int64_t totalDumpOutRows = 0;
 
 SDbInfo **dbInfos = NULL;
@@ -167,6 +182,7 @@ static struct argp_option options[] = {
   // input/output file
   {"outpath",       'o', "OUTPATH",     0,  "Output file path.",                                          1},
   {"inpath",        'i', "INPATH",      0,  "Input file path.",                                           1},
+  {"resultFile",    'r', "RESULTFILE",  0,  "DumpOut/In Result file path and name.",                      1},
   #ifdef _TD_POWER_
   {"config",        'c', "CONFIG_DIR",  0,  "Configure directory. Default is /etc/power/taos.cfg.",       1},
   #else
@@ -200,6 +216,8 @@ struct arguments {
   // output file
   char     outpath[TSDB_FILENAME_LEN+1];
   char     inpath[TSDB_FILENAME_LEN+1];
+  // result file
+  char    *resultFile;
   char    *encode;
   // dump unit option
   bool     all_databases;
@@ -274,6 +292,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
       tstrncpy(arguments->inpath, full_path.we_wordv[0], TSDB_FILENAME_LEN);
       wordfree(&full_path);
       break;
+    case 'r':
+      arguments->resultFile = arg;
+      break;
     case 'c':
       if (wordexp(arg, &full_path, 0) != 0) {
         fprintf(stderr, "Invalid path %s\n", arg);
@@ -343,16 +364,18 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 
 /* Our argp parser. */
 static struct argp argp = {options, parse_opt, args_doc, doc};
+static resultStatistics g_resultStatistics = {0};
+static FILE *g_fpOfResult = NULL;
 
 int taosDumpOut(struct arguments *arguments);
 int taosDumpIn(struct arguments *arguments);
 void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp);
 int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon);
-int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon);
-void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp);
-void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp);
-int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon);
-int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon);
+int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName);
+void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName);
+void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName);
+int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName);
+int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName);
 int taosCheckParam(struct arguments *arguments);
 void taosFreeDbInfos();
 static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t  numOfThread, char *dbName);
@@ -371,7 +394,8 @@ struct arguments tsArguments = {
   0,
   // outpath and inpath
   "", 
-  "", 
+  "",
+  "./dump_result.txt",
   NULL,
   // dump unit option
   false, 
@@ -392,18 +416,34 @@ struct arguments tsArguments = {
   0, 
   false
 };
-
-int queryDB(TAOS *taos, char *command) {
-  TAOS_RES *pSql = NULL;
+  
+static int queryDbImpl(TAOS *taos, char *command) {
+  int i;
+  TAOS_RES *res = NULL;
   int32_t   code = -1;
+
+  for (i = 0; i < 5; i++) {
+    if (NULL != res) {
+      taos_free_result(res);
+      res = NULL;
+    }
     
-  pSql = taos_query(taos, command);
-  code = taos_errno(pSql);
-  if (code) {
-    fprintf(stderr, "sql error: %s, reason:%s\n", command, taos_errstr(pSql));
-  } 
-  taos_free_result(pSql);
-  return code;
+    res = taos_query(taos, command);
+    code = taos_errno(res);
+    if (0 == code) {
+      break;
+    }
+  }
+
+  if (code != 0) {
+    fprintf(stderr, "Failed to run <%s>, reason: %s\n", command, taos_errstr(res));
+    taos_free_result(res);
+    //taos_close(taos);
+    return -1;
+  }
+
+  taos_free_result(res);
+  return 0;
 }
 
 int main(int argc, char *argv[]) {
@@ -430,6 +470,7 @@ int main(int argc, char *argv[]) {
     printf("mysqlFlag: %d\n", tsArguments.mysqlFlag);    
     printf("outpath: %s\n", tsArguments.outpath);
     printf("inpath: %s\n", tsArguments.inpath);
+    printf("resultFile: %s\n", tsArguments.resultFile);
     printf("encode: %s\n", tsArguments.encode);
     printf("all_databases: %d\n", tsArguments.all_databases);
     printf("databases: %d\n", tsArguments.databases);
@@ -459,13 +500,78 @@ int main(int argc, char *argv[]) {
   if (taosCheckParam(&tsArguments) < 0) {
     exit(EXIT_FAILURE);
   }
+  
+  g_fpOfResult = fopen(tsArguments.resultFile, "a");
+  if (NULL == g_fpOfResult) {
+    fprintf(stderr, "Failed to open %s for save result\n", tsArguments.resultFile);
+    return 1;
+  };
 
-  if (tsArguments.isDumpIn) {
-    if (taosDumpIn(&tsArguments) < 0) return -1;
-  } else {
-    if (taosDumpOut(&tsArguments) < 0) return -1;
+  fprintf(g_fpOfResult, "#############################################################################\n");
+  fprintf(g_fpOfResult, "============================== arguments config =============================\n");
+  {
+    fprintf(g_fpOfResult, "host: %s\n", tsArguments.host);
+    fprintf(g_fpOfResult, "user: %s\n", tsArguments.user);
+    fprintf(g_fpOfResult, "password: %s\n", tsArguments.password);
+    fprintf(g_fpOfResult, "port: %u\n", tsArguments.port);
+    fprintf(g_fpOfResult, "cversion: %s\n", tsArguments.cversion);    
+    fprintf(g_fpOfResult, "mysqlFlag: %d\n", tsArguments.mysqlFlag);    
+    fprintf(g_fpOfResult, "outpath: %s\n", tsArguments.outpath);
+    fprintf(g_fpOfResult, "inpath: %s\n", tsArguments.inpath);
+    fprintf(g_fpOfResult, "resultFile: %s\n", tsArguments.resultFile);
+    fprintf(g_fpOfResult, "encode: %s\n", tsArguments.encode);
+    fprintf(g_fpOfResult, "all_databases: %d\n", tsArguments.all_databases);
+    fprintf(g_fpOfResult, "databases: %d\n", tsArguments.databases);
+    fprintf(g_fpOfResult, "schemaonly: %d\n", tsArguments.schemaonly);
+    fprintf(g_fpOfResult, "with_property: %d\n", tsArguments.with_property);
+    fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", tsArguments.start_time);
+    fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", tsArguments.end_time);
+    fprintf(g_fpOfResult, "data_batch: %d\n", tsArguments.data_batch);
+    fprintf(g_fpOfResult, "max_sql_len: %d\n", tsArguments.max_sql_len);
+    fprintf(g_fpOfResult, "table_batch: %d\n", tsArguments.table_batch);
+    fprintf(g_fpOfResult, "thread_num: %d\n", tsArguments.thread_num);    
+    fprintf(g_fpOfResult, "allow_sys: %d\n", tsArguments.allow_sys);
+    fprintf(g_fpOfResult, "abort: %d\n", tsArguments.abort);
+    fprintf(g_fpOfResult, "isDumpIn: %d\n", tsArguments.isDumpIn);
+    fprintf(g_fpOfResult, "arg_list_len: %d\n", tsArguments.arg_list_len);
+
+    for (int32_t i = 0; i < tsArguments.arg_list_len; i++) {
+      fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, tsArguments.arg_list[i]);
+    }
   }
 
+  time_t tTime = time(NULL);
+  struct tm tm = *localtime(&tTime);
+
+  if (tsArguments.isDumpIn) {    
+    fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n");
+    fprintf(g_fpOfResult, "# DumpIn start time:                   %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
+          tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
+    if (taosDumpIn(&tsArguments) < 0) {
+      fprintf(g_fpOfResult, "\n");
+      fclose(g_fpOfResult);
+      return -1;
+    }
+  } else {
+    fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n");
+    fprintf(g_fpOfResult, "# DumpOut start time:                   %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
+          tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
+    if (taosDumpOut(&tsArguments) < 0) {
+      fprintf(g_fpOfResult, "\n");
+      fclose(g_fpOfResult);
+      return -1;
+    }
+
+    fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n");
+    fprintf(g_fpOfResult, "# total database count:     %d\n", g_resultStatistics.totalDatabasesOfDumpOut);
+    fprintf(g_fpOfResult, "# total super table count:  %d\n", g_resultStatistics.totalSuperTblsOfDumpOut);    
+    fprintf(g_fpOfResult, "# total child table count:  %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut);   
+    fprintf(g_fpOfResult, "# total row count:          %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut);    
+  }
+
+  fprintf(g_fpOfResult, "\n");
+  fclose(g_fpOfResult);
+
   return 0;
 }
 
@@ -700,7 +806,7 @@ int taosDumpOut(struct arguments *arguments) {
   int32_t code = taos_errno(result);
   
   if (code != 0) {
-    fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos));
+    fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(result));
     goto _exit_failure;
   }
 
@@ -736,27 +842,29 @@ int taosDumpOut(struct arguments *arguments) {
     }
 
     strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
-#if 0
     if (arguments->with_property) {
-      dbInfos[count]->tables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
+      dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
       dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);  
-      dbInfos[count]->replications = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
+      dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
       dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
-      dbInfos[count]->daysPerFile = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);  
-      dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]);
-      dbInfos[count]->daysToKeep1;
-      dbInfos[count]->daysToKeep2;
-      dbInfos[count]->cacheBlockSize = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
-      dbInfos[count]->totalBlocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
-      dbInfos[count]->minRowsPerFileBlock = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
-      dbInfos[count]->maxRowsPerFileBlock = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
-      dbInfos[count]->walLevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
-      dbInfos[count]->fsyncPeriod = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
-      dbInfos[count]->compression = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
-      dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]);
+      dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);  
+
+      strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);      
+      //dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]);
+      //dbInfos[count]->daysToKeep1;
+      //dbInfos[count]->daysToKeep2;
+      dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
+      dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
+      dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
+      dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
+      dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
+      dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
+      dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
+
+      strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);  
+      //dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]);
       dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
     }
-#endif
     count++;
 
     if (arguments->databases) {
@@ -781,6 +889,8 @@ int taosDumpOut(struct arguments *arguments) {
       taosDumpDb(dbInfos[0], arguments, fp, taos);
     } else {                                        // case: taosdump <db> tablex tabley ...
       taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp);
+      fprintf(g_fpOfResult, "\n#### database:                       %s\n", dbInfos[0]->name);
+      g_resultStatistics.totalDatabasesOfDumpOut++;
 
       sprintf(command, "use %s", dbInfos[0]->name);
       
@@ -796,6 +906,7 @@ int taosDumpOut(struct arguments *arguments) {
       int32_t totalNumOfThread = 1;  // 0: all normal talbe into .tables.tmp.0
       int  normalTblFd = -1;
       int32_t retCode;
+      int superTblCnt = 0 ;
       for (int i = 1; arguments->arg_list[i]; i++) {
         if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo, taos) < 0) {
           fprintf(stderr, "input the invalide table %s\n", arguments->arg_list[i]);
@@ -803,11 +914,17 @@ int taosDumpOut(struct arguments *arguments) {
         }
 
         if (tableRecordInfo.isMetric) {  // dump all table of this metric
-          (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos);
+          int ret = taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos, dbInfos[0]->name);
+          if (0 == ret) {
+            superTblCnt++;
+          }
           retCode = taosSaveTableOfMetricToTempFile(taos, tableRecordInfo.tableRecord.metric, arguments, &totalNumOfThread);
         } else {
           if (tableRecordInfo.tableRecord.metric[0] != '\0') {  // dump this sub table and it's metric
-            (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos);          
+            int ret = taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos, dbInfos[0]->name);
+            if (0 == ret) {
+              superTblCnt++;
+            }            
           }
           retCode = taosSaveAllNormalTableToTempFile(taos, tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, &normalTblFd);
         }
@@ -819,6 +936,10 @@ int taosDumpOut(struct arguments *arguments) {
           goto _clean_tmp_file;
         }
       }
+      
+      // TODO: save dump super table <superTblCnt> into result_output.txt
+      fprintf(g_fpOfResult, "# super table counter:               %d\n", superTblCnt);
+      g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt;
 
       if (-1 != normalTblFd){
         taosClose(normalTblFd);
@@ -855,41 +976,27 @@ _exit_failure:
   return -1;
 }
 
-int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) {
+int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) {
   TAOS_ROW row = NULL;
-  TAOS_RES *tmpResult = NULL;
+  TAOS_RES* res = NULL;
   int count = 0;
 
-  char* tempCommand = (char *)malloc(COMMAND_SIZE);
-  if (tempCommand == NULL) {
-    fprintf(stderr, "failed to allocate memory\n");
-    return -1;
-  }
-
-  char* tbuf = (char *)malloc(COMMAND_SIZE);
-  if (tbuf == NULL) {
-    fprintf(stderr, "failed to allocate memory\n");
-    free(tempCommand);
-    return -1;
-  }
-
-  sprintf(tempCommand, "describe %s", table);
+  char sqlstr[COMMAND_SIZE];
+  sprintf(sqlstr, "describe %s.%s;", dbName, table);
   
-  tmpResult = taos_query(taosCon, tempCommand);  
-  int32_t code = taos_errno(tmpResult);
+  res = taos_query(taosCon, sqlstr);  
+  int32_t code = taos_errno(res);
   if (code != 0) {
-    fprintf(stderr, "failed to run command %s\n", tempCommand);
-    free(tempCommand);
-    free(tbuf);
-    taos_free_result(tmpResult);
+    fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res));
+    taos_free_result(res);
     return -1;
   }
 
-  TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+  TAOS_FIELD *fields = taos_fetch_fields(res);
 
   tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN);
 
-  while ((row = taos_fetch_row(tmpResult)) != NULL) {
+  while ((row = taos_fetch_row(res)) != NULL) {
     strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
             fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
     strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
@@ -901,12 +1008,10 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe
     count++;
   }
 
-  taos_free_result(tmpResult);
-  tmpResult = NULL;
+  taos_free_result(res);
+  res = NULL;
 
   if (isSuperTable) {
-    free(tempCommand);
-    free(tbuf);
     return count;
   }
   
@@ -915,37 +1020,33 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe
     if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue;
 
 
-    sprintf(tempCommand, "select %s from %s", tableDes->cols[i].field, table);
+    sprintf(sqlstr, "select %s from %s.%s", tableDes->cols[i].field, dbName, table);
     
-    tmpResult = taos_query(taosCon, tempCommand);  
-    code = taos_errno(tmpResult);
+    res = taos_query(taosCon, sqlstr);  
+    code = taos_errno(res);
     if (code != 0) {
-      fprintf(stderr, "failed to run command %s\n", tempCommand);
-      free(tempCommand);
-      free(tbuf);
-      taos_free_result(tmpResult);
+      fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res));
+      taos_free_result(res);
       return -1;
     }
     
-    fields = taos_fetch_fields(tmpResult); 
+    fields = taos_fetch_fields(res); 
 
-    row = taos_fetch_row(tmpResult);
+    row = taos_fetch_row(res);
     if (NULL == row) {
-      fprintf(stderr, " fetch failed to run command %s\n", tempCommand);
-      free(tempCommand);
-      free(tbuf);
-      taos_free_result(tmpResult);
+      fprintf(stderr, " fetch failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res));
+      taos_free_result(res);
       return -1;
     }
 
     if (row[0] == NULL) {
       sprintf(tableDes->cols[i].note, "%s", "NULL");
-      taos_free_result(tmpResult);
-      tmpResult = NULL;
+      taos_free_result(res);
+      res = NULL;
       continue;
     }
     
-    int32_t* length = taos_fetch_lengths(tmpResult);
+    int32_t* length = taos_fetch_lengths(res);
 
     //int32_t* length = taos_fetch_lengths(tmpResult);
     switch (fields[0].type) {
@@ -970,18 +1071,22 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe
       case TSDB_DATA_TYPE_DOUBLE:
         sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0]));
         break;
-      case TSDB_DATA_TYPE_BINARY:
+      case TSDB_DATA_TYPE_BINARY: {
         memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note));
         tableDes->cols[i].note[0] = '\'';
+        char tbuf[COMMAND_SIZE];
         converStringToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE);
         char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf);
         *(pstr++) = '\'';
         break;
-      case TSDB_DATA_TYPE_NCHAR:
+      }
+      case TSDB_DATA_TYPE_NCHAR: {
         memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note));
+        char tbuf[COMMAND_SIZE];
         convertNCharToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE);
         sprintf(tableDes->cols[i].note, "\'%s\'", tbuf);
         break;
+      }
       case TSDB_DATA_TYPE_TIMESTAMP:
         sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
         #if 0
@@ -1001,17 +1106,14 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe
         break;
     }
   
-    taos_free_result(tmpResult);
-    tmpResult = NULL;    
+    taos_free_result(res);
+    res = NULL;    
   }
 
-  free(tempCommand);
-  free(tbuf);
-
   return count;
 }
 
-int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon) {
+int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName) {
   int count = 0;
 
   STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
@@ -1030,7 +1132,7 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI
     memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
     */
 
-    count = taosGetTableDes(table, tableDes, taosCon, false);
+    count = taosGetTableDes(dbName, table, tableDes, taosCon, false);
 
     if (count < 0) {
       free(tableDes);
@@ -1038,10 +1140,10 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI
     }
 
     // create child-table using super-table
-    taosDumpCreateMTableClause(tableDes, metric, count, fp);
+    taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName);
 
   } else {  // dump table definition
-    count = taosGetTableDes(table, tableDes, taosCon, false);
+    count = taosGetTableDes(dbName, table, tableDes, taosCon, false);
 
     if (count < 0) {
       free(tableDes);
@@ -1049,39 +1151,28 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI
     }
 
     // create normal-table or super-table
-    taosDumpCreateTableClause(tableDes, count, fp);
+    taosDumpCreateTableClause(tableDes, count, fp, dbName);
   }
 
   free(tableDes);
 
-  return taosDumpTableData(fp, table, arguments, taosCon);
+  return taosDumpTableData(fp, table, arguments, taosCon, dbName);
 }
 
 void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) {
+  char sqlstr[TSDB_MAX_SQL_LEN] = {0};
 
-  char* tmpCommand = (char *)malloc(COMMAND_SIZE);
-  if (tmpCommand == NULL) {
-    fprintf(stderr, "failed to allocate memory\n");
-    return;
-  }
-
-  char *pstr = tmpCommand;
-
-  pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s", dbInfo->name);
+  char *pstr = sqlstr;
+  pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name);
   if (isDumpProperty) {
-    #if 0
     pstr += sprintf(pstr,
-        "TABLES %d vgroups %d REPLICA %d quorum %d DAYS %d KEEP %d CACHE %d BLOCKS %d MINROWS %d MAXROWS %d WALLEVEL %d FYNC %d COMP %d PRECISION %s UPDATE %d",
-        dbInfo->tables, dbInfo->vgroups, dbInfo->replications, dbInfo->quorum, dbInfo->daysPerFile, dbInfo->daysToKeep, dbInfo->cacheBlockSize,
-        dbInfo->totalBlocks, dbInfo->minRowsPerFileBlock, dbInfo->maxRowsPerFileBlock,  dbInfo->walLevel, dbInfo->fsyncPeriod, dbInfo->compression, 
-        dbInfo->precision, dbInfo->update);
-    #endif
+        "TABLES %d VGROUPS %d REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d WALLEVEL %d FYNC %d COMP %d PRECISION '%s' UPDATE %d",
+        dbInfo->ntables, dbInfo->vgroups, dbInfo->replica, dbInfo->quorum, dbInfo->days, dbInfo->keeplist, dbInfo->cache,
+        dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows,  dbInfo->wallevel, dbInfo->fsync, dbInfo->comp, dbInfo->precision, dbInfo->update);
   }
 
   pstr += sprintf(pstr, ";");
-
-  fprintf(fp, "%s\n\n", tmpCommand);
-  free(tmpCommand);
+  fprintf(fp, "%s\n\n", sqlstr);
 }
 
 void* taosDumpOutWorkThreadFp(void *arg)
@@ -1131,7 +1222,13 @@ void* taosDumpOutWorkThreadFp(void *arg)
   while (1) {
     ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
     if (readLen <= 0) break;
-    taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon);
+
+    int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon, pThread->dbName);
+    if (ret >= 0) {
+      // TODO: sum table count and table rows by self
+      pThread->tablesOfDumpOut++;
+      pThread->rowsOfDumpOut += ret;
+    }
   }
 
   taos_free_result(tmpResult);
@@ -1147,13 +1244,15 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t  numOfTh
   SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj));
   for (int t = 0; t < numOfThread; ++t) {
     SThreadParaObj *pThread = threadObj + t;
+    pThread->rowsOfDumpOut = 0;
+    pThread->tablesOfDumpOut = 0;
     pThread->threadIndex = t;
     pThread->totalThreads = numOfThread;
     tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN);
     pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port);
                                 
     if (pThread->taosCon == NULL) {
-      fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon));
+      fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, reason:%s\n", pThread->threadIndex, taos_errstr(NULL));
       exit(0);
     }
 
@@ -1170,15 +1269,25 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t  numOfTh
     pthread_join(threadObj[t].threadID, NULL);
   }
 
+  // TODO: sum all thread dump table count and rows of per table, then save into result_output.txt  
+  int64_t   totalRowsOfDumpOut = 0;
+  int64_t   totalChildTblsOfDumpOut = 0;
   for (int32_t t = 0; t < numOfThread; ++t) {
     taos_close(threadObj[t].taosCon);
+    totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut;
+    totalRowsOfDumpOut      += threadObj[t].rowsOfDumpOut;
   }
+
+  fprintf(g_fpOfResult, "# child table counter:               %"PRId64"\n", totalChildTblsOfDumpOut);
+  fprintf(g_fpOfResult, "# row counter:                       %"PRId64"\n", totalRowsOfDumpOut);
+  g_resultStatistics.totalChildTblsOfDumpOut += totalChildTblsOfDumpOut;
+  g_resultStatistics.totalRowsOfDumpOut      += totalRowsOfDumpOut;
   free(threadObj);
 }
 
 
 
-int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) {
+int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName) {
   int count = 0;
 
   STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
@@ -1187,15 +1296,15 @@ int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) {
     exit(-1);
   }
 
-  count = taosGetTableDes(table, tableDes, taosCon, true);
+  count = taosGetTableDes(dbName, table, tableDes, taosCon, true);
 
   if (count < 0) {
     free(tableDes);
-    fprintf(stderr, "failed to get stable schema\n");
+    fprintf(stderr, "failed to get stable[%s] schema\n", table);
     exit(-1);
   }
 
-  taosDumpCreateTableClause(tableDes, count, fp);
+  taosDumpCreateTableClause(tableDes, count, fp, dbName);
 
   free(tableDes);
   return 0;
@@ -1207,38 +1316,19 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp)
   TAOS_ROW row;
   int fd = -1;
   STableRecord tableRecord;
+  char sqlstr[TSDB_MAX_SQL_LEN] = {0};
 
-  char* tmpCommand = (char *)malloc(COMMAND_SIZE);
-  if (tmpCommand == NULL) {
-    fprintf(stderr, "failed to allocate memory\n");
-    exit(-1);
-  }
-
-  sprintf(tmpCommand, "use %s", dbName);
+  sprintf(sqlstr, "show %s.stables", dbName);
   
-  TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);  
-  int32_t code = taos_errno(tmpResult);
+  TAOS_RES* res = taos_query(taosCon, sqlstr);  
+  int32_t  code = taos_errno(res);
   if (code != 0) {
-    fprintf(stderr, "invalid database %s, error: %s\n", dbName, taos_errstr(taosCon));
-    free(tmpCommand);
-    taos_free_result(tmpResult);
-    exit(-1);
-  }
-  
-  taos_free_result(tmpResult);
-
-  sprintf(tmpCommand, "show stables");
-  
-  tmpResult = taos_query(taosCon, tmpCommand);  
-  code = taos_errno(tmpResult);
-  if (code != 0) {
-    fprintf(stderr, "failed to run command %s, error: %s\n", tmpCommand, taos_errstr(taosCon));
-    free(tmpCommand);
-    taos_free_result(tmpResult);
+    fprintf(stderr, "failed to run command <%s>, reason: %s\n", sqlstr, taos_errstr(res));
+    taos_free_result(res);
     exit(-1);
   }
 
-  TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+  TAOS_FIELD *fields = taos_fetch_fields(res);
 
   char     tmpFileName[TSDB_FILENAME_LEN + 1];
   memset(tmpFileName, 0, TSDB_FILENAME_LEN);
@@ -1246,32 +1336,38 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp)
   fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
   if (fd == -1) {
     fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
-    taos_free_result(tmpResult);
-    free(tmpCommand);
+    taos_free_result(res);
     (void)remove(".stables.tmp");
     exit(-1);
   }
   
-  while ((row = taos_fetch_row(tmpResult)) != NULL) {  
+  while ((row = taos_fetch_row(res)) != NULL) {  
     memset(&tableRecord, 0, sizeof(STableRecord));
     strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
     taosWrite(fd, &tableRecord, sizeof(STableRecord));
   }  
   
-  taos_free_result(tmpResult);
+  taos_free_result(res);
   (void)lseek(fd, 0, SEEK_SET);
 
+  int superTblCnt = 0;
   while (1) {
     ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
     if (readLen <= 0) break;
     
-    (void)taosDumpStable(tableRecord.name, fp, taosCon);
+    int ret = taosDumpStable(tableRecord.name, fp, taosCon, dbName);
+    if (0 == ret) {
+      superTblCnt++;
+    }
   }
 
+  // TODO: save dump super table <superTblCnt> into result_output.txt
+  fprintf(g_fpOfResult, "# super table counter:               %d\n", superTblCnt);
+  g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt;
+
   close(fd);
   (void)remove(".stables.tmp");
   
-  free(tmpCommand);
   return 0;  
 }
 
@@ -1282,58 +1378,43 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
   STableRecord tableRecord;
 
   taosDumpCreateDbClause(dbInfo, arguments->with_property, fp);
-
-  char* tmpCommand = (char *)malloc(COMMAND_SIZE);
-  if (tmpCommand == NULL) {
-    fprintf(stderr, "failed to allocate memory\n");
-    return -1;
-  }
-
-  sprintf(tmpCommand, "use %s", dbInfo->name);
   
-  TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);  
-  int32_t code = taos_errno(tmpResult);
-  if (code != 0) {
-    fprintf(stderr, "invalid database %s\n", dbInfo->name);
-    free(tmpCommand);
-    taos_free_result(tmpResult);
-    return -1;
-  }
-  taos_free_result(tmpResult);
+  fprintf(g_fpOfResult, "\n#### database:                       %s\n", dbInfo->name);
+  g_resultStatistics.totalDatabasesOfDumpOut++;
+
+  char sqlstr[TSDB_MAX_SQL_LEN] = {0};
 
   fprintf(fp, "USE %s;\n\n", dbInfo->name);
   
   (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp);
 
-  sprintf(tmpCommand, "show tables");
+  sprintf(sqlstr, "show %s.tables", dbInfo->name);
   
-  tmpResult = taos_query(taosCon, tmpCommand);  
-  code = taos_errno(tmpResult);
+  TAOS_RES* res = taos_query(taosCon, sqlstr);  
+  int code = taos_errno(res);
   if (code != 0) {
-    fprintf(stderr, "failed to run command %s\n", tmpCommand);
-    free(tmpCommand);
-    taos_free_result(tmpResult);
+    fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res));
+    taos_free_result(res);
     return -1;
   }
 
-  TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+  TAOS_FIELD *fields = taos_fetch_fields(res);
 
   int32_t  numOfTable  = 0;
   int32_t  numOfThread = 0;
   char     tmpFileName[TSDB_FILENAME_LEN + 1];
-  while ((row = taos_fetch_row(tmpResult)) != NULL) {
+  while ((row = taos_fetch_row(res)) != NULL) {
     if (0 == numOfTable) {
       memset(tmpFileName, 0, TSDB_FILENAME_LEN);
       sprintf(tmpFileName, ".tables.tmp.%d", numOfThread);
       fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
       if (fd == -1) {
         fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
-        taos_free_result(tmpResult);
+        taos_free_result(res);
         for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
           sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
           (void)remove(tmpFileName);
         }
-        free(tmpCommand);
         return -1;
       }
 
@@ -1360,33 +1441,26 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
     fd = -1;
   }
   
-  taos_free_result(tmpResult);
+  taos_free_result(res);
 
   // start multi threads to dumpout
   taosStartDumpOutWorkThreads(arguments, numOfThread, dbInfo->name);
   for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
     sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
     (void)remove(tmpFileName);
-  }
-  
-  free(tmpCommand);
+  }  
 
   return 0;
 }
 
-void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) {
+void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName) {
   int counter = 0;
   int count_temp = 0;
+  char sqlstr[COMMAND_SIZE];
 
-  char* tmpBuf = (char *)malloc(COMMAND_SIZE);
-  if (tmpBuf == NULL) {
-    fprintf(stderr, "failed to allocate memory\n");
-    return;
-  }
+  char* pstr = sqlstr;
 
-  char* pstr = tmpBuf;
-
-  pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s", tableDes->name);
+  pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s", dbName, tableDes->name);
 
   for (; counter < numOfCols; counter++) {
     if (tableDes->cols[counter].note[0] != '\0') break;
@@ -1420,12 +1494,10 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) {
 
   pstr += sprintf(pstr, ");");
 
-  fprintf(fp, "%s\n", tmpBuf);
-
-  free(tmpBuf);
+  fprintf(fp, "%s\n\n", sqlstr);
 }
 
-void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp) {
+void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName) {
   int counter = 0;
   int count_temp = 0;
 
@@ -1438,7 +1510,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
   char *pstr = NULL;
   pstr = tmpBuf;
 
-  pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s USING %s TAGS (", tableDes->name, metric);
+  pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", dbName, tableDes->name, dbName, metric);
 
   for (; counter < numOfCols; counter++) {
     if (tableDes->cols[counter].note[0] != '\0') break;
@@ -1479,48 +1551,36 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
   free(tmpBuf);
 }
 
-int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon) {
+int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName) {
   /* char       temp[MAX_COMMAND_SIZE] = "\0"; */
   int64_t totalRows = 0;
   int count = 0;
   char *pstr = NULL;
   TAOS_ROW row = NULL;
   int numFields = 0;
-  char *tbuf = NULL;
-
-  char* tmpCommand = (char *)calloc(1, COMMAND_SIZE);
-  if (tmpCommand == NULL) {
-    fprintf(stderr, "failed to allocate memory\n");
-    return -1;
+  
+  if (arguments->schemaonly) {
+    return 0;
   }
 
   int32_t  sql_buf_len = arguments->max_sql_len;
   char* tmpBuffer = (char *)calloc(1, sql_buf_len + 128);
   if (tmpBuffer == NULL) {
     fprintf(stderr, "failed to allocate memory\n");
-    free(tmpCommand);
     return -1;
   }
 
   pstr = tmpBuffer;
 
-  if (arguments->schemaonly) {
-    free(tmpCommand);
-    free(tmpBuffer);
-    return 0;
-  }
+  char sqlstr[1024] = {0};
+  sprintf(sqlstr, 
+          "select * from %s.%s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc;", 
+          dbName, tbname, arguments->start_time, arguments->end_time);
   
-  sprintf(tmpCommand, 
-          "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", 
-          tbname, 
-          arguments->start_time,
-          arguments->end_time);
-  
-  TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);  
+  TAOS_RES* tmpResult = taos_query(taosCon, sqlstr);  
   int32_t code = taos_errno(tmpResult);
   if (code != 0) {
-    fprintf(stderr, "failed to run command %s, reason: %s\n", tmpCommand, taos_errstr(taosCon));
-    free(tmpCommand);
+    fprintf(stderr, "failed to run command %s, reason: %s\n", sqlstr, taos_errstr(tmpResult));
     free(tmpBuffer);
     taos_free_result(tmpResult);
     return -1;
@@ -1529,14 +1589,6 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
   numFields = taos_field_count(tmpResult);
   assert(numFields > 0);
   TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
-  tbuf = (char *)malloc(COMMAND_SIZE);
-  if (tbuf == NULL) {
-    fprintf(stderr, "No enough memory\n");
-    free(tmpCommand);
-    free(tmpBuffer);
-    taos_free_result(tmpResult);
-    return -1;
-  }
 
   int rowFlag = 0;
   int32_t  curr_sqlstr_len = 0;
@@ -1550,7 +1602,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
 
     if (count == 0) {
       total_sqlstr_len = 0;
-      curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s VALUES (", tbname);
+      curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s.%s VALUES (", dbName, tbname);
     } else {
       if (arguments->mysqlFlag) {
         if (0 == rowFlag) {
@@ -1594,17 +1646,21 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
         case TSDB_DATA_TYPE_DOUBLE:
           curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_DOUBLE_VAL(row[col]));
           break;
-        case TSDB_DATA_TYPE_BINARY:
+        case TSDB_DATA_TYPE_BINARY: {
+          char tbuf[COMMAND_SIZE] = {0};
           //*(pstr++) = '\'';
           converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE);
           //pstr = stpcpy(pstr, tbuf);
           //*(pstr++) = '\'';
           pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf);          
           break;
-        case TSDB_DATA_TYPE_NCHAR:
+        }
+        case TSDB_DATA_TYPE_NCHAR: {
+          char tbuf[COMMAND_SIZE] = {0};
           convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE);
           pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf);
           break;
+        }
         case TSDB_DATA_TYPE_TIMESTAMP:
           if (!arguments->mysqlFlag) {
             curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", *(int64_t *)row[col]);
@@ -1638,19 +1694,12 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
     //}
   }
 
+  fprintf(fp, "\n");  
   atomic_add_fetch_64(&totalDumpOutRows, totalRows);
-
-  fprintf(fp, "\n");
-
-  if (tbuf) {
-    free(tbuf);
-  }
   
   taos_free_result(tmpResult);
-  tmpResult = NULL;
-  free(tmpCommand);
   free(tmpBuffer);
-  return 0;
+  return totalRows;
 }
 
 int taosCheckParam(struct arguments *arguments) {
@@ -1986,159 +2035,6 @@ static FILE*  taosOpenDumpInFile(char *fptr) {
   return f;
 }
 
-int taosDumpInOneFile_old(TAOS     * taos, FILE* fp, char* fcharset, char* encode) {
-  char     *command       = NULL;
-  char     *lcommand      = NULL;
-  int       tsize         = 0;
-  char     *line          = NULL;
-  _Bool     isRun         = true;
-  size_t    line_size     = 0;
-  char     *pstr          = NULL;
-  char     *lstr          = NULL;
-  size_t    inbytesleft   = 0;
-  size_t    outbytesleft  = COMMAND_SIZE;
-  char     *tcommand      = NULL;
-  char     *charsetOfFile = NULL;
-  iconv_t   cd            = (iconv_t)(-1);
-
-  command  = (char *)malloc(COMMAND_SIZE);
-  lcommand = (char *)malloc(COMMAND_SIZE);
-  if (command == NULL || lcommand == NULL) {
-    fprintf(stderr, "failed to connect to allocate memory\n");
-    goto _dumpin_exit_failure;
-  }
-
-  // Resolve locale
-  if (*fcharset != '\0') {
-    charsetOfFile = fcharset;
-  } else {
-    charsetOfFile = encode;
-  }
-
-  if (charsetOfFile != NULL && strcasecmp(tsCharset, charsetOfFile) != 0) {
-    cd = iconv_open(tsCharset, charsetOfFile);
-    if (cd == ((iconv_t)(-1))) {
-      fprintf(stderr, "Failed to open iconv handle\n");
-      goto _dumpin_exit_failure;
-    }
-  }
-
-  pstr = command;
-  int64_t linenu = 0;
-  while (1) {
-    ssize_t size = getline(&line, &line_size, fp);
-    linenu++;
-    if (size <= 0) break;
-    if (size == 1) {
-      if (pstr != command) {
-        inbytesleft = pstr - command;
-        memset(lcommand, 0, COMMAND_SIZE);
-        pstr = command;
-        lstr = lcommand;
-        outbytesleft = COMMAND_SIZE;
-        if (cd != ((iconv_t)(-1))) {
-          iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
-          tcommand = lcommand;
-        } else {
-          tcommand = command;
-        }
-        
-        taosReplaceCtrlChar(tcommand);
-        
-        if (queryDB(taos, tcommand) != 0) {
-          fprintf(stderr, "error sql: linenu: %" PRId64 " failed\n", linenu);
-          exit(0);
-        }
-        
-        pstr = command;
-        pstr[0] = '\0';
-        tsize = 0;
-        isRun = true;
-      }
-
-      continue;
-    }
-
-    /* if (line[0] == '-' && line[1] == '-') continue; */
-
-    line[size - 1] = 0;
-
-    if (tsize + size - 1 > COMMAND_SIZE) {
-      fprintf(stderr, "command is too long\n");
-      goto _dumpin_exit_failure;
-    }
-
-    if (line[size - 2] == '\\') {
-      line[size - 2] = ' ';
-      isRun = false;
-    } else {
-      isRun = true;
-    }
-
-    memcpy(pstr, line, size - 1);
-    pstr += (size - 1);
-    *pstr = '\0';
-
-    if (!isRun) continue;
-
-    if (command != pstr && !isEmptyCommand(command)) {
-      inbytesleft = pstr - command;
-      memset(lcommand, 0, COMMAND_SIZE);
-      pstr = command;
-      lstr = lcommand;
-      outbytesleft = COMMAND_SIZE;
-      if (cd != ((iconv_t)(-1))) {
-        iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
-        tcommand = lcommand;
-      } else {
-        tcommand = command;
-      }
-      taosReplaceCtrlChar(tcommand);
-      if (queryDB(taos, tcommand) != 0) {
-        fprintf(stderr, "error sql: linenu:%" PRId64 " failed\n", linenu);
-        exit(0); 
-      }
-    }
-
-    pstr = command;
-    *pstr = '\0';
-    tsize = 0;
-  }
-
-  if (pstr != command) {
-    inbytesleft = pstr - command;
-    memset(lcommand, 0, COMMAND_SIZE);
-    pstr = command;
-    lstr = lcommand;
-    outbytesleft = COMMAND_SIZE;
-    if (cd != ((iconv_t)(-1))) {
-      iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
-      tcommand = lcommand;
-    } else {
-      tcommand = command;
-    }
-    taosReplaceCtrlChar(lcommand);
-    if (queryDB(taos, tcommand) != 0)
-      fprintf(stderr, "error sql: linenu:%" PRId64 " failed \n", linenu);
-  }
-
-  if (cd != ((iconv_t)(-1))) iconv_close(cd);
-  tfree(line);
-  tfree(command);
-  tfree(lcommand);
-  taos_close(taos);
-  fclose(fp);
-  return 0;
-
-_dumpin_exit_failure:
-  if (cd != ((iconv_t)(-1))) iconv_close(cd);
-  tfree(command);
-  tfree(lcommand);
-  taos_close(taos);
-  fclose(fp);
-  return -1;
-}
-
 int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, char* fileName) {
   int       read_len = 0;
   char *    cmd      = NULL;
@@ -2172,8 +2068,9 @@ int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, c
 
     memcpy(cmd + cmd_len, line, read_len);
     cmd[read_len + cmd_len]= '\0';
-    if (queryDB(taos, cmd)) {
+    if (queryDbImpl(taos, cmd)) {
       fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName);
+      fprintf(g_fpOfResult, "error sql: linenu:%d, file:%s\n", lineNo, fileName);
     }
 
     memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN);
@@ -2221,7 +2118,7 @@ static void taosStartDumpInWorkThreads(struct arguments *args)
     pThread->totalThreads = totalThreads;
     pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port);
     if (pThread->taosCon == NULL) {
-      fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon));
+      fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, reason:%s\n", pThread->threadIndex, taos_errstr(NULL));
       exit(0);
     }
 

From c8570bac1fb3eb7cd0a3599f6072617db113bb67 Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Wed, 9 Dec 2020 17:57:25 +0800
Subject: [PATCH 117/118] [TD-2368]<fix>: show 4 new variables: arbitrator,
 timezone, locale, charset

---
 src/common/src/tglobal.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index a912cdfd7f..f8d2ef173b 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -416,7 +416,7 @@ static void doInitGlobalConfig(void) {
   cfg.option = "arbitrator";
   cfg.ptr = tsArbitrator;
   cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
   cfg.ptrLength = TSDB_EP_LEN;
@@ -901,7 +901,7 @@ static void doInitGlobalConfig(void) {
   cfg.option = "timezone";
   cfg.ptr = tsTimezone;
   cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
   cfg.ptrLength = tListLen(tsTimezone);
@@ -911,7 +911,7 @@ static void doInitGlobalConfig(void) {
   cfg.option = "locale";
   cfg.ptr = tsLocale;
   cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
   cfg.ptrLength = tListLen(tsLocale);
@@ -921,7 +921,7 @@ static void doInitGlobalConfig(void) {
   cfg.option = "charset";
   cfg.ptr = tsCharset;
   cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
   cfg.ptrLength = tListLen(tsCharset);

From 937a2164b925434a0c8e9e6ba7baccad1cea0ce1 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Wed, 9 Dec 2020 19:17:16 +0800
Subject: [PATCH 118/118] TD-2381

---
 src/common/src/tglobal.c    | 8 ++++----
 src/dnode/src/dnodeMWrite.c | 4 ++--
 src/dnode/src/dnodePeer.c   | 4 ++--
 src/dnode/src/dnodeVMgmt.c  | 4 ++--
 src/util/src/tconfig.c      | 3 ---
 src/wal/src/walWrite.c      | 6 +++---
 6 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index a912cdfd7f..0cead82a09 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -203,10 +203,10 @@ int32_t tsVersion = 0;
 
 // log
 int32_t tsNumOfLogLines = 10000000;
-int32_t mDebugFlag = 135;
-int32_t sdbDebugFlag = 135;
+int32_t mDebugFlag = 131;
+int32_t sdbDebugFlag = 131;
 int32_t dDebugFlag = 135;
-int32_t vDebugFlag = 135;
+int32_t vDebugFlag = 131;
 int32_t cDebugFlag = 131;
 int32_t jniDebugFlag = 131;
 int32_t odbcDebugFlag = 131;
@@ -220,7 +220,7 @@ int32_t debugFlag = 0;
 int32_t sDebugFlag = 135;
 int32_t wDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
-int32_t cqDebugFlag = 135;
+int32_t cqDebugFlag = 131;
 
 int32_t (*monStartSystemFp)() = NULL;
 void (*monStopSystemFp)() = NULL;
diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c
index bc387e2171..414b66653d 100644
--- a/src/dnode/src/dnodeMWrite.c
+++ b/src/dnode/src/dnodeMWrite.c
@@ -121,7 +121,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
     dnodeSendRedirectMsg(pMsg, true);
   } else {
     SMnodeMsg *pWrite = mnodeCreateMsg(pMsg);
-    dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle,
+    dTrace("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle,
            taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
     taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
   }
@@ -130,7 +130,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
 }
 
 static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {
-  dDebug("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle,
+  dTrace("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle,
          taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
 
   mnodeCleanupMsg(pWrite);
diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c
index bf74e14963..de0c360c88 100644
--- a/src/dnode/src/dnodePeer.c
+++ b/src/dnode/src/dnodePeer.c
@@ -96,7 +96,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     rspMsg.code = TSDB_CODE_APP_NOT_READY;
     rpcSendResponse(&rspMsg);
     rpcFreeCont(pMsg->pCont);
-    dDebug("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
+    dTrace("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
     return;
   }
 
@@ -151,7 +151,7 @@ void dnodeCleanupClient() {
 static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) {
     if (pMsg == NULL || pMsg->pCont == NULL) return;
-    dDebug("msg:%p is ignored since dnode is stopping", pMsg);
+    dTrace("msg:%p is ignored since dnode is stopping", pMsg);
     rpcFreeCont(pMsg->pCont);
     return;
   }
diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c
index 87302026ec..3c975f5cf9 100644
--- a/src/dnode/src/dnodeVMgmt.c
+++ b/src/dnode/src/dnodeVMgmt.c
@@ -126,14 +126,14 @@ static void *dnodeProcessMgmtQueue(void *param) {
     }
 
     pMsg = &pMgmt->rpcMsg;
-    dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]);
+    dTrace("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]);
     if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
       rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
     } else {
       rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
     }
 
-    dDebug("msg:%p, is processed, code:0x%x", pMgmt, rsp.code);
+    dTrace("msg:%p, is processed, code:0x%x", pMgmt, rsp.code);
     if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) {
       rsp.handle = pMsg->handle;
       rsp.pCont = NULL;
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index 173de294cf..7c805072c1 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -240,9 +240,6 @@ void taosReadGlobalLogCfg() {
   int    olen, vlen;
   char   fileName[PATH_MAX] = {0};
 
-  mDebugFlag = 135;
-  sdbDebugFlag = 135;
-
   wordexp_t full_path;
   if ( 0 != wordexp(configDir, &full_path, 0)) {
     printf("\nconfig file: %s wordexp fail! reason:%s\n", configDir, strerror(errno));
diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c
index 10e1b4dd61..9174da3d03 100644
--- a/src/wal/src/walWrite.c
+++ b/src/wal/src/walWrite.c
@@ -166,14 +166,14 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
     char walName[WAL_FILE_LEN];
     snprintf(walName, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
 
-    wDebug("vgId:%d, file:%s, will be restored", pWal->vgId, walName);
+    wInfo("vgId:%d, file:%s, will be restored", pWal->vgId, walName);
     int32_t code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId);
     if (code != TSDB_CODE_SUCCESS) {
       wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code));
       continue;
     }
 
-    wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName);
+    wInfo("vgId:%d, file:%s, restore success", pWal->vgId, walName);
 
     count++;
   }
@@ -326,7 +326,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
 
     offset = offset + sizeof(SWalHead) + pHead->len;
 
-    wDebug("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId,
+    wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId,
            fileId, pHead->version, pWal->version, pHead->len);
 
     pWal->version = pHead->version;