From 5444fe6beb99a9c6739bf19fc8f981d77e9b078c Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 24 Dec 2020 10:12:00 +0800
Subject: [PATCH 01/13] tsVersion: refactor to correct format

---
 src/common/inc/tglobal.h |  2 +-
 src/common/src/tglobal.c | 15 ++++++++++-----
 src/util/inc/buildInfo.h |  7 -------
 3 files changed, 11 insertions(+), 13 deletions(-)
 delete mode 100644 src/util/inc/buildInfo.h

diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index bf1f22a4ee..e971a9c957 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -161,7 +161,7 @@ extern float   tsMinimalLogDirGB;
 extern float   tsReservedTmpDirectorySpace;
 extern float   tsMinimalDataDirGB;
 extern int32_t tsTotalMemoryMB;
-extern int32_t tsVersion;
+extern uint32_t tsVersion;
 
 // build info
 extern char version[];
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index f007a82f84..cab234c787 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -201,7 +201,7 @@ float   tsAvailDataDirGB = 0;
 float   tsReservedTmpDirectorySpace = 0.1f;
 float   tsMinimalDataDirGB = 0.5f;
 int32_t tsTotalMemoryMB = 0;
-int32_t tsVersion = 0;
+uint32_t tsVersion = 0;
 
 // log
 int32_t tsNumOfLogLines = 10000000;
@@ -1451,15 +1451,20 @@ int32_t taosCheckGlobalCfg() {
 
   // todo refactor
   tsVersion = 0;
-  for (int i = 0; i < 10; i++) {
+  for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) {
     if (version[i] >= '0' && version[i] <= '9') {
-      tsVersion = tsVersion * 10 + (version[i] - '0');
+      ver = ver * 10 + (version[i] - '0');
+    } else if (version[i] == '.') {
+      tsVersion |= ver & 0xFF;
+      tsVersion <<= 8;
+
+      ver = 0;
     } else if (version[i] == 0) {
+      tsVersion |= ver & 0xFF;
+
       break;
     }
   }
-  
-  tsVersion = 10 * tsVersion;
 
   tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL;  // udp[6035-6039] tcp[6035]
   tsDnodeDnodePort = tsServerPort + TSDB_PORT_DNODEDNODE;   // udp/tcp
diff --git a/src/util/inc/buildInfo.h b/src/util/inc/buildInfo.h
deleted file mode 100644
index 8d169d618d..0000000000
--- a/src/util/inc/buildInfo.h
+++ /dev/null
@@ -1,7 +0,0 @@
-#ifndef _TS_BUILD_H_
-#define _TS_BUILD_H_
-
-extern const char tsVersion[];
-extern const char tsBuildInfo[];
-
-#endif

From eef2d8f05e9aa51f4899eaa05095776325ccc966 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 24 Dec 2020 10:49:29 +0800
Subject: [PATCH 02/13] [TD-2502]<feature>: force version checking with rpc
 request messages

---
 src/inc/taoserror.h   | 1 +
 src/rpc/inc/rpcHead.h | 1 +
 src/rpc/src/rpcMain.c | 6 ++++++
 3 files changed, 8 insertions(+)

diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index 7c7e7ec31a..69c01e6763 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -67,6 +67,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE,    0, 0x0012, "Invalid re
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP,       0, 0x0013, "Client and server's time is not synchronized")
 TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY,                0, 0x0014, "Database not ready")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR,               0, 0x0015, "Unable to resolve FQDN")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VERSION,          0, 0x0016, "Invalid app version")
 
 //common & util
 TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT,          0, 0x0100, "Operation not supported")
diff --git a/src/rpc/inc/rpcHead.h b/src/rpc/inc/rpcHead.h
index 520edadc7d..95204f72f6 100644
--- a/src/rpc/inc/rpcHead.h
+++ b/src/rpc/inc/rpcHead.h
@@ -58,6 +58,7 @@ typedef struct {
   char     empty[1];  // reserved
   uint8_t  msgType;   // message type  
   int32_t  msgLen;    // message length including the header iteslf
+  uint32_t msgVer;
   int32_t  code;      // code in response message
   uint8_t  content[0]; // message body starts from here
 } SRpcHead;
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 1d394d8795..8226620c1b 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -959,6 +959,11 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
     terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL;
   }
 
+  if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) {
+    tDebug("%s sid:%d, invalid client version:%d", pRpc->label, sid, htonl(pHead->msgVer));
+    terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL;
+  }
+
   pConn = rpcGetConnObj(pRpc, sid, pRecv);
   if (pConn == NULL) {
     tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); 
@@ -1282,6 +1287,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
 
   // set the message header  
   pHead->version = 1;
+  pHead->msgVer = htonl(tsVersion >> 8);
   pHead->msgType = msgType;
   pHead->encrypt = 0;
   pConn->tranId++;

From 87d2a0ab34bc3be5151f92bba5349341e6856c6c Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 25 Dec 2020 01:41:53 +0000
Subject: [PATCH 03/13] fix bug

---
 src/client/src/tscServer.c | 6 +++---
 src/query/src/qExecutor.c  | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index ded04388f4..1b16da9115 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -888,13 +888,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) {
       SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j);
   
-      *((int16_t *)pMsg) = pCol->colId;
+      *((int16_t *)pMsg) = htons(pCol->colId);
       pMsg += sizeof(pCol->colId);
 
-      *((int16_t *)pMsg) += pCol->colIndex;
+      *((int16_t *)pMsg) += htons(pCol->colIndex);
       pMsg += sizeof(pCol->colIndex);
 
-      *((int16_t *)pMsg) += pCol->flag;
+      *((int16_t *)pMsg) += htons(pCol->flag);
       pMsg += sizeof(pCol->flag);
       
       memcpy(pMsg, pCol->name, tListLen(pCol->name));
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 2b992a931d..d74aba215f 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -6211,13 +6211,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
     }
 
     for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) {
-      (*groupbyCols)[i].colId = *(int16_t *)pMsg;
+      (*groupbyCols)[i].colId = htons(*(int16_t *)pMsg);
       pMsg += sizeof((*groupbyCols)[i].colId);
 
-      (*groupbyCols)[i].colIndex = *(int16_t *)pMsg;
+      (*groupbyCols)[i].colIndex = htons(*(int16_t *)pMsg);
       pMsg += sizeof((*groupbyCols)[i].colIndex);
 
-      (*groupbyCols)[i].flag = *(int16_t *)pMsg;
+      (*groupbyCols)[i].flag = htons(*(int16_t *)pMsg);
       pMsg += sizeof((*groupbyCols)[i].flag);
 
       memcpy((*groupbyCols)[i].name, pMsg, tListLen(groupbyCols[i]->name));

From c7c4572c15b41a0384d7e4d2cb3723a8fcded6fb Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 25 Dec 2020 13:06:22 +0800
Subject: [PATCH 04/13] rpc: remove duplicate macro definitions

---
 src/rpc/inc/rpcHead.h | 1 +
 src/rpc/src/rpcMain.c | 7 -------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/rpc/inc/rpcHead.h b/src/rpc/inc/rpcHead.h
index 95204f72f6..dc0cac0143 100644
--- a/src/rpc/inc/rpcHead.h
+++ b/src/rpc/inc/rpcHead.h
@@ -20,6 +20,7 @@
 extern "C" {
 #endif
 
+// server:0 client:1  tcp:2 udp:0
 #define RPC_CONN_UDPS   0
 #define RPC_CONN_UDPC   1
 #define RPC_CONN_TCPS   2
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 8226620c1b..deaf30f42e 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -137,13 +137,6 @@ static int     tsRpcRefId = -1;
 static int32_t tsRpcNum = 0;
 //static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
 
-// server:0 client:1  tcp:2 udp:0
-#define RPC_CONN_UDPS   0
-#define RPC_CONN_UDPC   1
-#define RPC_CONN_TCPS   2
-#define RPC_CONN_TCPC   3
-#define RPC_CONN_TCP    2
-
 void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = {
     taosInitUdpConnection,
     taosInitUdpConnection,

From dbfa9a0b309966532457516ed1fc48bbe619c0c2 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Sat, 26 Dec 2020 08:23:51 +0000
Subject: [PATCH 05/13] add retry when receiving reconfigure table error

---
 src/client/src/tscParseInsert.c | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index ded12f64ea..e89d6f8a9f 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -1413,13 +1413,25 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
   if (taos_errno(pSql) != TSDB_CODE_SUCCESS) {  // handle error
     assert(taos_errno(pSql) == code);
 
-    taos_free_result(pSql);
-    tfree(pSupporter);
-    fclose(fp);
+    do {
+      if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+        assert(pSql->res.numOfRows == 0);
+        int32_t errc = fseek(fp, 0, SEEK_SET);
+        if (errc < 0) {
+          tscError("%p failed to seek SEEK_SET since:%s", pSql, tstrerror(errno));
+        } else {
+          break;
+        }
+      }
 
-    pParentSql->res.code = code;
-    tscQueueAsyncRes(pParentSql);
-    return;
+      taos_free_result(pSql);
+      tfree(pSupporter);
+      fclose(fp);
+
+      pParentSql->res.code = code;
+      tscQueueAsyncRes(pParentSql);
+      return;
+    } while (0);
   }
 
   // accumulate the total submit records

From 4046affaff918849c6c143935e565fc099d2ce84 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sat, 26 Dec 2020 23:01:45 +0800
Subject: [PATCH 06/13] TD-2270

---
 src/balance/src/bnMain.c         |  85 ++++++-
 tests/script/jenkins/basic.txt   |   2 +
 tests/script/jenkins/unique.txt  |   2 +
 tests/script/unique/dnode/m2.sim | 367 +++++++++++++++++++++++++++++++
 tests/script/unique/dnode/m3.sim | 359 ++++++++++++++++++++++++++++++
 5 files changed, 809 insertions(+), 6 deletions(-)
 create mode 100644 tests/script/unique/dnode/m2.sim
 create mode 100644 tests/script/unique/dnode/m3.sim

diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c
index a323050216..11576c1135 100644
--- a/src/balance/src/bnMain.c
+++ b/src/balance/src/bnMain.c
@@ -66,6 +66,77 @@ static bool bnCheckFree(SDnodeObj *pDnode) {
   return true;
 }
 
+static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
+  SVnodeGid tmp = *pVnodeGid1;
+  *pVnodeGid1 = *pVnodeGid2;
+  *pVnodeGid2 = tmp;
+}
+
+static void bnAdjustVnodeIndex(SVgObj *pInVg) {
+  int32_t d0Id = pInVg->vnodeGid[0].dnodeId;
+  int32_t d1Id = pInVg->vnodeGid[1].dnodeId;
+  int32_t d2Id = pInVg->vnodeGid[2].dnodeId;
+
+  int32_t vgId = pInVg->vgId;
+  int32_t d0Num = 0;
+  int32_t d1Num = 0;
+  int32_t d2Num = 0;
+
+  void *pIter = NULL;
+  while (1) {
+    SVgObj *pVgroup = NULL;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    if (pVgroup->vgId != vgId) {
+      if (pVgroup->vnodeGid[0].dnodeId == d0Id) d0Num++;
+      if (pVgroup->vnodeGid[0].dnodeId == d1Id) d1Num++;
+      if (pVgroup->vnodeGid[0].dnodeId == d2Id) d2Num++;
+    }
+
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  if (pInVg->numOfVnodes == 1) {
+  }
+
+  if (pInVg->numOfVnodes == 2) {
+    mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num);
+    if (d0Num > d1Num) {
+      mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId);
+      bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]);
+    }
+  }
+
+  if (pInVg->numOfVnodes >= 3) {
+    mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num, d2Id, d2Num);
+    if (d0Num <= d1Num && d0Num <= d2Num) {
+      if (d1Num > d2Num) {
+        mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
+        bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
+      }
+    } else if (d1Num <= d2Num && d1Num <= d0Num) {
+      mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId);
+      bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]);
+      if (d0Num > d2Num) {
+        mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
+        bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
+      }
+    } else {
+      mDebug("vgId:%d, adjust vnode index 0 to 2", pInVg->vgId);
+      bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[2]);
+      if (d1Num > d0Num) {
+        mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
+        bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
+      }
+    }
+  }
+
+  for (int i = 0; i < pInVg->numOfVnodes; ++i) {
+    mDebug("vgId:%d index:%d dnodeId:%d", pInVg->vgId, i, pInVg->vnodeGid[i].dnodeId);
+  }
+}
+
 static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
   mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
 
@@ -88,15 +159,10 @@ static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
   memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
   pVgroup->numOfVnodes = numOfVnodes;
 
+  bnAdjustVnodeIndex(pVgroup);
   mnodeUpdateVgroup(pVgroup);
 }
 
-static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
-  SVnodeGid tmp = *pVnodeGid1;
-  *pVnodeGid1 = *pVnodeGid2;
-  *pVnodeGid2 = tmp;
-}
-
 int32_t bnAllocVnodes(SVgObj *pVgroup) {
   int32_t dnode = 0;
   int32_t vnodes = 0;
@@ -147,6 +213,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) {
     }
   }
 
+  bnAdjustVnodeIndex(pVgroup);
   bnReleaseDnodes();
   bnUnLock();
   return TSDB_CODE_SUCCESS;
@@ -233,6 +300,7 @@ static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDes
   vnodeGids[numOfVnodes].pDnode = pDestDnode;
   numOfVnodes++;
 
+  // move the src vnode to the end
   for (int32_t v = 0; v < numOfVnodes; ++v) {
     if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) {
       bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]);
@@ -241,6 +309,11 @@ static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDes
     }
   }
 
+  // adjust the vgroup postion
+  if (pSrcDnode == NULL) {
+    bnAdjustVnodeIndex(pVgroup);
+  }
+
   memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA);
   pVgroup->numOfVnodes = numOfVnodes;
   atomic_add_fetch_32(&pDestDnode->openVnodes, 1);
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 2fc355c40c..e3555e700a 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -286,6 +286,8 @@ cd ../../../debug; make
 ./test.sh -f unique/dnode/balance3.sim
 ./test.sh -f unique/dnode/balancex.sim
 ./test.sh -f unique/dnode/data1.sim
+./test.sh -f unique/dnode/m2.sim
+./test.sh -f unique/dnode/m3.sim
 ./test.sh -f unique/dnode/offline1.sim
 ./test.sh -f unique/dnode/offline2.sim
 ./test.sh -f unique/dnode/offline3.sim
diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt
index 3be8123a42..8e3988e79e 100644
--- a/tests/script/jenkins/unique.txt
+++ b/tests/script/jenkins/unique.txt
@@ -29,6 +29,8 @@ cd ../../../debug; make
 ./test.sh -f unique/dnode/balance2.sim
 ./test.sh -f unique/dnode/balance3.sim
 ./test.sh -f unique/dnode/balancex.sim
+./test.sh -f unique/dnode/m2.sim
+./test.sh -f unique/dnode/m3.sim
 ./test.sh -f unique/dnode/offline1.sim
 ./test.sh -f unique/dnode/offline2.sim
 ./test.sh -f unique/dnode/offline3.sim
diff --git a/tests/script/unique/dnode/m2.sim b/tests/script/unique/dnode/m2.sim
new file mode 100644
index 0000000000..5fdf3b7400
--- /dev/null
+++ b/tests/script/unique/dnode/m2.sim
@@ -0,0 +1,367 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4
+
+print ========== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql create dnode $hostname2
+sql create dnode $hostname3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+print ========== step2
+
+sql create database d1 replica 2  
+sql create table d1.t1 (t timestamp, i int) 
+sql insert into d1.t1 values(now+1s, 15) 
+sql insert into d1.t1 values(now+2s, 14)
+sql insert into d1.t1 values(now+3s, 13)
+sql insert into d1.t1 values(now+4s, 12)
+sql insert into d1.t1 values(now+5s, 11)
+
+sql create database d2 replica 2  
+sql create table d2.t2 (t timestamp, i int) 
+sql insert into d2.t2 values(now+1s, 25)
+sql insert into d2.t2 values(now+2s, 24)
+sql insert into d2.t2 values(now+3s, 23)
+sql insert into d2.t2 values(now+4s, 22)
+sql insert into d2.t2 values(now+5s, 21)
+
+sql create database d3 replica 2 
+sql create table d3.t3 (t timestamp, i int) 
+sql insert into d3.t3 values(now+1s, 35)
+sql insert into d3.t3 values(now+2s, 34)
+sql insert into d3.t3 values(now+3s, 33)
+sql insert into d3.t3 values(now+4s, 32)
+sql insert into d3.t3 values(now+5s, 31)
+
+sql create database d4 replica 2 
+sql create table d4.t4 (t timestamp, i int) 
+sql insert into d4.t4 values(now+1s, 45)
+sql insert into d4.t4 values(now+2s, 44)
+sql insert into d4.t4 values(now+3s, 43)
+sql insert into d4.t4 values(now+4s, 42)
+sql insert into d4.t4 values(now+5s, 41)
+
+print ========== step2.1
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+
+if $data2_1 != 0 then
+	return -1
+endi
+if $data2_2 != 4 then
+	return -1
+endi
+if $data2_3 != 4 then
+	return -1
+endi
+
+sql show d1.vgroups;
+print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d2.vgroups;
+print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d3.vgroups;
+print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d4.vgroups;
+print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+print ========== step3
+sql create dnode $hostname4
+system sh/exec.sh -n dnode4 -s start
+sql create dnode $hostname5
+system sh/exec.sh -n dnode5 -s start
+
+
+$x = 0
+show3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then 
+	  return -1
+	endi
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+print dnode5 openVnodes $data2_5
+
+if $data2_2 != 2 then
+	goto show3
+endi
+if $data2_3 != 2 then
+	goto show3
+endi
+if $data2_4 != 2 then
+	goto show3
+endi
+if $data2_5 != 2 then
+	goto show3
+endi
+
+sql show d1.vgroups;
+print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show3
+endi
+
+sql show d2.vgroups;
+print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show3
+endi
+
+sql show d3.vgroups;
+print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show3
+endi
+
+sql show d4.vgroups;
+print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show3
+endi
+
+print ========== step4
+sql drop dnode $hostname2
+
+$x = 0
+show4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then 
+	  return -1
+	endi
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+print dnode5 openVnodes $data2_5
+
+if $data2_2 != null then
+	goto show4
+endi
+
+sql show d1.vgroups;
+print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show4
+endi
+
+sql show d2.vgroups;
+print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show4
+endi
+
+sql show d3.vgroups;
+print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show4
+endi
+
+sql show d4.vgroups;
+print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show4
+endi
+
+sql reset query cache
+sleep 100
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+
+print ========== step5
+sql drop dnode $hostname3
+
+$x = 0
+show5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then 
+	  return -1
+	endi
+
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+print dnode5 openVnodes $data2_5
+
+if $data2_2 != null then
+	goto show5
+endi
+if $data2_3 != null then
+	goto show5
+endi
+if $data2_4 != 4 then
+	goto show5
+endi
+if $data2_5 != 4 then
+	goto show4
+endi
+
+sql show d1.vgroups;
+print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show5
+endi
+
+sql show d2.vgroups;
+print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show5
+endi
+
+sql show d3.vgroups;
+print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show5
+endi
+
+sql show d4.vgroups;
+print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data03 != 2 then
+	goto show5
+endi
+
+sql reset query cache
+sleep 100
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+
+print ========== step6
+sql select * from d1.t1 order by t desc
+print $data01 $data11 $data21 $data31 $data41 
+if $data01 != 11 then
+  return -1
+endi
+if $data11 != 12 then
+  return -1
+endi
+if $data21 != 13 then
+  return -1
+endi
+if $data31 != 14 then
+  return -1
+endi
+if $data41 != 15 then
+  return -1
+endi
+
+sql select * from d2.t2 order by t desc
+print $data01 $data11 $data21 $data31 $data41 
+if $data01 != 21 then
+  return -1
+endi
+if $data11 != 22 then
+  return -1
+endi
+if $data21 != 23 then
+  return -1
+endi
+if $data31 != 24 then
+  return -1
+endi
+if $data41 != 25 then
+  return -1
+endi
+
+sql select * from d3.t3 order by t desc
+print $data01 $data11 $data21 $data31 $data41  
+if $data01 != 31 then
+  return -1
+endi
+if $data11 != 32 then
+  return -1
+endi
+if $data21 != 33 then
+  return -1
+endi
+if $data31 != 34 then
+  return -1
+endi
+if $data41 != 35 then
+  return -1
+endi
+
+sql select * from d4.t4 order by t desc
+print $data01 $data11 $data21 $data31 $data41  
+if $data01 != 41 then
+  return -1
+endi
+if $data11 != 42 then
+  return -1
+endi
+if $data21 != 43 then
+  return -1
+endi
+if $data31 != 44 then
+  return -1
+endi
+if $data41 != 45 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
\ No newline at end of file
diff --git a/tests/script/unique/dnode/m3.sim b/tests/script/unique/dnode/m3.sim
new file mode 100644
index 0000000000..5850147d04
--- /dev/null
+++ b/tests/script/unique/dnode/m3.sim
@@ -0,0 +1,359 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4
+
+print ========== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql create dnode $hostname2
+sql create dnode $hostname3
+sql create dnode $hostname4
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+if $data4_4 != ready then
+  goto step1
+endi
+
+print ========== step2
+
+sql create database d1 replica 3  
+sql create table d1.t1 (t timestamp, i int) 
+sql insert into d1.t1 values(now+1s, 15) 
+sql insert into d1.t1 values(now+2s, 14)
+sql insert into d1.t1 values(now+3s, 13)
+sql insert into d1.t1 values(now+4s, 12)
+sql insert into d1.t1 values(now+5s, 11)
+
+sql create database d2 replica 3  
+sql create table d2.t2 (t timestamp, i int) 
+sql insert into d2.t2 values(now+1s, 25)
+sql insert into d2.t2 values(now+2s, 24)
+sql insert into d2.t2 values(now+3s, 23)
+sql insert into d2.t2 values(now+4s, 22)
+sql insert into d2.t2 values(now+5s, 21)
+
+sql create database d3 replica 3 
+sql create table d3.t3 (t timestamp, i int) 
+sql insert into d3.t3 values(now+1s, 35)
+sql insert into d3.t3 values(now+2s, 34)
+sql insert into d3.t3 values(now+3s, 33)
+sql insert into d3.t3 values(now+4s, 32)
+sql insert into d3.t3 values(now+5s, 31)
+
+sql create database d4 replica 3 
+sql create table d4.t4 (t timestamp, i int) 
+sql insert into d4.t4 values(now+1s, 45)
+sql insert into d4.t4 values(now+2s, 44)
+sql insert into d4.t4 values(now+3s, 43)
+sql insert into d4.t4 values(now+4s, 42)
+sql insert into d4.t4 values(now+5s, 41)
+
+print ========== step2.1
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+
+if $data2_1 != 0 then
+	return -1
+endi
+if $data2_2 != 4 then
+	return -1
+endi
+if $data2_3 != 4 then
+	return -1
+endi
+if $data2_4 != 4 then
+	return -1
+endi
+
+sql show d1.vgroups;
+print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 4 then
+	return -1
+endi
+
+sql show d2.vgroups;
+print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 3 then
+	return -1
+endi
+
+sql show d3.vgroups;
+print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 2 then
+	return -1
+endi
+
+sql show d4.vgroups;
+print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 4 then
+	return -1
+endi
+
+print ========== step3
+sql create dnode $hostname5
+system sh/exec.sh -n dnode5 -s start
+
+$x = 0
+show3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then 
+	  return -1
+	endi
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+print dnode5 openVnodes $data2_5
+
+if $data2_2 != 3 then
+	goto show3
+endi
+if $data2_3 != 3 then
+	goto show3
+endi
+if $data2_4 != 3 then
+	goto show3
+endi
+if $data2_5 != 3 then
+	goto show3
+endi
+
+sql show d1.vgroups;
+print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 5 then
+	return -1
+endi
+if $data03 != 3 then
+	goto show3
+endi
+
+sql show d2.vgroups;
+print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 5 then
+	return -1
+endi
+if $data03 != 3 then
+	goto show3
+endi
+
+sql show d3.vgroups;
+print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 2 then
+	return -1
+endi
+if $data03 != 3 then
+	goto show3
+endi
+
+sql show d4.vgroups;
+print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 4 then
+	return -1
+endi
+if $data03 != 3 then
+	goto show3
+endi
+
+print ========== step4
+sql drop dnode $hostname2
+
+$x = 0
+show4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then 
+	  return -1
+	endi
+
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+print dnode5 openVnodes $data2_5
+
+if $data2_2 != null then
+	goto show4
+endi
+if $data2_3 != 4 then
+	goto show4
+endi
+if $data2_4 != 4 then
+	goto show4
+endi
+if $data2_5 != 4 then
+	goto show4
+endi
+
+sql show d1.vgroups;
+print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 5 then
+	return -1
+endi
+if $data03 != 3 then
+	goto show4
+endi
+
+sql show d2.vgroups;
+print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 3 then
+	return -1
+endi
+if $data03 != 3 then
+	goto show4
+endi
+
+sql show d3.vgroups;
+print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 4 then
+	return -1
+endi
+if $data03 != 3 then
+	goto show4
+endi
+
+sql show d4.vgroups;
+print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $data04 != 4 then
+	return -1
+endi
+if $data03 != 3 then
+	goto show4
+endi
+
+sql reset query cache
+sleep 100
+
+print ========== step5
+sql select * from d1.t1 order by t desc
+print $data01 $data11 $data21 $data31 $data41 
+if $data01 != 11 then
+  return -1
+endi
+if $data11 != 12 then
+  return -1
+endi
+if $data21 != 13 then
+  return -1
+endi
+if $data31 != 14 then
+  return -1
+endi
+if $data41 != 15 then
+  return -1
+endi
+
+sql select * from d2.t2 order by t desc
+print $data01 $data11 $data21 $data31 $data41 
+if $data01 != 21 then
+  return -1
+endi
+if $data11 != 22 then
+  return -1
+endi
+if $data21 != 23 then
+  return -1
+endi
+if $data31 != 24 then
+  return -1
+endi
+if $data41 != 25 then
+  return -1
+endi
+
+sql select * from d3.t3 order by t desc
+print $data01 $data11 $data21 $data31 $data41  
+if $data01 != 31 then
+  return -1
+endi
+if $data11 != 32 then
+  return -1
+endi
+if $data21 != 33 then
+  return -1
+endi
+if $data31 != 34 then
+  return -1
+endi
+if $data41 != 35 then
+  return -1
+endi
+
+sql select * from d4.t4 order by t desc
+print $data01 $data11 $data21 $data31 $data41  
+if $data01 != 41 then
+  return -1
+endi
+if $data11 != 42 then
+  return -1
+endi
+if $data21 != 43 then
+  return -1
+endi
+if $data31 != 44 then
+  return -1
+endi
+if $data41 != 45 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
\ No newline at end of file

From 76f75ba806f53bb39b0f17a2d3f35f4023102bfb Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Sun, 27 Dec 2020 06:54:04 +0800
Subject: [PATCH 07/13] rpc: refactor macro definitions

---
 src/rpc/inc/rpcHead.h | 5 -----
 src/rpc/src/rpcMain.c | 6 ++++++
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/rpc/inc/rpcHead.h b/src/rpc/inc/rpcHead.h
index dc0cac0143..5b401ac54b 100644
--- a/src/rpc/inc/rpcHead.h
+++ b/src/rpc/inc/rpcHead.h
@@ -20,11 +20,6 @@
 extern "C" {
 #endif
 
-// server:0 client:1  tcp:2 udp:0
-#define RPC_CONN_UDPS   0
-#define RPC_CONN_UDPC   1
-#define RPC_CONN_TCPS   2
-#define RPC_CONN_TCPC   3
 #define RPC_CONN_TCP    2
 
 extern int tsRpcOverhead;
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index deaf30f42e..22838a3f7b 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -137,6 +137,12 @@ static int     tsRpcRefId = -1;
 static int32_t tsRpcNum = 0;
 //static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
 
+// server:0 client:1  tcp:2 udp:0
+#define RPC_CONN_UDPS   0
+#define RPC_CONN_UDPC   1
+#define RPC_CONN_TCPS   2
+#define RPC_CONN_TCPC   3
+
 void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = {
     taosInitUdpConnection,
     taosInitUdpConnection,

From 28efec5195b83f857730942bf2740ef9203cab92 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 27 Dec 2020 21:00:39 +0800
Subject: [PATCH 08/13] scripts

---
 .../migrate/mn2_vn2_repl2_rmMnodeDir.sim      | 36 +++++++++++--------
 .../migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim | 30 ++++++++--------
 .../migrate/mn2_vn2_repl2_rmVnodeDir.sim      | 30 ++++++++--------
 3 files changed, 54 insertions(+), 42 deletions(-)

diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim
index 12f0013191..9b02933cbf 100644
--- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim
+++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim
@@ -194,36 +194,35 @@ print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $dat
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
 print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4  $data9_4
-$d2v2status = $data5_4
-$d2v3status = $data5_2
-$d2v4status = $data5_3
 
-$d1v2status = $data7_4
-$d1v3status = $data7_2
-$d1v4status = $data7_3
-
-if $d2v2status != master then
+if $data5_4 != master then
+  print $data5_4
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d2v3status != master then
+if $data5_3 != slave then
+print $data5_2
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d2v4status != master then
+if $data5_2 != master then
+print $data5_3
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
 
-if $d1v2status != slave then
+if $data7_4 != slave then
+  print $data7_4
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d1v3status != slave then
+if $data7_3 != master then
+  print $data7_3
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d1v4status != slave then
+if $data7_2 != slave then
+  print $data7_2
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
@@ -264,4 +263,13 @@ sql select count(*) from $stb
 print data00 $data00
 if $data00 != $totalRows then
 	return -1
-endi
\ No newline at end of file
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
+system sh/exec.sh -n dnode6 -s stop  -x SIGINT
+system sh/exec.sh -n dnode7 -s stop  -x SIGINT
+system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file
diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim
index 29da1fd773..90183949e7 100644
--- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim
+++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim
@@ -196,36 +196,29 @@ print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $dat
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
 print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4  $data9_4
-$d2v2status = $data5_4
-$d2v3status = $data5_2
-$d2v4status = $data5_3
 
-$d1v2status = $data7_4
-$d1v3status = $data7_2
-$d1v4status = $data7_3
-
-if $d2v2status != master then
+if $data5_4 != master then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d2v3status != master then
+if $data5_3 != slave then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d2v4status != master then
+if $data5_2 != master then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
 
-if $d1v2status != slave then
+if $data7_4 != slave then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d1v3status != slave then
+if $data7_3 != master then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d1v4status != slave then
+if $data7_2 != slave then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
@@ -266,4 +259,13 @@ sql select count(*) from $stb
 print data00 $data00
 if $data00 != $totalRows then
 	return -1
-endi
\ No newline at end of file
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
+system sh/exec.sh -n dnode6 -s stop  -x SIGINT
+system sh/exec.sh -n dnode7 -s stop  -x SIGINT
+system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file
diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim
index 8a1132de2e..02e2cd02e1 100644
--- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim
+++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim
@@ -194,36 +194,29 @@ print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $dat
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
 print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4  $data9_4
-$d2v2status = $data5_4
-$d2v3status = $data5_2
-$d2v4status = $data5_3
 
-$d1v2status = $data7_4
-$d1v3status = $data7_2
-$d1v4status = $data7_3
-
-if $d2v2status != master then
+if $data5_4 != master then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d2v3status != master then
+if $data5_3 != slave then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d2v4status != master then
+if $data5_2 != master then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
 
-if $d1v2status != slave then
+if $data7_4 != slave then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d1v3status != slave then
+if $data7_3 != master then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
-if $d1v4status != slave then
+if $data7_2 != slave then
   sleep 2000
   goto wait_dnode1_vgroup_slave
 endi
@@ -264,4 +257,13 @@ sql select count(*) from $stb
 print data00 $data00
 if $data00 != $totalRows then
 	return -1
-endi
\ No newline at end of file
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
+system sh/exec.sh -n dnode6 -s stop  -x SIGINT
+system sh/exec.sh -n dnode7 -s stop  -x SIGINT
+system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

From 2be8db9d421301f0510760f76d1536a7123a99f6 Mon Sep 17 00:00:00 2001
From: root <root@slave3.jzpmcbu051oebgsmotycmb3lpe.xx.internal.cloudapp.net>
Date: Mon, 28 Dec 2020 02:10:29 +0800
Subject: [PATCH 09/13] rpc/log: refactor invalid client version log

---
 src/rpc/src/rpcMain.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index deaf30f42e..bdefecfba3 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -953,7 +953,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
   }
 
   if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) {
-    tDebug("%s sid:%d, invalid client version:%d", pRpc->label, sid, htonl(pHead->msgVer));
+    tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, taosMsg[pHead->msgType]);
     terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL;
   }
 

From 4e8a37ed982fcb129d9f338de06c6772c8f49871 Mon Sep 17 00:00:00 2001
From: stephenkgu <stephenk@gmx.us>
Date: Mon, 28 Dec 2020 02:45:11 +0800
Subject: [PATCH 10/13] rpcMain/fix: fix version checking with rpcSendReqHead

---
 src/rpc/src/rpcMain.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index a9346f55d3..fbc59d8ddf 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -1216,6 +1216,7 @@ static void rpcSendReqHead(SRpcConn *pConn) {
   pHead = (SRpcHead *)msg;
   pHead->version = 1;
   pHead->msgType = pConn->outType;
+  pHead->msgVer = htonl(tsVersion >> 8);
   pHead->spi = pConn->spi;
   pHead->encrypt = 0;
   pHead->tranId = pConn->outTranId;

From 11fa2a83a40122560789c86639f8a82232f05861 Mon Sep 17 00:00:00 2001
From: liuyq-617 <yqliu@taosdata.com>
Date: Mon, 28 Dec 2020 13:05:58 +0800
Subject: [PATCH 11/13] <test>remove test case temporarily

---
 tests/pytest/pytest_1.sh         | 4 ++--
 tests/script/jenkins/basic_3.txt | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh
index 05d6ec1cec..bcba331a89 100755
--- a/tests/pytest/pytest_1.sh
+++ b/tests/pytest/pytest_1.sh
@@ -221,8 +221,8 @@ python3 test.py -f tools/taosdumpTest.py
 python3 test.py -f tools/lowaTest.py
 
 # subscribe
-python3 test.py -f subscribe/singlemeter.py
+#python3 test.py -f subscribe/singlemeter.py
 #python3 test.py -f subscribe/stability.py
-python3 test.py -f subscribe/supertable.py
+#python3 test.py -f subscribe/supertable.py
 
 
diff --git a/tests/script/jenkins/basic_3.txt b/tests/script/jenkins/basic_3.txt
index 23d1e0a17d..83b10a371c 100644
--- a/tests/script/jenkins/basic_3.txt
+++ b/tests/script/jenkins/basic_3.txt
@@ -70,7 +70,7 @@
 
 ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim
 ./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim
-#./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim
+./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim
 ./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim
 ./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim
 ./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim

From 852f48912ffb05465200e697fb311d8b986e9c40 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Mon, 28 Dec 2020 14:04:13 +0800
Subject: [PATCH 12/13] scripts

---
 tests/script/unique/cluster/balance1.sim | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/script/unique/cluster/balance1.sim b/tests/script/unique/cluster/balance1.sim
index c82e96db50..c98687a81c 100644
--- a/tests/script/unique/cluster/balance1.sim
+++ b/tests/script/unique/cluster/balance1.sim
@@ -340,7 +340,7 @@ print dnode1 $data4_1
 print dnode2 $data4_2
 print dnode3 $data4_3
 print dnode4 $data4_4
-print dnode4 $data4_5
+print dnode5 $data4_5
 
 if $data4_5 != ready then
   goto step13
@@ -380,13 +380,13 @@ $dnode5Vnodes = $data2_5
 print dnode5 $dnode2Vnodes
 
 if $dnode1Vnodes != 2 then
-  return -1
+  goto step13
 endi
 if $dnode4Vnodes != 2 then
-  return -1
+  goto step13
 endi
 if $dnode5Vnodes != 2 then
-  return -1
+  goto step13
 endi
 
 print ============================== step14

From b451e6859de7d18fa6a05179ecc8baadf3c3f68e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 28 Dec 2020 14:34:34 +0800
Subject: [PATCH 13/13] [TD-2587]<fix>: fix bugs introduced by refactor.

---
 src/client/src/tscSub.c   |  6 +++++-
 src/query/src/qExecutor.c | 11 +++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index 52b74f7502..7f0b174ad3 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -69,14 +69,17 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
 }
 
 void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
-  if( sub == NULL)
+  if( sub == NULL) {
     return;
+  }
+
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = ts};
   SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
   if (p != NULL) {
     p->key = ts;
+    tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key);
   }
 }
 
@@ -502,6 +505,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
   if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single tabel subscription
     pQueryInfo->window.skey = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, 0))->key;
+    tscDebug("subscribe:%s set subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey);
   }
 
   if (pSub->pTimer == NULL) {
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index fb6f5f2044..fa9fc6d1f3 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -1722,6 +1722,10 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
     blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock);
   }
 
+  // update the lastkey of current table for projection/aggregation query
+  TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey;
+  pTableQueryInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
+
   // interval query with limit applied
   int32_t numOfRes = 0;
   if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) {
@@ -4299,7 +4303,9 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
   *(int32_t*)data = htonl(numOfTables);
   data += sizeof(int32_t);
 
+  int32_t total = 0;
   STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL);
+
   while(item) {
     STableIdInfo* pDst = (STableIdInfo*)data;
     pDst->uid = htobe64(item->uid);
@@ -4307,9 +4313,14 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
     pDst->key = htobe64(item->key);
 
     data += sizeof(STableIdInfo);
+    total++;
+
+    qDebug("QInfo:%p set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo, item->tid, item->uid, item->key);
     item = taosHashIterate(pQInfo->arrTableIdInfo, item);
   }
 
+  qDebug("QInfo:%p set %d subscribe info", pQInfo, total);
+
   // Check if query is completed or not for stable query or normal table query respectively.
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
     if (pQInfo->runtimeEnv.stableQuery) {