From 192c84897d2eae35b58a38d8813142401adae179 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 20 Jun 2020 12:55:11 +0800
Subject: [PATCH 01/23] [td-225] update output log

---
 src/client/src/tscParseInsert.c | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index b03972bbfa..e65acc2483 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -1048,7 +1048,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
     str = pCmd->curSql;
   }
   
-  tscTrace("%p create data block list for submit data:%p, curSql:%p, pTableList:%p", pSql, pSql->cmd.pDataBlocks, pCmd->curSql, pCmd->pTableList);
+  tscTrace("%p create data block list for submit data:%p, pTableList:%p", pSql, pCmd->pDataBlocks, pCmd->pTableList);
 
   while (1) {
     int32_t   index = 0;
@@ -1088,22 +1088,16 @@ int tsParseInsertSql(SSqlObj *pSql) {
       goto _error;
     }
 
-    ptrdiff_t pos = pCmd->curSql - pSql->sqlstr;
-    
     if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) {
       /*
-       * For async insert, after get the table meta from server, the sql string will not be
-       * parsed using the new table meta to avoid the overhead cause by get table meta data information.
-       * And during the getMeterMetaCallback function, the sql string will be parsed from the
-       * interrupted position.
+       * After retrieving the table meta from server, the sql string will be parsed from the paused position.
+       * And during the getTableMetaCallback function, the sql string will be parsed from the paused position.
        */
       if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) {
-        tscTrace("%p waiting for get table meta during insert, then resume from offset: %" PRId64 ", %s", pSql,
-            pos, pCmd->curSql);
         return code;
       }
       
-      tscError("%p async insert parse error, code:%d, reason:%s", pSql, code, tstrerror(code));
+      tscError("%p async insert parse error, code:%s", pSql, tstrerror(code));
       pCmd->curSql = NULL;
       goto _error;
     }
@@ -1317,11 +1311,11 @@ int tsInsertInitialCheck(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 }
 
-int tsParseSql(SSqlObj *pSql, bool initialParse) {
+int tsParseSql(SSqlObj *pSql, bool initial) {
   int32_t ret = TSDB_CODE_SUCCESS;
   SSqlCmd* pCmd = &pSql->cmd;
 
-  if (!pCmd->parseFinished) {
+  if ((!pCmd->parseFinished) && (!initial)) {
     tscTrace("%p resume to parse sql: %s", pSql, pCmd->curSql);
   }
   
@@ -1330,12 +1324,12 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
      * Set the fp before parse the sql string, in case of getTableMeta failed, in which
      * the error handle callback function can rightfully restore the user-defined callback function (fp).
      */
-    if (initialParse && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
+    if (initial && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
       pSql->fetchFp = pSql->fp;
       pSql->fp = (void(*)())tscHandleMultivnodeInsert;
     }
     
-    if (initialParse && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) {
+    if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) {
       return ret;
     }
     

From 15636122903bdb4449bdde8837e71b8b83c36539 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 20 Jun 2020 13:09:20 +0800
Subject: [PATCH 02/23] [td-703]

---
 src/client/src/tscSQLParser.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 87af5fb5f7..f241e317cd 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -5755,6 +5755,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
   const char* msg7 = "illegal number of tables in from clause";
   const char* msg8 = "too many columns in selection clause";
   const char* msg9 = "TWA query requires both the start and end time";
+  const char* msg10= "too many tables in from clause";
 
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -5790,6 +5791,10 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
 
   pQueryInfo->command = TSDB_SQL_SELECT;
 
+  if (pQuerySql->from->nExpr > 2) {
+    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10);
+  }
+
   // set all query tables, which are maybe more than one.
   for (int32_t i = 0; i < pQuerySql->from->nExpr; ++i) {
     tVariant* pTableItem = &pQuerySql->from->a[i].pVar;

From 9d82654dfcf08aabe7f889b5743bccdbdbff046a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 20 Jun 2020 15:17:36 +0800
Subject: [PATCH 03/23] [td-225]

---
 src/client/inc/tscUtil.h     |  2 ++
 src/client/inc/tsclient.h    |  7 ++--
 src/client/src/tscServer.c   | 27 ++++++++-------
 src/client/src/tscSql.c      |  3 +-
 src/client/src/tscSubquery.c | 64 +++++++++++-------------------------
 src/client/src/tscUtil.c     | 42 ++++++++++++++++++-----
 6 files changed, 74 insertions(+), 71 deletions(-)

diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index d6562f008d..f954667287 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -247,6 +247,8 @@ void tscDoQuery(SSqlObj* pSql);
  * @param pPrevSql
  * @return
  */
+SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd);
+
 SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql);
 void     addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex);
 
diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index 90364987bb..27caa33ff1 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -296,7 +296,7 @@ typedef struct STscObj {
 typedef struct SSqlObj {
   void            *signature;
   STscObj         *pTscObj;
-  void            *SRpcReqContext;
+  void            *pRpcCtx;
   void            (*fp)();
   void            (*fetchFp)();
   void            *param;
@@ -308,8 +308,7 @@ typedef struct SSqlObj {
   char             retry;
   char             maxRetry;
   SRpcIpSet        ipList;
-  char             freed : 4;
-  char             listed : 4;
+  char             listed;
   tsem_t           rspSem;
   SSqlCmd          cmd;
   SSqlRes          res;
@@ -349,7 +348,7 @@ typedef struct SSqlStream {
 int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn);
 void    tscInitMsgsFp();
 
-int tsParseSql(SSqlObj *pSql, bool initialParse);
+int tsParseSql(SSqlObj *pSql, bool initial);
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet);
 int  tscProcessSql(SSqlObj *pSql);
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 6e3602b69d..b2e0c0107e 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -197,29 +197,34 @@ int tscSendMsgToServer(SSqlObj *pSql) {
       .code    = 0
   };
 
-  pSql->SRpcReqContext = rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
+  pSql->pRpcCtx = rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
   return TSDB_CODE_SUCCESS;
 }
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
   SSqlObj *pSql = (SSqlObj *)rpcMsg->handle;
-  if (pSql == NULL) {
+  if (pSql == NULL || pSql->signature != pSql) {
     tscError("%p sql is already released", pSql->signature);
     return;
   }
 
-  if (pSql->signature != pSql) {
-    tscError("%p sql is already released, signature:%p", pSql, pSql->signature);
+  STscObj *pObj = pSql->pTscObj;
+  SSqlRes *pRes = &pSql->res;
+  SSqlCmd *pCmd = &pSql->cmd;
+
+  if (pObj->signature != pObj) {
+    tscTrace("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
+
+    tscFreeSqlObj(pSql);
+    rpcFreeCont(rpcMsg->pCont);
     return;
   }
 
-  SSqlRes *pRes = &pSql->res;
-  SSqlCmd *pCmd = &pSql->cmd;
-  STscObj *pObj = pSql->pTscObj;
-
-  if (pObj->signature != pObj || pSql->freed == 1) {
-    tscTrace("%p sqlObj needs to be released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed,
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+  if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) {
+    tscTrace("%p sqlObj needs to be released or DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command,
              pObj, pObj->signature);
+
     tscFreeSqlObj(pSql);
     rpcFreeCont(rpcMsg->pCont);
     return;
@@ -421,7 +426,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
      * sub-queries not correctly released and master sql object of super table query reaches an abnormal state.
      */
     pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-    rpcCancelRequest(pSql->pSubs[i]->SRpcReqContext);
+    rpcCancelRequest(pSql->pSubs[i]->pRpcCtx);
   }
 
   /*
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 47a602a6fb..51b49304a8 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -489,7 +489,6 @@ static bool tscFreeQhandleInVnode(SSqlObj* pSql) {
 
     pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
     tscTrace("%p send msg to dnode to free qhandle ASAP, command:%s", pSql, sqlCmd[pCmd->command]);
-    pSql->freed = 1;
     tscProcessSql(pSql);
 
     // in case of sync model query, waits for response and then goes on
@@ -631,7 +630,7 @@ void taos_stop_query(TAOS_RES *res) {
     return;
   }
 
-  rpcCancelRequest(pSql->SRpcReqContext);
+  rpcCancelRequest(pSql->pRpcCtx);
   tscTrace("%p query is cancelled", res);
 }
 
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 6c6284b4a6..87c75bd7e6 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -1853,11 +1853,11 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
   SSubqueryState* pState = pSupporter->pState;
 
   // record the total inserted rows
-  if (numOfRows > 0 && tres != pParentObj) {
-      pParentObj->res.numOfRows += numOfRows;
+  if (numOfRows > 0) {
+    pParentObj->res.numOfRows += numOfRows;
   }
 
-  if (taos_errno(tres) != 0) {
+  if (taos_errno(tres) != TSDB_CODE_SUCCESS) {
     SSqlObj* pSql = (SSqlObj*) tres;
     assert(pSql != NULL && pSql->res.code == numOfRows);
     
@@ -1865,13 +1865,9 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
   }
 
   // it is not the initial sqlObj, free it
-  if (tres != pParentObj) {
-    taos_free_result(tres);
-  } else {
-    assert(pParentObj->pSubs[0] == tres);
-  }
-
+  taos_free_result(tres);
   tfree(pSupporter);
+
   if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
     return;
   }
@@ -1904,30 +1900,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   pState->numOfRemain = pSql->numOfSubs;
  
   pRes->code = TSDB_CODE_SUCCESS;
+  int32_t numOfSub = 0;
 
-  SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
-  pSupporter->pSql   = pSql;
-  pSupporter->pState = pState;
-
-  pSql->fp       = multiVnodeInsertFinalize;
-  pSql->param    = pSupporter;
-  pSql->pSubs[0] = pSql;  // the first sub insert points back to itself
-  tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pSql, 0);
-
-  int32_t numOfSub = 1;
-  int32_t code = tscCopyDataBlockToPayload(pSql, pDataBlocks->pData[0]);
-  if (code != TSDB_CODE_SUCCESS) {
-    tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%d", pSql, 0,
-             pDataBlocks->nSize, code);
-    goto _error;
-  }
-
-  for (; numOfSub < pSql->numOfSubs; ++numOfSub) {
-    SInsertSupporter* pSupporter1 = calloc(1, sizeof(SInsertSupporter));
-    pSupporter1->pSql = pSql;
-    pSupporter1->pState = pState;
+  while(numOfSub < pSql->numOfSubs) {
+    SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
+    pSupporter->pSql = pSql;
+    pSupporter->pState = pState;
     
-    SSqlObj *pNew = createSubqueryObj(pSql, 0, multiVnodeInsertFinalize, pSupporter1, TSDB_SQL_INSERT, NULL);
+    SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT);//createSubqueryObj(pSql, 0, multiVnodeInsertFinalize, pSupporter1, TSDB_SQL_INSERT, NULL);
     if (pNew == NULL) {
       tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, numOfSub, strerror(errno));
       goto _error;
@@ -1940,13 +1920,13 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
     pNew->fetchFp = pNew->fp;
     pSql->pSubs[numOfSub] = pNew;
 
-    code = tscCopyDataBlockToPayload(pNew, pDataBlocks->pData[numOfSub]);
-    if (code != TSDB_CODE_SUCCESS) {
-      tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%d", pSql, numOfSub,
-               pDataBlocks->nSize, code);
-      goto _error;
-    } else {
+    pRes->code = tscCopyDataBlockToPayload(pNew, pDataBlocks->pData[numOfSub++]);
+    if (pRes->code == TSDB_CODE_SUCCESS) {
       tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub);
+    } else {
+      tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub,
+               pDataBlocks->nSize, tstrerror(pRes->code));
+      goto _error;
     }
   }
   
@@ -1966,18 +1946,12 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 
   _error:
-  // restore the udf fp
-  pSql->fp = pSql->fetchFp;
-  pSql->pSubs[0] = NULL;
-
-  tfree(pState);
-  tfree(pSql->param);
-
-  for(int32_t j = 1; j < numOfSub; ++j) {
+  for(int32_t j = 0; j < numOfSub; ++j) {
     tfree(pSql->pSubs[j]->param);
     taos_free_result(pSql->pSubs[j]);
   }
 
+  tfree(pState);
   return TSDB_CODE_TSC_OUT_OF_MEMORY;
 }
 
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 110a435e57..521f08ff8a 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -364,7 +364,6 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   tscFreeSqlResult(pSql);
   
   tfree(pSql->pSubs);
-  pSql->freed = 0;
   pSql->numOfSubs = 0;
   
   tscResetSqlCmdObj(pCmd);
@@ -1653,6 +1652,38 @@ void tscResetForNextRetrieve(SSqlRes* pRes) {
   pRes->numOfRows = 0;
 }
 
+SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) {
+  SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
+  if (pNew == NULL) {
+    tscError("%p new subquery failed, tableIndex:%d", pSql, 0);
+    return NULL;
+  }
+
+  pNew->pTscObj = pSql->pTscObj;
+  pNew->signature = pNew;
+
+  SSqlCmd* pCmd = &pNew->cmd;
+  pCmd->command = cmd;
+
+  if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
+    tscFreeSqlObj(pNew);
+    return NULL;
+  }
+
+  pNew->fp = fp;
+  pNew->param = param;
+  pNew->maxRetry = TSDB_MAX_REPLICA_NUM;
+
+  SQueryInfo* pQueryInfo = NULL;
+  tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo);
+
+  assert(pSql->cmd.clauseIndex == 0);
+  STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
+
+  tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL);
+  return pNew;
+}
+
 SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) {
   SSqlCmd* pCmd = &pSql->cmd;
   SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
@@ -1666,14 +1697,6 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   pNew->pTscObj = pSql->pTscObj;
   pNew->signature = pNew;
 
-  pNew->sqlstr = strdup(pSql->sqlstr);
-  if (pNew->sqlstr == NULL) {
-    tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
-
-    free(pNew);
-    return NULL;
-  }
-
   SSqlCmd* pnCmd = &pNew->cmd;
   memcpy(pnCmd, pCmd, sizeof(SSqlCmd));
   
@@ -1795,6 +1818,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   if (pPrevSql == NULL) {
     STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta);  // get by name may failed due to the cache cleanup
     assert(pTableMeta != NULL);
+
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
   } else {  // transfer the ownership of pTableMeta to the newly create sql object.
     STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);

From 4c821664e46265c46bb913136214e9b4ff922669 Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Sat, 20 Jun 2020 15:18:38 +0800
Subject: [PATCH 04/23] [TD-533]

---
 src/mnode/src/mnodeVgroup.c                   |  34 +-
 tests/script/jenkins/basic.txt                |   4 +-
 .../arbitrator/dn3_mn1_r2_vnode_delDir.sim    | 357 ++++++++++++++++++
 .../arbitrator/dn3_mn1_r3_vnode_delDir.sim    | 357 ++++++++++++++++++
 .../arbitrator/dn3_mn1_vnode_delDir.sim       | 249 ++++++++++--
 tests/script/unique/arbitrator/testSuite.sim  |   6 +-
 6 files changed, 970 insertions(+), 37 deletions(-)
 create mode 100644 tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
 create mode 100644 tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim

diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 8df28aeec6..9553ade3aa 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -258,7 +258,37 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
   mnodeSendCreateVgroupMsg(pVgroup, NULL);
 }
 
-void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {}
+/*
+  Traverse all vgroups on mnode, if there no such vgId on a dnode, so send msg to this dnode for re-creating this vgId/vnode 
+*/
+void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {
+  SVnodeLoad *pNextV = NULL;
+
+  void *pIter = NULL;
+  while (1) {
+    SVgObj *pVgroup;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    pNextV = pVloads;
+    int32_t i;
+    for (i = 0; i < openVnodes; ++i) {
+      if ((pVgroup->vnodeGid[i].pDnode == pDnode) && (pVgroup->vgId == pNextV->vgId)) {
+        break;
+      }
+      pNextV++;
+    }
+
+    if (i == openVnodes) {
+      mnodeSendCreateVgroupMsg(pVgroup, NULL);
+    }
+    
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+  return;
+}
 
 void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
   bool dnodeExist = false;
@@ -861,4 +891,4 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) {
   sdbFreeIter(pIter);
 
   mPrint("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups);
-}
\ No newline at end of file
+}
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 79cb1551e1..9d02b1041b 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -337,7 +337,9 @@ cd ../../../debug; make
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
-#./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
 ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
 ./test.sh -f unique/arbitrator/insert_duplicationTs.sim
diff --git a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
new file mode 100644
index 0000000000..807709b675
--- /dev/null
+++ b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
@@ -0,0 +1,357 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+
+$totalTableNum = 10
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 2 maxTables $totalTableNum
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 int) tags(t1 int)
+$rowNum = 100
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1420041600000
+
+$i = 0
+while $i < $tblNum
+  $tb = tb . $i
+  sql create table $tb using $stb tags( $i )
+ 
+  $x = 0
+  while $x < $rowNum
+    $ts = $tsStart + $x
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x ) 
+    $x = $x + 60
+  endw 
+  $totalRows = $totalRows + $x
+  print       info: inserted $x rows into $tb and totalRows: $totalRows
+  $i = $i + 1
+endw 
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step3: stop dnode3, and remove its vnodeX subdirector 
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode3_offline_0
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode3Status != offline then
+  sleep 2000
+  goto wait_dnode3_offline_0
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode3_vgroup_offline
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+if $dnode3Vtatus != offline then
+  sleep 2000
+  goto wait_dnode3_vgroup_offline
+endi
+if $dnode2Vtatus != master then
+  sleep 2000
+  goto wait_dnode3_vgroup_offline
+endi
+
+system rm -rf ../../../sim/dnode3/data
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step4: restart dnode3, waiting sync end 
+system sh/exec.sh -n dnode3 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode3_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode3_reready
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode3_vgroup_slave
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+print dnode2Vtatus: $dnode3Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode3Vtatus != slave then
+  sleep 2000
+  goto wait_dnode3_vgroup_slave
+endi
+if $dnode2Vtatus != master then
+  sleep 2000
+  goto wait_dnode3_vgroup_slave
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: stop dnode2, and remove its vnode
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data
+#system rm -rf ../../../sim/dnode2/data/vnode/*
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+
+print ============== step6: restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+
+$loopCnt = 0
+wait_dnode2_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode3Vtatus != master then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+if $dnode2Vtatus != slave then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
\ No newline at end of file
diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
new file mode 100644
index 0000000000..95480e9d96
--- /dev/null
+++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
@@ -0,0 +1,357 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+sql create dnode $hostname4
+sleep 3000
+
+$totalTableNum = 10
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 3 maxTables $totalTableNum
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 int) tags(t1 int)
+$rowNum = 100
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1420041600000
+
+$i = 0
+while $i < $tblNum
+  $tb = tb . $i
+  sql create table $tb using $stb tags( $i )
+ 
+  $x = 0
+  while $x < $rowNum
+    $ts = $tsStart + $x
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x ) 
+    $x = $x + 60
+  endw 
+  $totalRows = $totalRows + $x
+  print       info: inserted $x rows into $tb and totalRows: $totalRows
+  $i = $i + 1
+endw 
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step3: stop dnode4, and remove its vnodeX subdirector 
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode4_offline_0
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode4Status != offline then
+  sleep 2000
+  goto wait_dnode4_offline_0
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode4_vgroup_offline
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+if $dnode4Vtatus != offline then
+  sleep 2000
+  goto wait_dnode4_vgroup_offline
+endi
+if $dnode3Vtatus != master then
+  sleep 2000
+  goto wait_dnode4_vgroup_offline
+endi
+
+system rm -rf ../../../sim/dnode4/data
+#system rm -rf ../../../sim/dnode4/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step4: restart dnode4, waiting sync end 
+system sh/exec.sh -n dnode4 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode4_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode4_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode4_reready
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode4_vgroup_slave
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode4Vtatus != slave then
+  sleep 2000
+  goto wait_dnode4_vgroup_slave
+endi
+if $dnode3Vtatus != master then
+  sleep 2000
+  goto wait_dnode4_vgroup_slave
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: stop dnode3/dnode2, and remove its vnode
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode23_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode23_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode23_offline
+endi
+if $dnode3Status != offline then
+  sleep 2000
+  goto wait_dnode23_offline
+endi
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode23_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data
+system rm -rf ../../../sim/dnode3/data
+#system rm -rf ../../../sim/dnode2/data/vnode/*
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+print ============== step6: restart dnode2/dnode3, and check rows
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode23_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode23_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode23_reready
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode23_reready
+endi
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode23_reready
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode4_vgroup_master
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode4Vtatus != master then
+  sleep 2000
+  goto wait_dnode4_vgroup_master
+endi
+if $dnode3Vtatus != slave then
+  sleep 2000
+  goto wait_dnode4_vgroup_master
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim
index 82f2aad07b..cf20122d4c 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim
@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -46,7 +47,7 @@ system sh/exec.sh -n dnode1 -s start
 sleep 3000
 sql connect
 
-print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
 system sh/exec.sh -n dnode4 -s start
@@ -96,7 +97,14 @@ endi
 print ============== step3: stop dnode4, and remove its vnodeX subdirector 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -105,21 +113,25 @@ endi
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode3Status = $data4_3  
 $dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
 if $dnode4Status != offline then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
 
+
+$loopCnt = 0
 wait_dnode4_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -141,14 +153,19 @@ if $dnode3Vtatus != master then
   goto wait_dnode4_vgroup_offline
 endi
 
-
 system rm -rf ../../../sim/dnode4/data/vnode/*
-sleep 1000
 
 print ============== step4: restart dnode4, waiting sync end 
 system sh/exec.sh -n dnode4 -s start
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode4_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -157,21 +174,24 @@ endi
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode3Status = $data4_3  
 $dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
 if $dnode4Status != ready then
   sleep 2000
   goto wait_dnode4_reready
 endi
 
+$loopCnt = 0
 wait_dnode4_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -195,47 +215,87 @@ if $dnode3Vtatus != master then
   goto wait_dnode4_vgroup_slave
 endi
 
-print ============== step5: stop dnode3/dnode2, and check rows
-system sh/exec.sh -n dnode2 -s stop
-system sh/exec.sh -n dnode3 -s stop
+print ============== step5: stop dnode3, and remove its vnodeX subdirector 
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
 sleep $sleepTimer
 
-wait_dnode23_offline:
+$loopCnt = 0
+wait_dnode3_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000
-  goto wait_dnode23_offline
+  goto wait_dnode3_offline
 endi
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
 print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
+$dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode3Status = $data4_3  
 $dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
-if $dnode2Status != offline then
+if $dnode2Status != ready then
   sleep 2000
-  goto wait_dnode23_offline
+  goto wait_dnode3_offline
 endi
 if $dnode3Status != offline then
   sleep 2000
-  goto wait_dnode23_offline
+  goto wait_dnode3_offline
 endi
 if $dnode4Status != ready then
   sleep 2000
-  goto wait_dnode23_offline
+  goto wait_dnode3_offline
+endi
+
+system rm -rf ../../../sim/dnode3/data/vnode/*
+
+print ============== step6: restart dnode3, and check rows
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode3_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode3_reready
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
 endi
 
-wait_dnode4_vgroup_master:
 sql show vgroups
 if $rows != 1 then
   sleep 2000
-  goto wait_dnode4_vgroup_master
+  goto wait_dnode3_vgroup_slave
 endi
 print show vgroups:
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
@@ -248,11 +308,11 @@ print dnode4Vtatus: $dnode4Vtatus
 print dnode3Vtatus: $dnode3Vtatus 
 if $dnode4Vtatus != master then
   sleep 2000
-  goto wait_dnode4_vgroup_master
+  goto wait_dnode3_vgroup_slave
 endi
-if $dnode3Vtatus != offline then
+if $dnode3Vtatus != slave then
   sleep 2000
-  goto wait_dnode4_vgroup_master
+  goto wait_dnode3_vgroup_slave
 endi
 
 # check using select
@@ -271,3 +331,128 @@ if $data00 != $totalRows then
 	return -1
 endi
 
+print ============== step7: stop dnode2, and remove its vnodeX subdirector 
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data/vnode/*
+
+print ============== step8: restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+
+$loopCnt = 0
+wait_dnode2_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode4Vtatus != master then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+if $dnode3Vtatus != slave then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/script/unique/arbitrator/testSuite.sim b/tests/script/unique/arbitrator/testSuite.sim
index 53160812b0..b667863327 100644
--- a/tests/script/unique/arbitrator/testSuite.sim
+++ b/tests/script/unique/arbitrator/testSuite.sim
@@ -12,7 +12,9 @@ run unique/arbitrator/dn3_mn1_vnode_change.sim
 run unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim 
 run unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
 run unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
-####run unique/arbitrator/dn3_mn1_vnode_delDir.sim                  # unsupport
+run unique/arbitrator/dn3_mn1_vnode_delDir.sim
+run unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim 
+run unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim 
 run unique/arbitrator/dn3_mn1_vnode_nomaster.sim
 run unique/arbitrator/dn3_mn2_killDnode.sim
 run unique/arbitrator/insert_duplicationTs.sim
@@ -34,4 +36,4 @@ run unique/arbitrator/sync_replica2_dropTable.sim
 run unique/arbitrator/sync_replica3_alterTable_add.sim
 run unique/arbitrator/sync_replica3_alterTable_drop.sim
 run unique/arbitrator/sync_replica3_dropDb.sim
-run unique/arbitrator/sync_replica3_dropTable.sim
\ No newline at end of file
+run unique/arbitrator/sync_replica3_dropTable.sim

From b3f0c7caeadb5973f22fe3193a3ee8f89e10c00a Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Sat, 20 Jun 2020 15:39:37 +0800
Subject: [PATCH 05/23] [modify sim case]

---
 .../arbitrator/dn3_mn1_r3_vnode_delDir.sim    | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
index 95480e9d96..a05681ece2 100644
--- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
+++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
@@ -355,3 +355,53 @@ if $data00 != $totalRows then
 	return -1
 endi
 
+
+print ============== step7: stop dnode3/dnode2, and cluster unable to provide services
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep 3000
+sql_error select count(*) from $stb
+
+print ============== step8: restart dnode2, and cluster Still unable to provide services
+system sh/exec.sh -n dnode2 -s start
+sleep 3000
+sql_error select count(*) from $stb
+
+print ============== step9: restart dnode3, and cluster Resume service delivery
+system sh/exec.sh -n dnode3 -s start
+
+$loopCnt = 0
+wait_dnode4_vgroup_master_2:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode4_vgroup_master_2
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode4Vtatus != master then
+  sleep 2000
+  goto wait_dnode4_vgroup_master_2
+endi
+if $dnode3Vtatus != slave then
+  sleep 2000
+  goto wait_dnode4_vgroup_master_2
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi

From 7062f0e02e0696a0a94fe639d0651a963862a595 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 20 Jun 2020 15:50:29 +0800
Subject: [PATCH 06/23] [td-225]

---
 src/client/src/tscUtil.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 521f08ff8a..7fc05c561d 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -1674,6 +1674,14 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
   pNew->param = param;
   pNew->maxRetry = TSDB_MAX_REPLICA_NUM;
 
+  pNew->sqlstr = strdup(pSql->sqlstr);
+  if (pNew->sqlstr == NULL) {
+    tscError("%p new subquery failed", pSql);
+
+    free(pNew);
+    return NULL;
+  }
+
   SQueryInfo* pQueryInfo = NULL;
   tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo);
 
@@ -1697,6 +1705,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   pNew->pTscObj = pSql->pTscObj;
   pNew->signature = pNew;
 
+  pNew->sqlstr = strdup(pSql->sqlstr);
+  if (pNew->sqlstr == NULL) {
+    tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
+
+    free(pNew);
+    return NULL;
+  }
+
   SSqlCmd* pnCmd = &pNew->cmd;
   memcpy(pnCmd, pCmd, sizeof(SSqlCmd));
   

From e9ffcf85d061f944924cfbcee2eef0cf36d3be06 Mon Sep 17 00:00:00 2001
From: liu0x54 <tliu@taosdata.com>
Date: Sat, 20 Jun 2020 07:54:21 +0000
Subject: [PATCH 07/23] [TD-680] store the nextColId

---
 src/mnode/inc/mnodeDef.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h
index 67159ecea7..ae1ba98a0f 100644
--- a/src/mnode/inc/mnodeDef.h
+++ b/src/mnode/inc/mnodeDef.h
@@ -82,7 +82,8 @@ typedef struct STableObj {
 
 typedef struct SSuperTableObj {
   STableObj  info; 
-  int8_t     reserved0[3]; // for fill struct STableObj to 4byte align
+  int8_t     reserved0[1]; // for fill struct STableObj to 4byte align
+  int16_t    nextColId;
   int32_t    sversion;
   uint64_t   uid;
   int64_t    createdTime;
@@ -95,13 +96,13 @@ typedef struct SSuperTableObj {
   int32_t    numOfTables;
   SSchema *  schema;
   void *     vgHash;
-  int16_t    nextColId;
   int8_t     reserved2[6];
 } SSuperTableObj;
 
 typedef struct {
   STableObj  info;  
-  int8_t     reserved0[3]; // for fill struct STableObj to 4byte align
+  int8_t     reserved0[1]; // for fill struct STableObj to 4byte align
+  int16_t    nextColId;    //used by normal table
   int32_t    sversion;     //used by normal table  
   uint64_t   uid;
   uint64_t   suid;
@@ -112,7 +113,6 @@ typedef struct {
   int32_t    sqlLen;
   int8_t     updateEnd[1];
   int8_t     reserved1[1]; 
-  int16_t    nextColId;    //used by normal table
   int32_t    refCount;
   char*      sql;          //used by normal table
   SSchema*   schema;       //used by normal table

From 337418f440798bee50ab8a9753ea76dd1e9a1c8b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 20 Jun 2020 15:54:46 +0800
Subject: [PATCH 08/23] [td-686]

---
 src/client/src/tscUtil.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 7fc05c561d..c6b3de69b6 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -1664,6 +1664,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
 
   SSqlCmd* pCmd = &pNew->cmd;
   pCmd->command = cmd;
+  pCmd->parseFinished = 1;
 
   if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);
@@ -1724,6 +1725,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   pnCmd->numOfClause = 0;
   pnCmd->clauseIndex = 0;
   pnCmd->pDataBlocks = NULL;
+  pnCmd->parseFinished = 1;
 
   if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);

From 62d35aade26e005f52fd4eba0a0c95dc1306fe10 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sat, 20 Jun 2020 07:59:48 +0000
Subject: [PATCH 09/23] [TD-695] let normal table alter be a sync operation

---
 src/mnode/src/mnodeTable.c             | 79 +++++++++++++++++++++-----
 src/vnode/src/vnodeWrite.c             |  7 +++
 tests/script/general/alter/insert2.sim |  8 ---
 3 files changed, 71 insertions(+), 23 deletions(-)

diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 7019cc942c..565ce9910e 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -1753,11 +1753,43 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col
   return -1;
 }
 
-static int32_t mnodeAddNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
+static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
   SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
-  mLPrint("app:%p:%p, ctable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
-          tstrerror(code));
-  return code;
+  if (code != TSDB_CODE_SUCCESS) {
+    mError("app:%p:%p, ctable %s, failed to alter column, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+           tstrerror(code));
+    return code;
+  }
+
+  SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(NULL, pTable);
+  if (pMDCreate == NULL) {
+    return terrno;
+  }
+
+  if (pMsg->pVgroup == NULL) {
+    pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
+    if (pMsg->pVgroup == NULL) {
+      rpcFreeCont(pMDCreate);
+      mError("app:%p:%p, ctable %s, vgId:%d not exist in mnode", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+             pTable->vgId);
+      return TSDB_CODE_MND_VGROUP_NOT_EXIST;
+    }
+  }
+
+  SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup);
+  SRpcMsg rpcMsg = {
+      .handle  = pMsg,
+      .pCont   = pMDCreate,
+      .contLen = htonl(pMDCreate->contLen),
+      .code    = 0,
+      .msgType = TSDB_MSG_TYPE_MD_ALTER_TABLE
+  };
+
+  mTrace("app:%p:%p, ctable %s, send alter column msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+         pMsg->pVgroup->vgId);
+
+  dnodeSendMsgToDnode(&ipSet, &rpcMsg);
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
 static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) {
@@ -1802,7 +1834,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
     .table = tsChildTableSdb,
     .pObj = pTable,
     .pMsg = pMsg,
-    .cb = mnodeAddNormalTableColumnCb
+    .cb = mnodeAlterNormalTableColumnCb
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -1813,13 +1845,6 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
   return code;
 }
 
-static int32_t mnodeDropNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
-  SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
-  mLPrint("app:%p:%p, ctable %s, drop column result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
-          tstrerror(code));
-  return code;
-}
-
 static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
   SDbObj *pDb = pMsg->pDb;
   SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
@@ -1847,7 +1872,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
     .table = tsChildTableSdb,
     .pObj = pTable,
     .pMsg = pMsg,
-    .cb = mnodeDropNormalTableColumnCb
+    .cb = mnodeAlterNormalTableColumnCb
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -2185,9 +2210,33 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
   }
 }
 
-// not implemented yet
 static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) {
-  mTrace("alter table rsp received, handle:%p code:%s", rpcMsg->handle, tstrerror(rpcMsg->code));
+  if (rpcMsg->handle == NULL) return;
+
+  SMnodeMsg *mnodeMsg = rpcMsg->handle;
+  mnodeMsg->received++;
+
+  SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
+  assert(pTable);
+
+  if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
+    mTrace("app:%p:%p, ctable:%s, altered in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+           pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
+
+    dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+  } else {
+    if (mnodeMsg->retry++ < 3) {
+      mTrace("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p",
+             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
+             mnodeMsg->rpcMsg.handle);
+
+      dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
+    } else {
+      mError("app:%p:%p, table:%s, failed to alter in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
+             pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+      dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+    }
+  }
 }
 
 static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index 6d2b050163..769ef03bf6 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -133,6 +133,13 @@ static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
 }
 
 static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
+  // TODO: disposed in tsdb
+  // STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont);
+  // if (pCfg == NULL) return terrno;
+  // if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno;
+
+  // tsdbClearTableCfg(pCfg);
+  vTrace("vgId:%d, alter table msg is received", pVnode->vgId);
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/tests/script/general/alter/insert2.sim b/tests/script/general/alter/insert2.sim
index c3283c856b..28948b69d2 100644
--- a/tests/script/general/alter/insert2.sim
+++ b/tests/script/general/alter/insert2.sim
@@ -180,11 +180,9 @@ if $data28 != null then
 endi
 
 print ======== step4
-sleep 2500
 sql alter table tb drop column d
 sql alter table tb drop column e
 sql insert into tb values(now-19d, -19, 6, 3, 0)
-sleep 3000
 sql select * from tb order by ts desc
 if $rows != 4 then
   return -1
@@ -287,10 +285,8 @@ if $data38 != null then
 endi
 
 print ======== step5
-sleep 2500
 sql alter table tb drop column g
 sql insert into tb values(now-16d, -16, 9, 5)
-sleep 3000
 sql select count(f) from tb
 if $data00 != 5 then
   return -1
@@ -421,10 +417,8 @@ if $data48 != null then
 endi
 
 print ======== step6
-sleep 2500
 sql alter table tb drop column f
 sql insert into tb values(now-13d, -13, 7)
-sleep 3000
 sql select * from tb order by ts desc
 if $rows != 6 then
   return -1
@@ -551,10 +545,8 @@ if $data58 != null then
 endi
 
 print ======== step7
-sleep 2500
 sql alter table tb drop column h
 sql insert into tb values(now-10d, -10)
-sleep 3000
 sql select * from tb order by ts desc
 if $rows != 7 then
   return -1

From 7e0c97423e1a90b8d697c2fc8d7593500cbaf799 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sat, 20 Jun 2020 16:09:53 +0800
Subject: [PATCH 10/23] move stream scripts to the endof basic.txt

---
 tests/script/jenkins/basic.txt | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 9d02b1041b..9cb9dc594e 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -159,21 +159,6 @@ cd ../../../debug; make
 ./test.sh -f general/stable/values.sim
 ./test.sh -f general/stable/vnode3.sim
 
-./test.sh -f general/stream/metrics_1.sim
-./test.sh -f general/stream/metrics_del.sim
-./test.sh -f general/stream/metrics_n.sim
-./test.sh -f general/stream/metrics_replica1_vnoden.sim
-#./test.sh -f general/stream/new_stream.sim
-./test.sh -f general/stream/restart_stream.sim
-./test.sh -f general/stream/stream_1.sim
-./test.sh -f general/stream/stream_2.sim
-./test.sh -f general/stream/stream_3.sim
-./test.sh -f general/stream/stream_restart.sim
-./test.sh -f general/stream/table_1.sim
-./test.sh -f general/stream/table_del.sim
-./test.sh -f general/stream/table_n.sim
-./test.sh -f general/stream/table_replica1_vnoden.sim
-
 ./test.sh -f general/table/autocreate.sim
 ./test.sh -f general/table/basic1.sim
 ./test.sh -f general/table/basic2.sim
@@ -323,6 +308,21 @@ cd ../../../debug; make
 ./test.sh -f unique/vnode/replica3_repeat.sim
 ./test.sh -f unique/vnode/replica3_vgroup.sim
 
+./test.sh -f general/stream/metrics_1.sim
+./test.sh -f general/stream/metrics_del.sim
+./test.sh -f general/stream/metrics_n.sim
+./test.sh -f general/stream/metrics_replica1_vnoden.sim
+#./test.sh -f general/stream/new_stream.sim
+./test.sh -f general/stream/restart_stream.sim
+./test.sh -f general/stream/stream_1.sim
+./test.sh -f general/stream/stream_2.sim
+./test.sh -f general/stream/stream_3.sim
+./test.sh -f general/stream/stream_restart.sim
+./test.sh -f general/stream/table_1.sim
+./test.sh -f general/stream/table_del.sim
+./test.sh -f general/stream/table_n.sim
+./test.sh -f general/stream/table_replica1_vnoden.sim
+
 ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim
 ./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim

From e8877f5f2165a57c51e312e79fda6c043788173e Mon Sep 17 00:00:00 2001
From: liu0x54 <tliu@taosdata.com>
Date: Sat, 20 Jun 2020 08:42:49 +0000
Subject: [PATCH 11/23] [TD-711] fix the token define conflicts

---
 src/inc/ttokendef.h    | 10 ++++++++++
 src/util/inc/tstoken.h |  9 +--------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h
index d388bc9dbe..7648aadc60 100644
--- a/src/inc/ttokendef.h
+++ b/src/inc/ttokendef.h
@@ -221,6 +221,16 @@
 #define TK_INTO                           203
 #define TK_VALUES                         204
 
+
+#define TK_SPACE      300
+#define TK_COMMENT    301
+#define TK_ILLEGAL    302
+#define TK_HEX        303   // hex number  0x123
+#define TK_OCT        304   // oct number
+#define TK_BIN        305   // bin format data 0b111
+#define TK_FILE       306
+#define TK_QUESTION   307   // denoting the placeholder of "?",when invoking statement bind query
+
 #endif
 
 
diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h
index 74687e9c18..7b0e498d76 100644
--- a/src/util/inc/tstoken.h
+++ b/src/util/inc/tstoken.h
@@ -24,14 +24,7 @@ extern "C" {
 #include "tutil.h"
 #include "ttokendef.h"
 
-#define TK_SPACE      200
-#define TK_COMMENT    201
-#define TK_ILLEGAL    202
-#define TK_HEX        203   // hex number  0x123
-#define TK_OCT        204   // oct number
-#define TK_BIN        205   // bin format data 0b111
-#define TK_FILE       206
-#define TK_QUESTION   207   // denoting the placeholder of "?",when invoking statement bind query
+
 
 #define TSQL_TBNAME   "TBNAME"
 #define TSQL_TBNAME_L "tbname"

From de8c9fc46fe0d13a1d060822b9cc8cdf991bdbb2 Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Sat, 20 Jun 2020 08:47:41 +0000
Subject: [PATCH 12/23] close UDP thread

---
 src/rpc/src/rpcTcp.c |  2 +-
 src/rpc/src/rpcUdp.c | 13 ++++++-------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c
index 674d560952..5d156492c7 100644
--- a/src/rpc/src/rpcTcp.c
+++ b/src/rpc/src/rpcTcp.c
@@ -437,7 +437,7 @@ static void *taosProcessTcpData(void *param) {
   while (1) {
     int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1);
     if (pThreadObj->stop) {
-      tTrace("%s, tcp thread get stop event, exiting...", pThreadObj->label);
+      tTrace("%s TCP thread get stop event, exiting...", pThreadObj->label);
       break;
     }
     if (fdNum < 0) continue;
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index 279cf7ed49..41446f87fb 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -142,16 +142,15 @@ void taosCleanUpUdpConnection(void *handle) {
     pConn = pSet->udpConn + i;
     pConn->signature = NULL;
 
-    // shutdown to signal the thread to exit
-    if ( pConn->fd >=0) shutdown(pConn->fd, SHUT_RD);
+    if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR);
+    if (pConn->fd >=0) taosCloseSocket(pConn->fd);
   }
 
   for (int i = 0; i < pSet->threads; ++i) {
     pConn = pSet->udpConn + i;
     if (pConn->thread) pthread_join(pConn->thread, NULL);
-    if (pConn->fd >=0) taosCloseSocket(pConn->fd);
     tfree(pConn->buffer);
-    tTrace("UDP chandle:%p is closed", pConn);
+    tTrace("%s UDP thread is closed, inedx:%d", pConn->label, i);
   }
 
   tfree(pSet);
@@ -185,15 +184,15 @@ static void *taosRecvUdpData(void *param) {
 
   while (1) {
     dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
-    if(dataLen == 0) {
-      tTrace("data length is 0, socket was closed, exiting");
+    if(dataLen <= 0) {
+      tTrace("%s UDP socket was closed, exiting", pConn->label);
       break;
     }
 
     port = ntohs(sourceAdd.sin_port);
 
     if (dataLen < sizeof(SRpcHead)) {
-      tError("%s recvfrom failed, reason:%s\n", pConn->label, strerror(errno));
+      tError("%s recvfrom failed(%s)", pConn->label, strerror(errno));
       continue;
     }
 

From 8bb37ee8305007e8e31f26ccc13a9e4f38328624 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sat, 20 Jun 2020 08:50:07 +0000
Subject: [PATCH 13/23] [TD-710] the deleted object should also store full
 information in sdb

---
 src/mnode/src/mnodeSdb.c | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index d8fc9aee5a..193ec1924b 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -678,29 +678,16 @@ int32_t sdbDeleteRow(SSdbOper *pOper) {
     return TSDB_CODE_SUCCESS;
   }
 
-  void *  key = sdbGetObjKey(pTable, pOper->pObj);
-  int32_t keySize = 0;
-  switch (pTable->keyType) {
-    case SDB_KEY_STRING:
-    case SDB_KEY_VAR_STRING:
-      keySize = strlen((char *)key) + 1;
-      break;
-    case SDB_KEY_INT:
-    case SDB_KEY_AUTO:
-      keySize = sizeof(uint32_t);
-      break;
-    default:
-      return TSDB_CODE_MND_SDB_INVAID_KEY_TYPE;
-  }
-
-  int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + keySize + SDB_SYNC_HACK;
+  int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
   SSdbOper *pNewOper = taosAllocateQitem(size);
 
   SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
   pHead->version = 0;
-  pHead->len = keySize;
   pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
-  memcpy(pHead->cont, key, keySize);
+  
+  pOper->rowData = pHead->cont;
+  (*pTable->encodeFp)(pOper);
+  pHead->len = pOper->rowSize;
 
   memcpy(pNewOper, pOper, sizeof(SSdbOper));
 

From dba77aa8e3cc600847ccf34ebce39f34203209c7 Mon Sep 17 00:00:00 2001
From: liu0x54 <tliu@taosdata.com>
Date: Sat, 20 Jun 2020 08:51:06 +0000
Subject: [PATCH 14/23] [TD-711] fix the token define conflicts

---
 src/query/src/qtokenizer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/src/qtokenizer.c b/src/query/src/qtokenizer.c
index aa9f6fddca..80d59a384e 100644
--- a/src/query/src/qtokenizer.c
+++ b/src/query/src/qtokenizer.c
@@ -25,7 +25,7 @@
 // All the keywords of the SQL language are stored in a hash table
 typedef struct SKeyword {
   const char* name;  // The keyword name
-  uint8_t     type;  // type
+  uint16_t     type;  // type
   uint8_t     len;   // length
 } SKeyword;
 

From f5caa7465d66655f4d0251e8e398c697a10246e0 Mon Sep 17 00:00:00 2001
From: Bomin Zhang <bmzhang@taosdata.com>
Date: Sat, 20 Jun 2020 16:52:37 +0800
Subject: [PATCH 15/23] fix subscription failure

---
 src/client/src/tscSub.c   | 20 ++++++++++++++------
 tests/examples/c/stream.c |  0
 2 files changed, 14 insertions(+), 6 deletions(-)
 mode change 100755 => 100644 tests/examples/c/stream.c

diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index bbde276c3b..30a0de76fc 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -182,21 +182,25 @@ static SArray* getTableList( SSqlObj* pSql ) {
   char* sql = alloca(strlen(p) + 32);
   sprintf(sql, "select tbid(tbname)%s", p);
   
-  SSqlObj* pSql1 = taos_query(pSql->pTscObj, sql);
-  if (terrno != TSDB_CODE_SUCCESS) {
-    tscError("failed to retrieve table id: %s", tstrerror(terrno));
+  SSqlObj* pNew = taos_query(pSql->pTscObj, sql);
+  if (pNew == NULL) {
+    tscError("failed to retrieve table id: cannot create new sql object.");
+    return NULL;
+
+  } else if (taos_errno(pNew) != TSDB_CODE_SUCCESS) {
+    tscError("failed to retrieve table id: %s", tstrerror(taos_errno(pNew)));
     return NULL;
   }
 
   TAOS_ROW row;
   SArray* result = taosArrayInit( 128, sizeof(STidTags) );
-  while ((row = taos_fetch_row(pSql1))) {
+  while ((row = taos_fetch_row(pNew))) {
     STidTags tags;
     memcpy(&tags, row[0], sizeof(tags));
     taosArrayPush(result, &tags);
   }
 
-  taos_free_result(pSql1);
+  taos_free_result(pNew);
   
   return result;
 }
@@ -222,6 +226,9 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   }
 
   SArray* tables = getTableList(pSql);
+  if (tables == NULL) {
+    return 0;
+  }
   size_t numOfTables = taosArrayGetSize(tables);
 
   SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress));
@@ -242,6 +249,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   }
   taosArrayDestroy(tables);
 
+  TSDB_QUERY_SET_TYPE(tscGetQueryInfoDetail(pCmd, 0)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
   return 1;
 }
 
@@ -413,7 +421,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
   }
 
   if (pRes->code != TSDB_CODE_SUCCESS) {
-    tscError("failed to query data, error code=%d", pRes->code);
+    tscError("failed to query data: %s", tstrerror(pRes->code));
     tscRemoveFromSqlList(pSql);
     return NULL;
   }
diff --git a/tests/examples/c/stream.c b/tests/examples/c/stream.c
old mode 100755
new mode 100644

From c9d13a431688b49138439ee77f9ac9f00c6bdd46 Mon Sep 17 00:00:00 2001
From: liu0x54 <tliu@taosdata.com>
Date: Sat, 20 Jun 2020 08:55:30 +0000
Subject: [PATCH 16/23] [TD-711] fix the token define conflicts

---
 src/inc/ttokendef.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h
index 7648aadc60..b313d6ebb1 100644
--- a/src/inc/ttokendef.h
+++ b/src/inc/ttokendef.h
@@ -222,14 +222,14 @@
 #define TK_VALUES                         204
 
 
-#define TK_SPACE      300
-#define TK_COMMENT    301
-#define TK_ILLEGAL    302
-#define TK_HEX        303   // hex number  0x123
-#define TK_OCT        304   // oct number
-#define TK_BIN        305   // bin format data 0b111
-#define TK_FILE       306
-#define TK_QUESTION   307   // denoting the placeholder of "?",when invoking statement bind query
+#define TK_SPACE                          300
+#define TK_COMMENT                        301
+#define TK_ILLEGAL                        302
+#define TK_HEX                            303   // hex number  0x123
+#define TK_OCT                            304   // oct number
+#define TK_BIN                            305   // bin format data 0b111
+#define TK_FILE                           306
+#define TK_QUESTION                       307   // denoting the placeholder of "?",when invoking statement bind query
 
 #endif
 

From c0ae4e75767e3dd9d2068044063f181b775abcc3 Mon Sep 17 00:00:00 2001
From: Hui Li <huili@taosdata.com>
Date: Sat, 20 Jun 2020 17:08:23 +0800
Subject: [PATCH 17/23] [TD-598]

---
 src/client/src/tscParseInsert.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index e65acc2483..69bc4a3e20 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -986,14 +986,16 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
   return code;
 }
 
-int validateTableName(char *tblName, int len) {
+int validateTableName(char *tblName, int len, SSQLToken* psTblToken) {
   char buf[TSDB_TABLE_ID_LEN] = {0};
   tstrncpy(buf, tblName, sizeof(buf));
 
-  SSQLToken token = {.n = len, .type = TK_ID, .z = buf};
-  tSQLGetToken(buf, &token.type);
+  psTblToken->n    = len;
+  psTblToken->type = TK_ID;
+  psTblToken->z    = buf;
+  tSQLGetToken(buf, &psTblToken->type);
 
-  return tscValidateName(&token);
+  return tscValidateName(psTblToken);
 }
 
 static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) {
@@ -1077,14 +1079,14 @@ int tsParseInsertSql(SSqlObj *pSql) {
     }
 
     pCmd->curSql = sToken.z;
-
+    SSQLToken sTblToken;
     // Check if the table name available or not
-    if (validateTableName(sToken.z, sToken.n) != TSDB_CODE_SUCCESS) {
+    if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) {
       code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z);
       goto _error;
     }
 
-    if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
+    if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) {
       goto _error;
     }
 

From e90d241eeb682102dbdeb7da7cbb1ace62b6a5f2 Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Sat, 20 Jun 2020 11:41:12 +0000
Subject: [PATCH 18/23] stop UDP/TCP connection first, then close all
 connections, then clean up

---
 src/rpc/inc/rpcTcp.h  |  1 +
 src/rpc/src/rpcMain.c | 30 ++++++++++++++++++++++++++----
 src/rpc/src/rpcTcp.c  | 13 +++++++++----
 src/rpc/src/rpcUdp.c  |  8 ++------
 4 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/src/rpc/inc/rpcTcp.h b/src/rpc/inc/rpcTcp.h
index 40fab00056..0b0e5bd0fb 100644
--- a/src/rpc/inc/rpcTcp.h
+++ b/src/rpc/inc/rpcTcp.h
@@ -21,6 +21,7 @@ extern "C" {
 #endif
 
 void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
+void taosStopTcpServer(void *param);
 void taosCleanUpTcpServer(void *param);
 
 void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle);
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 007a511adf..780ec5ccba 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -147,12 +147,19 @@ void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, vo
 };
 
 void (*taosCleanUpConn[])(void *thandle) = {
-    taosCleanUpUdpConnection, 
-    taosCleanUpUdpConnection, 
+    NULL, 
+    NULL,
     taosCleanUpTcpServer,
     taosCleanUpTcpClient
 };
 
+void (*taosStopConn[])(void *thandle) = {
+    taosCleanUpUdpConnection, 
+    taosCleanUpUdpConnection, 
+    taosStopTcpServer,
+    NULL 
+};
+
 int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
     taosSendUdpData, 
     taosSendUdpData, 
@@ -289,14 +296,26 @@ void *rpcOpen(const SRpcInit *pInit) {
 void rpcClose(void *param) {
   SRpcInfo *pRpc = (SRpcInfo *)param;
 
+  // stop connection to outside first
+  if (taosStopConn[pRpc->connType | RPC_CONN_TCP])
+    (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
+
+  if (taosStopConn[pRpc->connType])
+    (*taosStopConn[pRpc->connType])(pRpc->udphandle);
+
+  // close all connections 
   for (int i = 0; i < pRpc->sessions; ++i) {
     if (pRpc->connList && pRpc->connList[i].user[0]) {
       rpcCloseConn((void *)(pRpc->connList + i));
     }
   }
 
-  (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
-  (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
+  // clean up
+  if (taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])
+    (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
+
+  if (taosCleanUpConn[pRpc->connType])
+    (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
 
   tTrace("%s rpc is closed", pRpc->label);
   rpcDecRef(pRpc);
@@ -588,6 +607,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
   pConn->inTranId = 0;
   pConn->outTranId = 0;
   pConn->secured = 0;
+  pConn->peerId = 0;
   pConn->peerIp = 0;
   pConn->peerPort = 0;
   pConn->pReqMsg = NULL;
@@ -627,6 +647,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
     pConn->spi = pRpc->spi;
     pConn->encrypt = pRpc->encrypt;
     if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN);
+    tTrace("%s %p client connection is allocated", pRpc->label, pConn);
   }
 
   return pConn;
@@ -681,6 +702,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
     }
   
     taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
+    tTrace("%s %p server connection is allocated", pRpc->label, pConn);
   }
 
   return pConn;
diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c
index 5d156492c7..11d2c57dda 100644
--- a/src/rpc/src/rpcTcp.c
+++ b/src/rpc/src/rpcTcp.c
@@ -190,14 +190,19 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
   }
 }
 
+void taosStopTcpServer(void *handle) {
+  SServerObj *pServerObj = handle;
+
+  tTrace("TCP:%s, stop accept new connections", pServerObj->label);
+  if (pServerObj == NULL) return;
+  if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD);
+  if(pServerObj->thread) pthread_join(pServerObj->thread, NULL);
+}
 
 void taosCleanUpTcpServer(void *handle) {
   SServerObj *pServerObj = handle;
   SThreadObj *pThreadObj;
-
   if (pServerObj == NULL) return;
-  if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD);
-  if(pServerObj->thread) pthread_join(pServerObj->thread, NULL);
 
   for (int i = 0; i < pServerObj->numOfThreads; ++i) {
     pThreadObj = pServerObj->pThreadObj + i;
@@ -226,7 +231,7 @@ static void *taosAcceptTcpConnection(void *arg) {
     connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen);
     if (connFd == -1) {
       if (errno == EINVAL) {
-        tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label);
+        tTrace("%s TCP server stop accepting new connections, exiting", pServerObj->label);
         break;
       }
 
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index 41446f87fb..35e06e633b 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -30,7 +30,6 @@
 #define RPC_MAX_UDP_SIZE 65480
 
 typedef struct {
-  void           *signature;
   int             index;
   int             fd;
   uint16_t        port;       // peer port
@@ -111,7 +110,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
     pConn->processData = fp;
     pConn->index = i;
     pConn->pSet = pSet;
-    pConn->signature = pConn;
 
     int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn);
     if (code != 0) {
@@ -140,8 +138,6 @@ void taosCleanUpUdpConnection(void *handle) {
 
   for (int i = 0; i < pSet->threads; ++i) {
     pConn = pSet->udpConn + i;
-    pConn->signature = NULL;
-
     if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR);
     if (pConn->fd >=0) taosCloseSocket(pConn->fd);
   }
@@ -185,7 +181,7 @@ static void *taosRecvUdpData(void *param) {
   while (1) {
     dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
     if(dataLen <= 0) {
-      tTrace("%s UDP socket was closed, exiting", pConn->label);
+      tTrace("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno));
       break;
     }
 
@@ -221,7 +217,7 @@ static void *taosRecvUdpData(void *param) {
 int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) {
   SUdpConn *pConn = (SUdpConn *)chandle;
 
-  if (pConn == NULL || pConn->signature != pConn) return -1;
+  if (pConn == NULL) return -1;
 
   struct sockaddr_in destAdd;
   memset(&destAdd, 0, sizeof(destAdd));

From a3bb1d5f2b7eae6128379349dcf185dcb8b5869b Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Sat, 20 Jun 2020 12:51:14 +0000
Subject: [PATCH 19/23] tune the code

---
 src/rpc/inc/rpcTcp.h  |  1 +
 src/rpc/inc/rpcUdp.h  |  1 +
 src/rpc/src/rpcMain.c | 24 +++++++++---------------
 src/rpc/src/rpcTcp.c  | 14 +++++++++++---
 src/rpc/src/rpcUdp.c  | 19 +++++++++++++++++--
 5 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/src/rpc/inc/rpcTcp.h b/src/rpc/inc/rpcTcp.h
index 0b0e5bd0fb..6ef8fc2d92 100644
--- a/src/rpc/inc/rpcTcp.h
+++ b/src/rpc/inc/rpcTcp.h
@@ -25,6 +25,7 @@ void taosStopTcpServer(void *param);
 void taosCleanUpTcpServer(void *param);
 
 void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle);
+void taosStopTcpClient(void *chandle);
 void taosCleanUpTcpClient(void *chandle);
 void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
 
diff --git a/src/rpc/inc/rpcUdp.h b/src/rpc/inc/rpcUdp.h
index fd60f4a089..c1da6a9240 100644
--- a/src/rpc/inc/rpcUdp.h
+++ b/src/rpc/inc/rpcUdp.h
@@ -23,6 +23,7 @@ extern "C" {
 #include "taosdef.h"
 
 void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle);
+void  taosStopUdpConnection(void *handle);
 void  taosCleanUpUdpConnection(void *handle);
 int   taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle);
 void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 780ec5ccba..f48d207d7b 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -147,17 +147,17 @@ void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, vo
 };
 
 void (*taosCleanUpConn[])(void *thandle) = {
-    NULL, 
-    NULL,
+    taosCleanUpUdpConnection, 
+    taosCleanUpUdpConnection, 
     taosCleanUpTcpServer,
     taosCleanUpTcpClient
 };
 
 void (*taosStopConn[])(void *thandle) = {
-    taosCleanUpUdpConnection, 
-    taosCleanUpUdpConnection, 
+    taosStopUdpConnection, 
+    taosStopUdpConnection, 
     taosStopTcpServer,
-    NULL 
+    taosStopTcpClient,
 };
 
 int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
@@ -297,11 +297,8 @@ void rpcClose(void *param) {
   SRpcInfo *pRpc = (SRpcInfo *)param;
 
   // stop connection to outside first
-  if (taosStopConn[pRpc->connType | RPC_CONN_TCP])
-    (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
-
-  if (taosStopConn[pRpc->connType])
-    (*taosStopConn[pRpc->connType])(pRpc->udphandle);
+  (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
+  (*taosStopConn[pRpc->connType])(pRpc->udphandle);
 
   // close all connections 
   for (int i = 0; i < pRpc->sessions; ++i) {
@@ -311,11 +308,8 @@ void rpcClose(void *param) {
   }
 
   // clean up
-  if (taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])
-    (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
-
-  if (taosCleanUpConn[pRpc->connType])
-    (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
+  (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
+  (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
 
   tTrace("%s rpc is closed", pRpc->label);
   rpcDecRef(pRpc);
diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c
index 11d2c57dda..aa94accceb 100644
--- a/src/rpc/src/rpcTcp.c
+++ b/src/rpc/src/rpcTcp.c
@@ -193,10 +193,11 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
 void taosStopTcpServer(void *handle) {
   SServerObj *pServerObj = handle;
 
-  tTrace("TCP:%s, stop accept new connections", pServerObj->label);
   if (pServerObj == NULL) return;
   if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD);
   if(pServerObj->thread) pthread_join(pServerObj->thread, NULL);
+
+  tTrace("%s TCP server is stopped", pServerObj->label);
 }
 
 void taosCleanUpTcpServer(void *handle) {
@@ -210,7 +211,7 @@ void taosCleanUpTcpServer(void *handle) {
     pthread_mutex_destroy(&(pThreadObj->mutex));
   }
 
-  tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label);
+  tTrace("%s TCP server is cleaned up", pServerObj->label);
 
   tfree(pServerObj->pThreadObj);
   tfree(pServerObj);
@@ -309,12 +310,19 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *
   return pThreadObj;
 }
 
+void taosStopTcpClient(void *chandle) {
+  SThreadObj *pThreadObj = chandle;
+  if (pThreadObj == NULL) return;
+
+  tTrace ("%s TCP client is stopped", pThreadObj->label);
+}
+
 void taosCleanUpTcpClient(void *chandle) {
   SThreadObj *pThreadObj = chandle;
   if (pThreadObj == NULL) return;
 
   taosStopTcpThread(pThreadObj);
-  tTrace ("%s, all connections are cleaned up", pThreadObj->label);
+  tTrace ("%s TCP client is cleaned up", pThreadObj->label);
 
   tfree(pThreadObj);
 }
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index 35e06e633b..f88db3a226 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -130,7 +130,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
   return pSet;
 }
 
-void taosCleanUpUdpConnection(void *handle) {
+void taosStopUdpConnection(void *handle) {
   SUdpConnSet *pSet = (SUdpConnSet *)handle;
   SUdpConn    *pConn;
 
@@ -146,9 +146,24 @@ void taosCleanUpUdpConnection(void *handle) {
     pConn = pSet->udpConn + i;
     if (pConn->thread) pthread_join(pConn->thread, NULL);
     tfree(pConn->buffer);
-    tTrace("%s UDP thread is closed, inedx:%d", pConn->label, i);
+    // tTrace("%s UDP thread is closed, index:%d", pConn->label, i);
   }
 
+  tTrace("%s UDP is stopped", pSet->label);
+}
+
+void taosCleanUpUdpConnection(void *handle) {
+  SUdpConnSet *pSet = (SUdpConnSet *)handle;
+  SUdpConn    *pConn;
+
+  if (pSet == NULL) return;
+
+  for (int i = 0; i < pSet->threads; ++i) {
+    pConn = pSet->udpConn + i;
+    if (pConn->fd >=0) taosCloseSocket(pConn->fd);
+  }
+
+  tTrace("%s UDP is cleaned up", pSet->label);
   tfree(pSet);
 }
 

From f95d577615f78c4a57b34d1699c766f0c58911e5 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 21 Jun 2020 02:10:51 +0000
Subject: [PATCH 20/23] add loop.sh for test

---
 tests/script/loop.sh | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100755 tests/script/loop.sh

diff --git a/tests/script/loop.sh b/tests/script/loop.sh
new file mode 100755
index 0000000000..d41fe8d4aa
--- /dev/null
+++ b/tests/script/loop.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+##################################################
+# 
+# Do simulation test 
+#
+##################################################
+
+set +e
+#set -x
+
+CMD_NAME=
+LOOP_TIMES=5
+
+while getopts "f:t:" arg
+do
+  case $arg in
+    f)
+      CMD_NAME=$OPTARG
+      ;;
+    t)
+      LOOP_TIMES=$OPTARG
+      ;;
+    ?)
+      echo "unknow argument"
+      ;;
+  esac
+done
+
+echo LOOP_TIMES ${LOOP_TIMES}
+echo CMD_NAME ${CMD_NAME}
+
+for (( i=0; i<$LOOP_TIMES; i++ ))
+do
+    echo loop $i
+    echo cmd $CMD_NAME
+    $CMD_NAME
+done

From 06cc1ec244f1ae6f5a8e23065399626d40605c12 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 21 Jun 2020 02:12:31 +0000
Subject: [PATCH 21/23] loop.sh

---
 tests/script/loop.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/script/loop.sh b/tests/script/loop.sh
index d41fe8d4aa..adafe2ed60 100755
--- a/tests/script/loop.sh
+++ b/tests/script/loop.sh
@@ -6,7 +6,7 @@
 #
 ##################################################
 
-set +e
+set -e
 #set -x
 
 CMD_NAME=
@@ -30,9 +30,10 @@ done
 echo LOOP_TIMES ${LOOP_TIMES}
 echo CMD_NAME ${CMD_NAME}
 
-for (( i=0; i<$LOOP_TIMES; i++ ))
+for ((i=0; i<$LOOP_TIMES; i++ ))
 do
     echo loop $i
     echo cmd $CMD_NAME
     $CMD_NAME
+    sleep 2
 done

From 811bd5d2f2a266831c45a472f0b90dde9da9cadb Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 21 Jun 2020 02:44:15 +0000
Subject: [PATCH 22/23] fix bug while mnode replica

---
 src/mnode/src/mnodeDnode.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 0f2bbb8235..e16195499b 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -348,7 +348,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
   pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
   pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
-  mnodeGetMnodeInfos(&pRsp->mnodes);
   SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
 
   for (int32_t j = 0; j < openVnodes; ++j) {
@@ -392,6 +391,10 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   }
 
   pDnode->lastAccess = tsAccessSquence;
+
+  //this func should be called after sdb replica changed
+  mnodeGetMnodeInfos(&pRsp->mnodes);
+  
   mnodeDecDnodeRef(pDnode);
 
   pMsg->rpcRsp.len = contLen;

From 1721714f56f2618c47ca0df00860afa32339baac Mon Sep 17 00:00:00 2001
From: Jeff Tao <jhtao@taosdata.com>
Date: Sun, 21 Jun 2020 02:57:07 +0000
Subject: [PATCH 23/23] add debug info

---
 src/rpc/src/rpcMain.c | 18 +++++++-----------
 src/rpc/src/rpcUdp.c  |  2 +-
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index f48d207d7b..f62df56771 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -964,11 +964,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
   terrno = 0;
   pConn = rpcProcessMsgHead(pRpc, pRecv);
 
-  if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) {
-    tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
+  tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x",
         pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, 
         pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code);
-  }
 
   int32_t code = terrno;
   if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) {
@@ -1196,16 +1194,14 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
   msgLen = rpcAddAuthPart(pConn, msg, msgLen);
 
   if ( rpcIsReq(pHead->msgType)) {
-    if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
-      tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
-             pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, 
-             msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
+    tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
+           pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, 
+           msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   } else {
     if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
-    if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
-      tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
-          pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, 
-          htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
+    tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
+           pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, 
+           htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 
   //tTrace("connection type is: %d", pConn->connType);
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index f88db3a226..a4c7d6c145 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -175,7 +175,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
   SUdpConn *pConn = pSet->udpConn + pSet->index;
   pConn->port = port;
 
-  tTrace("%s UDP connection is setup, ip:%x:%hu", pConn->label, ip, port);
+  tTrace("%s UDP connection is setup, ip:%x:%hu localPort:%hu", pConn->label, ip, port, pConn->localPort);
 
   return pConn;
 }