Merge branch 'develop' of https://github.com/taosdata/TDengine into develop

src/dnode/src/dnodeModule.c

@@ -16,11 +16,13 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosdef.h"
+#include "taosmsg.h"
 #include "tglobal.h"
 #include "mnode.h"
 #include "http.h"
 #include "tmqtt.h"
 #include "monitor.h"
+#include "dnode.h"
 #include "dnodeInt.h"
 #include "dnodeModule.h"
 
@@ -129,17 +131,32 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
   for (int32_t module = TSDB_MOD_MNODE; module < TSDB_MOD_HTTP; ++module) {
     bool enableModule = moduleStatus & (1 << module);
     if (!tsModule[module].enable && enableModule) {
-      dInfo("module status:%u is received, start %s module", tsModuleStatus, tsModule[module].name);
+      dInfo("module status:%u is set, start %s module", moduleStatus, tsModule[module].name);
       tsModule[module].enable = true;
       dnodeSetModuleStatus(module);
       (*tsModule[module].startFp)();
     }
 
     if (tsModule[module].enable && !enableModule) {
-      dInfo("module status:%u is received, stop %s module", tsModuleStatus, tsModule[module].name);
+      dInfo("module status:%u is set, stop %s module", moduleStatus, tsModule[module].name);
       tsModule[module].enable = false;
       dnodeUnSetModuleStatus(module);
       (*tsModule[module].stopFp)();
     }
   }
 }
+
+void dnodeCheckModules() {
+  if (tsModuleStatus & TSDB_MOD_MNODE) return;
+
+  SDMMnodeInfos *mnodes = dnodeGetMnodeInfos();
+  for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
+    SDMMnodeInfo *node = &mnodes->nodeInfos[i];
+    if (node->nodeId == dnodeGetDnodeId()) {
+      uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE);;
+      dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus);
+      dnodeProcessModuleStatus(moduleStatus);
+      break;
+    }
+  }
+}

src/inc/dnode.h

@@ -43,6 +43,7 @@ void    dnodeGetMnodeEpSetForPeer(void *epSet);
 void    dnodeGetMnodeEpSetForShell(void *epSet);
 void *  dnodeGetMnodeInfos();
 int32_t dnodeGetDnodeId();
+void    dnodeCheckModules();
 
 void  dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
 void  dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);

src/inc/tsync.h

@@ -103,6 +103,9 @@ typedef struct {
 
 typedef void* tsync_h;
 
+int32_t syncInit();
+void    syncCleanUp();
+
 tsync_h syncStart(const SSyncInfo *);
 void    syncStop(tsync_h shandle);
 int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);

src/mnode/src/mnodeSdb.c

@@ -22,6 +22,7 @@
 #include "tqueue.h"
 #include "twal.h"
 #include "tsync.h"
+#include "ttimer.h"
 #include "tglobal.h"
 #include "dnode.h"
 #include "mnode.h"
@@ -88,6 +89,8 @@ typedef struct {
   SSdbWriteWorker *writeWorker;
 } SSdbWriteWorkerPool;
 
+extern void *     tsMnodeTmr;
+static void *     tsUpdateSyncTmr;
 static SSdbObject tsSdbObj = {0};
 static taos_qset  tsSdbWriteQset;
 static taos_qall  tsSdbWriteQall;
@@ -290,11 +293,16 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
   taosFreeQitem(pOper);
 }
 
+static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(); }
+
 void sdbUpdateSync() {
   if (!mnodeIsRunning()) {
     mDebug("mnode not start yet, update sync info later");
+    dnodeCheckModules();
+    taosTmrReset(sdbUpdateSyncTmrFp, 1000, NULL, tsMnodeTmr, &tsUpdateSyncTmr);
     return;
   }
+  mDebug("update sync info in sdb");
 
   SSyncCfg syncCfg = {0};
   int32_t  index = 0;
@@ -387,8 +395,6 @@ int32_t sdbInit() {
     tsSdbObj.role = TAOS_SYNC_ROLE_MASTER;
   }
 
-  sdbUpdateSync();
-
   tsSdbObj.status = SDB_STATUS_SERVING;
   return TSDB_CODE_SUCCESS;
 }

src/sync/src/syncMain.c

@@ -35,18 +35,14 @@ int       tsSyncTcpThreads = 2;
 int tsMaxWatchFiles = 500;
 int tsMaxFwdInfo = 200;
 int tsSyncTimer = 1;
-//int       sDebugFlag = 135;
-//char      tsArbitrator[TSDB_FQDN_LEN] = {0};
 
 // module global, not configurable
 int  tsSyncNum;  // number of sync in process in whole system
 char tsNodeFqdn[TSDB_FQDN_LEN];
 
-static int            tsNodeNum;    // number of nodes in system
 static ttpool_h tsTcpPool;
 static void *   syncTmrCtrl = NULL;
 static void *   vgIdHash;
-static pthread_once_t syncModuleInit = PTHREAD_ONCE_INIT;
 
 // local functions
 static void  syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
@@ -75,7 +71,7 @@ char* syncRole[] = {
   "master"
 };
 
-static void syncModuleInitFunc() {
+int32_t syncInit() {
   SPoolInfo info;
 
   info.numOfThreads = tsSyncTcpThreads;
@@ -87,25 +83,52 @@ static void syncModuleInitFunc() {
   info.processIncomingConn = syncProcessIncommingConnection;
 
   tsTcpPool = taosOpenTcpThreadPool(&info);
-  if (tsTcpPool == NULL) return;
+  if (tsTcpPool == NULL) {
+    sError("failed to init tcpPool");
+    return -1;
+  }
 
   syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
   if (syncTmrCtrl == NULL) {
+    sError("failed to init tmrCtrl");
     taosCloseTcpThreadPool(tsTcpPool);
     tsTcpPool = NULL;
-    return;
+    return -1;
   }
 
   vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
   if (vgIdHash == NULL) {
+    sError("failed to init vgIdHash");
     taosTmrCleanUp(syncTmrCtrl);
     taosCloseTcpThreadPool(tsTcpPool);
     tsTcpPool = NULL;
     syncTmrCtrl = NULL;
-    return;
+    return -1;
   }
 
   tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn));
+  sInfo("sync module initialized successfully");
+
+  return 0;
+}
+
+void syncCleanUp() {
+  if (tsTcpPool) {
+    taosCloseTcpThreadPool(tsTcpPool);
+    tsTcpPool = NULL;
+  }
+
+  if (syncTmrCtrl) {
+    taosTmrCleanUp(syncTmrCtrl);
+    syncTmrCtrl = NULL;
+  }
+
+  if (vgIdHash) {
+    taosHashCleanup(vgIdHash);
+    vgIdHash = NULL;
+  }
+
+  sInfo("sync module is cleaned up");
 }
 
 void *syncStart(const SSyncInfo *pInfo) {
@@ -118,15 +141,6 @@ void *syncStart(const SSyncInfo *pInfo) {
     return NULL;
   }
 
-  pthread_once(&syncModuleInit, syncModuleInitFunc); 
-  if (tsTcpPool == NULL) {
-    free(pNode);
-    syncModuleInit = PTHREAD_ONCE_INIT;
-    sError("failed to init sync module(%s)", tstrerror(errno));
-    return NULL;
-  }
-
-  atomic_add_fetch_32(&tsNodeNum, 1);
   tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
   pthread_mutex_init(&pNode->mutex, NULL);
 
@@ -148,9 +162,10 @@ void *syncStart(const SSyncInfo *pInfo) {
   for (int i = 0; i < pCfg->replica; ++i) {
     const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
     pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
-    if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) 
+    if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) {
       pNode->selfIndex = i;
     }
+  }
 
   if (pNode->selfIndex < 0) {
     sInfo("vgId:%d, this node is not configured", pNode->vgId);
@@ -182,8 +197,9 @@ void *syncStart(const SSyncInfo *pInfo) {
   syncAddNodeRef(pNode);
   taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
 
-  if (pNode->notifyRole) 
+  if (pNode->notifyRole) {
     (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  }
 
   return pNode;
 }
@@ -218,8 +234,8 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
   int        i, j;
 
   if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
-  sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], 
+  sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
-         pNewCfg->replica, pNode->replica);
+        pNode->replica);
 
   pthread_mutex_lock(&(pNode->mutex));
 
@@ -252,17 +268,19 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
       newPeers[i] = pNode->peerInfo[j];
     }
 
-    if ((strcmp(pNewNode->nodeFqdn, tsNodeFqdn) == 0) && (pNewNode->nodePort == tsSyncPort)) 
+    if ((strcmp(pNewNode->nodeFqdn, tsNodeFqdn) == 0) && (pNewNode->nodePort == tsSyncPort)) {
       pNode->selfIndex = i;
     }
+  }
 
   pNode->replica = pNewCfg->replica;
   pNode->quorum = pNewCfg->quorum;
   if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
   memcpy(pNode->peerInfo, newPeers, sizeof(SSyncPeer *) * pNewCfg->replica);
 
-  for (i = pNewCfg->replica; i < TAOS_SYNC_MAX_REPLICA; ++i)
+  for (i = pNewCfg->replica; i < TAOS_SYNC_MAX_REPLICA; ++i) {
     pNode->peerInfo[i] = NULL;
+  }
 
   syncAddArbitrator(pNode);
 
@@ -274,7 +292,8 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
 
   pthread_mutex_unlock(&(pNode->mutex));
 
-  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]);
+  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum,
+        syncRole[nodeRole]);
   syncBroadcastStatus(pNode);
 
   return 0;
@@ -423,29 +442,16 @@ static void syncAddArbitrator(SSyncNode *pNode) {
   pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
 }
 
-static void syncAddNodeRef(SSyncNode *pNode)
+static void syncAddNodeRef(SSyncNode *pNode) {
-{
   atomic_add_fetch_8(&pNode->refCount, 1);
 }
 
-static void syncDecNodeRef(SSyncNode *pNode)
+static void syncDecNodeRef(SSyncNode *pNode) {
-{
   if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) {
     pthread_mutex_destroy(&pNode->mutex);
     taosTFree(pNode->pRecv);
     taosTFree(pNode->pSyncFwds);
     taosTFree(pNode);
-
-    if (atomic_sub_fetch_32(&tsNodeNum, 1) == 0) { 
-      if (tsTcpPool) taosCloseTcpThreadPool(tsTcpPool);
-      if (syncTmrCtrl) taosTmrCleanUp(syncTmrCtrl);
-      if (vgIdHash) taosHashCleanup(vgIdHash);
-      syncTmrCtrl = NULL;
-      tsTcpPool = NULL;
-      vgIdHash = NULL;
-      syncModuleInit = PTHREAD_ONCE_INIT;
-      sDebug("sync module is cleaned up");
-    }
   }
 }
 
@@ -506,7 +512,9 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
   int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
   if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
     sDebug("%s, start to check peer connection", pPeer->id);
-    taosTmrReset(syncCheckPeerConnection, 100 + (pNode->vgId*10)%100, pPeer, syncTmrCtrl, &pPeer->timer);
+    int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
+    if (pNode->vgId) checkMs = tsStatusInterval * 2000 + 100;
+    taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer);
   }
 
   syncAddNodeRef(pNode);
@@ -542,18 +550,20 @@ static void syncChooseMaster(SSyncNode *pNode) {
   sDebug("vgId:%d, choose master", pNode->vgId);
 
   for (int i = 0; i < pNode->replica; ++i) {
-    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE)
+    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
       onlineNum++;
     }
+  }
 
   if (onlineNum == pNode->replica) {
     // if all peers are online, peer with highest version shall be master
     index = 0;
     for (int i = 1; i < pNode->replica; ++i) {
-      if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version)
+      if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version) {
         index = i;
       }
     }
+  }
 
   // add arbitrator connection
   SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
@@ -568,11 +578,12 @@ static void syncChooseMaster(SSyncNode *pNode) {
       //slave with highest version shall be master
       pPeer = pNode->peerInfo[i];
       if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) {
-        if (index < 0 || pPeer->version > pNode->peerInfo[index]->version)
+        if (index < 0 || pPeer->version > pNode->peerInfo[index]->version) {
           index = i;
         }
       }
     }
+  }
 
   if (index >= 0) {
     if (index == pNode->selfIndex) {
@@ -595,9 +606,10 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
   int replica = pNode->replica;
 
   for (int i = 0; i < pNode->replica; ++i) {
-    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) 
+    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
       onlineNum++;
     }
+  }
 
   // add arbitrator connection
   SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
@@ -661,8 +673,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne
   pNode->peerInfo[pNode->selfIndex]->version = nodeVersion;
   pPeer->role = newRole;
 
-  sDebug("%s, own role:%s, new peer role:%s", pPeer->id, 
+  sDebug("%s, own role:%s, new peer role:%s", pPeer->id, syncRole[nodeRole], syncRole[pPeer->role]);
-          syncRole[nodeRole], syncRole[pPeer->role]);  
 
   SSyncPeer *pMaster = syncCheckMaster(pNode);
 
@@ -699,20 +710,23 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne
       if (pNode->replica == 2) consistent = 1;
     }
 
-    if (consistent)
+    if (consistent) {
       syncChooseMaster(pNode);
     }
+  }
 
   if (syncRequired) {
     syncRecoverFromMaster(pMaster);
   }
 
-  if (peerOldRole != newRole || nodeRole != selfOldRole)
+  if (peerOldRole != newRole || nodeRole != selfOldRole) {
     syncBroadcastStatus(pNode);
+  }
 
-  if (nodeRole != TAOS_SYNC_ROLE_MASTER) 
+  if (nodeRole != TAOS_SYNC_ROLE_MASTER) {
     syncResetFlowCtrl(pNode);
   }
+}
 
 static void syncRestartPeer(SSyncPeer *pPeer) {
   sDebug("%s, restart connection", pPeer->id);
@@ -722,9 +736,10 @@ static void syncRestartPeer(SSyncPeer *pPeer) {
   pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
 
   int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
-  if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort))
+  if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) {
     taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
   }
+}
 
 void syncRestartConnection(SSyncPeer *pPeer) {
   if (pPeer->ip == 0) return;
@@ -805,7 +820,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
     return;
   }
 
-  sDebug("%s, try to sync", pPeer->id)
+  sDebug("%s, try to sync", pPeer->id);
 
   SFirstPkt firstPkt;
   memset(&firstPkt, 0, sizeof(firstPkt));
@@ -822,8 +837,6 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
     nodeSStatus = TAOS_SYNC_STATUS_START;
     sInfo("%s, sync-req is sent", pPeer->id);
   }
-
-  return;
 }
 
 static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) {
@@ -877,9 +890,10 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
   pPeer->version = pPeersStatus->version;
   syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role);
 
-  if (pPeersStatus->ack)
+  if (pPeersStatus->ack) {
     syncSendPeersStatusMsgToPeer(pPeer, 0);
   }
+}
 
 static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
   if (pPeer->peerFd < 0) return -1;
@@ -1065,8 +1079,7 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) {
   SSyncPeer *pPeer;
   for (i = 0; i < pNode->replica; ++i) {
     pPeer = pNode->peerInfo[i];
-    if (pPeer && (strcmp(pPeer->fqdn, firstPkt.fqdn) == 0) && (pPeer->port == firstPkt.port))
+    if (pPeer && (strcmp(pPeer->fqdn, firstPkt.fqdn) == 0) && (pPeer->port == firstPkt.port)) break;
-      break;
   }
 
   pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL;
@@ -1091,8 +1104,6 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) {
   }
 
   pthread_mutex_unlock(&(pNode->mutex));
-
-  return;
 }
 
 static void syncProcessBrokenLink(void *param) {
@@ -1123,8 +1134,10 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
     pSyncFwds->fwds--;
   }
 
-  if (pSyncFwds->fwds > 0) 
+  if (pSyncFwds->fwds > 0) {
     pSyncFwds->last = (pSyncFwds->last + 1) % tsMaxFwdInfo;
+  }
+
   SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last;
   pFwdInfo->version = version;
   pFwdInfo->mhandle = mhandle;
@@ -1159,13 +1172,15 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
 
   if (code == 0) {
     pFwdInfo->acks++;
-    if (pFwdInfo->acks >= pNode->quorum-1) 
+    if (pFwdInfo->acks >= pNode->quorum - 1) {
       confirm = 1;
+    }
   } else {
     pFwdInfo->nacks++;
-    if (pFwdInfo->nacks > pNode->replica-pNode->quorum) 
+    if (pFwdInfo->nacks > pNode->replica - pNode->quorum) {
       confirm = 1;
     }
+  }
 
   if (confirm && pFwdInfo->confirmed == 0) {
     sDebug("vgId:%d, forward is confirmed, ver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code);

src/vnode/src/vnodeMain.c

@@ -57,6 +57,9 @@ void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
 #endif
 
 int32_t vnodeInitResources() {
+  int code = syncInit();
+  if (code != 0) return code;
+
   vnodeInitWriteFp();
   vnodeInitReadFp();
 
@@ -70,11 +73,12 @@ int32_t vnodeInitResources() {
 }
 
 void vnodeCleanupResources() {
-
   if (tsDnodeVnodesHash != NULL) {
     taosHashCleanup(tsDnodeVnodesHash);
     tsDnodeVnodesHash = NULL;
   }
+
+  syncCleanUp();
 }
 
 int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {

tests/script/unique/cluster/balance3.sim

@@ -105,6 +105,15 @@ if $dnode4Vnodes != null then
   goto show1
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step2
 print ========= start dnode4
 sql create dnode $hostname4
@@ -132,6 +141,15 @@ if $dnode4Vnodes != 2 then
   goto show2
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step3
 print ========= drop dnode2
 sql drop dnode $hostname2
@@ -167,6 +185,15 @@ if $dnode4Vnodes != 3 then
   goto show3
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 
 print ============================== step4
@@ -195,6 +222,15 @@ if $dnode5Vnodes != 2 then
   goto show4
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step5
 print ========= drop dnode3
 sql drop dnode $hostname3
@@ -232,6 +268,15 @@ endi
 
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step6
 sql create dnode $hostname6
 system sh/exec.sh -n dnode6 -s start
@@ -258,6 +303,15 @@ if $dnode6Vnodes != 2 then
   goto show6
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step7
 print ========= drop dnode4
 sql drop dnode $hostname4
@@ -294,6 +348,14 @@ if $dnode4Vnodes != null then
 endi
 
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
 
 print ============================== step8
 sql create dnode $hostname7
@@ -321,6 +383,15 @@ if $dnode7Vnodes != 2 then
   goto show8
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step9
 print ========= drop dnode1
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
@@ -335,15 +406,20 @@ sql show mnodes
 $dnode1Role = $data2_1
 $dnode4Role = $data2_4
 $dnode5Role = $data2_5
-print dnode1 ==> $dnode1Role
+print dnode1 ==> $data2_1
-print dnode4 ==> $dnode4Role
+print dnode2 ==> $data2_2
-print dnode5 ==> $dnode5Role
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
 
 if $dnode1Role != offline then
   return -1
 endi
 
 print ============================== step9.1
+sleep 2000
 system sh/exec.sh -n dnode1 -s start
 
 $x = 0
@@ -353,6 +429,19 @@ show9:
 	if $x == 20 then
 	  return -1
 	endi
+
+sql show mnodes
+$dnode1Role = $data2_1
+$dnode4Role = $data2_4
+$dnode5Role = $data2_5
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 sql show dnodes -x show9
 $dnode5Vnodes = $data2_5
 print dnode5 $dnode5Vnodes
@@ -374,6 +463,15 @@ endi
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 sleep 5000
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step11
 print ========= add db4
 

tests/tsim/src/simExe.c

@@ -667,7 +667,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
 
   TAOS_RES* pSql = NULL;
   
-  for (int attempt = 0; attempt < 3; ++attempt) {
+  for (int attempt = 0; attempt < 10; ++attempt) {
     simLogSql(rest, false);
     pSql = taos_query(script->taos, rest);
     ret = taos_errno(pSql);