enh: force drop dnode

source/dnode/mnode/impl/inc/mndMnode.h

@@ -28,7 +28,7 @@ SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId);
 void       mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj);
 bool       mndIsMnode(SMnode *pMnode, int32_t dnodeId);
 void       mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet);
-int32_t    mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj);
+int32_t    mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndQnode.h

@@ -29,7 +29,7 @@ void       mndCleanupQnode(SMnode *pMnode);
 SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId);
 void       mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj);
 int32_t    mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit);
-int32_t    mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj);
+int32_t    mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj, bool force);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndSnode.h

@@ -27,7 +27,7 @@ void       mndCleanupSnode(SMnode *pMnode);
 SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t qnodeId);
 void       mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj);
 SEpSet     mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode);
-int32_t    mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj);
+int32_t    mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -42,8 +42,7 @@ int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *p
 int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup);
 int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType);
 int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo);
-int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray);
+int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId, bool force);
-int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId);
 int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
                                   SArray *pArray);
 

source/dnode/mnode/impl/src/mndDnode.c

@@ -670,7 +670,7 @@ _OVER:
 }
 
 static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj, SQnodeObj *pQObj,
-                            SSnodeObj *pSObj, int32_t numOfVnodes) {
+                            SSnodeObj *pSObj, int32_t numOfVnodes, bool force) {
   int32_t  code = -1;
   SSdbRaw *pRaw = NULL;
   STrans  *pTrans = NULL;
@@ -678,7 +678,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
   pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-dnode");
   if (pTrans == NULL) goto _OVER;
   mndTransSetSerial(pTrans);
-  mInfo("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id);
+  mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force);
 
   pRaw = mndDnodeActionEncode(pDnode);
   if (pRaw == NULL) goto _OVER;
@@ -694,22 +694,22 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
 
   if (pMObj != NULL) {
     mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
-    if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER;
+    if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force) != 0) goto _OVER;
   }
 
   if (pQObj != NULL) {
     mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
-    if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj) != 0) goto _OVER;
+    if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force) != 0) goto _OVER;
   }
 
   if (pSObj != NULL) {
     mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id);
-    if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj) != 0) goto _OVER;
+    if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force) != 0) goto _OVER;
   }
 
   if (numOfVnodes > 0) {
     mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
-    if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER;
+    if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force) != 0) goto _OVER;
   }
 
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -767,16 +767,16 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
   }
 
   int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id);
-  if (numOfVnodes > 0 || pMObj != NULL) {
+  if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !dropReq.force) {
     if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
       terrno = TSDB_CODE_NODE_OFFLINE;
-      mError("dnode:%d, failed to drop since %s, has_mnode:%d numOfVnodes:%d", pDnode->id, terrstr(), pMObj != NULL,
+      mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(),
-             numOfVnodes);
+             numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
       goto _OVER;
     }
   }
 
-  code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes);
+  code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes, dropReq.force);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
 _OVER:

source/dnode/mnode/impl/src/mndMnode.c

@@ -472,7 +472,7 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO
   return 0;
 }
 
-static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
+static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj, bool force) {
   SSdb          *pSdb = pMnode->pSdb;
   void          *pIter = NULL;
   SDDropMnodeReq dropReq = {0};
@@ -485,12 +485,21 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
 
   int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE);
   if (totalMnodes == 2) {
+    if (force) {
+      mError("cant't force drop dnode, since a mnode on it and replica is 2");
+      terrno = TSDB_CODE_NODE_OFFLINE;
+      return -1;
+    }
     mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
     if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
-    if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
+    if (!force) {
+      if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
+    }
   } else if (totalMnodes == 3) {
     mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
-    if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
+    if (!force) {
+      if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
+    }
     if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
   } else {
     return -1;
@@ -499,9 +508,9 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
   return 0;
 }
 
-int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
+int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) {
   if (pObj == NULL) return 0;
-  if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1;
+  if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force) != 0) return -1;
   if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1;
   return 0;
 }
@@ -515,7 +524,7 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) {
   mndTransSetSerial(pTrans);
   mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
 
-  if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER;
+  if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

source/dnode/mnode/impl/src/mndQnode.c

@@ -353,11 +353,13 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn
   return 0;
 }
 
-int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj) {
+int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj, bool force) {
   if (pObj == NULL) return 0;
   if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) return -1;
   if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) return -1;
-  if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
+  if (!force) {
+    if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
+  }
   return 0;
 }
 
@@ -368,7 +370,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
   if (pTrans == NULL) goto _OVER;
 
   mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
-  if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER;
+  if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

source/dnode/mnode/impl/src/mndSnode.c

@@ -360,11 +360,13 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn
   return 0;
 }
 
-int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj) {
+int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) {
   if (pObj == NULL) return 0;
   if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) return -1;
   if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) return -1;
-  if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
+  if (!force) {
+    if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
+  }
   return 0;
 }
 
@@ -375,7 +377,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
   if (pTrans == NULL) goto _OVER;
 
   mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
-  if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER;
+  if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

source/dnode/mnode/impl/src/mndVgroup.c

@@ -1071,7 +1071,7 @@ int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb
 }
 
 int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vnIndex,
-                                    SArray *pArray) {
+                                    SArray *pArray, bool force) {
   SVgObj newVg = {0};
   memcpy(&newVg, pVgroup, sizeof(SVgObj));
 
@@ -1080,24 +1080,45 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
     mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
   }
 
-  mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId);
+  if (!force) {
-  if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
+    mInfo("vgId:%d, will add 1 vnode", pVgroup->vgId);
-  if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1]) != 0) return -1;
+    if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
-  for (int32_t i = 0; i < newVg.replica - 1; ++i) {
+    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1]) != 0) return -1;
-    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
+    for (int32_t i = 0; i < newVg.replica - 1; ++i) {
-  }
+      if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
-  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
+    }
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
 
-  mInfo("vgId:%d, will remove 1 vnodes", pVgroup->vgId);
+    mInfo("vgId:%d, will remove 1 vnode", pVgroup->vgId);
-  newVg.replica--;
+    newVg.replica--;
-  SVnodeGid del = newVg.vnodeGid[vnIndex];
+    SVnodeGid del = newVg.vnodeGid[vnIndex];
-  newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
+    newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
-  memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
+    memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
-  if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
+    if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
-  for (int32_t i = 0; i < newVg.replica; ++i) {
+    for (int32_t i = 0; i < newVg.replica; ++i) {
-    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
+      if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
+    }
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
+  } else {
+    mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId);
+    if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
+    newVg.replica--;
+    SVnodeGid del = newVg.vnodeGid[vnIndex];
+    newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
+    memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
+
+    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex]) != 0) return -1;
+    for (int32_t i = 0; i < newVg.replica; ++i) {
+      if (i != vnIndex) {
+        if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
+      }
+    }
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
+
+    if (newVg.replica == 1) {
+      mInfo("vgId:%d, all data is dropped since replica=1 and dnode force dropped, stable should be recreated in this vgroup, but we not support it yet", pVgroup->vgId);
+    }
   }
-  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
 
   {
     SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
@@ -1120,7 +1141,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
   return 0;
 }
 
-int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId) {
+int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId, bool force) {
   int32_t code = 0;
   SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId);
   if (pArray == NULL) return -1;
@@ -1141,9 +1162,9 @@ int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t del
 
     code = 0;
     if (vnIndex != -1) {
-      mInfo("vgId:%d, vnode:%d will be removed from dnode:%d", pVgroup->vgId, vnIndex, delDnodeId);
+      mInfo("vgId:%d, vnode:%d will be removed from dnode:%d, force:%d", pVgroup->vgId, vnIndex, delDnodeId, force);
       SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
-      code = mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray);
+      code = mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray, force);
       mndReleaseDb(pMnode, pDb);
     }
 

source/util/src/tconfig.c

@@ -915,7 +915,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
   int32_t olen, vlen, vlen2, vlen3;
   int32_t code = 0;
   if (url == NULL || strlen(url) == 0) {
-    uInfo("fail to load apoll url");
+    uInfo("apoll url not load");
     return 0;
   }
 

tests/script/jenkins/basic.txt

@@ -48,6 +48,7 @@
 ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
+./test.sh -f tsim/dnode/drop_dnode_force.sim
 ./test.sh -f tsim/dnode/offline_reason.sim
 ./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
 ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim

tests/script/tsim/dnode/drop_dnode_force.sim

@@ -0,0 +1,257 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+
+print =============== step1 create dnode2 dnode3 dnode4 dnode 5
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+sql create dnode $hostname port 7500
+
+$x = 0
+step1:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql select * from information_schema.ins_dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 5 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != ready then
+  goto step1
+endi
+if $data(4)[4] != ready then
+  goto step1
+endi
+if $data(5)[4] != offline then
+  goto step1
+endi
+
+print =============== step2 create database
+sql create database d1 vgroups 1 replica 3
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c0 using st tags(0)
+sql create table d1.c1 using st tags(1)
+sql create table d1.c2 using st tags(2)
+sql create table d1.c3 using st tags(3)
+sql create table d1.c4 using st tags(4)
+sql create table d1.c5 using st tags(5)
+sql create table d1.c6 using st tags(6)
+sql create table d1.c7 using st tags(7)
+sql create table d1.c8 using st tags(8)
+sql create table d1.c9 using st tags(9)
+sql show d1.tables
+if $rows != 10 then
+  return -1
+endi
+
+sql create database d2 vgroups 3 replica 1
+sql use d2
+sql create table d2.st1 (ts timestamp, i int) tags (j int)
+sql create table d2.c10 using st1 tags(0)
+sql create table d2.c11 using st1 tags(1)
+sql create table d2.c12 using st1 tags(2)
+sql create table d2.c13 using st1 tags(3)
+sql create table d2.c14 using st1 tags(4)
+sql create table d2.c15 using st1 tags(5)
+sql create table d2.c16 using st1 tags(6)
+sql create table d2.c17 using st1 tags(7)
+sql create table d2.c18 using st1 tags(8)
+sql create table d2.c19 using st1 tags(9)
+sql show d2.tables
+if $rows != 10 then
+  return -1
+endi
+sql create table d2.st2 (ts timestamp, i int) tags (j int)
+sql create table d2.c20 using st2 tags(0)
+sql create table d2.c21 using st2 tags(1)
+sql create table d2.c22 using st2 tags(2)
+sql create table d2.c23 using st2 tags(3)
+sql create table d2.c24 using st2 tags(4)
+sql create table d2.c25 using st2 tags(5)
+sql create table d2.c26 using st2 tags(6)
+sql create table d2.c27 using st2 tags(7)
+sql create table d2.c28 using st2 tags(8)
+sql create table d2.c29 using st2 tags(9)
+sql show d2.tables
+if $rows != 20 then
+  return -1
+endi
+
+print =============== step3: create qnode snode on dnode 2
+sql create qnode on dnode 2
+sql create snode on dnode 2
+sql select * from information_schema.ins_qnodes
+if $rows != 1 then
+  return -1
+endi
+sql show snodes
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step4: create mnode on dnode 2
+sql create mnode on dnode 3
+sql create mnode on dnode 2
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql select * from information_schema.ins_mnodes -x step4
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[1][4] 
+#if $data(1)[2] != leader then
+#  goto step4
+#endi
+if $data(2)[2] != follower then
+  goto step4
+endi
+#if $data(3)[2] != follower then
+#  goto step4
+#endi
+
+print =============== step5: create dnode 5
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode5 -s start
+$x = 0
+step5:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql select * from information_schema.ins_dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 5 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step5
+endi
+if $data(2)[4] != offline then
+  goto step5
+endi
+if $data(3)[4] != ready then
+  goto step5
+endi
+if $data(4)[4] != ready then
+  goto step5
+endi
+if $data(5)[4] != ready then
+  goto step5
+endi
+
+print =============== step5: drop dnode 2
+sql_error drop dnode 2
+sql drop dnode 2 force
+
+print select * from information_schema.ins_dnodes; 
+sql select * from information_schema.ins_dnodes;
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+if $rows != 4 then 
+  return -1
+endi
+
+print select * from information_schema.ins_mnodes; 
+sql select * from information_schema.ins_mnodes
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[2] != leader then
+  return -1
+endi
+
+sql select * from information_schema.ins_qnodes
+if $rows != 0 then
+  return -1
+endi
+sql show snodes
+if $rows != 0 then
+  return -1
+endi
+
+print =============== step6: check d1
+sql show d1.tables
+if $rows != 10 then
+  return -1
+endi
+
+print =============== step7: check d2
+sql show d2.tables
+print ===> rows: $rows remained
+if $rows > 10 then
+  return -1
+endi
+$remain = $rows
+
+return 
+print =============== step8: alter stable
+sql alter table d2.s1 add column b smallint
+sql create table d2.c30 using st tags(0)
+sql create table d2.c31 using st tags(1)
+sql create table d2.c32 using st tags(2)
+sql create table d2.c33 using st tags(3)
+sql create table d2.c34 using st tags(4)
+sql create table d2.c35 using st tags(5)
+sql create table d2.c36 using st tags(6)
+sql create table d2.c37 using st tags(7)
+sql create table d2.c38 using st tags(8)
+sql create table d2.c39 using st tags(9)
+sql show d2.tables
+if $rows != 10 then
+  return -1
+endi
+
+print =============== step9: drop stable and recreate it
+sql create table d2.st2 (ts timestamp, i int) tags (j int)
+sql create table d2.c20 using st2 tags(0)
+sql create table d2.c21 using st2 tags(1)
+sql create table d2.c22 using st2 tags(2)
+sql create table d2.c23 using st2 tags(3)
+sql create table d2.c24 using st2 tags(4)
+sql create table d2.c25 using st2 tags(5)
+sql create table d2.c26 using st2 tags(6)
+sql create table d2.c27 using st2 tags(7)
+sql create table d2.c28 using st2 tags(8)
+sql create table d2.c29 using st2 tags(9)
+sql show d2.tables
+if $rows != 20 then
+  return -1
+endi
+
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT