diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index a7238d7f37..f6a08f8198 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -36,6 +36,7 @@ extern int8_t tsEnableVnodeBak; extern int8_t tsEnableTelemetryReporting; extern char tsEmail[]; extern char tsArbitrator[]; +extern int8_t tsArbOnline; // common extern int tsRpcTimer; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 63edb6d206..551fb96195 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -41,6 +41,7 @@ int32_t tsStatusInterval = 1; // second int32_t tsNumOfMnodes = 3; int8_t tsEnableVnodeBak = 1; int8_t tsEnableTelemetryReporting = 1; +int8_t tsArbOnline = 0; char tsEmail[TSDB_FQDN_LEN] = {0}; // common diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 14d1fa5816..792e41dd5b 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -810,6 +810,10 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC } pShow->numOfRows = mnodeGetDnodesNum(); + if (tsArbitrator[0] != 0) { + pShow->numOfRows++; + } + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pIter = NULL; @@ -821,7 +825,7 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; int32_t cols = 0; - SDnodeObj *pDnode = NULL; + SDnodeObj *pDnode = NULL; char *pWrite; while (numOfRows < rows) { @@ -864,10 +868,49 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); cols++; - numOfRows++; + numOfRows++; mnodeDecDnodeRef(pDnode); } + if (tsArbitrator[0] != 0) { + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tsArbitrator, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char *status = dnodeStatus[tsArbOnline > 0 ? TAOS_DN_STATUS_READY : TAOS_DN_STATUS_OFFLINE]; + STR_TO_VARSTR(pWrite, status); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, "arb"); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, "-"); + cols++; + + numOfRows++; + } + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 47090cfa0c..eef687d647 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -87,6 +87,7 @@ typedef struct SsyncPeer { int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process int32_t refCount; + int8_t isArb; int64_t rid; void * timer; void * pConn; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index db100250a8..98100fbdd8 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -475,7 +475,13 @@ static void syncAddArbitrator(SSyncNode *pNode) { } } - pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo); + pPeer = syncAddPeer(pNode, &nodeInfo); + if (pPeer != NULL) { + pPeer->isArb = 1; + sInfo("%s, is added as arbitrator", pPeer->id); + } + + pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = pPeer; } static void syncFreeNode(void *param) { @@ -651,9 +657,14 @@ static void syncChooseMaster(SSyncNode *pNode) { // add arbitrator connection SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; - if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) { - onlineNum++; - replica = pNode->replica + 1; + if (pArb) { + if (pArb->role != TAOS_SYNC_ROLE_OFFLINE) { + onlineNum++; + replica = pNode->replica + 1; + sDebug("vgId:%d, arb:%s is used while choose master", pNode->vgId, pArb->id); + } else { + sError("vgId:%d, arb:%s is not used while choose master for its offline", pNode->vgId, pArb->id); + } } if (index < 0 && onlineNum > replica / 2.0) { @@ -1118,6 +1129,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); + if (pPeer->isArb) tsArbOnline = 1; } else { sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); taosClose(connFd); @@ -1196,7 +1208,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } - sDebug("vgId:%d, sync connection is incomming, tranId:%u", vgId, msg.tranId); + sDebug("vgId:%d, sync connection is incoming, tranId:%u", vgId, msg.tranId); SSyncNode *pNode = *ppNode; pthread_mutex_lock(&pNode->mutex); @@ -1243,6 +1255,9 @@ static void syncProcessBrokenLink(int64_t rid) { sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd); pPeer->peerFd = -1; + if (pPeer->isArb) { + tsArbOnline = 0; + } syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index 11f79fb372..6889843530 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -44,8 +44,6 @@ bool vnodeSetReadyStatus(SVnodeObj* pVnode) { pVnode->status == TAOS_VN_STATUS_UPDATING || pVnode->status == TAOS_VN_STATUS_RESET) { pVnode->status = TAOS_VN_STATUS_READY; set = true; - } else { - vDebug("vgId:%d, cannot set status:ready, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]); } qQueryMgmtReOpen(pVnode->qMgmt); @@ -61,8 +59,6 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) { pVnode->status = TAOS_VN_STATUS_CLOSING; set = true; - } else { - vTrace("vgId:%d, cannot set status:closing, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]); } pthread_mutex_unlock(&pVnode->statusMutex); @@ -70,11 +66,8 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { } bool vnodeSetClosingStatus(SVnodeObj* pVnode) { - int32_t i = 0; while (!vnodeSetClosingStatusImp(pVnode)) { - if (++i % 1000 == 0) { - sched_yield(); - } + taosMsleep(1); } // release local resources only after cutting off outside connections @@ -92,8 +85,6 @@ bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) { if (pVnode->status == TAOS_VN_STATUS_READY) { pVnode->status = TAOS_VN_STATUS_UPDATING; set = true; - } else { - vDebug("vgId:%d, cannot set status:updating, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]); } pthread_mutex_unlock(&pVnode->statusMutex); @@ -107,8 +98,6 @@ static bool vnodeSetResetStatusImp(SVnodeObj* pVnode) { if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) { pVnode->status = TAOS_VN_STATUS_RESET; set = true; - } else { - vDebug("vgId:%d, cannot set status:reset, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]); } pthread_mutex_unlock(&pVnode->statusMutex); @@ -116,11 +105,8 @@ static bool vnodeSetResetStatusImp(SVnodeObj* pVnode) { } bool vnodeSetResetStatus(SVnodeObj* pVnode) { - int32_t i = 0; while (!vnodeSetResetStatusImp(pVnode)) { - if (++i % 1000 == 0) { - sched_yield(); - } + taosMsleep(1); } // release local resources only after cutting off outside connections @@ -167,18 +153,6 @@ bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode) { return in; } -bool vnodeInClosingStatus(SVnodeObj* pVnode) { - bool in = false; - pthread_mutex_lock(&pVnode->statusMutex); - - if (pVnode->status == TAOS_VN_STATUS_CLOSING) { - in = true; - } - - pthread_mutex_unlock(&pVnode->statusMutex); - return in; -} - bool vnodeInResetStatus(SVnodeObj* pVnode) { bool in = false; pthread_mutex_lock(&pVnode->statusMutex); diff --git a/tests/script/issue/TD-2680.sim b/tests/script/issue/TD-2680.sim new file mode 100644 index 0000000000..631332160f --- /dev/null +++ b/tests/script/issue/TD-2680.sim @@ -0,0 +1,202 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 + +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/cfg.sh -n dnode2 -c walLevel -v 2 +system sh/cfg.sh -n dnode3 -c walLevel -v 2 +system sh/cfg.sh -n dnode4 -c walLevel -v 2 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c role -v 1 +system sh/cfg.sh -n dnode2 -c role -v 2 +system sh/cfg.sh -n dnode3 -c role -v 2 +system sh/cfg.sh -n dnode4 -c role -v 2 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator + +print ============== step0 +system sh/exec_tarbitrator.sh -s start + +print ============== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi + +print ============== step2 +sql show dnodes +if $rows != 4 then + return -1 +endi + +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 +print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 + +if $data30 != 0 then + return -1 +endi + +if $data32 != 0 then + return -1 +endi + +if $data33 != 0 then + return -1 +endi + +if $data34 != ready then + return -1 +endi + +if $data35 != arb then + return -1 +endi + +if $data37 != - then + return -1 +endi + +print ============== step4 +system sh/exec_tarbitrator.sh -s stop + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show dnodes +if $rows != 4 then + return -1 +endi + +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 +print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 + +if $data30 != 0 then + return -1 +endi + +if $data32 != 0 then + return -1 +endi + +if $data33 != 0 then + return -1 +endi + +if $data34 != offline then + goto step4 +endi + +if $data35 != arb then + return -1 +endi + +if $data37 != - then + return -1 +endi + +print ============== step5 +system sh/exec_tarbitrator.sh -s start + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show dnodes +if $rows != 4 then + return -1 +endi + +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 +print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 + +if $data30 != 0 then + return -1 +endi + +if $data32 != 0 then + return -1 +endi + +if $data33 != 0 then + return -1 +endi + +if $data34 != ready then + goto step5 +endi + +if $data35 != arb then + return -1 +endi + +if $data37 != - then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/issue/TD-2713.sim b/tests/script/issue/TD-2713.sim new file mode 100644 index 0000000000..b66c55b9b9 --- /dev/null +++ b/tests/script/issue/TD-2713.sim @@ -0,0 +1,145 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c slaveQuery -v 1 +system sh/cfg.sh -n dnode2 -c slaveQuery -v 1 +system sh/cfg.sh -n dnode3 -c slaveQuery -v 1 +system sh/cfg.sh -n dnode4 -c slaveQuery -v 1 + +print ========= step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi + +print ========= step2 +sql create database d1 replica 3 +sql create table d1.t1 (ts timestamp, i int) +sql insert into d1.t1 values(now, 1) + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show d1.vgroups +print online vgroups: $data03 +if $data03 != 3 then + goto step2 +endi +sleep 1000 + +print ========= step3 +$i = 0 +while $i < 100 + $i = $i + 1 + sql select * from d1.t1 + print d1.t1 rows: $rows + if $rows != 1 then + return -1 + endi +endw + +print ========= step4 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +system rm -rf ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/* +system rm -rf ../../../sim/dnode3/data/vnode/vnode2/version.json + +system sh/exec.sh -n dnode1 -s start -x SIGINT +system sh/exec.sh -n dnode2 -s start -x SIGINT +system sh/exec.sh -n dnode3 -s start -x SIGINT + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 30 then + return -1 + endi + +sql show d1.vgroups +print online vgroups: $data03 +if $data03 != 3 then + goto step4 +endi +sleep 1000 + +print ========= step5 +$i = 0 +while $i < 100 + $i = $i + 1 + sql select * from d1.t1 + if $rows != 1 then + return -1 + endi + print d1.t1 rows: $rows +endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 733b01f895..73996a67e9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,6 +1,9 @@ cd ../../../debug; cmake .. cd ../../../debug; make +./test.sh -f issue/TD-2680.sim +./test.sh -f issue/TD-2713.sim + ./test.sh -f general/alter/cached_schema_after_alter.sim ./test.sh -f general/alter/count.sim ./test.sh -f general/alter/dnode.sim diff --git a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim index 1b22db9d82..915fef47da 100644 --- a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim +++ b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim @@ -115,7 +115,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 7 then +if $rows != 8 then sleep 2000 goto wait_dnode_created endi @@ -172,7 +172,7 @@ print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode_offline_overtime_dropped endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim index aaf0da8553..318a89f96b 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim @@ -103,7 +103,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode3_offline_0 endi @@ -170,7 +170,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode3_reready endi @@ -237,7 +237,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_offline endi @@ -282,7 +282,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_reready endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index c9c85cb45d..9fc19d588a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -105,7 +105,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -172,7 +172,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_reready endi @@ -239,7 +239,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode23_offline endi @@ -283,7 +283,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode23_reready endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim index dd868e9eed..08d2db207b 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim @@ -107,7 +107,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_dropped endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim index 3582a3b7b2..df41e4df36 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim @@ -106,7 +106,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -167,7 +167,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_reready endi @@ -227,7 +227,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode3_offline endi @@ -267,7 +267,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode3_reready endi @@ -343,7 +343,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode2_offline endi @@ -383,7 +383,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode2_reready endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim index 4040318801..d814398d06 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim @@ -107,7 +107,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode3_offline_0 endi @@ -198,7 +198,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode3_reready endi @@ -276,7 +276,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_offline_0 endi @@ -357,7 +357,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode23_reready_2 endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index b3bd853ffc..8f2cd9169b 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -110,7 +110,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -219,7 +219,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode3_offline_0 endi diff --git a/tests/script/unique/arbitrator/insert_duplicationTs.sim b/tests/script/unique/arbitrator/insert_duplicationTs.sim index 7c6c6e6e92..d58f903dcf 100644 --- a/tests/script/unique/arbitrator/insert_duplicationTs.sim +++ b/tests/script/unique/arbitrator/insert_duplicationTs.sim @@ -132,7 +132,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_offline endi @@ -190,7 +190,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode2_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim index c11bfa62f9..9527d230e4 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim @@ -108,7 +108,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -174,7 +174,7 @@ if $loopCnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim index bed5cefa76..1da8d749d5 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -204,7 +204,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim index 9fdb70142b..791ba76a8d 100644 --- a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -190,7 +190,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim index 6752e0345d..0d5abb2b91 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim index 1280cfe7a0..0c59ee8525 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -176,7 +176,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim index 27c2ba4328..ce7151f78e 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -178,7 +178,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim index 86fb51cfa9..e29f8267c3 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -204,7 +204,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim index 4dcc5977f3..8f8b996c09 100644 --- a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim @@ -108,7 +108,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -191,7 +191,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim index 4f4cf6a263..f3779cf20a 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim index 98946c4b7c..f70ef1fc8c 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi @@ -176,7 +176,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_ready endi diff --git a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim index 3715be5fa9..8d1a508ef0 100644 --- a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim +++ b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim @@ -115,7 +115,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode2_ready endi @@ -161,7 +161,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode_ready endi diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim index dc8b1d015d..12db84cbb3 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim index d2c82f98ff..06d67a1cc9 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim index 96aec2dcf8..b388bc73a6 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim index a1d7d18c94..2e20d7b5d4 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 3 then +if $rows != 4 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index d4eb360efb..69cdb9f942 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim index 9b918acc8e..491b858500 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim @@ -107,7 +107,7 @@ if $cnt == 20 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim index 4c7bb3d26c..7a5966f60c 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim index 7ac424666a..abd2d8a788 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim @@ -107,7 +107,7 @@ if $cnt == 10 then return -1 endi sql show dnodes -if $rows != 4 then +if $rows != 5 then sleep 2000 goto wait_dnode4_offline_0 endi diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim index 9b02933cbf..e5f2928748 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim @@ -112,7 +112,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_offline endi @@ -159,7 +159,7 @@ if $loopCnt == 20 then endi sql show dnodes -x wait_dnode1_ready -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_ready endi @@ -238,7 +238,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode2_offline endi diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim index 90183949e7..8d063020e7 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim @@ -112,7 +112,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_offline endi @@ -161,7 +161,7 @@ if $loopCnt == 20 then endi sql show dnodes -x wait_dnode1_ready -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_ready endi @@ -234,7 +234,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode2_offline endi diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim index 691b2cb568..69e83a2c00 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim @@ -126,7 +126,7 @@ if $loopCnt == 20 then endi sql show dnodes -x wait_dnode1_ready -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_ready endi diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index 02e2cd02e1..6c9e92502c 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -112,7 +112,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_offline endi