TD-2513

src/balance/src/bnMain.c

@@ -489,6 +489,7 @@ void bnCheckStatus() {
         mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
               pDnode->lastAccess, pDnode->status);
         bnSetVgroupOffline(pDnode);
+        bnStartTimer(3000);
       }
     }
     mnodeDecDnodeRef(pDnode);

src/balance/src/bnThread.c

@@ -104,8 +104,8 @@ static void bnProcessTimer(void *handle, void *tmrId) {
   tsBnThread.timer = NULL;
   tsAccessSquence++;
 
-  bnCheckStatus();
   bnStartTimer(-1);
+  bnCheckStatus();
 
   if (handle == NULL) {
     if (tsAccessSquence % tsBalanceInterval == 0) {
@@ -124,6 +124,7 @@ void bnStartTimer(int64_t mseconds) {
 
   bool updateSoon = (mseconds != -1);
   if (updateSoon) {
+    mTrace("balance function will be called after %" PRId64 " ms", mseconds);
     taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
   } else {
     taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);

src/common/src/tglobal.c

@@ -543,7 +543,7 @@ static void doInitGlobalConfig(void) {
   cfg.ptr = &tsOfflineThreshold;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 5;
+  cfg.minValue = 3;
   cfg.maxValue = 7200000;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_SECOND;

src/sync/src/syncMain.c

@@ -480,7 +480,7 @@ static void syncFreeNode(void *param) {
   SSyncNode *pNode = param;
 
   int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1);
-  sDebug("vgId:%d, snode is freed, refCount:%d", pNode->vgId, refCount);
+  sDebug("vgId:%d, syncnode is freed, refCount:%d", pNode->vgId, refCount);
 
   pthread_mutex_destroy(&pNode->mutex);
   tfree(pNode->pRecv);
@@ -491,10 +491,10 @@ static void syncFreeNode(void *param) {
 SSyncNode *syncAcquireNode(int64_t rid) {
   SSyncNode *pNode = taosAcquireRef(tsNodeRefId, rid);
   if (pNode == NULL) {
-    sDebug("failed to acquire snode from refId:%" PRId64, rid);
+    sDebug("failed to acquire syncnode from refId:%" PRId64, rid);
   } else {
     int32_t refCount = atomic_add_fetch_32(&pNode->refCount, 1);
-    sTrace("vgId:%d, acquire snode refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount);
+    sTrace("vgId:%d, acquire syncnode refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount);
   }
 
   return pNode;
@@ -502,7 +502,7 @@ SSyncNode *syncAcquireNode(int64_t rid) {
 
 void syncReleaseNode(SSyncNode *pNode) {
   int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1);
-  sTrace("vgId:%d, dec snode refId:%" PRId64 " refCount:%d", pNode->vgId, pNode->rid, refCount);
+  sTrace("vgId:%d, dec syncnode refId:%" PRId64 " refCount:%d", pNode->vgId, pNode->rid, refCount);
 
   taosReleaseRef(tsNodeRefId, pNode->rid);
 }
@@ -716,8 +716,8 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
   if (onlineNum <= replica * 0.5) {
     if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
       nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-      (*pNode->notifyRole)(pNode->vgId, nodeRole);
       sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
+      (*pNode->notifyRole)(pNode->vgId, nodeRole);
     }
   } else {
     for (int32_t index = 0; index < pNode->replica; ++index) {

tests/script/jenkins/basic.txt

@@ -288,6 +288,7 @@ cd ../../../debug; make
 ./test.sh -f unique/dnode/data1.sim
 ./test.sh -f unique/dnode/offline1.sim
 ./test.sh -f unique/dnode/offline2.sim
+./test.sh -f unique/dnode/offline3.sim
 ./test.sh -f unique/dnode/reason.sim
 ./test.sh -f unique/dnode/remove1.sim
 ./test.sh -f unique/dnode/remove2.sim

tests/script/jenkins/unique.txt

@@ -31,6 +31,8 @@ cd ../../../debug; make
 ./test.sh -f unique/dnode/balancex.sim
 ./test.sh -f unique/dnode/offline1.sim
 ./test.sh -f unique/dnode/offline2.sim
+./test.sh -f unique/dnode/offline3.sim
+./test.sh -f unique/dnode/reason.sim
 ./test.sh -f unique/dnode/remove1.sim
 ./test.sh -f unique/dnode/remove2.sim
 ./test.sh -f unique/dnode/vnode_clean.sim

tests/script/unique/dnode/offline3.sim

@@ -0,0 +1,111 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c offlineThreshold -v 3
+system sh/cfg.sh -n dnode2 -c offlineThreshold -v 3
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 3
+system sh/cfg.sh -n dnode4 -c offlineThreshold -v 3
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 300
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 300
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 300
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 300
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+
+print ========== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sql create dnode $hostname2
+system sh/exec.sh -n dnode2 -s start
+sql create dnode $hostname3
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname4
+system sh/exec.sh -n dnode4 -s start
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+if $data4_4 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+if $data2_1 != master then
+  goto step1
+endi
+
+print ========== step2
+sql create database db replica 3
+sql use db
+sql create table tb (ts timestamp, value int)
+sql insert into tb values (now, 1)
+sql insert into tb values (now, 2)
+
+sql show vgroups;
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+print ========== step2
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+sql show vgroups;
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+if $data4_1 != ready then
+  goto step2
+endi
+if $data4_2 != ready then
+  goto step2
+endi
+if $data4_3 != ready then
+  goto step2
+endi
+if $data4_4 != null then
+  goto step2
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT