diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 3680a46d85..a7b3bfd024 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -117,7 +117,7 @@ int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg); TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid); -uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size); +uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size); // the TSDB repository info typedef struct STsdbRepoInfo { diff --git a/src/inc/tsync.h b/src/inc/tsync.h index fcf26d22c3..972db294f6 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -21,6 +21,7 @@ extern "C" { #endif #define TAOS_SYNC_MAX_REPLICA 5 +#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF typedef enum _TAOS_SYNC_ROLE { TAOS_SYNC_ROLE_OFFLINE, @@ -53,11 +54,16 @@ typedef struct { uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; int role[TAOS_SYNC_MAX_REPLICA]; } SNodesRole; - -// if name is empty(name[0] is zero), get the file from index or after, used by master -// if name is provided(name[0] is not zero), get the named file at the specified index, used by unsynced node -// it returns the file magic number and size, if file not there, magic shall be 0. -typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion); + +/* + if name is empty(name[0] is zero), get the file from index or after, but not larger than eindex. If a file + is found between index and eindex, index shall be updated, name shall be set, size shall be set to + file size, and file magic number shall be returned. + + if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return + zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated. +*/ +typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 6638fba53b..51b5fb0d2d 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -200,7 +200,7 @@ void sdbUpdateMnodeRoles() { mnodeUpdateMnodeIpSet(); } -static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) { +static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { sdbUpdateMnodeRoles(); return 0; } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index e1b85ae99d..8b3d9f135b 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -508,6 +508,8 @@ void tsdbAdjustCacheBlocks(STsdbCache *pCache); int32_t tsdbGetMetaFileName(char *rootDir, char *fname); int tsdbUpdateFileHeader(SFile *pFile, uint32_t version); +int compFGroupKey(const void *key, const void *fgroup); + #ifdef __cplusplus } #endif diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 413b4cff13..f8c00033b2 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -35,7 +35,6 @@ const char *tsdbFileSuffix[] = { ".last" // TSDB_FILE_TYPE_LAST }; -static int compFGroupKey(const void *key, const void *fgroup); static int compFGroup(const void *arg1, const void *arg2); static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid); @@ -285,7 +284,7 @@ int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInf return 0; } -static int compFGroupKey(const void *key, const void *fgroup) { +int compFGroupKey(const void *key, const void *fgroup) { int fid = *(int *)key; SFileGroup *pFGroup = (SFileGroup *)fgroup; if (fid == pFGroup->fileId) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index e79873d1b9..ef5b9f7261 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -1199,46 +1199,71 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); } -uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size) { - // TODO: need to refactor this function - +#define TSDB_META_FILE_INDEX 10000000 +uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) { STsdbRepo *pRepo = (STsdbRepo *)repo; // STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; - uint32_t magic = 0; - char fname[256] = "\0"; + uint32_t magic = 0; + char fname[256] = "\0"; struct stat fState; - char *spath = strdup(pRepo->rootDir); - char *prefixDir = dirname(spath); - if (name[0] == 0) { - // Map index to the file name + tsdbTrace("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); + ASSERT(*index <= eindex); + + char *sdup = strdup(pRepo->rootDir); + char *prefix = dirname(sdup); + + if (name[0] == 0) { // get the file from index or after, but not larger than eindex int fid = (*index) / 3; - if (fid >= pFileH->numOfFGroups) { - // return meta data file - if ((*index) % 3 > 0) { // it is finished - tfree(spath); - return 0; - } else { + if (pFileH->numOfFGroups == 0 || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) { + if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { tsdbGetMetaFileName(pRepo->rootDir, fname); + *index = TSDB_META_FILE_INDEX; + } else { + tfree(sdup); + return 0; } } else { - // return data file name - strcpy(fname, pFileH->fGroup[fid].files[(*index) % 3].fname); + SFileGroup *pFGroup = + taosbsearch(&fid, pFileH->fGroup, pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey, TD_GE); + if (pFGroup->fileId == fid) { + strcpy(fname, pFGroup->files[(*index) % 3].fname); + } else { + if (pFGroup->fileId * 3 + 2 < eindex) { + strcpy(fname, pFGroup->files[0].fname); + *index = pFGroup->fileId * 3; + } else { + tfree(sdup); + return 0; + } + } + } + strcpy(name, fname + strlen(prefix)); + } else { // get the named file at the specified index. If not there, return 0 + if (*index == TSDB_META_FILE_INDEX) { // get meta file + tsdbGetMetaFileName(pRepo->rootDir, fname); + } else { + int fid = (*index) / 3; + SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid); + if (pFGroup == NULL) { // not found + tfree(sdup); + return 0; + } + + SFile *pFile = &pFGroup->files[(*index) % 3]; + strcpy(fname, pFile->fname); } - strcpy(name, fname + strlen(spath)); - } else { - // Name is provided, need to get the file info - sprintf(fname, "%s/%s", prefixDir, name); } if (stat(fname, &fState) < 0) { - tfree(spath); + tfree(sdup); return 0; } + tfree(sdup); *size = fState.st_size; magic = *size; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index aa2ddc7776..a3386ecfd9 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -39,7 +39,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode); static int32_t vnodeSaveVersion(SVnodeObj *pVnode); static int32_t vnodeReadVersion(SVnodeObj *pVnode); static int vnodeProcessTsdbStatus(void *arg, int status); -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion); +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); @@ -433,10 +433,10 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) { +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { SVnodeObj *pVnode = ahandle; *fversion = pVnode->fversion; - return tsdbGetFileInfo(pVnode->tsdb, name, index, size); + return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); } static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim new file mode 100644 index 0000000000..f34322a255 --- /dev/null +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -0,0 +1,413 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 + +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/cfg.sh -n dnode2 -c walLevel -v 2 +system sh/cfg.sh -n dnode3 -c walLevel -v 2 +system sh/cfg.sh -n dnode4 -c walLevel -v 2 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 + +system sh/cfg.sh -n dnode1 -c alternativeRole -v 1 +system sh/cfg.sh -n dnode2 -c alternativeRole -v 2 +system sh/cfg.sh -n dnode3 -c alternativeRole -v 2 +system sh/cfg.sh -n dnode4 -c alternativeRole -v 2 + +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 + +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator + +print ============== step0: start tarbitrator +system sh/exec_tarbitrator.sh -s start + +print ============== step1: start dnode1, only deploy mnode +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect + +print ============== step2: start dnode2 and add into cluster , then create database with replica 1, and create table, insert data +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname2 +sleep 3000 + +$totalTableNum = 10000 +$sleepTimer = 10000 + +$db = db +sql create database $db replica 1 maxTables $totalTableNum +sql use $db + +# create table , insert data +$stb = stb +sql create table $stb (ts timestamp, c1 int) tags(t1 int) +$rowNum = 100 +$tblNum = $totalTableNum +$totalRows = 0 +$tsStart = 1420041600000 + +$i = 0 +while $i < $tblNum + $tb = tb . $i + sql create table $tb using $stb tags( $i ) + + $x = 0 + while $x < $rowNum + $ts = $tsStart + $x + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x ) + $x = $x + 60 + endw + $totalRows = $totalRows + $x + print info: inserted $x rows into $tb and totalRows: $totalRows + $i = $i + 1 +endw + +sql select count(*) from $stb +sleep 1000 +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname3 +sleep 3000 + +sql alter database $db replica 2 +sleep $sleepTimer + +wait_dnode3_ready: +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode3_ready +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +#$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +#$dnode4Status = $data4_4 +#$dnode5Status = $data4_5 + +if $dnode2Status != ready then + sleep 2000 + goto wait_dnode3_ready +endi +if $dnode3Status != ready then + sleep 2000 + goto wait_dnode3_ready +endi + +sleep $sleepTimer +# check using select +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + + +print ============== step4: stop dnode2 for checking if sync success +system sh/exec.sh -n dnode2 -s stop +sleep $sleepTimer + +wait_dnode2_offline: +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode2_offline +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +#$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +#$dnode4Status = $data4_4 +#$dnode5Status = $data4_5 + +if $dnode2Status != offline then + sleep 2000 + goto wait_dnode2_offline +endi +if $dnode3Status != ready then + sleep 2000 + goto wait_dnode2_offline +endi + +sleep $sleepTimer # waitting for move master vnode of dnode2 to dnode3 +# check using select +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +print ============== step5: restart dnode2 +system sh/exec.sh -n dnode2 -s start +sleep 3000 + +wait_dnode2_ready: +sql show dnodes +if $rows != 3 then + sleep 2000 + goto wait_dnode2_ready +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +#$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +#$dnode4Status = $data4_4 +#$dnode5Status = $data4_5 + +if $dnode2Status != ready then + sleep 2000 + goto wait_dnode2_ready +endi + +sleep $sleepTimer +# check using select +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +print ============== step6: start dnode4 and add into cluster , then alter replica from 2 to 3, and waiting sync +system sh/exec.sh -n dnode4 -s start +sql create dnode $hostname4 +sleep 3000 + +sql alter database $db replica 3 +sleep $sleepTimer + +wait_dnode4_ready: +sql show dnodes +if $rows != 4 then + sleep 2000 + goto wait_dnode4_ready +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +#$dnode1Status = $data4_1 +#$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +$dnode4Status = $data4_4 +#$dnode5Status = $data4_5 + +if $dnode4Status != ready then + sleep 2000 + goto wait_dnode4_ready +endi + +sleep $sleepTimer +# check using select +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +print ============== step7: alter replica from 3 to 2, and waiting sync +sql alter database $db replica 2 +sleep $sleepTimer + +wait_vgroups_replic_to_2: +sql show vgroups +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 +#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +$thirdDnode_2 = $data8_1 +$thirdDnode_3 = $data8_2 +$thirdDnode_4 = $data8_3 +$thirdDnode_5 = $data8_4 + +if $thirdDnode_2 != null then + sleep 2000 + goto wait_vgroups_replic_to_2 +endi +if $thirdDnode_3 != null then + sleep 2000 + goto wait_vgroups_replic_to_2 +endi +if $thirdDnode_4 != null then + sleep 2000 + goto wait_vgroups_replic_to_2 +endi +if $thirdDnode_5 != null then + sleep 2000 + goto wait_vgroups_replic_to_2 +endi + +sleep $sleepTimer #waiting del one replica data +# check using select +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +print ============== step8: alter replica from 2 to 1, and waiting sync +sql alter database $db replica 1 +sleep $sleepTimer + +wait_vgroups_replic_to_1: +sql show vgroups +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 +#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +$sencodDnode_2 = $data5_1 +$sencodDnode_3 = $data5_2 +$sencodDnode_4 = $data5_3 +$sencodDnode_5 = $data5_4 + +if $sencodDnode_2 != null then + sleep 2000 + goto wait_vgroups_replic_to_1 +endi +if $sencodDnode_3 != null then + sleep 2000 + goto wait_vgroups_replic_to_1 +endi +if $sencodDnode_4 != null then + sleep 2000 + goto wait_vgroups_replic_to_1 +endi +if $sencodDnode_5 != null then + sleep 2000 + goto wait_vgroups_replic_to_1 +endi + +all_dnodes_ready: +sql show dnodes +if $rows != 4 then + sleep 2000 + goto all_dnodes_ready +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +$dnode4Status = $data4_4 +#$dnode5Status = $data4_5 + +if $dnode1Status != ready then + sleep 2000 + goto all_dnodes_ready +endi +if $dnode2Status != ready then + sleep 2000 + goto all_dnodes_ready +endi +if $dnode3Status != ready then + sleep 2000 + goto all_dnodes_ready +endi +if $dnode4Status != ready then + sleep 2000 + goto all_dnodes_ready +endi + +sleep $sleepTimer #waiting del one replica data +# check using select +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi + +print ============== step9: drop dnode2/dnode3 +sql drop dnode $hostname2 +sql drop dnode $hostname3 +sleep $sleepTimer + +wait_dnode23_dropped: +sql show dnodes +if $rows != 2 then + sleep 2000 + goto wait_dnode23_dropped +endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +$dnode1Status = $data4_1 +$dnode2Status = $data4_2 +$dnode3Status = $data4_3 +$dnode4Status = $data4_4 + +if $dnode2Status != null then + sleep 2000 + goto wait_dnode23_dropped +endi +if $dnode3Status != null then + sleep 2000 + goto wait_dnode23_dropped +endi +if $dnode4Status != ready then + return -1 +endi + +sleep $sleepTimer #waiting move vnode from dnode3/dnode3 to dnode4 +# check using select +sql select count(*) from $stb +print data00 $data00 +if $data00 != $totalRows then + return -1 +endi