xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor retry

This commit is contained in:
Yihao Deng 2024-01-19 06:20:16 +00:00
parent 9d82fd8c78
commit 6c732a14a3
2 changed files with 260 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndMnode.h" #include "mndMnode.h"
#include "audit.h"
#include "mndCluster.h" #include "mndCluster.h"
#include "mndDnode.h" #include "mndDnode.h"
#include "mndPrivilege.h" #include "mndPrivilege.h"
@ -22,7 +23,6 @@
#include "mndSync.h" #include "mndSync.h"
#include "mndTrans.h" #include "mndTrans.h"
#include "tmisce.h" #include "tmisce.h"
#include "audit.h"
#define MNODE_VER_NUMBER 2 #define MNODE_VER_NUMBER 2
#define MNODE_RESERVE_SIZE 64 #define MNODE_RESERVE_SIZE 64
@ -241,6 +241,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
} }
void *pIter = NULL; void *pIter = NULL;
pEpSet->inUse = 0;
while (1) { while (1) {
SMnodeObj *pObj = NULL; SMnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
@ -250,7 +251,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
if (mndIsLeader(pMnode)) { if (mndIsLeader(pMnode)) {
pEpSet->inUse = pEpSet->numOfEps; pEpSet->inUse = pEpSet->numOfEps;
} else { } else {
pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes; // pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes;
} }
} }
if (pObj->pDnode != NULL) { if (pObj->pDnode != NULL) {
@ -320,8 +321,8 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p
return 0; return 0;
} }
static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq,
SDAlterMnodeTypeReq *pAlterMnodeTypeReq, SEpSet *pAlterMnodeTypeEpSet) { SEpSet *pAlterMnodeTypeEpSet) {
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq); tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq);
@ -401,8 +402,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; createReq.replicas[numOfReplicas].port = pMObj->pDnode->port;
memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
numOfReplicas++; numOfReplicas++;
} } else {
else{
createReq.learnerReplicas[numOfLearnerReplicas].id = pMObj->id; createReq.learnerReplicas[numOfLearnerReplicas].id = pMObj->id;
createReq.learnerReplicas[numOfLearnerReplicas].port = pMObj->pDnode->port; createReq.learnerReplicas[numOfLearnerReplicas].port = pMObj->pDnode->port;
memcpy(createReq.learnerReplicas[numOfLearnerReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createReq.learnerReplicas[numOfLearnerReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
@ -451,8 +451,7 @@ int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createReq.replicas[createReq.replica].port = pMObj->pDnode->port; createReq.replicas[createReq.replica].port = pMObj->pDnode->port;
memcpy(createReq.replicas[createReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createReq.replicas[createReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
createReq.replica++; createReq.replica++;
} } else {
else{
createReq.learnerReplicas[createReq.learnerReplica].id = pMObj->id; createReq.learnerReplicas[createReq.learnerReplica].id = pMObj->id;
createReq.learnerReplicas[createReq.learnerReplica].port = pMObj->pDnode->port; createReq.learnerReplicas[createReq.learnerReplica].port = pMObj->pDnode->port;
memcpy(createReq.learnerReplicas[createReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createReq.learnerReplicas[createReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
@ -495,8 +494,7 @@ static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
alterReq.replicas[alterReq.replica].port = pMObj->pDnode->port; alterReq.replicas[alterReq.replica].port = pMObj->pDnode->port;
memcpy(alterReq.replicas[alterReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); memcpy(alterReq.replicas[alterReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
alterReq.replica++; alterReq.replica++;
} } else {
else{
alterReq.learnerReplicas[alterReq.learnerReplica].id = pMObj->id; alterReq.learnerReplicas[alterReq.learnerReplica].id = pMObj->id;
alterReq.learnerReplicas[alterReq.learnerReplica].port = pMObj->pDnode->port; alterReq.learnerReplicas[alterReq.learnerReplica].port = pMObj->pDnode->port;
memcpy(alterReq.learnerReplicas[alterReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); memcpy(alterReq.learnerReplicas[alterReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
@ -544,8 +542,7 @@ int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
alterReq.replicas[alterReq.replica].port = pMObj->pDnode->port; alterReq.replicas[alterReq.replica].port = pMObj->pDnode->port;
memcpy(alterReq.replicas[alterReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); memcpy(alterReq.replicas[alterReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
alterReq.replica++; alterReq.replica++;
} } else {
else{
alterReq.learnerReplicas[alterReq.learnerReplica].id = pMObj->id; alterReq.learnerReplicas[alterReq.learnerReplica].id = pMObj->id;
alterReq.learnerReplicas[alterReq.learnerReplica].port = pMObj->pDnode->port; alterReq.learnerReplicas[alterReq.learnerReplica].port = pMObj->pDnode->port;
memcpy(alterReq.learnerReplicas[alterReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); memcpy(alterReq.learnerReplicas[alterReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
@ -959,7 +956,10 @@ static void mndReloadSyncConfig(SMnode *pMnode) {
void *pIter = NULL; void *pIter = NULL;
int32_t updatingMnodes = 0; int32_t updatingMnodes = 0;
int32_t readyMnodes = 0; int32_t readyMnodes = 0;
SSyncCfg cfg = {.myIndex = -1, .lastIndex = 0,}; SSyncCfg cfg = {
.myIndex = -1,
.lastIndex = 0,
};
SyncIndex maxIndex = 0; SyncIndex maxIndex = 0;
while (1) { while (1) {
@ -1023,8 +1023,8 @@ static void mndReloadSyncConfig(SMnode *pMnode) {
} }
if (pMnode->syncMgmt.sync > 0) { if (pMnode->syncMgmt.sync > 0) {
mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", cfg.totalReplicaNum, cfg.replicaNum,
cfg.totalReplicaNum, cfg.replicaNum, cfg.myIndex); cfg.myIndex);
for (int32_t i = 0; i < cfg.totalReplicaNum; ++i) { for (int32_t i = 0; i < cfg.totalReplicaNum; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i]; SNodeInfo *pNode = &cfg.nodeInfo[i];

166
source/libs/sync/src/syncMain.c
View File

@ -589,6 +589,7 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) {
} }
if (pEpSet->numOfEps > 0) { if (pEpSet->numOfEps > 0) {
pEpSet->inUse = (pSyncNode->raftCfg.cfg.myIndex + 1) % pEpSet->numOfEps; pEpSet->inUse = (pSyncNode->raftCfg.cfg.myIndex + 1) % pEpSet->numOfEps;
pEpSet->inUse = 0;
} }
sInfo("vgId:%d, sync get retry epset numOfEps:%d inUse:%d", pSyncNode->vgId, pEpSet->numOfEps, pEpSet->inUse); sInfo("vgId:%d, sync get retry epset numOfEps:%d inUse:%d", pSyncNode->vgId, pEpSet->numOfEps, pEpSet->inUse);
@ -695,11 +696,11 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_
sTrace("vgId:%d, propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, sTrace("vgId:%d, propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex,
TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
return 1; return 1;
} } else {
else{ sTrace("vgId:%d, propose optimized msg, return to normal, index:%" PRId64
sTrace("vgId:%d, propose optimized msg, return to normal, index:%" PRId64 " type:%s, " " type:%s, "
"handle:%p", pSyncNode->vgId, retIndex, "handle:%p",
TMSG_INFO(pMsg->msgType), pMsg->info.handle); pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType), pMsg->info.handle);
return 0; return 0;
} }
} else { } else {
@ -856,14 +857,12 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) {
sInfo("vgId:%d, use sync config from sync cfg file", pSyncNode->vgId); sInfo("vgId:%d, use sync config from sync cfg file", pSyncNode->vgId);
pSyncInfo->syncCfg = pSyncNode->raftCfg.cfg; pSyncInfo->syncCfg = pSyncNode->raftCfg.cfg;
} }
} } else {
else{ sInfo("vgId:%d, skip save sync cfg file since request ver:%d <= file ver:%d", pSyncNode->vgId, vnodeVersion,
sInfo("vgId:%d, skip save sync cfg file since request ver:%d <= file ver:%d", pSyncInfo->syncCfg.changeVersion);
pSyncNode->vgId, vnodeVersion, pSyncInfo->syncCfg.changeVersion);
} }
} }
// init by SSyncInfo // init by SSyncInfo
pSyncNode->vgId = pSyncInfo->vgId; pSyncNode->vgId = pSyncInfo->vgId;
SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg; SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg;
@ -2346,7 +2345,8 @@ void syncBuildConfigFromReq(SAlterVnodeReplicaReq *pReq, SSyncCfg *cfg){//TODO S
tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; pNode->nodeRole = TAOS_SYNC_ROLE_VOTER;
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
sInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d nodeRole:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->nodeRole); sInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d nodeRole:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId, pNode->nodeRole);
cfg->replicaNum++; cfg->replicaNum++;
} }
if (pReq->selfIndex != -1) { if (pReq->selfIndex != -1) {
@ -2359,7 +2359,8 @@ void syncBuildConfigFromReq(SAlterVnodeReplicaReq *pReq, SSyncCfg *cfg){//TODO S
pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER;
tstrncpy(pNode->nodeFqdn, pReq->learnerReplicas[cfg->totalReplicaNum].fqdn, sizeof(pNode->nodeFqdn)); tstrncpy(pNode->nodeFqdn, pReq->learnerReplicas[cfg->totalReplicaNum].fqdn, sizeof(pNode->nodeFqdn));
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
sInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d nodeRole:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->nodeRole); sInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d nodeRole:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId, pNode->nodeRole);
cfg->totalReplicaNum++; cfg->totalReplicaNum++;
} }
cfg->totalReplicaNum += pReq->replica; cfg->totalReplicaNum += pReq->replica;
@ -2389,8 +2390,8 @@ int32_t syncNodeCheckChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry){
if (cfg.totalReplicaNum >= 1 && ths->state == TAOS_SYNC_STATE_LEADER) { if (cfg.totalReplicaNum >= 1 && ths->state == TAOS_SYNC_STATE_LEADER) {
bool incfg = false; bool incfg = false;
for (int32_t j = 0; j < cfg.totalReplicaNum; ++j) { for (int32_t j = 0; j < cfg.totalReplicaNum; ++j) {
if(strcmp(ths->myNodeInfo.nodeFqdn, cfg.nodeInfo[j].nodeFqdn) == 0 if (strcmp(ths->myNodeInfo.nodeFqdn, cfg.nodeInfo[j].nodeFqdn) == 0 &&
&& ths->myNodeInfo.nodePort == cfg.nodeInfo[j].nodePort){ ths->myNodeInfo.nodePort == cfg.nodeInfo[j].nodePort) {
incfg = true; incfg = true;
break; break;
} }
@ -2406,20 +2407,19 @@ int32_t syncNodeCheckChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry){
} }
void syncNodeLogConfigInfo(SSyncNode* ths, SSyncCfg* cfg, char* str) { void syncNodeLogConfigInfo(SSyncNode* ths, SSyncCfg* cfg, char* str) {
sInfo("vgId:%d, %s. SyncNode, replicaNum:%d, peersNum:%d, lastConfigIndex:%" PRId64 ", changeVersion:%d, " sInfo("vgId:%d, %s. SyncNode, replicaNum:%d, peersNum:%d, lastConfigIndex:%" PRId64
", changeVersion:%d, "
"restoreFinish:%d", "restoreFinish:%d",
ths->vgId, str, ths->vgId, str, ths->replicaNum, ths->peersNum, ths->raftCfg.lastConfigIndex, ths->raftCfg.cfg.changeVersion,
ths->replicaNum, ths->peersNum, ths->raftCfg.lastConfigIndex, ths->raftCfg.cfg.changeVersion,
ths->restoreFinish); ths->restoreFinish);
sInfo("vgId:%d, %s, myNodeInfo, clusterId:%" PRId64 ", nodeId:%d, Fqdn:%s, port:%d, role:%d", sInfo("vgId:%d, %s, myNodeInfo, clusterId:%" PRId64 ", nodeId:%d, Fqdn:%s, port:%d, role:%d", ths->vgId, str,
ths->vgId, str, ths->myNodeInfo.clusterId, ths->myNodeInfo.nodeId, ths->myNodeInfo.nodeFqdn, ths->myNodeInfo.clusterId, ths->myNodeInfo.nodeId, ths->myNodeInfo.nodeFqdn, ths->myNodeInfo.nodePort,
ths->myNodeInfo.nodePort, ths->myNodeInfo.nodeRole); ths->myNodeInfo.nodeRole);
for (int32_t i = 0; i < ths->peersNum; ++i) { for (int32_t i = 0; i < ths->peersNum; ++i) {
sInfo("vgId:%d, %s, peersNodeInfo%d, clusterId:%" PRId64 ", nodeId:%d, Fqdn:%s, port:%d, role:%d", sInfo("vgId:%d, %s, peersNodeInfo%d, clusterId:%" PRId64 ", nodeId:%d, Fqdn:%s, port:%d, role:%d", ths->vgId, str,
ths->vgId, str, i, ths->peersNodeInfo[i].clusterId, i, ths->peersNodeInfo[i].clusterId, ths->peersNodeInfo[i].nodeId, ths->peersNodeInfo[i].nodeFqdn,
ths->peersNodeInfo[i].nodeId, ths->peersNodeInfo[i].nodeFqdn,
ths->peersNodeInfo[i].nodePort, ths->peersNodeInfo[i].nodeRole); ths->peersNodeInfo[i].nodePort, ths->peersNodeInfo[i].nodeRole);
} }
@ -2434,27 +2434,23 @@ void syncNodeLogConfigInfo(SSyncNode* ths, SSyncCfg *cfg, char *str){
} }
n += snprintf(buf + n, len - n, "%s", "}"); n += snprintf(buf + n, len - n, "%s", "}");
sInfo("vgId:%d, %s, peersEpset%d, %s, inUse:%d", sInfo("vgId:%d, %s, peersEpset%d, %s, inUse:%d", ths->vgId, str, i, buf, ths->peersEpset->inUse);
ths->vgId, str, i, buf, ths->peersEpset->inUse);
} }
for (int32_t i = 0; i < ths->peersNum; ++i) { for (int32_t i = 0; i < ths->peersNum; ++i) {
sInfo("vgId:%d, %s, peersId%d, addr:%"PRId64, sInfo("vgId:%d, %s, peersId%d, addr:%" PRId64, ths->vgId, str, i, ths->peersId[i].addr);
ths->vgId, str, i, ths->peersId[i].addr);
} }
for (int32_t i = 0; i < ths->raftCfg.cfg.totalReplicaNum; ++i) { for (int32_t i = 0; i < ths->raftCfg.cfg.totalReplicaNum; ++i) {
sInfo("vgId:%d, %s, nodeInfo%d, clusterId:%" PRId64 ", nodeId:%d, Fqdn:%s, port:%d, role:%d", sInfo("vgId:%d, %s, nodeInfo%d, clusterId:%" PRId64 ", nodeId:%d, Fqdn:%s, port:%d, role:%d", ths->vgId, str, i,
ths->vgId, str, i, ths->raftCfg.cfg.nodeInfo[i].clusterId, ths->raftCfg.cfg.nodeInfo[i].clusterId, ths->raftCfg.cfg.nodeInfo[i].nodeId,
ths->raftCfg.cfg.nodeInfo[i].nodeId, ths->raftCfg.cfg.nodeInfo[i].nodeFqdn, ths->raftCfg.cfg.nodeInfo[i].nodeFqdn, ths->raftCfg.cfg.nodeInfo[i].nodePort,
ths->raftCfg.cfg.nodeInfo[i].nodePort, ths->raftCfg.cfg.nodeInfo[i].nodeRole); ths->raftCfg.cfg.nodeInfo[i].nodeRole);
} }
for (int32_t i = 0; i < ths->raftCfg.cfg.totalReplicaNum; ++i) { for (int32_t i = 0; i < ths->raftCfg.cfg.totalReplicaNum; ++i) {
sInfo("vgId:%d, %s, replicasId%d, addr:%" PRId64, sInfo("vgId:%d, %s, replicasId%d, addr:%" PRId64, ths->vgId, str, i, ths->replicasId[i].addr);
ths->vgId, str, i, ths->replicasId[i].addr);
} }
} }
int32_t syncNodeRebuildPeerAndCfg(SSyncNode* ths, SSyncCfg* cfg) { int32_t syncNodeRebuildPeerAndCfg(SSyncNode* ths, SSyncCfg* cfg) {
@ -2463,8 +2459,8 @@ int32_t syncNodeRebuildPeerAndCfg(SSyncNode* ths, SSyncCfg *cfg){
// change peersNodeInfo // change peersNodeInfo
i = 0; i = 0;
for (int32_t j = 0; j < cfg->totalReplicaNum; ++j) { for (int32_t j = 0; j < cfg->totalReplicaNum; ++j) {
if(!(strcmp(ths->myNodeInfo.nodeFqdn, cfg->nodeInfo[j].nodeFqdn) == 0 if (!(strcmp(ths->myNodeInfo.nodeFqdn, cfg->nodeInfo[j].nodeFqdn) == 0 &&
&& ths->myNodeInfo.nodePort == cfg->nodeInfo[j].nodePort)){ ths->myNodeInfo.nodePort == cfg->nodeInfo[j].nodePort)) {
ths->peersNodeInfo[i].nodeRole = cfg->nodeInfo[j].nodeRole; ths->peersNodeInfo[i].nodeRole = cfg->nodeInfo[j].nodeRole;
ths->peersNodeInfo[i].clusterId = cfg->nodeInfo[j].clusterId; ths->peersNodeInfo[i].clusterId = cfg->nodeInfo[j].clusterId;
tstrncpy(ths->peersNodeInfo[i].nodeFqdn, cfg->nodeInfo[j].nodeFqdn, TSDB_FQDN_LEN); tstrncpy(ths->peersNodeInfo[i].nodeFqdn, cfg->nodeInfo[j].nodeFqdn, TSDB_FQDN_LEN);
@ -2495,8 +2491,8 @@ int32_t syncNodeRebuildPeerAndCfg(SSyncNode* ths, SSyncCfg *cfg){
tstrncpy(ths->raftCfg.cfg.nodeInfo[i].nodeFqdn, cfg->nodeInfo[j].nodeFqdn, TSDB_FQDN_LEN); tstrncpy(ths->raftCfg.cfg.nodeInfo[i].nodeFqdn, cfg->nodeInfo[j].nodeFqdn, TSDB_FQDN_LEN);
ths->raftCfg.cfg.nodeInfo[i].nodeId = cfg->nodeInfo[j].nodeId; ths->raftCfg.cfg.nodeInfo[i].nodeId = cfg->nodeInfo[j].nodeId;
ths->raftCfg.cfg.nodeInfo[i].nodePort = cfg->nodeInfo[j].nodePort; ths->raftCfg.cfg.nodeInfo[i].nodePort = cfg->nodeInfo[j].nodePort;
if((strcmp(ths->myNodeInfo.nodeFqdn, cfg->nodeInfo[j].nodeFqdn) == 0 if ((strcmp(ths->myNodeInfo.nodeFqdn, cfg->nodeInfo[j].nodeFqdn) == 0 &&
&& ths->myNodeInfo.nodePort == cfg->nodeInfo[j].nodePort)){ ths->myNodeInfo.nodePort == cfg->nodeInfo[j].nodePort)) {
ths->raftCfg.cfg.myIndex = i; ths->raftCfg.cfg.myIndex = i;
} }
i++; i++;
@ -2510,8 +2506,8 @@ void syncNodeChangePeerAndCfgToVoter(SSyncNode* ths, SSyncCfg *cfg){
// change peersNodeInfo // change peersNodeInfo
for (int32_t i = 0; i < ths->peersNum; ++i) { for (int32_t i = 0; i < ths->peersNum; ++i) {
for (int32_t j = 0; j < cfg->totalReplicaNum; ++j) { for (int32_t j = 0; j < cfg->totalReplicaNum; ++j) {
if(strcmp(ths->peersNodeInfo[i].nodeFqdn, cfg->nodeInfo[j].nodeFqdn) == 0 if (strcmp(ths->peersNodeInfo[i].nodeFqdn, cfg->nodeInfo[j].nodeFqdn) == 0 &&
&& ths->peersNodeInfo[i].nodePort == cfg->nodeInfo[j].nodePort){ ths->peersNodeInfo[i].nodePort == cfg->nodeInfo[j].nodePort) {
if (cfg->nodeInfo[j].nodeRole == TAOS_SYNC_ROLE_VOTER) { if (cfg->nodeInfo[j].nodeRole == TAOS_SYNC_ROLE_VOTER) {
ths->peersNodeInfo[i].nodeRole = TAOS_SYNC_ROLE_VOTER; ths->peersNodeInfo[i].nodeRole = TAOS_SYNC_ROLE_VOTER;
} }
@ -2523,8 +2519,8 @@ void syncNodeChangePeerAndCfgToVoter(SSyncNode* ths, SSyncCfg *cfg){
ths->raftCfg.cfg.replicaNum = 0; ths->raftCfg.cfg.replicaNum = 0;
for (int32_t i = 0; i < ths->raftCfg.cfg.totalReplicaNum; ++i) { for (int32_t i = 0; i < ths->raftCfg.cfg.totalReplicaNum; ++i) {
for (int32_t j = 0; j < cfg->totalReplicaNum; ++j) { for (int32_t j = 0; j < cfg->totalReplicaNum; ++j) {
if(strcmp(ths->raftCfg.cfg.nodeInfo[i].nodeFqdn, cfg->nodeInfo[j].nodeFqdn) == 0 if (strcmp(ths->raftCfg.cfg.nodeInfo[i].nodeFqdn, cfg->nodeInfo[j].nodeFqdn) == 0 &&
&& ths->raftCfg.cfg.nodeInfo[i].nodePort == cfg->nodeInfo[j].nodePort){ ths->raftCfg.cfg.nodeInfo[i].nodePort == cfg->nodeInfo[j].nodePort) {
if (cfg->nodeInfo[j].nodeRole == TAOS_SYNC_ROLE_VOTER) { if (cfg->nodeInfo[j].nodeRole == TAOS_SYNC_ROLE_VOTER) {
ths->raftCfg.cfg.nodeInfo[i].nodeRole = TAOS_SYNC_ROLE_VOTER; ths->raftCfg.cfg.nodeInfo[i].nodeRole = TAOS_SYNC_ROLE_VOTER;
ths->raftCfg.cfg.replicaNum++; ths->raftCfg.cfg.replicaNum++;
@ -2545,7 +2541,6 @@ int32_t syncNodeRebuildAndCopyIfExist(SSyncNode* ths, int32_t oldtotalReplicaNum
syncUtilNodeInfo2RaftId(&ths->raftCfg.cfg.nodeInfo[i], ths->vgId, &ths->replicasId[i]); syncUtilNodeInfo2RaftId(&ths->raftCfg.cfg.nodeInfo[i], ths->vgId, &ths->replicasId[i]);
} }
// 2.rebuild MatchIndex, remove deleted one // 2.rebuild MatchIndex, remove deleted one
SSyncIndexMgr* oldIndex = ths->pMatchIndex; SSyncIndexMgr* oldIndex = ths->pMatchIndex;
@ -2555,7 +2550,6 @@ int32_t syncNodeRebuildAndCopyIfExist(SSyncNode* ths, int32_t oldtotalReplicaNum
syncIndexMgrDestroy(oldIndex); syncIndexMgrDestroy(oldIndex);
// 3.rebuild NextIndex, remove deleted one // 3.rebuild NextIndex, remove deleted one
SSyncIndexMgr* oldNextIndex = ths->pNextIndex; SSyncIndexMgr* oldNextIndex = ths->pNextIndex;
@ -2565,16 +2559,14 @@ int32_t syncNodeRebuildAndCopyIfExist(SSyncNode* ths, int32_t oldtotalReplicaNum
syncIndexMgrDestroy(oldNextIndex); syncIndexMgrDestroy(oldNextIndex);
// 4.rebuild pVotesGranted, pVotesRespond, no need to keep old vote state, only rebuild // 4.rebuild pVotesGranted, pVotesRespond, no need to keep old vote state, only rebuild
voteGrantedUpdate(ths->pVotesGranted, ths); voteGrantedUpdate(ths->pVotesGranted, ths);
votesRespondUpdate(ths->pVotesRespond, ths); votesRespondUpdate(ths->pVotesRespond, ths);
// 5.rebuild logReplMgr // 5.rebuild logReplMgr
for (int i = 0; i < oldtotalReplicaNum; ++i) { for (int i = 0; i < oldtotalReplicaNum; ++i) {
sDebug("vgId:%d, old logReplMgrs i:%d, peerId:%d, restoreed:%d, [%" PRId64 " %" PRId64 ", %" PRId64 ")", ths->vgId, i, sDebug("vgId:%d, old logReplMgrs i:%d, peerId:%d, restoreed:%d, [%" PRId64 " %" PRId64 ", %" PRId64 ")", ths->vgId,
ths->logReplMgrs[i]->peerId, ths->logReplMgrs[i]->restored, ths->logReplMgrs[i]->startIndex, i, ths->logReplMgrs[i]->peerId, ths->logReplMgrs[i]->restored, ths->logReplMgrs[i]->startIndex,
ths->logReplMgrs[i]->matchIndex, ths->logReplMgrs[i]->endIndex); ths->logReplMgrs[i]->matchIndex, ths->logReplMgrs[i]->endIndex);
} }
@ -2601,15 +2593,15 @@ int32_t syncNodeRebuildAndCopyIfExist(SSyncNode* ths, int32_t oldtotalReplicaNum
} }
for (int i = 0; i < ths->totalReplicaNum; ++i) { for (int i = 0; i < ths->totalReplicaNum; ++i) {
sDebug("vgId:%d, new logReplMgrs i:%d, peerId:%d, restoreed:%d, [%" PRId64 " %" PRId64 ", %" PRId64 ")" , ths->vgId, i, sDebug("vgId:%d, new logReplMgrs i:%d, peerId:%d, restoreed:%d, [%" PRId64 " %" PRId64 ", %" PRId64 ")", ths->vgId,
ths->logReplMgrs[i]->peerId, ths->logReplMgrs[i]->restored, ths->logReplMgrs[i]->startIndex, i, ths->logReplMgrs[i]->peerId, ths->logReplMgrs[i]->restored, ths->logReplMgrs[i]->startIndex,
ths->logReplMgrs[i]->matchIndex, ths->logReplMgrs[i]->endIndex); ths->logReplMgrs[i]->matchIndex, ths->logReplMgrs[i]->endIndex);
} }
// 6.rebuild sender // 6.rebuild sender
for (int i = 0; i < oldtotalReplicaNum; ++i) { for (int i = 0; i < oldtotalReplicaNum; ++i) {
sDebug("vgId:%d, old sender i:%d, replicaIndex:%d, lastSendTime:%" PRId64, sDebug("vgId:%d, old sender i:%d, replicaIndex:%d, lastSendTime:%" PRId64, ths->vgId, i,
ths->vgId, i, ths->senders[i]->replicaIndex, ths->senders[i]->lastSendTime) ths->senders[i]->replicaIndex, ths->senders[i]->lastSendTime)
} }
for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) {
@ -2634,11 +2626,10 @@ int32_t syncNodeRebuildAndCopyIfExist(SSyncNode* ths, int32_t oldtotalReplicaNum
} }
for (int i = 0; i < ths->totalReplicaNum; i++) { for (int i = 0; i < ths->totalReplicaNum; i++) {
sDebug("vgId:%d, new sender i:%d, replicaIndex:%d, lastSendTime:%" PRId64, sDebug("vgId:%d, new sender i:%d, replicaIndex:%d, lastSendTime:%" PRId64, ths->vgId, i,
ths->vgId, i, ths->senders[i]->replicaIndex, ths->senders[i]->lastSendTime) ths->senders[i]->replicaIndex, ths->senders[i]->lastSendTime)
} }
// 7.rebuild synctimer // 7.rebuild synctimer
syncNodeStopHeartbeatTimer(ths); syncNodeStopHeartbeatTimer(ths);
@ -2648,7 +2639,6 @@ int32_t syncNodeRebuildAndCopyIfExist(SSyncNode* ths, int32_t oldtotalReplicaNum
syncNodeStartHeartbeatTimer(ths); syncNodeStartHeartbeatTimer(ths);
// 8.rebuild peerStates // 8.rebuild peerStates
SPeerState oldState[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA] = {0}; SPeerState oldState[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA] = {0};
for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) { for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) {
@ -2722,36 +2712,41 @@ int32_t syncNodeChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry, char* str){
syncBuildConfigFromReq(&req, &cfg); syncBuildConfigFromReq(&req, &cfg);
if (cfg.changeVersion <= ths->raftCfg.cfg.changeVersion) { if (cfg.changeVersion <= ths->raftCfg.cfg.changeVersion) {
sInfo("vgId:%d, skip conf change entry since lower version. " sInfo(
"this entry, index:%" PRId64 ", term:%" PRId64 ", totalReplicaNum:%d, changeVersion:%d; " "vgId:%d, skip conf change entry since lower version. "
"this entry, index:%" PRId64 ", term:%" PRId64
", totalReplicaNum:%d, changeVersion:%d; "
"current node, replicaNum:%d, peersNum:%d, lastConfigIndex:%" PRId64 ", changeVersion:%d", "current node, replicaNum:%d, peersNum:%d, lastConfigIndex:%" PRId64 ", changeVersion:%d",
ths->vgId, ths->vgId, pEntry->index, pEntry->term, cfg.totalReplicaNum, cfg.changeVersion, ths->replicaNum, ths->peersNum,
pEntry->index, pEntry->term, cfg.totalReplicaNum, cfg.changeVersion, ths->raftCfg.lastConfigIndex, ths->raftCfg.cfg.changeVersion);
ths->replicaNum, ths->peersNum, ths->raftCfg.lastConfigIndex, ths->raftCfg.cfg.changeVersion);
return 0; return 0;
} }
if (strcmp(str, "Commit") == 0) { if (strcmp(str, "Commit") == 0) {
sInfo("vgId:%d, change config from %s. " sInfo(
"this, i:%" PRId64 ", trNum:%d, vers:%d; " "vgId:%d, change config from %s. "
"this, i:%" PRId64
", trNum:%d, vers:%d; "
"node, rNum:%d, pNum:%d, trNum:%d, " "node, rNum:%d, pNum:%d, trNum:%d, "
"buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 "), " "buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64
"), "
"cond:(next i:%" PRId64 ", t:%" PRId64 " ==%s)", "cond:(next i:%" PRId64 ", t:%" PRId64 " ==%s)",
ths->vgId, str, pEntry->index - 1, cfg.totalReplicaNum, cfg.changeVersion, ths->vgId, str, pEntry->index - 1, cfg.totalReplicaNum, cfg.changeVersion, ths->replicaNum, ths->peersNum,
ths->replicaNum, ths->peersNum, ths->totalReplicaNum, ths->totalReplicaNum, ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex, ths->pLogBuf->matchIndex,
ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex, ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex, ths->pLogBuf->endIndex, pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType));
pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType)); } else {
} sInfo(
else{ "vgId:%d, change config from %s. "
sInfo("vgId:%d, change config from %s. " "this, i:%" PRId64 ", t:%" PRId64
"this, i:%" PRId64 ", t:%" PRId64 ", trNum:%d, vers:%d; " ", trNum:%d, vers:%d; "
"node, rNum:%d, pNum:%d, trNum:%d, " "node, rNum:%d, pNum:%d, trNum:%d, "
"buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 "), " "buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64
"), "
"cond:(pre i:%" PRId64 "==ci:%" PRId64 ", bci:%" PRId64 ")", "cond:(pre i:%" PRId64 "==ci:%" PRId64 ", bci:%" PRId64 ")",
ths->vgId, str, pEntry->index, pEntry->term, cfg.totalReplicaNum, cfg.changeVersion, ths->vgId, str, pEntry->index, pEntry->term, cfg.totalReplicaNum, cfg.changeVersion, ths->replicaNum,
ths->replicaNum, ths->peersNum, ths->totalReplicaNum, ths->peersNum, ths->totalReplicaNum, ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex,
ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex, ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex, ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex, pEntry->index - 1, ths->commitIndex,
pEntry->index -1, ths->commitIndex, ths->pLogBuf->commitIndex); ths->pLogBuf->commitIndex);
} }
syncNodeLogConfigInfo(ths, &cfg, "before config change"); syncNodeLogConfigInfo(ths, &cfg, "before config change");
@ -2762,8 +2757,8 @@ int32_t syncNodeChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry, char* str){
bool incfg = false; bool incfg = false;
for (int32_t j = 0; j < cfg.totalReplicaNum; ++j) { for (int32_t j = 0; j < cfg.totalReplicaNum; ++j) {
if(strcmp(ths->myNodeInfo.nodeFqdn, cfg.nodeInfo[j].nodeFqdn) == 0 if (strcmp(ths->myNodeInfo.nodeFqdn, cfg.nodeInfo[j].nodeFqdn) == 0 &&
&& ths->myNodeInfo.nodePort == cfg.nodeInfo[j].nodePort){ ths->myNodeInfo.nodePort == cfg.nodeInfo[j].nodePort) {
incfg = true; incfg = true;
break; break;
} }
@ -2781,8 +2776,7 @@ int32_t syncNodeChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry, char* str){
if (syncNodeRebuildAndCopyIfExist(ths, oldTotalReplicaNum) != 0) { if (syncNodeRebuildAndCopyIfExist(ths, oldTotalReplicaNum) != 0) {
return -1; return -1;
}; };
} } else { // remove myself
else{//remove myself
// no need to do anything actually, to change the following to reduce distruptive server chance // no need to do anything actually, to change the following to reduce distruptive server chance
syncNodeResetPeerAndCfg(ths); syncNodeResetPeerAndCfg(ths);
@ -2806,15 +2800,14 @@ int32_t syncNodeChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry, char* str){
} }
ths->restoreFinish = false; ths->restoreFinish = false;
} } else { // add replica, or change replica type
else{//add replica, or change replica type
if (ths->totalReplicaNum == 3) { // change replica type if (ths->totalReplicaNum == 3) { // change replica type
sInfo("vgId:%d, begin change replica type", ths->vgId); sInfo("vgId:%d, begin change replica type", ths->vgId);
// change myNodeInfo // change myNodeInfo
for (int32_t j = 0; j < cfg.totalReplicaNum; ++j) { for (int32_t j = 0; j < cfg.totalReplicaNum; ++j) {
if(strcmp(ths->myNodeInfo.nodeFqdn, cfg.nodeInfo[j].nodeFqdn) == 0 if (strcmp(ths->myNodeInfo.nodeFqdn, cfg.nodeInfo[j].nodeFqdn) == 0 &&
&& ths->myNodeInfo.nodePort == cfg.nodeInfo[j].nodePort){ ths->myNodeInfo.nodePort == cfg.nodeInfo[j].nodePort) {
if (cfg.nodeInfo[j].nodeRole == TAOS_SYNC_ROLE_VOTER) { if (cfg.nodeInfo[j].nodeRole == TAOS_SYNC_ROLE_VOTER) {
ths->myNodeInfo.nodeRole = TAOS_SYNC_ROLE_VOTER; ths->myNodeInfo.nodeRole = TAOS_SYNC_ROLE_VOTER;
} }
@ -2835,8 +2828,7 @@ int32_t syncNodeChangeConfig(SSyncNode* ths, SSyncRaftEntry* pEntry, char* str){
} }
ths->restoreFinish = false; ths->restoreFinish = false;
} } else { // add replica
else{//add replica
sInfo("vgId:%d, begin add replica", ths->vgId); sInfo("vgId:%d, begin add replica", ths->vgId);
// no need to change myNodeInfo // no need to change myNodeInfo