From 24b599f14bc1e6126dde16af6d7af84cd624dd55 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 12 Nov 2021 18:09:09 +0800 Subject: [PATCH] [TD-10645][raft]add restore process --- source/libs/sync/inc/sync_raft_progress.h | 2 +- source/libs/sync/inc/sync_raft_quorum_joint.h | 8 +++ source/libs/sync/inc/sync_raft_restore.h | 11 +++- .../libs/sync/src/sync_raft_config_change.c | 60 ++++++++++++++++++- source/libs/sync/src/sync_raft_progress.c | 2 +- source/libs/sync/src/sync_raft_restore.c | 22 ++++--- 6 files changed, 91 insertions(+), 14 deletions(-) diff --git a/source/libs/sync/inc/sync_raft_progress.h b/source/libs/sync/inc/sync_raft_progress.h index b5017f963d..98edfc5e4f 100644 --- a/source/libs/sync/inc/sync_raft_progress.h +++ b/source/libs/sync/inc/sync_raft_progress.h @@ -214,7 +214,7 @@ bool syncRaftProgressIsUptodate(SSyncRaft* pRaft, SSyncRaftProgress* progress); void syncRaftProgressBecomeSnapshot(SSyncRaftProgress* progress, SyncIndex snapshotIndex); -void syncRaftProgressCopy(const SSyncRaftProgress* from, SSyncRaftProgress* to); +void syncRaftCopyProgress(const SSyncRaftProgress* from, SSyncRaftProgress* to); void syncRaftProgressMapCopy(const SSyncRaftProgressMap* from, SSyncRaftProgressMap* to); diff --git a/source/libs/sync/inc/sync_raft_quorum_joint.h b/source/libs/sync/inc/sync_raft_quorum_joint.h index 103a147de3..cdde6d24f7 100644 --- a/source/libs/sync/inc/sync_raft_quorum_joint.h +++ b/source/libs/sync/inc/sync_raft_quorum_joint.h @@ -55,4 +55,12 @@ static FORCE_INLINE bool syncRaftJointConfigInIncoming(const SSyncRaftQuorumJoin return syncRaftJointConfigInCluster(&config->incoming, id); } +static FORCE_INLINE const SSyncCluster* syncRaftJointConfigIncoming(const SSyncRaftQuorumJointConfig* config) { + return &config->incoming; +} + +static FORCE_INLINE const SSyncCluster* syncRaftJointConfigOutgoing(const SSyncRaftQuorumJointConfig* config) { + return &config->outgoing; +} + #endif /* _TD_LIBS_SYNC_RAFT_QUORUM_JOINT_H */ diff --git a/source/libs/sync/inc/sync_raft_restore.h b/source/libs/sync/inc/sync_raft_restore.h index fc65ae7440..f7c4ce67b5 100644 --- a/source/libs/sync/inc/sync_raft_restore.h +++ b/source/libs/sync/inc/sync_raft_restore.h @@ -19,7 +19,14 @@ #include "sync_type.h" #include "sync_raft_proto.h" -int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap); +// Restore takes a Changer (which must represent an empty configuration), and +// runs a sequence of changes enacting the configuration described in the +// ConfState. +// +// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure +// the Changer only needs a ProgressMap (not a whole Tracker) at which point +// this can just take LastIndex and MaxInflight directly instead and cook up +// the results from that alone. +int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs); #endif /* TD_SYNC_RAFT_RESTORE_H */ diff --git a/source/libs/sync/src/sync_raft_config_change.c b/source/libs/sync/src/sync_raft_config_change.c index e99da0e226..e9cad376aa 100644 --- a/source/libs/sync/src/sync_raft_config_change.c +++ b/source/libs/sync/src/sync_raft_config_change.c @@ -25,6 +25,7 @@ static int checkInvariants(SSyncRaftProgressTrackerConfig* config, SSyncRaftProg static bool hasJointConfig(const SSyncRaftProgressTrackerConfig* config); static int applyConfig(SSyncRaftChanger* changer, const SSyncRaftProgressTrackerConfig* config, const SSyncRaftProgressMap* progressMap, const SSyncConfChangeSingleArray* css); +static int symDiff(const SSyncCluster* l, const SSyncCluster* r); // Simple carries out a series of configuration changes that (in aggregate) // mutates the incoming majority config Voters[0] by at most one. This method @@ -49,9 +50,34 @@ int syncRaftChangerSimpleConfig(SSyncRaftChanger* changer, const SSyncConfChange return ret; } + int n = symDiff(syncRaftJointConfigIncoming(&changer->tracker->config.voters), + syncRaftJointConfigIncoming(&config->voters)); + if (n > 1) { + syncError("more than one voter changed without entering joint config"); + return -1; + } + return checkAndReturn(config, progressMap); } +// EnterJoint verifies that the outgoing (=right) majority config of the joint +// config is empty and initializes it with a copy of the incoming (=left) +// majority config. That is, it transitions from +// +// (1 2 3)&&() +// to +// (1 2 3)&&(1 2 3). +// +// The supplied changes are then applied to the incoming majority config, +// resulting in a joint configuration that in terms of the Raft thesis[1] +// (Section 4.3) corresponds to `C_{new,old}`. +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +int syncRaftChangerEnterJoint(SSyncRaftChanger* changer, const SSyncConfChangeSingleArray* css, + SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { + +} + // checkAndCopy copies the tracker's config and progress map (deeply enough for // the purposes of the Changer) and returns those copies. It returns an error // if checkInvariants does. @@ -63,7 +89,7 @@ static int checkAndCopy(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfi if (progress->id == SYNC_NON_NODE_ID) { continue; } - syncRaftProgressCopy(progress, &(progressMap->progress[i])); + syncRaftCopyProgress(progress, &(progressMap->progress[i])); } return checkAndReturn(config, progressMap); } @@ -151,4 +177,36 @@ static int applyConfig(SSyncRaftChanger* changer, const SSyncRaftProgressTracker } return 0; +} + +// symdiff returns the count of the symmetric difference between the sets of +// uint64s, i.e. len( (l - r) \union (r - l)). +static int symDiff(const SSyncCluster* l, const SSyncCluster* r) { + int n; + int i; + int j0, j1; + const SSyncCluster* pairs[2][2] = { + {l, r}, // count elems in l but not in r + {r, l}, // count elems in r but not in l + }; + + for (n = 0, i = 0; i < 2; ++i) { + const SSyncCluster** pp = pairs[i]; + + const SSyncCluster* p0 = pp[0]; + const SSyncCluster* p1 = pp[1]; + for (j0 = 0; j0 < p0->replica; ++j0) { + SyncNodeId id = p0->nodeInfo[j0].nodeId; + if (id == SYNC_NON_NODE_ID) { + continue; + } + for (j1 = 0; j1 < p1->replica; ++j1) { + if (p1->nodeInfo[j1].nodeId != SYNC_NON_NODE_ID && p1->nodeInfo[j1].nodeId != id) { + n+=1; + } + } + } + } + + return n; } \ No newline at end of file diff --git a/source/libs/sync/src/sync_raft_progress.c b/source/libs/sync/src/sync_raft_progress.c index e7de3fcf98..a3426aa999 100644 --- a/source/libs/sync/src/sync_raft_progress.c +++ b/source/libs/sync/src/sync_raft_progress.c @@ -149,7 +149,7 @@ void syncRaftProgressBecomeSnapshot(SSyncRaftProgress* progress, SyncIndex snaps progress->pendingSnapshotIndex = snapshotIndex; } -void syncRaftProgressCopy(const SSyncRaftProgress* progress, SSyncRaftProgress* out) { +void syncRaftCopyProgress(const SSyncRaftProgress* progress, SSyncRaftProgress* out) { } diff --git a/source/libs/sync/src/sync_raft_restore.c b/source/libs/sync/src/sync_raft_restore.c index c7dfaa07b4..b7d9cc6888 100644 --- a/source/libs/sync/src/sync_raft_restore.c +++ b/source/libs/sync/src/sync_raft_restore.c @@ -19,11 +19,21 @@ static int toConfChangeSingle(const SSyncConfigState* cs, SSyncConfChangeSingleArray* out, SSyncConfChangeSingleArray* in); -int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs, - SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) { +// Restore takes a Changer (which must represent an empty configuration), and +// runs a sequence of changes enacting the configuration described in the +// ConfState. +// +// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure +// the Changer only needs a ProgressMap (not a whole Tracker) at which point +// this can just take LastIndex and MaxInflight directly instead and cook up +// the results from that alone. +int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs) { SSyncConfChangeSingleArray outgoing; SSyncConfChangeSingleArray incoming; SSyncConfChangeSingleArray css; + SSyncRaftProgressTracker* tracker = changer->tracker; + SSyncRaftProgressTrackerConfig* config = &tracker->config; + SSyncRaftProgressMap* progressMap = &tracker->progressMap; int i, ret; ret = toConfChangeSingle(cs, &outgoing, &incoming); @@ -38,12 +48,10 @@ int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs, .n = 1, .changes = &incoming.changes[i], }; - ret = syncRaftChangerSimpleConfig(changer, &css, config, progressMap); + ret = syncRaftChangerSimpleConfig(changer, &css, &config, &progressMap); if (ret != 0) { goto out; } - syncRaftCloneTrackerConfig(config, &changer->tracker->config); - syncRaftProgressMapCopy(progressMap, &changer->tracker->progressMap); } } else { // The ConfState describes a joint configuration. @@ -60,16 +68,12 @@ int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs, if (ret != 0) { goto out; } - syncRaftCloneTrackerConfig(config, &changer->tracker->config); - syncRaftProgressMapCopy(progressMap, &changer->tracker->progressMap); } ret = syncRaftChangerEnterJoint(changer, &incoming, config, progressMap); if (ret != 0) { goto out; } - syncRaftCloneTrackerConfig(config, &changer->tracker->config); - syncRaftProgressMapCopy(progressMap, &changer->tracker->progressMap); } out: