[TD-10645][raft]<feature>add restore process

source/libs/sync/inc/sync_raft_progress.h

@@ -214,7 +214,7 @@ bool syncRaftProgressIsUptodate(SSyncRaft* pRaft, SSyncRaftProgress* progress);
 
 void syncRaftProgressBecomeSnapshot(SSyncRaftProgress* progress, SyncIndex snapshotIndex);
 
-void syncRaftProgressCopy(const SSyncRaftProgress* from, SSyncRaftProgress* to);
+void syncRaftCopyProgress(const SSyncRaftProgress* from, SSyncRaftProgress* to);
 
 void syncRaftProgressMapCopy(const SSyncRaftProgressMap* from, SSyncRaftProgressMap* to);
 

source/libs/sync/inc/sync_raft_quorum_joint.h

@@ -55,4 +55,12 @@ static FORCE_INLINE bool syncRaftJointConfigInIncoming(const SSyncRaftQuorumJoin
   return syncRaftJointConfigInCluster(&config->incoming, id);
 }
 
+static FORCE_INLINE const SSyncCluster* syncRaftJointConfigIncoming(const SSyncRaftQuorumJointConfig* config) {
+  return &config->incoming;
+}
+
+static FORCE_INLINE const SSyncCluster* syncRaftJointConfigOutgoing(const SSyncRaftQuorumJointConfig* config) {
+  return &config->outgoing;
+}
+
 #endif /* _TD_LIBS_SYNC_RAFT_QUORUM_JOINT_H */

source/libs/sync/inc/sync_raft_restore.h

@@ -19,7 +19,14 @@
 #include "sync_type.h"
 #include "sync_raft_proto.h"
 
-int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs,
-                          SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap);
+// Restore takes a Changer (which must represent an empty configuration), and
+// runs a sequence of changes enacting the configuration described in the
+// ConfState.
+//
+// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure
+// the Changer only needs a ProgressMap (not a whole Tracker) at which point
+// this can just take LastIndex and MaxInflight directly instead and cook up
+// the results from that alone.
+int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs);
 
 #endif /* TD_SYNC_RAFT_RESTORE_H */

source/libs/sync/src/sync_raft_config_change.c

@@ -25,6 +25,7 @@ static int checkInvariants(SSyncRaftProgressTrackerConfig* config, SSyncRaftProg
 static bool hasJointConfig(const SSyncRaftProgressTrackerConfig* config);
 static int applyConfig(SSyncRaftChanger* changer, const SSyncRaftProgressTrackerConfig* config,
                         const SSyncRaftProgressMap* progressMap, const SSyncConfChangeSingleArray* css);
+static int symDiff(const SSyncCluster* l, const SSyncCluster* r);
 
 // Simple carries out a series of configuration changes that (in aggregate)
 // mutates the incoming majority config Voters[0] by at most one. This method
@@ -49,9 +50,34 @@ int syncRaftChangerSimpleConfig(SSyncRaftChanger* changer, const SSyncConfChange
     return ret;
   }
 
+  int n = symDiff(syncRaftJointConfigIncoming(&changer->tracker->config.voters),
+                  syncRaftJointConfigIncoming(&config->voters));
+  if (n > 1) {
+    syncError("more than one voter changed without entering joint config");
+    return -1;
+  }
+
   return checkAndReturn(config, progressMap); 
 }
 
+// EnterJoint verifies that the outgoing (=right) majority config of the joint
+// config is empty and initializes it with a copy of the incoming (=left)
+// majority config. That is, it transitions from
+//
+//     (1 2 3)&&()
+// to
+//     (1 2 3)&&(1 2 3).
+//
+// The supplied changes are then applied to the incoming majority config,
+// resulting in a joint configuration that in terms of the Raft thesis[1]
+// (Section 4.3) corresponds to `C_{new,old}`.
+//
+// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf
+int syncRaftChangerEnterJoint(SSyncRaftChanger* changer, const SSyncConfChangeSingleArray* css,
+                            SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) {
+
+}
+
 // checkAndCopy copies the tracker's config and progress map (deeply enough for
 // the purposes of the Changer) and returns those copies. It returns an error
 // if checkInvariants does.
@@ -63,7 +89,7 @@ static int checkAndCopy(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfi
     if (progress->id == SYNC_NON_NODE_ID) {
       continue;
     }
-    syncRaftProgressCopy(progress, &(progressMap->progress[i]));
+    syncRaftCopyProgress(progress, &(progressMap->progress[i]));
   }
   return checkAndReturn(config, progressMap);
 }
@@ -151,4 +177,36 @@ static int applyConfig(SSyncRaftChanger* changer, const SSyncRaftProgressTracker
   }
 
   return 0;
+}
+
+// symdiff returns the count of the symmetric difference between the sets of
+// uint64s, i.e. len( (l - r) \union (r - l)).
+static int symDiff(const SSyncCluster* l, const SSyncCluster* r) {
+  int n;
+  int i;
+  int j0, j1;
+  const SSyncCluster* pairs[2][2] = {
+    {l, r}, // count elems in l but not in r
+    {r, l}, // count elems in r but not in l
+  };
+
+  for (n = 0, i = 0; i < 2; ++i) {
+    const SSyncCluster** pp = pairs[i];
+
+    const SSyncCluster* p0 = pp[0];
+    const SSyncCluster* p1 = pp[1];
+    for (j0 = 0; j0 < p0->replica; ++j0) {
+      SyncNodeId id = p0->nodeInfo[j0].nodeId;
+      if (id == SYNC_NON_NODE_ID) {
+        continue;
+      }
+      for (j1 = 0; j1 < p1->replica; ++j1) {
+        if (p1->nodeInfo[j1].nodeId != SYNC_NON_NODE_ID && p1->nodeInfo[j1].nodeId != id) {
+          n+=1;
+        }
+      }
+    }
+  }
+
+  return n;
 }

source/libs/sync/src/sync_raft_progress.c

@@ -149,7 +149,7 @@ void syncRaftProgressBecomeSnapshot(SSyncRaftProgress* progress, SyncIndex snaps
   progress->pendingSnapshotIndex = snapshotIndex;
 }
 
-void syncRaftProgressCopy(const SSyncRaftProgress* progress, SSyncRaftProgress* out) {
+void syncRaftCopyProgress(const SSyncRaftProgress* progress, SSyncRaftProgress* out) {
 
 }
 

source/libs/sync/src/sync_raft_restore.c

@@ -19,11 +19,21 @@
 
 static int toConfChangeSingle(const SSyncConfigState* cs, SSyncConfChangeSingleArray* out, SSyncConfChangeSingleArray* in);
 
-int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs,
-                          SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) {
+// Restore takes a Changer (which must represent an empty configuration), and
+// runs a sequence of changes enacting the configuration described in the
+// ConfState.
+//
+// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure
+// the Changer only needs a ProgressMap (not a whole Tracker) at which point
+// this can just take LastIndex and MaxInflight directly instead and cook up
+// the results from that alone.
+int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs) {
   SSyncConfChangeSingleArray outgoing;
   SSyncConfChangeSingleArray incoming;
   SSyncConfChangeSingleArray css;
+  SSyncRaftProgressTracker* tracker = changer->tracker;
+  SSyncRaftProgressTrackerConfig* config = &tracker->config;
+  SSyncRaftProgressMap* progressMap = &tracker->progressMap;
   int i, ret;
 
   ret = toConfChangeSingle(cs, &outgoing, &incoming);
@@ -38,12 +48,10 @@ int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs,
         .n = 1,
         .changes = &incoming.changes[i],
       };
-      ret = syncRaftChangerSimpleConfig(changer, &css, config, progressMap);
+      ret = syncRaftChangerSimpleConfig(changer, &css, &config, &progressMap);
       if (ret != 0) {
         goto out;
       }
-      syncRaftCloneTrackerConfig(config, &changer->tracker->config);
-      syncRaftProgressMapCopy(progressMap, &changer->tracker->progressMap);
     }
   } else {
 		// The ConfState describes a joint configuration.
@@ -60,16 +68,12 @@ int syncRaftRestoreConfig(SSyncRaftChanger* changer, const SSyncConfigState* cs,
       if (ret != 0) {
         goto out;
       }
-      syncRaftCloneTrackerConfig(config, &changer->tracker->config);
-      syncRaftProgressMapCopy(progressMap, &changer->tracker->progressMap);
     }
 
     ret = syncRaftChangerEnterJoint(changer, &incoming, config, progressMap);
     if (ret != 0) {
       goto out;
     }
-    syncRaftCloneTrackerConfig(config, &changer->tracker->config);
-    syncRaftProgressMapCopy(progressMap, &changer->tracker->progressMap);
   }
 
 out: