From be532bd5ebdbceda0dcf26b6f5408cd809b36c39 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Mar 2022 12:29:44 +0800 Subject: [PATCH 01/15] sync refactor --- source/libs/sync/inc/syncEnv.h | 20 +++++-- source/libs/sync/src/syncEnv.c | 71 ++++++++++++++++-------- source/libs/sync/test/syncEnvTest.cpp | 26 ++++----- source/libs/sync/test/syncRpcMsgTest.cpp | 14 ++--- 4 files changed, 82 insertions(+), 49 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 9fbea03265..c3679636e6 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -29,6 +29,7 @@ extern "C" { #include "ttimer.h" #define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 #define PING_TIMER_MS 1000 #define ELECT_TIMER_MS_MIN 150 #define ELECT_TIMER_MS_MAX 300 @@ -38,17 +39,28 @@ extern "C" { #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) typedef struct SSyncEnv { - tmr_h pEnvTickTimer; + // tick timer + tmr_h pEnvTickTimer; + int32_t envTickTimerMS; + uint64_t envTickTimerLogicClock; + uint64_t envTickTimerLogicClockUser; + TAOS_TMR_CALLBACK FpEnvTickTimer; // Timer Fp + uint64_t envTickTimerCounter; + + // timer manager tmr_h pTimerManager; - char name[128]; + + // other resources shared by SyncNodes + // ... + } SSyncEnv; extern SSyncEnv* gSyncEnv; int32_t syncEnvStart(); int32_t syncEnvStop(); -tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void* param); -void syncEnvStopTimer(tmr_h* pTimer); +int32_t syncEnvStartTimer(); +int32_t syncEnvStopTimer(); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index cb38b3f6f8..2830d2d4aa 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -19,19 +19,18 @@ SSyncEnv *gSyncEnv = NULL; // local function ----------------- -static void syncEnvTick(void *param, void *tmrId); -static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv); -static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); -static tmr_h doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int mseconds, void *param); -static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer); +static SSyncEnv *doSyncEnvStart(); +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); +static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv); +static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv); +static void syncEnvTick(void *param, void *tmrId); // -------------------------------- int32_t syncEnvStart() { - int32_t ret; + int32_t ret = 0; taosSeedRand(taosGetTimestampSec()); - gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + gSyncEnv = doSyncEnvStart(gSyncEnv); assert(gSyncEnv != NULL); - ret = doSyncEnvStart(gSyncEnv); return ret; } @@ -40,31 +39,46 @@ int32_t syncEnvStop() { return ret; } -tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void *param) { - return doSyncEnvStartTimer(gSyncEnv, fp, mseconds, param); +int32_t syncEnvStartTimer() { + int32_t ret = doSyncEnvStartTimer(gSyncEnv); + return ret; } -void syncEnvStopTimer(tmr_h *pTimer) { doSyncEnvStopTimer(gSyncEnv, pTimer); } +int32_t syncEnvStopTimer() { + int32_t ret = doSyncEnvStopTimer(gSyncEnv); + return ret; +} // local function ----------------- static void syncEnvTick(void *param, void *tmrId) { SSyncEnv *pSyncEnv = (SSyncEnv *)param; - sTrace("syncEnvTick ... name:%s ", pSyncEnv->name); + if (atomic_load_64(&pSyncEnv->envTickTimerLogicClockUser) <= atomic_load_64(&pSyncEnv->envTickTimerLogicClock)) { + ++(pSyncEnv->envTickTimerCounter); + sTrace( + "syncEnvTick ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " + "envTickTimerMS:%d", + pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, + pSyncEnv->envTickTimerMS); - pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager); + // do something, tick ... + taosTmrReset(syncEnvTick, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, &pSyncEnv->pEnvTickTimer); + } } -static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { - snprintf(pSyncEnv->name, sizeof(pSyncEnv->name), "SyncEnv_%p", pSyncEnv); +static SSyncEnv *doSyncEnvStart() { + SSyncEnv *pSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + assert(pSyncEnv != NULL); + memset(pSyncEnv, 0, sizeof(pSyncEnv)); + + pSyncEnv->envTickTimerCounter = 0; + pSyncEnv->envTickTimerMS = ENV_TICK_TIMER_MS; + pSyncEnv->FpEnvTickTimer = syncEnvTick; + atomic_store_64(&pSyncEnv->envTickTimerLogicClock, 0); + atomic_store_64(&pSyncEnv->envTickTimerLogicClockUser, 0); // start tmr thread pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); - - // pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager); - - sTrace("SyncEnv start ok, name:%s", pSyncEnv->name); - - return 0; + return pSyncEnv; } static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { @@ -72,8 +86,17 @@ static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { return 0; } -static tmr_h doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int mseconds, void *param) { - return taosTmrStart(fp, mseconds, pSyncEnv, pSyncEnv->pTimerManager); +static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv) { + int32_t ret = 0; + pSyncEnv->pEnvTickTimer = + taosTmrStart(pSyncEnv->FpEnvTickTimer, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager); + atomic_store_64(&pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerLogicClockUser); + return ret; } -static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer) {} +static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv) { + atomic_add_fetch_64(&pSyncEnv->envTickTimerLogicClockUser, 1); + taosTmrStop(pSyncEnv->pEnvTickTimer); + pSyncEnv->pEnvTickTimer = NULL; + return 0; +} diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index 101c0efe9a..a7a819e046 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -14,15 +14,6 @@ void logTest() { sFatal("--- sync log test: fatal"); } -void *pTimer = NULL; -void *pTimerMgr = NULL; -int g = 300; - -static void timerFp(void *param, void *tmrId) { - printf("param:%p, tmrId:%p, pTimer:%p, pTimerMgr:%p \n", param, tmrId, pTimer, pTimerMgr); - taosTmrReset(timerFp, 1000, param, pTimerMgr, &pTimer); -} - int main() { // taosInitLog((char*)"syncEnvTest.log", 100000, 10); tsAsyncLog = 0; @@ -34,13 +25,20 @@ int main() { ret = syncEnvStart(); assert(ret == 0); - // timer - pTimerMgr = taosTmrInit(1000, 50, 10000, "SYNC-ENV-TEST"); - taosTmrStart(timerFp, 1000, &g, pTimerMgr); + for (int i = 0; i < 5; ++i) { + ret = syncEnvStartTimer(); + assert(ret == 0); - while (1) { - taosMsleep(1000); + taosMsleep(5000); + + ret = syncEnvStopTimer(); + assert(ret == 0); + + taosMsleep(5000); } + ret = syncEnvStop(); + assert(ret == 0); + return 0; } diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 0331a29f22..61edbd3012 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -100,7 +100,7 @@ SyncAppendEntriesReply *createSyncAppendEntriesReply() { void test1() { SyncTimeout *pMsg = createSyncTimeout(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncTimeout2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test1", &rpcMsg); syncTimeoutDestroy(pMsg); @@ -108,7 +108,7 @@ void test1() { void test2() { SyncPing *pMsg = createSyncPing(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test2", &rpcMsg); syncPingDestroy(pMsg); @@ -116,7 +116,7 @@ void test2() { void test3() { SyncPingReply *pMsg = createSyncPingReply(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test3", &rpcMsg); syncPingReplyDestroy(pMsg); @@ -132,7 +132,7 @@ void test4() { void test5() { SyncRequestVoteReply *pMsg = createSyncRequestVoteReply(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test5", &rpcMsg); syncRequestVoteReplyDestroy(pMsg); @@ -140,7 +140,7 @@ void test5() { void test6() { SyncAppendEntries *pMsg = createSyncAppendEntries(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test6", &rpcMsg); syncAppendEntriesDestroy(pMsg); @@ -148,7 +148,7 @@ void test6() { void test7() { SyncAppendEntriesReply *pMsg = createSyncAppendEntriesReply(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test7", &rpcMsg); syncAppendEntriesReplyDestroy(pMsg); @@ -156,7 +156,7 @@ void test7() { void test8() { SyncClientRequest *pMsg = createSyncClientRequest(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pMsg, &rpcMsg); syncRpcMsgPrint2((char *)"test8", &rpcMsg); syncClientRequestDestroy(pMsg); From 81be154e0d66492cbe8bd1d0b01378993f863836 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Mar 2022 17:02:15 +0800 Subject: [PATCH 02/15] sync refactor --- source/libs/sync/inc/syncEnv.h | 2 +- source/libs/sync/inc/syncIO.h | 18 +- source/libs/sync/inc/syncMessage.h | 1 + source/libs/sync/src/syncEnv.c | 19 +- source/libs/sync/src/syncIO.c | 332 +++++++++++-------- source/libs/sync/src/syncMessage.c | 24 +- source/libs/sync/test/syncIOTickPingTest.cpp | 11 +- source/libs/sync/test/syncIOTickQTest.cpp | 15 +- source/libs/sync/test/syncUtilTest.cpp | 1 + 9 files changed, 266 insertions(+), 157 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index c3679636e6..40ff79287b 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -42,7 +42,7 @@ typedef struct SSyncEnv { // tick timer tmr_h pEnvTickTimer; int32_t envTickTimerMS; - uint64_t envTickTimerLogicClock; + uint64_t envTickTimerLogicClock; // if use queue, should pass logic clock into queue item uint64_t envTickTimerLogicClockUser; TAOS_TMR_CALLBACK FpEnvTickTimer; // Timer Fp uint64_t envTickTimerCounter; diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 160fefd086..352d30c8d7 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -29,6 +29,9 @@ extern "C" { #include "tqueue.h" #include "trpc.h" +#define TICK_Q_TIMER_MS 1000 +#define TICK_Ping_TIMER_MS 1000 + typedef struct SSyncIO { STaosQueue *pMsgQ; STaosQset * pQset; @@ -38,9 +41,11 @@ typedef struct SSyncIO { void * clientRpc; SEpSet myAddr; - void *ioTimerTickQ; - void *ioTimerTickPing; - void *ioTimerManager; + tmr_h qTimer; + int32_t qTimerMS; + tmr_h pingTimer; + int32_t pingTimerMS; + tmr_h timerMgr; void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); @@ -59,11 +64,14 @@ extern SSyncIO *gSyncIO; int32_t syncIOStart(char *host, uint16_t port); int32_t syncIOStop(); -int32_t syncIOTickQ(); -int32_t syncIOTickPing(); int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg); int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg); +int32_t syncIOQTimerStart(); +int32_t syncIOQTimerStop(); +int32_t syncIOPingTimerStart(); +int32_t syncIOPingTimerStop(); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 2876577410..9bb5b6195e 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -39,6 +39,7 @@ typedef enum ESyncMessageType { SYNC_REQUEST_VOTE_REPLY = 111, SYNC_APPEND_ENTRIES = 113, SYNC_APPEND_ENTRIES_REPLY = 115, + SYNC_RESPONSE = 119, } ESyncMessageType; diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 2830d2d4aa..dd7161800d 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -55,13 +55,19 @@ static void syncEnvTick(void *param, void *tmrId) { if (atomic_load_64(&pSyncEnv->envTickTimerLogicClockUser) <= atomic_load_64(&pSyncEnv->envTickTimerLogicClock)) { ++(pSyncEnv->envTickTimerCounter); sTrace( - "syncEnvTick ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " - "envTickTimerMS:%d", + "syncEnvTick do ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " + "envTickTimerMS:%d, tmrId:%p", pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, - pSyncEnv->envTickTimerMS); + pSyncEnv->envTickTimerMS, tmrId); // do something, tick ... taosTmrReset(syncEnvTick, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, &pSyncEnv->pEnvTickTimer); + } else { + sTrace( + "syncEnvTick pass ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, " + "envTickTimerMS:%d, tmrId:%p", + pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, + pSyncEnv->envTickTimerMS, tmrId); } } @@ -88,15 +94,16 @@ static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv) { int32_t ret = 0; - pSyncEnv->pEnvTickTimer = - taosTmrStart(pSyncEnv->FpEnvTickTimer, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager); + taosTmrReset(pSyncEnv->FpEnvTickTimer, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, + &pSyncEnv->pEnvTickTimer); atomic_store_64(&pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerLogicClockUser); return ret; } static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv) { + int32_t ret = 0; atomic_add_fetch_64(&pSyncEnv->envTickTimerLogicClockUser, 1); taosTmrStop(pSyncEnv->pEnvTickTimer); pSyncEnv->pEnvTickTimer = NULL; - return 0; + return ret; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index af97c4663c..c8745ca138 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -16,6 +16,7 @@ #include "syncIO.h" #include #include "syncMessage.h" +#include "syncUtil.h" #include "tglobal.h" #include "ttimer.h" #include "tutil.h" @@ -23,33 +24,36 @@ SSyncIO *gSyncIO = NULL; // local function ------------ -static int32_t syncIOStartInternal(SSyncIO *io); -static int32_t syncIOStopInternal(SSyncIO *io); static SSyncIO *syncIOCreate(char *host, uint16_t port); static int32_t syncIODestroy(SSyncIO *io); +static int32_t syncIOStartInternal(SSyncIO *io); +static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); -static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); -static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void *syncIOConsumerFunc(void *param); +static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); -static int32_t syncIOTickQInternal(SSyncIO *io); -static void syncIOTickQFunc(void *param, void *tmrId); -static int32_t syncIOTickPingInternal(SSyncIO *io); -static void syncIOTickPingFunc(void *param, void *tmrId); +static int32_t syncIOStartQ(SSyncIO *io); +static int32_t syncIOStopQ(SSyncIO *io); +static int32_t syncIOStartPing(SSyncIO *io); +static int32_t syncIOStopPing(SSyncIO *io); +static void syncIOTickQ(void *param, void *tmrId); +static void syncIOTickPing(void *param, void *tmrId); // ---------------------------- // public function ------------ int32_t syncIOStart(char *host, uint16_t port) { + int32_t ret = 0; gSyncIO = syncIOCreate(host, port); assert(gSyncIO != NULL); taosSeedRand(taosGetTimestampSec()); - int32_t ret = syncIOStartInternal(gSyncIO); + ret = syncIOStartInternal(gSyncIO); assert(ret == 0); - sTrace("syncIOStart ok, gSyncIO:%p gSyncIO->clientRpc:%p", gSyncIO, gSyncIO->clientRpc); - return 0; + sTrace("syncIOStart ok, gSyncIO:%p", gSyncIO); + return ret; } int32_t syncIOStop() { @@ -61,37 +65,25 @@ int32_t syncIOStop() { return ret; } -int32_t syncIOTickQ() { - int32_t ret = syncIOTickQInternal(gSyncIO); - assert(ret == 0); - return ret; -} - -int32_t syncIOTickPing() { - int32_t ret = syncIOTickPingInternal(gSyncIO); - assert(ret == 0); - return ret; -} - int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { - sTrace( - "<--- syncIOSendMsg ---> clientRpc:%p, numOfEps:%d, inUse:%d, destAddr:%s-%u, pMsg->ahandle:%p, pMsg->handle:%p, " - "pMsg->msgType:%d, pMsg->contLen:%d", - clientRpc, pEpSet->numOfEps, pEpSet->inUse, pEpSet->eps[0].fqdn, pEpSet->eps[0].port, pMsg->ahandle, pMsg->handle, - pMsg->msgType, pMsg->contLen); - { - cJSON *pJson = syncRpcMsg2Json(pMsg); - char * serialized = cJSON_Print(pJson); - sTrace("process syncMessage send: pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } + assert(pEpSet->inUse == 0); + assert(pEpSet->numOfEps == 1); + + int32_t ret = 0; + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port); + syncRpcMsgPrint2(logBuf, pMsg); + pMsg->handle = NULL; rpcSendRequest(clientRpc, pEpSet, pMsg, NULL); - return 0; + return ret; } int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) { + int32_t ret = 0; + char logBuf[128]; + syncRpcMsgPrint2((char *)"==syncIOEqMsg==", pMsg); + SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); @@ -99,11 +91,75 @@ int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) { STaosQueue *pMsgQ = queue; taosWriteQitem(pMsgQ, pTemp); - return 0; + return ret; +} + +int32_t syncIOQTimerStart() { + int32_t ret = syncIOStartQ(gSyncIO); + assert(ret == 0); + return ret; +} + +int32_t syncIOQTimerStop() { + int32_t ret = syncIOStopQ(gSyncIO); + assert(ret == 0); + return ret; +} + +int32_t syncIOPingTimerStart() { + int32_t ret = syncIOStartPing(gSyncIO); + assert(ret == 0); + return ret; +} + +int32_t syncIOPingTimerStop() { + int32_t ret = syncIOStopPing(gSyncIO); + assert(ret == 0); + return ret; } // local function ------------ +static SSyncIO *syncIOCreate(char *host, uint16_t port) { + SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + memset(io, 0, sizeof(*io)); + + io->pMsgQ = taosOpenQueue(); + io->pQset = taosOpenQset(); + taosAddIntoQset(io->pQset, io->pMsgQ, NULL); + + io->myAddr.inUse = 0; + io->myAddr.numOfEps = 0; + addEpIntoEpSet(&io->myAddr, host, port); + + io->qTimerMS = TICK_Q_TIMER_MS; + io->pingTimerMS = TICK_Ping_TIMER_MS; + + return io; +} + +static int32_t syncIODestroy(SSyncIO *io) { + int32_t ret = 0; + int8_t start = atomic_load_8(&io->isStart); + assert(start == 0); + + if (io->serverRpc != NULL) { + rpcClose(io->serverRpc); + io->serverRpc = NULL; + } + + if (io->clientRpc != NULL) { + rpcClose(io->clientRpc); + io->clientRpc = NULL; + } + + taosCloseQueue(io->pMsgQ); + taosCloseQset(io->pQset); + + return ret; +} + static int32_t syncIOStartInternal(SSyncIO *io) { + int32_t ret = 0; taosBlockSIGPIPE(); rpcInit(); @@ -163,58 +219,24 @@ static int32_t syncIOStartInternal(SSyncIO *io) { } // start tmr thread - io->ioTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); + io->timerMgr = taosTmrInit(1000, 50, 10000, "SYNC-IO"); - return 0; + atomic_store_8(&io->isStart, 1); + return ret; } static int32_t syncIOStopInternal(SSyncIO *io) { + int32_t ret = 0; atomic_store_8(&io->isStart, 0); pthread_join(io->consumerTid, NULL); - return 0; -} - -static SSyncIO *syncIOCreate(char *host, uint16_t port) { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); - memset(io, 0, sizeof(*io)); - - io->pMsgQ = taosOpenQueue(); - io->pQset = taosOpenQset(); - taosAddIntoQset(io->pQset, io->pMsgQ, NULL); - - io->myAddr.inUse = 0; - addEpIntoEpSet(&io->myAddr, host, port); - - return io; -} - -static int32_t syncIODestroy(SSyncIO *io) { - int8_t start = atomic_load_8(&io->isStart); - assert(start == 0); - - if (io->serverRpc != NULL) { - free(io->serverRpc); - io->serverRpc = NULL; - } - - if (io->clientRpc != NULL) { - free(io->clientRpc); - io->clientRpc = NULL; - } - - taosCloseQueue(io->pMsgQ); - taosCloseQset(io->pQset); - - return 0; + taosTmrCleanUp(io->timerMgr); + return ret; } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; - + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; - int type; - + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -226,77 +248,67 @@ static void *syncIOConsumerFunc(void *param) { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); + syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg); - char *s = syncRpcMsg2Str(pRpcMsg); - sTrace("syncIOConsumerFunc get item from queue: msgType:%d contLen:%d msg:%s", pRpcMsg->msgType, pRpcMsg->contLen, - s); - free(s); - + // use switch case instead of if else if (pRpcMsg->msgType == SYNC_PING) { if (io->FpOnSyncPing != NULL) { - SyncPing *pSyncMsg; + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + io->FpOnSyncPing(io->pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + /* pSyncMsg = syncPingBuild(pRpcMsg->contLen); syncPingFromRpcMsg(pRpcMsg, pSyncMsg); // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); io->FpOnSyncPing(io->pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); + */ } } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { if (io->FpOnSyncPingReply != NULL) { - SyncPingReply *pSyncMsg; - pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); - syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) { if (io->FpOnSyncRequestVote != NULL) { - SyncRequestVote *pSyncMsg; - pSyncMsg = syncRequestVoteBuild(pRpcMsg->contLen); - syncRequestVoteFromRpcMsg(pRpcMsg, pSyncMsg); + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); io->FpOnSyncRequestVote(io->pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) { if (io->FpOnSyncRequestVoteReply != NULL) { - SyncRequestVoteReply *pSyncMsg; - pSyncMsg = syncRequestVoteReplyBuild(); - syncRequestVoteReplyFromRpcMsg(pRpcMsg, pSyncMsg); + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); io->FpOnSyncRequestVoteReply(io->pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) { if (io->FpOnSyncAppendEntries != NULL) { - SyncAppendEntries *pSyncMsg; - pSyncMsg = syncAppendEntriesBuild(pRpcMsg->contLen); - syncAppendEntriesFromRpcMsg(pRpcMsg, pSyncMsg); + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); io->FpOnSyncAppendEntries(io->pSyncNode, pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) { if (io->FpOnSyncAppendEntriesReply != NULL) { - SyncAppendEntriesReply *pSyncMsg; - pSyncMsg = syncAppendEntriesReplyBuild(); - syncAppendEntriesReplyFromRpcMsg(pRpcMsg, pSyncMsg); + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); io->FpOnSyncAppendEntriesReply(io->pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_TIMEOUT) { if (io->FpOnSyncTimeout != NULL) { - SyncTimeout *pSyncMsg; - pSyncMsg = syncTimeoutBuild(); - syncTimeoutFromRpcMsg(pRpcMsg, pSyncMsg); + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); io->FpOnSyncTimeout(io->pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); } } else { - ; + sTrace("unknown msgType:%d, no operator", pRpcMsg->msgType); } } @@ -306,15 +318,16 @@ static void *syncIOConsumerFunc(void *param) { rpcFreeCont(pRpcMsg->pCont); if (pRpcMsg->handle != NULL) { - int msgSize = 128; + int msgSize = 32; memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.msgType = SYNC_RESPONSE; rpcMsg.pCont = rpcMallocCont(msgSize); rpcMsg.contLen = msgSize; snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); rpcMsg.handle = pRpcMsg->handle; rpcMsg.code = 0; - sTrace("syncIOConsumerFunc rpcSendResponse ... msgType:%d contLen:%d", pRpcMsg->msgType, rpcMsg.contLen); + syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); rpcSendResponse(&rpcMsg); } @@ -326,60 +339,102 @@ static void *syncIOConsumerFunc(void *param) { return NULL; } -static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { - // app shall retrieve the auth info based on meterID from DB or a data file - // demo code here only for simple demo - int ret = 0; - return ret; -} - static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - sTrace("<-- syncIOProcessRequest --> type:%d, contLen:%d, cont:%s", pMsg->msgType, pMsg->contLen, - (char *)pMsg->pCont); - + syncRpcMsgPrint2((char *)"==syncIOProcessRequest==", pMsg); SSyncIO *io = pParent; SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - taosWriteQitem(io->pMsgQ, pTemp); } static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - sTrace("syncIOProcessReply: type:%d, contLen:%d msg:%s", pMsg->msgType, pMsg->contLen, (char *)pMsg->pCont); + if (pMsg->msgType == SYNC_RESPONSE) { + sTrace("==syncIOProcessReply=="); + } else { + syncRpcMsgPrint2((char *)"==syncIOProcessReply==", pMsg); + } rpcFreeCont(pMsg->pCont); } -static int32_t syncIOTickQInternal(SSyncIO *io) { - io->ioTimerTickQ = taosTmrStart(syncIOTickQFunc, 1000, io, io->ioTimerManager); - return 0; +static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int32_t ret = 0; + return ret; } -static void syncIOTickQFunc(void *param, void *tmrId) { +static int32_t syncIOStartQ(SSyncIO *io) { + int32_t ret = 0; + taosTmrReset(syncIOTickQ, io->qTimerMS, io, io->timerMgr, &io->qTimer); + return ret; +} + +static int32_t syncIOStopQ(SSyncIO *io) { + int32_t ret = 0; + taosTmrStop(io->qTimer); + io->qTimer = NULL; + return ret; +} + +static int32_t syncIOStartPing(SSyncIO *io) { + int32_t ret = 0; + taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); + return ret; +} + +static int32_t syncIOStopPing(SSyncIO *io) { + int32_t ret = 0; + taosTmrStop(io->pingTimer); + io->pingTimer = NULL; + return ret; +} + +static void syncIOTickQ(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; - sTrace("<-- syncIOTickQFunc -->"); + + SRaftId srcId, destId; + srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + srcId.vgId = -1; + destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + destId.vgId = -1; + SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOTickQ"); SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOTickQ"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 55; - + syncPingReply2RpcMsg(pMsg, &rpcMsg); SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); - + syncRpcMsgPrint2((char *)"==syncIOTickQ==", &rpcMsg); taosWriteQitem(io->pMsgQ, pTemp); - taosTmrReset(syncIOTickQFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickQ); + syncPingReplyDestroy(pMsg); + + taosTmrReset(syncIOTickQ, io->qTimerMS, io, io->timerMgr, &io->qTimer); } -static int32_t syncIOTickPingInternal(SSyncIO *io) { - io->ioTimerTickPing = taosTmrStart(syncIOTickPingFunc, 1000, io, io->ioTimerManager); - return 0; +static void syncIOTickPing(void *param, void *tmrId) { + SSyncIO *io = (SSyncIO *)param; + + SRaftId srcId, destId; + srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + srcId.vgId = -1; + destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + destId.vgId = -1; + SyncPing *pMsg = syncPingBuild2(&srcId, &destId, "syncIOTickPing"); + // SyncPing *pMsg = syncPingBuild3(&srcId, &destId); + + SRpcMsg rpcMsg; + syncPing2RpcMsg(pMsg, &rpcMsg); + syncRpcMsgPrint2((char *)"==syncIOTickPing==", &rpcMsg); + rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); + syncPingDestroy(pMsg); + + taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); } +#if 0 + + static void syncIOTickPingFunc(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; sTrace("<-- syncIOTickPingFunc -->"); @@ -393,4 +448,5 @@ static void syncIOTickPingFunc(void *param, void *tmrId) { rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); taosTmrReset(syncIOTickPingFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickPing); -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 5a55bbc11f..8d2ff6a9a5 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -65,10 +65,32 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { pRoot = syncAppendEntriesReply2Json(pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == SYNC_RESPONSE) { + pRoot = cJSON_CreateObject(); + char* s; + s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + free(s); + s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + free(s); + } else { pRoot = syncRpcUnknownMsg2Json(); + char* s; + s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + free(s); + s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + free(s); } + cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); + cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); + cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); + cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); + cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); return pJson; @@ -77,7 +99,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { cJSON* syncRpcUnknownMsg2Json() { cJSON* pRoot = cJSON_CreateObject(); cJSON_AddNumberToObject(pRoot, "msgType", SYNC_UNKNOWN); - cJSON_AddStringToObject(pRoot, "data", "known message"); + cJSON_AddStringToObject(pRoot, "data", "unknown message"); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot); diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 8be93e6fc0..9c2342828e 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -25,8 +25,15 @@ int main() { ret = syncIOStart((char*)"127.0.0.1", 7010); assert(ret == 0); - ret = syncIOTickPing(); - assert(ret == 0); + for (int i = 0; i < 3; ++i) { + ret = syncIOPingTimerStart(); + assert(ret == 0); + taosMsleep(5000); + + ret = syncIOPingTimerStop(); + assert(ret == 0); + taosMsleep(5000); + } while (1) { taosSsleep(1); diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 76f5e33e82..64b65f25c8 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -25,11 +25,18 @@ int main() { ret = syncIOStart((char*)"127.0.0.1", 7010); assert(ret == 0); - ret = syncIOTickQ(); + for (int i = 0; i < 3; ++i) { + ret = syncIOQTimerStart(); + assert(ret == 0); + taosMsleep(5000); + + ret = syncIOQTimerStop(); + assert(ret == 0); + taosMsleep(5000); + } + + ret = syncIOStop(); assert(ret == 0); - while (1) { - taosSsleep(1); - } return 0; } diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index 9a1c113620..663db3a7b3 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -26,6 +26,7 @@ int main() { tsAsyncLog = 0; sDebugFlag = 143 + 64; logTest(); + electRandomMSTest(); return 0; From 2796c0d171ed245e86660adc4bf0ea603569ac0c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Mar 2022 17:13:49 +0800 Subject: [PATCH 03/15] sync comment --- source/libs/sync/src/syncIO.c | 6 +++--- source/libs/sync/src/syncMessage.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index c8745ca138..327e692d21 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 8d2ff6a9a5..1e40df4419 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -89,7 +89,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); - cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); + // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); @@ -1305,4 +1305,4 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); free(serialized); -} \ No newline at end of file +} From 7607169b17f4b5f2463fbce4c27e03923e165b54 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 12 Mar 2022 17:19:54 +0800 Subject: [PATCH 04/15] sync io --- source/libs/sync/src/syncIO.c | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 327e692d21..91dea79640 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void * syncIOConsumerFunc(void *param); +static void *syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO * io = param; + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -430,23 +430,4 @@ static void syncIOTickPing(void *param, void *tmrId) { syncPingDestroy(pMsg); taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); -} - -#if 0 - - -static void syncIOTickPingFunc(void *param, void *tmrId) { - SSyncIO *io = (SSyncIO *)param; - sTrace("<-- syncIOTickPingFunc -->"); - - SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOTickPing"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 77; - - rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL); - taosTmrReset(syncIOTickPingFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickPing); -} -#endif \ No newline at end of file +} \ No newline at end of file From 9bac1deeb9b3a9b2526491c798cea661a1a225db Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 13 Mar 2022 16:19:54 +0800 Subject: [PATCH 05/15] sync io --- source/libs/sync/src/syncIO.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 91dea79640..c307ec5068 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { From 1336e08ad0ec5d157db2ec3256f360ce49c1fa39 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 13 Mar 2022 16:27:14 +0800 Subject: [PATCH 06/15] sync io --- source/libs/sync/test/syncEnqTest.cpp | 83 ++++++++------- source/libs/sync/test/syncIOSendMsgTest.cpp | 111 +++++++++++++++----- 2 files changed, 131 insertions(+), 63 deletions(-) diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index e1706bb40b..57315f40ec 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -1,9 +1,10 @@ +#include #include #include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" -#include "syncMessage.h" #include "syncRaftStore.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -14,64 +15,69 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[3] = {7010, 7110, 7210}; +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 5; +int32_t myIndex = 0; -SSyncNode* doSync(int myIndex) { - SSyncFSM* pFsm; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; - SSyncInfo syncInfo; - syncInfo.vgId = 1; +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; syncInfo.rpcClient = gSyncIO->clientRpc; syncInfo.FpSendMsg = syncIOSendMsg; syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; - pCfg->replicaNum = 3; + pCfg->replicaNum = replicaNum; - pCfg->nodeInfo[0].nodePort = ports[0]; - snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - - pCfg->nodeInfo[1].nodePort = ports[1]; - snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodePort = ports[2]; - snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; return pSyncNode; } -void timerPingAll(void* param, void* tmrId) { - SSyncNode* pSyncNode = (SSyncNode*)param; - syncNodePingAll(pSyncNode); +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } } int main(int argc, char** argv) { - // taosInitLog((char*)"syncPingTest.log", 100000, 10); + // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; - logTest(); - - int myIndex = 0; + myIndex = 0; if (argc >= 2) { myIndex = atoi(argv[1]); - if (myIndex > 2 || myIndex < 0) { - fprintf(stderr, "myIndex:%d error. should be 0 - 2", myIndex); - return 1; - } } int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); @@ -80,21 +86,22 @@ int main(int argc, char** argv) { ret = syncEnvStart(); assert(ret == 0); - SSyncNode* pSyncNode = doSync(myIndex); - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"syncInitTest", pSyncNode); + + initRaftId(pSyncNode); + + //-------------------------------------------------------------- for (int i = 0; i < 10; ++i) { - SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); + SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncEnqTest"); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); taosMsleep(1000); } - while (1) { - taosMsleep(1000); - } - return 0; } diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index ed88fbb03e..0fc3ebfe4c 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,8 +1,10 @@ #include #include +#include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -13,37 +15,96 @@ void logTest() { sFatal("--- sync log test: fatal"); } -int main() { +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 5; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; - logTest(); - - int32_t ret; - - ret = syncIOStart((char *)"127.0.0.1", 7010); - assert(ret == 0); - - for (int i = 0; i < 10; ++i) { - SEpSet epSet; - epSet.inUse = 0; - epSet.numOfEps = 0; - addEpIntoEpSet(&epSet, "127.0.0.1", 7010); - - SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf((char *)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 77; - - syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg); - taosSsleep(1); + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); } - while (1) { - taosSsleep(1); + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"syncInitTest", pSyncNode); + + initRaftId(pSyncNode); + + //-------------------------------------------------------------- + + for (int i = 0; i < 10; ++i) { + SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncIOSendMsgTest"); + SRpcMsg rpcMsg; + syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); + + SEpSet epSet; + syncUtilnodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, &rpcMsg); + + taosMsleep(1000); } return 0; From 51ceed983f49e971450f45057e205f503ea62ae8 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 13 Mar 2022 16:36:32 +0800 Subject: [PATCH 07/15] sync io test --- source/libs/sync/test/CMakeLists.txt | 20 ++++++++-------- ...MsgClientTest.cpp => syncIOClientTest.cpp} | 24 +++++++++---------- ...MsgServerTest.cpp => syncIOServerTest.cpp} | 0 3 files changed, 22 insertions(+), 22 deletions(-) rename source/libs/sync/test/{syncIOSendMsgClientTest.cpp => syncIOClientTest.cpp} (62%) rename source/libs/sync/test/{syncIOSendMsgServerTest.cpp => syncIOServerTest.cpp} (100%) diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 2a47b53945..6ade78936d 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -4,8 +4,8 @@ add_executable(syncPingTimerTest "") add_executable(syncIOTickQTest "") add_executable(syncIOTickPingTest "") add_executable(syncIOSendMsgTest "") -add_executable(syncIOSendMsgClientTest "") -add_executable(syncIOSendMsgServerTest "") +add_executable(syncIOClientTest "") +add_executable(syncIOServerTest "") add_executable(syncRaftStoreTest "") add_executable(syncEnqTest "") add_executable(syncIndexTest "") @@ -51,13 +51,13 @@ target_sources(syncIOSendMsgTest PRIVATE "syncIOSendMsgTest.cpp" ) -target_sources(syncIOSendMsgClientTest +target_sources(syncIOClientTest PRIVATE - "syncIOSendMsgClientTest.cpp" + "syncIOClientTest.cpp" ) -target_sources(syncIOSendMsgServerTest +target_sources(syncIOServerTest PRIVATE - "syncIOSendMsgServerTest.cpp" + "syncIOServerTest.cpp" ) target_sources(syncRaftStoreTest PRIVATE @@ -167,12 +167,12 @@ target_include_directories(syncIOSendMsgTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncIOSendMsgClientTest +target_include_directories(syncIOClientTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncIOSendMsgServerTest +target_include_directories(syncIOServerTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" @@ -298,11 +298,11 @@ target_link_libraries(syncIOSendMsgTest sync gtest_main ) -target_link_libraries(syncIOSendMsgClientTest +target_link_libraries(syncIOClientTest sync gtest_main ) -target_link_libraries(syncIOSendMsgServerTest +target_link_libraries(syncIOServerTest sync gtest_main ) diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOClientTest.cpp similarity index 62% rename from source/libs/sync/test/syncIOSendMsgClientTest.cpp rename to source/libs/sync/test/syncIOClientTest.cpp index 250054fd5a..dffa8b5cb9 100644 --- a/source/libs/sync/test/syncIOSendMsgClientTest.cpp +++ b/source/libs/sync/test/syncIOClientTest.cpp @@ -2,7 +2,8 @@ #include #include "syncIO.h" #include "syncInt.h" -#include "syncRaftStore.h" +#include "syncMessage.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -22,7 +23,7 @@ int main() { int32_t ret; - ret = syncIOStart((char *)"127.0.0.1", 7010); + ret = syncIOStart((char*)"127.0.0.1", 7010); assert(ret == 0); for (int i = 0; i < 10; ++i) { @@ -31,20 +32,19 @@ int main() { epSet.numOfEps = 0; addEpIntoEpSet(&epSet, "127.0.0.1", 7030); - SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf((char *)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 77; + SRaftId srcId, destId; + srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + srcId.vgId = 100; + destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + destId.vgId = 100; + + SyncPingReply* pSyncMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOClientTest"); + SRpcMsg rpcMsg; + syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg); taosSsleep(1); } - while (1) { - taosSsleep(1); - } - return 0; } diff --git a/source/libs/sync/test/syncIOSendMsgServerTest.cpp b/source/libs/sync/test/syncIOServerTest.cpp similarity index 100% rename from source/libs/sync/test/syncIOSendMsgServerTest.cpp rename to source/libs/sync/test/syncIOServerTest.cpp From 1a7891776766e2b358dabb4823aa9724764f70c2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 14:05:40 +0800 Subject: [PATCH 08/15] sync index --- source/libs/sync/inc/syncRaftLog.h | 3 ++ source/libs/sync/inc/syncUtil.h | 2 + source/libs/sync/src/syncMain.c | 54 +++++++++++----------- source/libs/sync/src/syncRaftLog.c | 2 +- source/libs/sync/src/syncReplication.c | 19 ++++++++ source/libs/sync/src/syncUtil.c | 10 ++++ source/libs/sync/test/syncLogStoreTest.cpp | 11 ++++- 7 files changed, 72 insertions(+), 29 deletions(-) diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index 59b5fa94db..d979e0df15 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -27,6 +27,9 @@ extern "C" { #include "syncRaftEntry.h" #include "taosdef.h" +#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_INVALID -1 + typedef struct SSyncLogStoreData { SSyncNode* pSyncNode; SWal* pWal; diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 2bbc1948dd..bc38acdfe6 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -52,6 +52,8 @@ const char* syncUtilState2String(ESyncState state); bool syncUtilCanPrint(char c); char* syncUtilprintBin(char* ptr, uint32_t len); char* syncUtilprintBin2(char* ptr, uint32_t len); +SyncIndex syncUtilMinIndex(SyncIndex a, SyncIndex b); +SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 860dd96cdf..63fb4062e0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -327,33 +327,6 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } -// for debug -------------- -void syncNodePrint(SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - printf("syncNodePrint | len:%lu | %s \n", strlen(serialized), serialized); - fflush(NULL); - free(serialized); -} - -void syncNodePrint2(char* s, SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - printf("syncNodePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); - fflush(NULL); - free(serialized); -} - -void syncNodeLog(SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); -} - -void syncNodeLog2(char* s, SSyncNode* pObj) { - char* serialized = syncNode2Str(pObj); - sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); -} - int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); @@ -491,6 +464,33 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { return 0; } +// for debug -------------- +void syncNodePrint(SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + printf("syncNodePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + free(serialized); +} + +void syncNodePrint2(char* s, SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + printf("syncNodePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + free(serialized); +} + +void syncNodeLog(SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); + free(serialized); +} + +void syncNodeLog2(char* s, SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + free(serialized); +} + // ------ local funciton --------- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { int32_t ret = 0; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 27c8a26154..6ebeba1991 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -130,7 +130,7 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastIndex(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index dfbe5db0ed..d4c57630ad 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -14,7 +14,9 @@ */ #include "syncReplication.h" +#include "syncIndexMgr.h" #include "syncMessage.h" +#include "syncRaftEntry.h" // TLA+ Spec // AppendEntries(i, j) == @@ -42,7 +44,24 @@ // /\ UNCHANGED <> // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId* pDestId = &(pSyncNode->peersId[i]); + SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + SyncIndex preLogIndex = nextIndex - 1; + SyncTerm preLogTerm = 0; + if (preLogIndex >= 0) { + SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preLogIndex); + preLogTerm = pPreEntry->term; + } else { + preLogTerm = 0; + } + // SyncTerm lastIndex = + // pSyncNode->pLogStore->getLastIndex < nextIndex ? pSyncNode->pLogStore->getLastIndex : nextIndex; + } + return ret; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index b78971bf37..4e56a9670f 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -184,4 +184,14 @@ char* syncUtilprintBin2(char* ptr, uint32_t len) { p += n; } return s; +} + +SyncIndex syncUtilMinIndex(SyncIndex a, SyncIndex b) { + SyncIndex r = a < b ? a : b; + return r; +} + +SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b) { + SyncIndex r = a > b ? a : b; + return r; } \ No newline at end of file diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 602fdee8c2..1b05f76fa2 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -81,7 +81,10 @@ SSyncNode* syncNodeInit() { SSyncNode* syncInitTest() { return syncNodeInit(); } void logStoreTest() { - logStorePrint(pSyncNode->pLogStore); + logStorePrint2((char*)"logStoreTest2", pSyncNode->pLogStore); + + assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_INVALID); + for (int i = 0; i < 5; ++i) { int32_t dataLen = 10; SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); @@ -97,6 +100,10 @@ void logStoreTest() { // syncEntryPrint2((char*)"write entry:", pEntry); pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry); syncEntryDestory(pEntry); + + if (i == 0) { + assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_BEGIN); + } } logStorePrint(pSyncNode->pLogStore); @@ -129,6 +136,8 @@ int main(int argc, char** argv) { ret = syncEnvStart(); assert(ret == 0); + taosRemoveDir("./wal_test"); + pSyncNode = syncInitTest(); assert(pSyncNode != NULL); From 7d897369d5259aad38e41dcfa3ab0cc2094d2ea4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 16:27:25 +0800 Subject: [PATCH 09/15] sync index --- source/libs/sync/inc/syncInt.h | 28 ++- source/libs/sync/src/syncMain.c | 408 ++++++++++++++++---------------- 2 files changed, 220 insertions(+), 216 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index df277e2d7e..8866603b8e 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -67,9 +67,6 @@ extern "C" { } \ } -struct SRaft; -typedef struct SRaft SRaft; - struct SyncTimeout; typedef struct SyncTimeout SyncTimeout; @@ -117,8 +114,10 @@ typedef struct SSyncNode { SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; char raftStorePath[TSDB_FILENAME_LEN * 2]; - SWal* pWal; - void* rpcClient; + + // sync io + SWal* pWal; + void* rpcClient; int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); void* queue; int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg); @@ -164,7 +163,7 @@ typedef struct SSyncNode { int32_t pingTimerMS; uint64_t pingTimerLogicClock; uint64_t pingTimerLogicClockUser; - TAOS_TMR_CALLBACK FpPingTimer; // Timer Fp + TAOS_TMR_CALLBACK FpPingTimerCB; // Timer Fp uint64_t pingTimerCounter; // elect timer @@ -172,7 +171,7 @@ typedef struct SSyncNode { int32_t electTimerMS; uint64_t electTimerLogicClock; uint64_t electTimerLogicClockUser; - TAOS_TMR_CALLBACK FpElectTimer; // Timer Fp + TAOS_TMR_CALLBACK FpElectTimerCB; // Timer Fp uint64_t electTimerCounter; // heartbeat timer @@ -180,7 +179,7 @@ typedef struct SSyncNode { int32_t heartbeatTimerMS; uint64_t heartbeatTimerLogicClock; uint64_t heartbeatTimerLogicClockUser; - TAOS_TMR_CALLBACK FpHeartbeatTimer; // Timer Fp + TAOS_TMR_CALLBACK FpHeartbeatTimerCB; // Timer Fp uint64_t heartbeatTimerCounter; // callback @@ -194,16 +193,17 @@ typedef struct SSyncNode { } SSyncNode; +// open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeClose(SSyncNode* pSyncNode); -int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); +// ping -------------- int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg); -int32_t syncNodePingAll(SSyncNode* pSyncNode); -int32_t syncNodePingPeers(SSyncNode* pSyncNode); int32_t syncNodePingSelf(SSyncNode* pSyncNode); +int32_t syncNodePingPeers(SSyncNode* pSyncNode); +int32_t syncNodePingAll(SSyncNode* pSyncNode); +// timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms); @@ -211,6 +211,10 @@ int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); + +// utils -------------- +int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); +int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 63fb4062e0..4caa376599 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -31,13 +31,16 @@ static int32_t tsNodeRefId = -1; // ------ local funciton --------- +// enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId); static void syncNodeEqElectTimer(void* param, void* tmrId); static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); +// on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); +// raft algorithm ---- static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); @@ -48,16 +51,20 @@ static void syncNodeCandidate2Follower(SSyncNode* pSyncNode); // --------------------------------- int32_t syncInit() { - sTrace("syncInit ok"); - return 0; + int32_t ret = syncEnvStart(); + return ret; } -void syncCleanUp() { sTrace("syncCleanUp ok"); } +void syncCleanUp() { + int32_t ret = syncEnvStop(); + assert(ret == 0); +} int64_t syncStart(const SSyncInfo* pSyncInfo) { + int32_t ret = 0; SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); assert(pSyncNode != NULL); - return 0; + return ret; } void syncStop(int64_t rid) { @@ -65,9 +72,13 @@ void syncStop(int64_t rid) { syncNodeClose(pSyncNode); } -int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; } +int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { + int32_t ret = 0; + return ret; +} int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + int32_t ret = 0; SSyncNode* pSyncNode = NULL; // get pointer from rid if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak); @@ -75,11 +86,13 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); syncClientRequestDestroy(pSyncMsg); + ret = 0; + } else { sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state)); - return -1; // need define err code !! + ret = -1; // need define err code !! } - return 0; + return ret; } ESyncState syncGetMyRole(int64_t rid) { @@ -89,6 +102,7 @@ ESyncState syncGetMyRole(int64_t rid) { void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} +// open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); @@ -162,7 +176,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->pingTimerMS = PING_TIMER_MS; atomic_store_64(&pSyncNode->pingTimerLogicClock, 0); atomic_store_64(&pSyncNode->pingTimerLogicClockUser, 0); - pSyncNode->FpPingTimer = syncNodeEqPingTimer; + pSyncNode->FpPingTimerCB = syncNodeEqPingTimer; pSyncNode->pingTimerCounter = 0; // init elect timer @@ -170,7 +184,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->electTimerMS = syncUtilElectRandomMS(); atomic_store_64(&pSyncNode->electTimerLogicClock, 0); atomic_store_64(&pSyncNode->electTimerLogicClockUser, 0); - pSyncNode->FpElectTimer = syncNodeEqElectTimer; + pSyncNode->FpElectTimerCB = syncNodeEqElectTimer; pSyncNode->electTimerCounter = 0; // init heartbeat timer @@ -178,7 +192,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->heartbeatTimerMS = HEARTBEAT_TIMER_MS; atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, 0); atomic_store_64(&pSyncNode->heartbeatTimerLogicClockUser, 0); - pSyncNode->FpHeartbeatTimer = syncNodeEqHeartbeatTimer; + pSyncNode->FpHeartbeatTimerCB = syncNodeEqHeartbeatTimer; pSyncNode->heartbeatTimerCounter = 0; // init callback @@ -194,10 +208,146 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { } void syncNodeClose(SSyncNode* pSyncNode) { + int32_t ret; assert(pSyncNode != NULL); + + ret = raftStoreClose(pSyncNode->pRaftStore); + assert(ret == 0); + + voteGrantedDestroy(pSyncNode->pVotesGranted); + votesRespondDestory(pSyncNode->pVotesRespond); + syncIndexMgrDestroy(pSyncNode->pNextIndex); + syncIndexMgrDestroy(pSyncNode->pMatchIndex); + logStoreDestory(pSyncNode->pLogStore); + + syncNodeStopPingTimer(pSyncNode); + syncNodeStopElectTimer(pSyncNode); + syncNodeStopHeartbeatTimer(pSyncNode); + free(pSyncNode); } +// ping -------------- +int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { + syncPingLog2((char*)"==syncNodePing==", pMsg); + int32_t ret = 0; + + SRpcMsg rpcMsg; + syncPing2RpcMsg(pMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg); + + ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); + return ret; +} + +int32_t syncNodePingSelf(SSyncNode* pSyncNode) { + int32_t ret = 0; + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); + ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); + assert(ret == 0); + + syncPingDestroy(pMsg); + return ret; +} + +int32_t syncNodePingPeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId destId; + syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); + ret = syncNodePing(pSyncNode, &destId, pMsg); + assert(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + +int32_t syncNodePingAll(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { + SRaftId destId; + syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); + ret = syncNodePing(pSyncNode, &destId, pMsg); + assert(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + +// timer control -------------- +int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pPingTimer); + atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); + return ret; +} + +int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + atomic_add_fetch_64(&pSyncNode->pingTimerLogicClockUser, 1); + taosTmrStop(pSyncNode->pPingTimer); + pSyncNode->pPingTimer = NULL; + return ret; +} + +int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) { + int32_t ret = 0; + pSyncNode->electTimerMS = ms; + taosTmrReset(pSyncNode->FpElectTimerCB, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pElectTimer); + atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); + return ret; +} + +int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + atomic_add_fetch_64(&pSyncNode->electTimerLogicClockUser, 1); + taosTmrStop(pSyncNode->pElectTimer); + pSyncNode->pElectTimer = NULL; + return ret; +} + +int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { + int32_t ret = 0; + syncNodeStopElectTimer(pSyncNode); + syncNodeStartElectTimer(pSyncNode, ms); + return ret; +} + +int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pHeartbeatTimer); + atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); + return ret; +} + +int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + atomic_add_fetch_64(&pSyncNode->heartbeatTimerLogicClockUser, 1); + taosTmrStop(pSyncNode->pHeartbeatTimer); + pSyncNode->pHeartbeatTimer = NULL; + return ret; +} + +// utils -------------- +int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { + SEpSet epSet; + syncUtilraftId2EpSet(destRaftId, &epSet); + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + return 0; +} + +int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { + SEpSet epSet; + syncUtilnodeInfo2EpSet(nodeInfo, &epSet); + pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); + return 0; +} + cJSON* syncNode2Json(const SSyncNode* pSyncNode) { char u64buf[128]; cJSON* pRoot = cJSON_CreateObject(); @@ -253,12 +403,22 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // tla+ server vars cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); + char tmpBuf[RAFT_STORE_BLOCK_SIZE]; + raftStoreSerialize(pSyncNode->pRaftStore, tmpBuf, sizeof(tmpBuf)); + cJSON_AddStringToObject(pRoot, "pRaftStore", tmpBuf); // tla+ candidate vars + cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); + cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); // tla+ leader vars + cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); + cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); // tla+ log vars + cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%ld", pSyncNode->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); // ping timer snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); @@ -268,8 +428,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimer); - cJSON_AddStringToObject(pRoot, "FpPingTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); + cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerCounter); cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); @@ -281,8 +441,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimer); - cJSON_AddStringToObject(pRoot, "FpElectTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); + cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerCounter); cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); @@ -294,8 +454,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimer); - cJSON_AddStringToObject(pRoot, "FpHeartbeatTimer", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); + cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerCounter); cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); @@ -327,143 +487,6 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } -int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { - SEpSet epSet; - syncUtilraftId2EpSet(destRaftId, &epSet); - pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); - return 0; -} - -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { - SEpSet epSet; - syncUtilnodeInfo2EpSet(nodeInfo, &epSet); - pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg); - return 0; -} - -int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { - sTrace("syncNodePing pSyncNode:%p ", pSyncNode); - int32_t ret = 0; - - SRpcMsg rpcMsg; - syncPing2RpcMsg(pMsg, &rpcMsg); - syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); - - { - cJSON* pJson = syncPing2Json(pMsg); - char* serialized = cJSON_Print(pJson); - sTrace("syncNodePing pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } - - { - SyncPing* pMsg2 = rpcMsg.pCont; - cJSON* pJson = syncPing2Json(pMsg2); - char* serialized = cJSON_Print(pJson); - sTrace("syncNodePing rpcMsg.pCont:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } - - return ret; -} - -int32_t syncNodePingAll(SSyncNode* pSyncNode) { - sTrace("syncNodePingAll pSyncNode:%p ", pSyncNode); - int32_t ret = 0; - for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) { - SRaftId destId; - syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); - ret = syncNodePing(pSyncNode, &destId, pMsg); - assert(ret == 0); - syncPingDestroy(pMsg); - } -} - -int32_t syncNodePingPeers(SSyncNode* pSyncNode) { - int32_t ret = 0; - for (int i = 0; i < pSyncNode->peersNum; ++i) { - SRaftId destId; - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId); - ret = syncNodePing(pSyncNode, &destId, pMsg); - assert(ret == 0); - syncPingDestroy(pMsg); - } -} - -int32_t syncNodePingSelf(SSyncNode* pSyncNode) { - int32_t ret; - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId); - ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); - assert(ret == 0); - syncPingDestroy(pMsg); -} - -int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { - atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); - pSyncNode->pingTimerMS = PING_TIMER_MS; - if (pSyncNode->pPingTimer == NULL) { - pSyncNode->pPingTimer = - taosTmrStart(pSyncNode->FpPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager); - } else { - taosTmrReset(pSyncNode->FpPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pPingTimer); - } - return 0; -} - -int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { - atomic_add_fetch_64(&pSyncNode->pingTimerLogicClockUser, 1); - pSyncNode->pingTimerMS = TIMER_MAX_MS; - return 0; -} - -int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) { - pSyncNode->electTimerMS = ms; - atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); - if (pSyncNode->pElectTimer == NULL) { - pSyncNode->pElectTimer = - taosTmrStart(pSyncNode->FpElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager); - } else { - taosTmrReset(pSyncNode->FpElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pElectTimer); - } - return 0; -} - -int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode) { - atomic_add_fetch_64(&pSyncNode->electTimerLogicClockUser, 1); - pSyncNode->electTimerMS = TIMER_MAX_MS; - return 0; -} - -int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { - syncNodeStopElectTimer(pSyncNode); - syncNodeStartElectTimer(pSyncNode, ms); - return 0; -} - -int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { - atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); - if (pSyncNode->pHeartbeatTimer == NULL) { - pSyncNode->pHeartbeatTimer = - taosTmrStart(pSyncNode->FpHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager); - } else { - taosTmrReset(pSyncNode->FpHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pHeartbeatTimer); - } - return 0; -} - -int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { - atomic_add_fetch_64(&pSyncNode->heartbeatTimerLogicClockUser, 1); - pSyncNode->heartbeatTimerMS = TIMER_MAX_MS; - return 0; -} - // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); @@ -492,61 +515,23 @@ void syncNodeLog2(char* s, SSyncNode* pObj) { } // ------ local funciton --------- -static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { - int32_t ret = 0; - sTrace("<-- syncNodeOnPingCb -->"); - - { - cJSON* pJson = syncPing2Json(pMsg); - char* serialized = cJSON_Print(pJson); - sTrace("process syncMessage recv: syncNodeOnPingCb pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } - - SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId); - SRpcMsg rpcMsg; - syncPingReply2RpcMsg(pMsgReply, &rpcMsg); - syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); - - return ret; -} - -static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { - int32_t ret = 0; - sTrace("<-- syncNodeOnPingReplyCb -->"); - - { - cJSON* pJson = syncPingReply2Json(pMsg); - char* serialized = cJSON_Print(pJson); - sTrace("process syncMessage recv: syncNodeOnPingReplyCb pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } - - return ret; -} - +// enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId) { - sTrace("<-- syncNodeEqPingTimer -->"); - SSyncNode* pSyncNode = (SSyncNode*)param; if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, atomic_load_64(&pSyncNode->pingTimerLogicClock), pSyncNode->pingTimerMS, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); syncTimeoutDestroy(pSyncMsg); - // reset timer ms - // pSyncNode->pingTimerMS += 100; - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { - sTrace("syncNodeEqPingTimer: pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu", pSyncNode->pingTimerLogicClock, - pSyncNode->pingTimerLogicClockUser); + sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu", + pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } } @@ -555,19 +540,18 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { if (atomic_load_64(&pSyncNode->electTimerLogicClockUser) <= atomic_load_64(&pSyncNode->electTimerLogicClock)) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_ELECTION, atomic_load_64(&pSyncNode->electTimerLogicClock), pSyncNode->electTimerMS, pSyncNode); - - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodeEqElectTimer==", &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); syncTimeoutDestroy(pSyncMsg); // reset timer ms pSyncNode->electTimerMS = syncUtilElectRandomMS(); - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { - sTrace("syncNodeEqElectTimer: electTimerLogicClock:%lu, electTimerLogicClockUser:%lu", + sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%lu, electTimerLogicClockUser:%lu", pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); } } @@ -579,23 +563,39 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock), pSyncNode->heartbeatTimerMS, pSyncNode); - SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); syncTimeoutDestroy(pSyncMsg); - // reset timer ms - // pSyncNode->heartbeatTimerMS += 100; - taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); } else { - sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu", + sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu", pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } } +// on message ---- +static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { + int32_t ret = 0; + syncPingLog2("==syncNodeOnPingCb==", pMsg); + SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId); + SRpcMsg rpcMsg; + syncPingReply2RpcMsg(pMsgReply, &rpcMsg); + syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); + + return ret; +} + +static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { + int32_t ret = 0; + syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg); + return ret; +} + +// raft algorithm ---- static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { pSyncNode->pRaftStore->currentTerm = term; From a471152b9696701db65cc78c6cf82160a57b95d2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 16:34:01 +0800 Subject: [PATCH 10/15] sync index --- source/libs/sync/src/syncMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4caa376599..15369cc32d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -40,7 +40,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); -// raft algorithm ---- +// raft state change ---- static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); @@ -595,7 +595,7 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { return ret; } -// raft algorithm ---- +// raft state change ---- static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { pSyncNode->pRaftStore->currentTerm = term; From 9734b9b0ec59f853a9828a638d4b14de6d805e55 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 17:02:43 +0800 Subject: [PATCH 11/15] sync index --- source/libs/sync/src/syncMain.c | 47 ++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 15369cc32d..7f183021cf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -17,11 +17,13 @@ #include "sync.h" #include "syncAppendEntries.h" #include "syncAppendEntriesReply.h" +#include "syncElection.h" #include "syncEnv.h" #include "syncIndexMgr.h" #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncReplication.h" #include "syncRequestVote.h" #include "syncRequestVoteReply.h" #include "syncTimeout.h" @@ -41,13 +43,15 @@ static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); // raft state change ---- -static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term); +static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); static void syncNodeBecomeFollower(SSyncNode* pSyncNode); static void syncNodeBecomeLeader(SSyncNode* pSyncNode); -static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); + static void syncNodeCandidate2Leader(SSyncNode* pSyncNode); +static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); static void syncNodeLeader2Follower(SSyncNode* pSyncNode); static void syncNodeCandidate2Follower(SSyncNode* pSyncNode); + // --------------------------------- int32_t syncInit() { @@ -596,12 +600,15 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { } // raft state change ---- -static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { +static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { pSyncNode->pRaftStore->currentTerm = term; + raftStorePersist(pSyncNode->pRaftStore); + + syncNodeBecomeFollower(pSyncNode); + pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID; raftStorePersist(pSyncNode->pRaftStore); - syncNodeBecomeFollower(pSyncNode); } } @@ -610,9 +617,11 @@ static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { pSyncNode->leaderCache = EMPTY_RAFT_ID; } + pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncNodeStopHeartbeatTimer(pSyncNode); + int32_t electMS = syncUtilElectRandomMS(); - syncNodeStartElectTimer(pSyncNode, electMS); + syncNodeRestartElectTimer(pSyncNode, electMS); } // TLA+ Spec @@ -637,13 +646,23 @@ static void syncNodeBecomeLeader(SSyncNode* pSyncNode) { pSyncNode->state = TAOS_SYNC_STATE_LEADER; pSyncNode->leaderCache = pSyncNode->myRaftId; - // next Index +=1 - // match Index = 0; + for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { + pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + } + + for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { + pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; + } syncNodeStopElectTimer(pSyncNode); syncNodeStartHeartbeatTimer(pSyncNode); + syncNodeReplicate(pSyncNode); +} - // appendEntries; +static void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + assert(voteGrantedMajority(pSyncNode->pVotesGranted)); + syncNodeBecomeLeader(pSyncNode); } static void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { @@ -651,8 +670,12 @@ static void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; } -static void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {} +static void syncNodeLeader2Follower(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + syncNodeBecomeFollower(pSyncNode); +} -static void syncNodeLeader2Follower(SSyncNode* pSyncNode) {} - -static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) {} +static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + syncNodeBecomeFollower(pSyncNode); +} From 8ae9fb6ae133f33efaf659e8732864b3c89c542b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 18:22:39 +0800 Subject: [PATCH 12/15] sync index --- source/libs/sync/inc/syncRaftStore.h | 6 +++++ source/libs/sync/inc/syncUtil.h | 1 + source/libs/sync/src/syncMain.c | 34 +++++++++++++++++++++++----- source/libs/sync/src/syncRaftStore.c | 29 ++++++++++++++++++++++++ source/libs/sync/src/syncUtil.c | 2 ++ 5 files changed, 66 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 4058d3bd1c..30f7c5d9f7 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -43,6 +43,12 @@ int32_t raftStorePersist(SRaftStore *pRaftStore); int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); +bool raftStoreHasVoted(SRaftStore *pRaftStore); +void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId); +void raftStoreClearVote(SRaftStore *pRaftStore); +void raftStoreNextTerm(SRaftStore *pRaftStore); +void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); + // for debug ------------------- void raftStorePrint(SRaftStore *pObj); void raftStorePrint2(char *s, SRaftStore *pObj); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index bc38acdfe6..1b702c2528 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -34,6 +34,7 @@ void syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet); void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId); bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); +bool syncUtilEmptyId(const SRaftId* pId); // ---- SSyncBuffer ---- void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7f183021cf..da1286bd32 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -21,6 +21,7 @@ #include "syncEnv.h" #include "syncIndexMgr.h" #include "syncInt.h" +#include "syncMessage.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncReplication.h" @@ -52,6 +53,9 @@ static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); static void syncNodeLeader2Follower(SSyncNode* pSyncNode); static void syncNodeCandidate2Follower(SSyncNode* pSyncNode); +// raft vote ---- +static void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); +static void syncNodeVoteForSelf(SSyncNode* pSyncNode); // --------------------------------- int32_t syncInit() { @@ -602,13 +606,9 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { // raft state change ---- static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { - pSyncNode->pRaftStore->currentTerm = term; - raftStorePersist(pSyncNode->pRaftStore); - + raftStoreSetTerm(pSyncNode->pRaftStore, term); syncNodeBecomeFollower(pSyncNode); - - pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID; - raftStorePersist(pSyncNode->pRaftStore); + raftStoreClearVote(pSyncNode->pRaftStore); } } @@ -679,3 +679,25 @@ static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); syncNodeBecomeFollower(pSyncNode); } + +// raft vote ---- +static void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { + assert(term == pSyncNode->pRaftStore->currentTerm); + assert(!raftStoreHasVoted(pSyncNode->pRaftStore)); + + raftStoreVote(pSyncNode->pRaftStore, pRaftId); +} + +static void syncNodeVoteForSelf(SSyncNode* pSyncNode) { + syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId)); + + SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = pSyncNode->myRaftId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->voteGranted = true; + + voteGrantedVote(pSyncNode->pVotesGranted, pMsg); + votesRespondAdd(pSyncNode->pVotesRespond, pMsg); + syncRequestVoteReplyDestroy(pMsg); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 3a26caa161..5ad618b9c0 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -15,6 +15,8 @@ #include "syncRaftStore.h" #include "cJSON.h" +#include "syncEnv.h" +#include "syncUtil.h" // private function static int32_t raftStoreInit(SRaftStore *pRaftStore); @@ -135,6 +137,33 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0; } +bool raftStoreHasVoted(SRaftStore *pRaftStore) { + bool b = syncUtilEmptyId(&(pRaftStore->voteFor)); + return b; +} + +void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId) { + assert(!raftStoreHasVoted(pRaftStore)); + assert(!syncUtilEmptyId(pRaftId)); + pRaftStore->voteFor = *pRaftId; + raftStorePersist(pRaftStore); +} + +void raftStoreClearVote(SRaftStore *pRaftStore) { + pRaftStore->voteFor = EMPTY_RAFT_ID; + raftStorePersist(pRaftStore); +} + +void raftStoreNextTerm(SRaftStore *pRaftStore) { + ++(pRaftStore->currentTerm); + raftStorePersist(pRaftStore); +} + +void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term) { + pRaftStore->currentTerm = term; + raftStorePersist(pRaftStore); +} + // for debug ------------------- void raftStorePrint(SRaftStore *pObj) { char serialized[RAFT_STORE_BLOCK_SIZE]; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 4e56a9670f..ba8a76c190 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -74,6 +74,8 @@ bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { return ret; } +bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); } + // ---- SSyncBuffer ----- void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len) { syncBuf->len = len; From 7c868aea85f1255f093ad397b3a41cf477512e1e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 18:44:53 +0800 Subject: [PATCH 13/15] sync index --- source/libs/sync/inc/syncInt.h | 15 ++ source/libs/sync/src/syncElection.c | 13 ++ source/libs/sync/src/syncMain.c | 219 ++++++++++++++-------------- source/libs/sync/src/syncTimeout.c | 10 +- 4 files changed, 135 insertions(+), 122 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8866603b8e..4a1557addd 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -209,6 +209,7 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); +int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); @@ -218,6 +219,20 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); +// raft state change -------------- +void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); +void syncNodeBecomeFollower(SSyncNode* pSyncNode); +void syncNodeBecomeLeader(SSyncNode* pSyncNode); + +void syncNodeCandidate2Leader(SSyncNode* pSyncNode); +void syncNodeFollower2Candidate(SSyncNode* pSyncNode); +void syncNodeLeader2Follower(SSyncNode* pSyncNode); +void syncNodeCandidate2Follower(SSyncNode* pSyncNode); + +// raft vote -------------- +void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); +void syncNodeVoteForSelf(SSyncNode* pSyncNode); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 223431336e..d06253c049 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -16,6 +16,7 @@ #include "syncElection.h" #include "syncMessage.h" #include "syncRaftStore.h" +#include "syncVoteMgr.h" // TLA+ Spec // RequestVote(i, j) == @@ -49,10 +50,22 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { } int32_t syncNodeElect(SSyncNode* pSyncNode) { + if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { + syncNodeFollower2Candidate(pSyncNode); + } assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); // start election + raftStoreNextTerm(pSyncNode->pRaftStore); + raftStoreClearVote(pSyncNode->pRaftStore); + voteGrantedReset(pSyncNode->pVotesGranted, pSyncNode->pRaftStore->currentTerm); + votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->pRaftStore->currentTerm); + + syncNodeVoteForSelf(pSyncNode); int32_t ret = syncNodeRequestVotePeers(pSyncNode); + assert(ret == 0); + syncNodeResetElectTimer(pSyncNode); + return ret; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index da1286bd32..5ec9df1e70 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -42,20 +42,6 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); // on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); - -// raft state change ---- -static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); -static void syncNodeBecomeFollower(SSyncNode* pSyncNode); -static void syncNodeBecomeLeader(SSyncNode* pSyncNode); - -static void syncNodeCandidate2Leader(SSyncNode* pSyncNode); -static void syncNodeFollower2Candidate(SSyncNode* pSyncNode); -static void syncNodeLeader2Follower(SSyncNode* pSyncNode); -static void syncNodeCandidate2Follower(SSyncNode* pSyncNode); - -// raft vote ---- -static void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); -static void syncNodeVoteForSelf(SSyncNode* pSyncNode); // --------------------------------- int32_t syncInit() { @@ -325,6 +311,13 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { return ret; } +int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { + int32_t ret = 0; + int32_t electMS = syncUtilElectRandomMS(); + ret = syncNodeRestartElectTimer(pSyncNode, electMS); + return ret; +} + int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { int32_t ret = 0; taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, @@ -495,6 +488,105 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } +// raft state change -------------- +void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { + if (term > pSyncNode->pRaftStore->currentTerm) { + raftStoreSetTerm(pSyncNode->pRaftStore, term); + syncNodeBecomeFollower(pSyncNode); + raftStoreClearVote(pSyncNode->pRaftStore); + } +} + +void syncNodeBecomeFollower(SSyncNode* pSyncNode) { + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + pSyncNode->leaderCache = EMPTY_RAFT_ID; + } + + pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + syncNodeStopHeartbeatTimer(pSyncNode); + + int32_t electMS = syncUtilElectRandomMS(); + syncNodeRestartElectTimer(pSyncNode, electMS); +} + +// TLA+ Spec +// \* Candidate i transitions to leader. +// BecomeLeader(i) == +// /\ state[i] = Candidate +// /\ votesGranted[i] \in Quorum +// /\ state' = [state EXCEPT ![i] = Leader] +// /\ nextIndex' = [nextIndex EXCEPT ![i] = +// [j \in Server |-> Len(log[i]) + 1]] +// /\ matchIndex' = [matchIndex EXCEPT ![i] = +// [j \in Server |-> 0]] +// /\ elections' = elections \cup +// {[eterm |-> currentTerm[i], +// eleader |-> i, +// elog |-> log[i], +// evotes |-> votesGranted[i], +// evoterLog |-> voterLog[i]]} +// /\ UNCHANGED <> +// +void syncNodeBecomeLeader(SSyncNode* pSyncNode) { + pSyncNode->state = TAOS_SYNC_STATE_LEADER; + pSyncNode->leaderCache = pSyncNode->myRaftId; + + for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { + pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + } + + for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { + pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; + } + + syncNodeStopElectTimer(pSyncNode); + syncNodeStartHeartbeatTimer(pSyncNode); + syncNodeReplicate(pSyncNode); +} + +void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + assert(voteGrantedMajority(pSyncNode->pVotesGranted)); + syncNodeBecomeLeader(pSyncNode); +} + +void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); + pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; +} + +void syncNodeLeader2Follower(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + syncNodeBecomeFollower(pSyncNode); +} + +void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + syncNodeBecomeFollower(pSyncNode); +} + +// raft vote -------------- +void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { + assert(term == pSyncNode->pRaftStore->currentTerm); + assert(!raftStoreHasVoted(pSyncNode->pRaftStore)); + + raftStoreVote(pSyncNode->pRaftStore, pRaftId); +} + +void syncNodeVoteForSelf(SSyncNode* pSyncNode) { + syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId)); + + SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = pSyncNode->myRaftId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->voteGranted = true; + + voteGrantedVote(pSyncNode->pVotesGranted, pMsg); + votesRespondAdd(pSyncNode->pVotesRespond, pMsg); + syncRequestVoteReplyDestroy(pMsg); +} + // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); @@ -602,102 +694,3 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg); return ret; } - -// raft state change ---- -static void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { - if (term > pSyncNode->pRaftStore->currentTerm) { - raftStoreSetTerm(pSyncNode->pRaftStore, term); - syncNodeBecomeFollower(pSyncNode); - raftStoreClearVote(pSyncNode->pRaftStore); - } -} - -static void syncNodeBecomeFollower(SSyncNode* pSyncNode) { - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - pSyncNode->leaderCache = EMPTY_RAFT_ID; - } - - pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; - syncNodeStopHeartbeatTimer(pSyncNode); - - int32_t electMS = syncUtilElectRandomMS(); - syncNodeRestartElectTimer(pSyncNode, electMS); -} - -// TLA+ Spec -// \* Candidate i transitions to leader. -// BecomeLeader(i) == -// /\ state[i] = Candidate -// /\ votesGranted[i] \in Quorum -// /\ state' = [state EXCEPT ![i] = Leader] -// /\ nextIndex' = [nextIndex EXCEPT ![i] = -// [j \in Server |-> Len(log[i]) + 1]] -// /\ matchIndex' = [matchIndex EXCEPT ![i] = -// [j \in Server |-> 0]] -// /\ elections' = elections \cup -// {[eterm |-> currentTerm[i], -// eleader |-> i, -// elog |-> log[i], -// evotes |-> votesGranted[i], -// evoterLog |-> voterLog[i]]} -// /\ UNCHANGED <> -// -static void syncNodeBecomeLeader(SSyncNode* pSyncNode) { - pSyncNode->state = TAOS_SYNC_STATE_LEADER; - pSyncNode->leaderCache = pSyncNode->myRaftId; - - for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { - pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; - } - - for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { - pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; - } - - syncNodeStopElectTimer(pSyncNode); - syncNodeStartHeartbeatTimer(pSyncNode); - syncNodeReplicate(pSyncNode); -} - -static void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); - assert(voteGrantedMajority(pSyncNode->pVotesGranted)); - syncNodeBecomeLeader(pSyncNode); -} - -static void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); - pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; -} - -static void syncNodeLeader2Follower(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); - syncNodeBecomeFollower(pSyncNode); -} - -static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); - syncNodeBecomeFollower(pSyncNode); -} - -// raft vote ---- -static void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { - assert(term == pSyncNode->pRaftStore->currentTerm); - assert(!raftStoreHasVoted(pSyncNode->pRaftStore)); - - raftStoreVote(pSyncNode->pRaftStore, pRaftId); -} - -static void syncNodeVoteForSelf(SSyncNode* pSyncNode) { - syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId)); - - SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(); - pMsg->srcId = pSyncNode->myRaftId; - pMsg->destId = pSyncNode->myRaftId; - pMsg->term = pSyncNode->pRaftStore->currentTerm; - pMsg->voteGranted = true; - - voteGrantedVote(pSyncNode->pVotesGranted, pMsg); - votesRespondAdd(pSyncNode->pVotesRespond, pMsg); - syncRequestVoteReplyDestroy(pMsg); -} \ No newline at end of file diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 7cbfd6d40a..3a48b0cbb3 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -19,15 +19,7 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) { int32_t ret = 0; - sTrace("<-- syncNodeOnTimeoutCb -->"); - - { - cJSON* pJson = syncTimeout2Json(pMsg); - char* serialized = cJSON_Print(pJson); - sTrace("process syncMessage recv: syncNodeOnTimeoutCb pMsg:%s ", serialized); - free(serialized); - cJSON_Delete(pJson); - } + syncTimeoutLog2("==syncNodeOnTimeoutCb==", pMsg); if (pMsg->timeoutType == SYNC_TIMEOUT_PING) { if (atomic_load_64(&ths->pingTimerLogicClockUser) <= pMsg->logicClock) { From f924c1b879f8c38013e31de19e124e96c1d82b4d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Mar 2022 20:43:35 +0800 Subject: [PATCH 14/15] sync index --- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/inc/syncMessage.h | 3 +- source/libs/sync/src/syncAppendEntries.c | 7 ++- source/libs/sync/src/syncAppendEntriesReply.c | 45 ++++++++++++++++++- source/libs/sync/src/syncElection.c | 2 +- source/libs/sync/src/syncMain.c | 2 + source/libs/sync/src/syncMessage.c | 8 ++-- source/libs/sync/src/syncReplication.c | 29 +++++++++--- source/libs/sync/src/syncRequestVote.c | 36 ++++++++++++++- source/libs/sync/src/syncRequestVoteReply.c | 35 ++++++++++++++- source/libs/sync/test/syncRequestVoteTest.cpp | 2 +- source/libs/sync/test/syncRpcMsgTest.cpp | 2 +- 12 files changed, 155 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 4a1557addd..15c719b76e 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -232,6 +232,7 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode); // raft vote -------------- void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForSelf(SSyncNode* pSyncNode); +void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 9bb5b6195e..7dfea31f5c 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -196,7 +196,7 @@ typedef struct SyncRequestVote { SRaftId srcId; SRaftId destId; // private data - SyncTerm currentTerm; + SyncTerm term; SyncIndex lastLogIndex; SyncTerm lastLogTerm; } SyncRequestVote; @@ -287,6 +287,7 @@ typedef struct SyncAppendEntriesReply { SRaftId srcId; SRaftId destId; // private data + SyncTerm term; bool success; SyncIndex matchIndex; } SyncAppendEntriesReply; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index ba10234a1d..55d369a115 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -80,4 +80,9 @@ // /\ UNCHANGED <> // /\ UNCHANGED <> // -int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {} +int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = 0; + syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + + return ret; +} diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 0a5120c8dc..61eb4884e2 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -14,6 +14,12 @@ */ #include "syncAppendEntriesReply.h" +#include "syncIndexMgr.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" // TLA+ Spec // HandleAppendEntriesResponse(i, j, m) == @@ -28,4 +34,41 @@ // /\ Discard(m) // /\ UNCHANGED <> // -int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {} +int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = 0; + syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg); + + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("DropStaleResponse, receive term:%lu, current term:%lu", pMsg->term, ths->pRaftStore->currentTerm); + return ret; + } + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + assert(pMsg->term == ths->pRaftStore->currentTerm); + + if (pMsg->success) { + // nextIndex = reply.matchIndex + 1 + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); + + // matchIndex = reply.matchIndex + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + + // maybe commit + syncNodeMaybeAdvanceCommitIndex(ths); + + } else { + SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + if (nextIndex > SYNC_INDEX_BEGIN) { + --nextIndex; + } else { + nextIndex = SYNC_INDEX_BEGIN; + } + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + } + + return ret; +} diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index d06253c049..77c3d07698 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -38,7 +38,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { SyncRequestVote* pMsg = syncRequestVoteBuild(); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = pSyncNode->peersId[i]; - pMsg->currentTerm = pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSyncNode->pRaftStore->currentTerm; pMsg->lastLogIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); pMsg->lastLogTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5ec9df1e70..aaf6535f40 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -587,6 +587,8 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { syncRequestVoteReplyDestroy(pMsg); } +void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {} + // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 1e40df4419..2447ea2218 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -820,8 +820,8 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->currentTerm); - cJSON_AddStringToObject(pRoot, "currentTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); @@ -1264,9 +1264,11 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "success", pMsg->success); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); - cJSON_AddStringToObject(pRoot, "match_index", u64buf); + cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncAppendEntriesReply", pRoot); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index d4c57630ad..b935943a1d 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -17,6 +17,8 @@ #include "syncIndexMgr.h" #include "syncMessage.h" #include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncUtil.h" // TLA+ Spec // AppendEntries(i, j) == @@ -50,16 +52,31 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId* pDestId = &(pSyncNode->peersId[i]); SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + SyncIndex preLogIndex = nextIndex - 1; - SyncTerm preLogTerm = 0; - if (preLogIndex >= 0) { + + SyncTerm preLogTerm = 0; + if (preLogIndex >= SYNC_INDEX_BEGIN) { SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preLogIndex); preLogTerm = pPreEntry->term; - } else { - preLogTerm = 0; } - // SyncTerm lastIndex = - // pSyncNode->pLogStore->getLastIndex < nextIndex ? pSyncNode->pLogStore->getLastIndex : nextIndex; + + SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); + assert(nextIndex == lastIndex); + + SSyncRaftEntry* pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex); + assert(pEntry != NULL); + + SyncAppendEntries* pMsg = syncAppendEntriesBuild(pEntry->bytes); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = *pDestId; + pMsg->prevLogIndex = preLogIndex; + pMsg->prevLogTerm = preLogTerm; + pMsg->commitIndex = pSyncNode->commitIndex; + pMsg->dataLen = pEntry->bytes; + // add pEntry into msg + + syncNodeAppendEntries(pSyncNode, pDestId, pMsg); } return ret; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 354c559a90..be4f40aaad 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -14,6 +14,10 @@ */ #include "syncRequestVote.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" // TLA+ Spec // HandleRequestVoteRequest(i, j, m) == @@ -37,4 +41,34 @@ // m) // /\ UNCHANGED <> // -int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {} +int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = 0; + syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg); + + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + assert(pMsg->term <= ths->pRaftStore->currentTerm); + + bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || + ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && + (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore))); + bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && + ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + if (grant) { + raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + } + + SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->voteGranted = grant; + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncRequestVoteReplyDestroy(pReply); + + return ret; +} diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 72223ea83c..7cdeace166 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -14,6 +14,10 @@ */ #include "syncRequestVoteReply.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" // TLA+ Spec // HandleRequestVoteResponse(i, j, m) == @@ -32,4 +36,33 @@ // /\ Discard(m) // /\ UNCHANGED <> // -int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) {} +int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = 0; + syncRequestVoteReplyLog2("==syncNodeOnRequestVoteReplyCb==", pMsg); + + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("DropStaleResponse, receive term:%lu, current term:%lu", pMsg->term, ths->pRaftStore->currentTerm); + return ret; + } + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + assert(pMsg->term == ths->pRaftStore->currentTerm); + + if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { + votesRespondAdd(ths->pVotesRespond, pMsg); + if (pMsg->voteGranted) { + voteGrantedVote(ths->pVotesGranted, pMsg); + if (voteGrantedMajority(ths->pVotesGranted)) { + if (ths->pVotesGranted->toLeader) { + syncNodeCandidate2Leader(ths); + ths->pVotesGranted->toLeader = true; + } + } + } + } + return ret; +} diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 7f75ee937b..22f47046de 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -20,7 +20,7 @@ SyncRequestVote *createMsg() { pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; - pMsg->currentTerm = 11; + pMsg->term = 11; pMsg->lastLogIndex = 22; pMsg->lastLogTerm = 33; return pMsg; diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 61edbd3012..ee6f6c0800 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -57,7 +57,7 @@ SyncRequestVote *createSyncRequestVote() { pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; - pMsg->currentTerm = 11; + pMsg->term = 11; pMsg->lastLogIndex = 22; pMsg->lastLogTerm = 33; return pMsg; From 17e607103366f515f0410e55dccedb788ea7ea2d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 15 Mar 2022 10:47:11 +0800 Subject: [PATCH 15/15] sync index --- source/libs/sync/inc/syncMessage.h | 1 + source/libs/sync/src/syncAppendEntries.c | 117 +++++++++++++++++++++++ source/libs/sync/src/syncMessage.c | 3 + 3 files changed, 121 insertions(+) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 7dfea31f5c..5785089a20 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -255,6 +255,7 @@ typedef struct SyncAppendEntries { SRaftId srcId; SRaftId destId; // private data + SyncTerm term; SyncIndex prevLogIndex; SyncTerm prevLogTerm; SyncIndex commitIndex; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 55d369a115..87d6669f59 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -14,6 +14,11 @@ */ #include "syncAppendEntries.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == @@ -84,5 +89,117 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + assert(pMsg->term <= ths->pRaftStore->currentTerm); + + if (pMsg->term == ths->pRaftStore->currentTerm) { + ths->leaderCache = pMsg->srcId; + syncNodeResetElectTimer(ths); + } + assert(pMsg->dataLen >= 0); + + SyncTerm localPreLogTerm = 0; + if (pMsg->prevLogTerm >= SYNC_INDEX_BEGIN && pMsg->prevLogTerm <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm); + assert(pEntry != NULL); + localPreLogTerm = pEntry->term; + syncEntryDestory(pEntry); + } + + bool logOK = + (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || + ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && + (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogIndex == localPreLogTerm)); + + // reject + if ((pMsg->term < ths->pRaftStore->currentTerm) || + ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + + // return to follower state + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + syncNodeBecomeFollower(ths); + } + + // accept request + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { + bool matchSuccess = false; + if (pMsg->prevLogIndex == SYNC_INDEX_INVALID && + ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) { + matchSuccess = true; + } + if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm); + assert(pEntry != NULL); + if (pMsg->prevLogTerm == pEntry->term) { + matchSuccess = true; + } + syncEntryDestory(pEntry); + } + + if (matchSuccess) { + // delete conflict entries + if (ths->pLogStore->getLastIndex(ths->pLogStore) > pMsg->prevLogIndex) { + SyncIndex fromIndex = pMsg->prevLogIndex + 1; + ths->pLogStore->truncate(ths->pLogStore, fromIndex); + } + + // append one entry + if (pMsg->dataLen > 0) { + SSyncRaftEntry* pEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + syncEntryDestory(pEntry); + } + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + pReply->matchIndex = pMsg->prevLogIndex + 1; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + + syncAppendEntriesReplyDestroy(pReply); + } else { + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + } + + if (pMsg->commitIndex > ths->commitIndex) { + if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + // commit + ths->commitIndex = pMsg->commitIndex; + ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + } + } + } + return ret; } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 2447ea2218..509ede274b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1108,6 +1108,9 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf);