From dca01dda58c8c4b369130ea0ff6803cfa30e4f23 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 22 Feb 2022 11:52:49 +0800 Subject: [PATCH 01/19] add test --- source/libs/sync/CMakeLists.txt | 6 +++++- source/libs/sync/test/CMakeLists.txt | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 source/libs/sync/test/CMakeLists.txt diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index 37ee5194c8..cb38d7e363 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -13,4 +13,8 @@ target_include_directories( sync PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) \ No newline at end of file +) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt new file mode 100644 index 0000000000..f24a1a9a46 --- /dev/null +++ b/source/libs/sync/test/CMakeLists.txt @@ -0,0 +1,20 @@ +add_executable(syncTest "") +target_sources(syncTest + PRIVATE + "syncTest.cpp" +) +target_include_directories(syncTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries(syncTest + sync + gtest_main +) +enable_testing() +add_test( + NAME sync_test + COMMAND syncTest +) From 3d9d5240a520b0fdc206eb361a08323c49f0e333 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Feb 2022 09:54:44 +0800 Subject: [PATCH 02/19] add sync code --- include/libs/sync/sync.h | 2 ++ source/libs/sync/inc/syncEnv.h | 33 ++++++++++++++++++++++++++++ source/libs/sync/inc/syncIO.h | 33 ++++++++++++++++++++++++++++ source/libs/sync/inc/syncInt.h | 9 ++++++++ source/libs/sync/inc/syncRaftEntry.h | 6 ++--- source/libs/sync/src/syncEnv.c | 26 ++++++++++++++++++++++ source/libs/sync/src/syncIO.c | 16 ++++++++++++++ source/libs/sync/test/syncTest.cpp | 19 ++++++++++++++++ 8 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 source/libs/sync/inc/syncEnv.h create mode 100644 source/libs/sync/inc/syncIO.h create mode 100644 source/libs/sync/src/syncEnv.c create mode 100644 source/libs/sync/src/syncIO.c diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 0ec741ec3e..3d875a4be8 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -153,6 +153,8 @@ int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); ESyncState syncGetMyRole(int64_t rid); void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); +int32_t syncStartEnv(); + extern int32_t sDebugFlag; #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h new file mode 100644 index 0000000000..a6ab03e1ee --- /dev/null +++ b/source/libs/sync/inc/syncEnv.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_ENV_H +#define _TD_LIBS_SYNC_ENV_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" +#include "trpc.h" + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_ENV_H*/ diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h new file mode 100644 index 0000000000..ebe9fb32d1 --- /dev/null +++ b/source/libs/sync/inc/syncIO.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_IO_H +#define _TD_LIBS_IO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_IO_H*/ diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 551ce83122..c1aba0518c 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -24,6 +24,15 @@ extern "C" { #include #include #include "taosdef.h" +#include "sync.h" +#include "tlog.h" + +extern int32_t sDebugFlag; + +#define sLog(...) \ + { \ + taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); \ + } #define sFatal(...) \ { \ diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index adc82f2c5d..65f77f3759 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_LIBS_TPL_H -#define _TD_LIBS_TPL_H +#ifndef _TD_LIBS_SYNC_RAFT_ENTRY_H +#define _TD_LIBS_SYNC_RAFT_ENTRY_H #ifdef __cplusplus extern "C" { @@ -37,4 +37,4 @@ typedef struct SSyncRaftEntry { } #endif -#endif /*_TD_LIBS_TPL_H*/ +#endif /*_TD_LIBS_SYNC_RAFT_ENTRY_H*/ diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c new file mode 100644 index 0000000000..a11ed86dc0 --- /dev/null +++ b/source/libs/sync/src/syncEnv.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncEnv.h" +#include "sync.h" +#include "syncInt.h" + +int32_t syncStartEnv() { + sInfo("log: syncStartEnv \n"); + + if (rpcInit() != 0) { + return -1; + } +} \ No newline at end of file diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c new file mode 100644 index 0000000000..738fc4c5e1 --- /dev/null +++ b/source/libs/sync/src/syncIO.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sync.h" diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 47566d537e..e069091ad8 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,7 +1,26 @@ #include +#include "syncInt.h" int main() { printf("test \n"); + + syncStartEnv(); + + char temp[100]; + snprintf(temp, 100, "./debug.log"); + taosInitLog(temp, 10000, 1); + tsAsyncLog = 0; + + for (int i = 0; i < 100; i++) { + sDebug("log:%d -------- \n", i); + } + + fflush(NULL); + //taosCloseLog(); + + while(1) { + sleep(3); + } return 0; } From 0e0af1de054fdf9f1bcb324432340ddb250050b7 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Feb 2022 15:17:49 +0800 Subject: [PATCH 03/19] add sync code --- source/libs/sync/inc/syncIO.h | 29 ++- source/libs/sync/inc/syncInt.h | 8 +- source/libs/sync/src/syncAppendEntries.c | 183 +++++++++-------- source/libs/sync/src/syncAppendEntriesReply.c | 26 ++- source/libs/sync/src/syncIO.c | 189 +++++++++++++++++- source/libs/sync/src/syncRequestVote.c | 68 +++---- source/libs/sync/src/syncRequestVoteReply.c | 34 ++-- source/libs/sync/test/syncTest.cpp | 29 +-- 8 files changed, 379 insertions(+), 187 deletions(-) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index ebe9fb32d1..bf3b3d34c1 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -20,11 +20,38 @@ extern "C" { #endif +#include #include #include -#include +#include "os.h" +#include "syncInt.h" #include "taosdef.h" +#include "tqueue.h" +#include "trpc.h" +typedef struct SSyncIO { + void * serverRpc; + void * clientRpc; + STaosQueue *pMsgQ; + STaosQset * pQset; + pthread_t tid; + int8_t isStart; + + int32_t (*start)(struct SSyncIO *ths); + int32_t (*stop)(struct SSyncIO *ths); + int32_t (*ping)(struct SSyncIO *ths); + int32_t (*onMessage)(struct SSyncIO *ths, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); + int32_t (*destroy)(struct SSyncIO *ths); + +} SSyncIO; + +SSyncIO * syncIOCreate(); + +static int32_t syncIOStart(SSyncIO *io); +static int32_t syncIOStop(SSyncIO *io); +static int32_t syncIOPing(SSyncIO *io); +static int32_t syncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t syncIODestroy(SSyncIO *io); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index c1aba0518c..df8412ab23 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -23,16 +23,14 @@ extern "C" { #include #include #include -#include "taosdef.h" #include "sync.h" +#include "taosdef.h" #include "tlog.h" extern int32_t sDebugFlag; -#define sLog(...) \ - { \ - taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); \ - } +#define sLog(...) \ + { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); } #define sFatal(...) \ { \ diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 1286108664..65654564ab 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -17,101 +17,96 @@ #include "sync.h" void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { - -// TLA+ Spec -//AppendEntries(i, j) == -// /\ i /= j -// /\ state[i] = Leader -// /\ LET prevLogIndex == nextIndex[i][j] - 1 -// prevLogTerm == IF prevLogIndex > 0 THEN -// log[i][prevLogIndex].term -// ELSE -// 0 -// \* Send up to 1 entry, constrained by the end of the log. -// lastEntry == Min({Len(log[i]), nextIndex[i][j]}) -// entries == SubSeq(log[i], nextIndex[i][j], lastEntry) -// IN Send([mtype |-> AppendEntriesRequest, -// mterm |-> currentTerm[i], -// mprevLogIndex |-> prevLogIndex, -// mprevLogTerm |-> prevLogTerm, -// mentries |-> entries, -// \* mlog is used as a history variable for the proof. -// \* It would not exist in a real implementation. -// mlog |-> log[i], -// mcommitIndex |-> Min({commitIndex[i], lastEntry}), -// msource |-> i, -// mdest |-> j]) -// /\ UNCHANGED <> - + // TLA+ Spec + // AppendEntries(i, j) == + // /\ i /= j + // /\ state[i] = Leader + // /\ LET prevLogIndex == nextIndex[i][j] - 1 + // prevLogTerm == IF prevLogIndex > 0 THEN + // log[i][prevLogIndex].term + // ELSE + // 0 + // \* Send up to 1 entry, constrained by the end of the log. + // lastEntry == Min({Len(log[i]), nextIndex[i][j]}) + // entries == SubSeq(log[i], nextIndex[i][j], lastEntry) + // IN Send([mtype |-> AppendEntriesRequest, + // mterm |-> currentTerm[i], + // mprevLogIndex |-> prevLogIndex, + // mprevLogTerm |-> prevLogTerm, + // mentries |-> entries, + // \* mlog is used as a history variable for the proof. + // \* It would not exist in a real implementation. + // mlog |-> log[i], + // mcommitIndex |-> Min({commitIndex[i], lastEntry}), + // msource |-> i, + // mdest |-> j]) + // /\ UNCHANGED <> } void onAppendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { - -// TLA+ Spec -//HandleAppendEntriesRequest(i, j, m) == -// LET logOk == \/ m.mprevLogIndex = 0 -// \/ /\ m.mprevLogIndex > 0 -// /\ m.mprevLogIndex <= Len(log[i]) -// /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term -// IN /\ m.mterm <= currentTerm[i] -// /\ \/ /\ \* reject request -// \/ m.mterm < currentTerm[i] -// \/ /\ m.mterm = currentTerm[i] -// /\ state[i] = Follower -// /\ \lnot logOk -// /\ Reply([mtype |-> AppendEntriesResponse, -// mterm |-> currentTerm[i], -// msuccess |-> FALSE, -// mmatchIndex |-> 0, -// msource |-> i, -// mdest |-> j], -// m) -// /\ UNCHANGED <> -// \/ \* return to follower state -// /\ m.mterm = currentTerm[i] -// /\ state[i] = Candidate -// /\ state' = [state EXCEPT ![i] = Follower] -// /\ UNCHANGED <> -// \/ \* accept request -// /\ m.mterm = currentTerm[i] -// /\ state[i] = Follower -// /\ logOk -// /\ LET index == m.mprevLogIndex + 1 -// IN \/ \* already done with request -// /\ \/ m.mentries = << >> -// \/ /\ m.mentries /= << >> -// /\ Len(log[i]) >= index -// /\ log[i][index].term = m.mentries[1].term -// \* This could make our commitIndex decrease (for -// \* example if we process an old, duplicated request), -// \* but that doesn't really affect anything. -// /\ commitIndex' = [commitIndex EXCEPT ![i] = -// m.mcommitIndex] -// /\ Reply([mtype |-> AppendEntriesResponse, -// mterm |-> currentTerm[i], -// msuccess |-> TRUE, -// mmatchIndex |-> m.mprevLogIndex + -// Len(m.mentries), -// msource |-> i, -// mdest |-> j], -// m) -// /\ UNCHANGED <> -// \/ \* conflict: remove 1 entry -// /\ m.mentries /= << >> -// /\ Len(log[i]) >= index -// /\ log[i][index].term /= m.mentries[1].term -// /\ LET new == [index2 \in 1..(Len(log[i]) - 1) |-> -// log[i][index2]] -// IN log' = [log EXCEPT ![i] = new] -// /\ UNCHANGED <> -// \/ \* no conflict: append entry -// /\ m.mentries /= << >> -// /\ Len(log[i]) = m.mprevLogIndex -// /\ log' = [log EXCEPT ![i] = -// Append(log[i], m.mentries[1])] -// /\ UNCHANGED <> -// /\ UNCHANGED <> -// - - + // TLA+ Spec + // HandleAppendEntriesRequest(i, j, m) == + // LET logOk == \/ m.mprevLogIndex = 0 + // \/ /\ m.mprevLogIndex > 0 + // /\ m.mprevLogIndex <= Len(log[i]) + // /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term + // IN /\ m.mterm <= currentTerm[i] + // /\ \/ /\ \* reject request + // \/ m.mterm < currentTerm[i] + // \/ /\ m.mterm = currentTerm[i] + // /\ state[i] = Follower + // /\ \lnot logOk + // /\ Reply([mtype |-> AppendEntriesResponse, + // mterm |-> currentTerm[i], + // msuccess |-> FALSE, + // mmatchIndex |-> 0, + // msource |-> i, + // mdest |-> j], + // m) + // /\ UNCHANGED <> + // \/ \* return to follower state + // /\ m.mterm = currentTerm[i] + // /\ state[i] = Candidate + // /\ state' = [state EXCEPT ![i] = Follower] + // /\ UNCHANGED <> + // \/ \* accept request + // /\ m.mterm = currentTerm[i] + // /\ state[i] = Follower + // /\ logOk + // /\ LET index == m.mprevLogIndex + 1 + // IN \/ \* already done with request + // /\ \/ m.mentries = << >> + // \/ /\ m.mentries /= << >> + // /\ Len(log[i]) >= index + // /\ log[i][index].term = m.mentries[1].term + // \* This could make our commitIndex decrease (for + // \* example if we process an old, duplicated request), + // \* but that doesn't really affect anything. + // /\ commitIndex' = [commitIndex EXCEPT ![i] = + // m.mcommitIndex] + // /\ Reply([mtype |-> AppendEntriesResponse, + // mterm |-> currentTerm[i], + // msuccess |-> TRUE, + // mmatchIndex |-> m.mprevLogIndex + + // Len(m.mentries), + // msource |-> i, + // mdest |-> j], + // m) + // /\ UNCHANGED <> + // \/ \* conflict: remove 1 entry + // /\ m.mentries /= << >> + // /\ Len(log[i]) >= index + // /\ log[i][index].term /= m.mentries[1].term + // /\ LET new == [index2 \in 1..(Len(log[i]) - 1) |-> + // log[i][index2]] + // IN log' = [log EXCEPT ![i] = new] + // /\ UNCHANGED <> + // \/ \* no conflict: append entry + // /\ m.mentries /= << >> + // /\ Len(log[i]) = m.mprevLogIndex + // /\ log' = [log EXCEPT ![i] = + // Append(log[i], m.mentries[1])] + // /\ UNCHANGED <> + // /\ UNCHANGED <> + // } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 4a9055e172..20235ef720 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -17,18 +17,16 @@ #include "sync.h" void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg) { - -// TLA+ Spec -//HandleAppendEntriesResponse(i, j, m) == -// /\ m.mterm = currentTerm[i] -// /\ \/ /\ m.msuccess \* successful -// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] -// /\ matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] -// \/ /\ \lnot m.msuccess \* not successful -// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = -// Max({nextIndex[i][j] - 1, 1})] -// /\ UNCHANGED <> -// /\ Discard(m) -// /\ UNCHANGED <> - + // TLA+ Spec + // HandleAppendEntriesResponse(i, j, m) == + // /\ m.mterm = currentTerm[i] + // /\ \/ /\ m.msuccess \* successful + // /\ nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] + // /\ matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] + // \/ /\ \lnot m.msuccess \* not successful + // /\ nextIndex' = [nextIndex EXCEPT ![i][j] = + // Max({nextIndex[i][j] - 1, 1})] + // /\ UNCHANGED <> + // /\ Discard(m) + // /\ UNCHANGED <> } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 738fc4c5e1..b20367cf56 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -13,4 +13,191 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncIO.h" +#include "syncOnMessage.h" + +void *syncConsumer(void *param) { + SSyncIO *io = param; + + STaosQall *qall; + SRpcMsg *pRpcMsg, rpcMsg; + int type; + + qall = taosAllocateQall(); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(io->pQset, qall, NULL, NULL); + sDebug("%d sync-io msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + sDebug("sync-io recv msg: %s", (char *)(pRpcMsg->pCont)); + } + + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + rpcFreeCont(pRpcMsg->pCont); + + /* + int msgSize = 128; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + */ + + taosFreeQitem(pRpcMsg); + } + } + + taosFreeQall(qall); +} + +static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int ret = 0; + return ret; +} + +static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + /* +// SInfo *pInfo = (SInfo *)pMsg->ahandle; +sDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + pMsg->code); + +if (pEpSet) pInfo->epSet = *pEpSet; + +rpcFreeCont(pMsg->pCont); +// tsem_post(&pInfo->rspSem); +tsem_post(&pInfo->rspSem); +*/ +} + +static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SSyncIO *io = pParent; + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + sDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(io->pMsgQ, pTemp); +} + +SSyncIO *syncIOCreate() { + SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + memset(io, 0, sizeof(*io)); + + io->pMsgQ = taosOpenQueue(); + io->pQset = taosOpenQset(); + taosAddIntoQset(io->pQset, io->pMsgQ, NULL); + + io->start = syncIOStart; + io->stop = syncIOStop; + io->ping = syncIOPing; + io->onMessage = syncIOOnMessage; + io->destroy = syncIODestroy; + + return io; +} + +static int32_t syncIOStart(SSyncIO *io) { + taosBlockSIGPIPE(); + + // cient rpc init + { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "SYNC-IO-CLIENT"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processResponse; + rpcInit.sessions = 100; + rpcInit.idleTime = 100; + rpcInit.user = "sync-io"; + rpcInit.secret = "sync-io"; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.connType = TAOS_CONN_CLIENT; + + io->clientRpc = rpcOpen(&rpcInit); + if (io->clientRpc == NULL) { + sError("failed to initialize RPC"); + return -1; + } + } + + // server rpc init + { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 38000; + rpcInit.label = "SYNC-IO-SERVER"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processRequestMsg; + rpcInit.sessions = 1000; + rpcInit.idleTime = 2 * 1500; + rpcInit.afp = retrieveAuthInfo; + rpcInit.parent = io; + rpcInit.connType = TAOS_CONN_SERVER; + + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + sError("failed to start RPC server"); + return -1; + } + } + + // start consumer thread + { + if (pthread_create(&io->tid, NULL, syncConsumer, NULL) != 0) { + sError("failed to create sync consumer thread since %s", strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } + + return 0; +} + +static int32_t syncIOStop(SSyncIO *io) { + atomic_store_8(&io->isStart, 0); + pthread_join(io->tid, NULL); + return 0; +} + +static int32_t syncIOPing(SSyncIO *io) { return 0; } + +static int32_t syncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } + +static int32_t syncIODestroy(SSyncIO *io) { + int8_t start = atomic_load_8(&io->isStart); + assert(start == 0); + + if (io->serverRpc != NULL) { + free(io->serverRpc); + io->serverRpc = NULL; + } + + if (io->clientRpc != NULL) { + free(io->clientRpc); + io->clientRpc = NULL; + } + + if (io->pMsgQ != NULL) { + free(io->pMsgQ); + io->pMsgQ = NULL; + } + + if (io->pQset != NULL) { + free(io->pQset); + io->pQset = NULL; + } + + return 0; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index c31ec0f34d..88056c95ff 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -17,43 +17,39 @@ #include "sync.h" void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { - -// TLA+ Spec -//RequestVote(i, j) == -// /\ state[i] = Candidate -// /\ j \notin votesResponded[i] -// /\ Send([mtype |-> RequestVoteRequest, -// mterm |-> currentTerm[i], -// mlastLogTerm |-> LastTerm(log[i]), -// mlastLogIndex |-> Len(log[i]), -// msource |-> i, -// mdest |-> j]) -// /\ UNCHANGED <> - + // TLA+ Spec + // RequestVote(i, j) == + // /\ state[i] = Candidate + // /\ j \notin votesResponded[i] + // /\ Send([mtype |-> RequestVoteRequest, + // mterm |-> currentTerm[i], + // mlastLogTerm |-> LastTerm(log[i]), + // mlastLogIndex |-> Len(log[i]), + // msource |-> i, + // mdest |-> j]) + // /\ UNCHANGED <> } void onRequestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { - -// TLA+ Spec -//HandleRequestVoteRequest(i, j, m) == -// LET logOk == \/ m.mlastLogTerm > LastTerm(log[i]) -// \/ /\ m.mlastLogTerm = LastTerm(log[i]) -// /\ m.mlastLogIndex >= Len(log[i]) -// grant == /\ m.mterm = currentTerm[i] -// /\ logOk -// /\ votedFor[i] \in {Nil, j} -// IN /\ m.mterm <= currentTerm[i] -// /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] -// \/ ~grant /\ UNCHANGED votedFor -// /\ Reply([mtype |-> RequestVoteResponse, -// mterm |-> currentTerm[i], -// mvoteGranted |-> grant, -// \* mlog is used just for the `elections' history variable for -// \* the proof. It would not exist in a real implementation. -// mlog |-> log[i], -// msource |-> i, -// mdest |-> j], -// m) -// /\ UNCHANGED <> - + // TLA+ Spec + // HandleRequestVoteRequest(i, j, m) == + // LET logOk == \/ m.mlastLogTerm > LastTerm(log[i]) + // \/ /\ m.mlastLogTerm = LastTerm(log[i]) + // /\ m.mlastLogIndex >= Len(log[i]) + // grant == /\ m.mterm = currentTerm[i] + // /\ logOk + // /\ votedFor[i] \in {Nil, j} + // IN /\ m.mterm <= currentTerm[i] + // /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] + // \/ ~grant /\ UNCHANGED votedFor + // /\ Reply([mtype |-> RequestVoteResponse, + // mterm |-> currentTerm[i], + // mvoteGranted |-> grant, + // \* mlog is used just for the `elections' history variable for + // \* the proof. It would not exist in a real implementation. + // mlog |-> log[i], + // msource |-> i, + // mdest |-> j], + // m) + // /\ UNCHANGED <> } diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index ba9787f00c..4ca1b1343f 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -17,22 +17,20 @@ #include "sync.h" void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg) { - -// TLA+ Spec -//HandleRequestVoteResponse(i, j, m) == -// \* This tallies votes even when the current state is not Candidate, but -// \* they won't be looked at, so it doesn't matter. -// /\ m.mterm = currentTerm[i] -// /\ votesResponded' = [votesResponded EXCEPT ![i] = -// votesResponded[i] \cup {j}] -// /\ \/ /\ m.mvoteGranted -// /\ votesGranted' = [votesGranted EXCEPT ![i] = -// votesGranted[i] \cup {j}] -// /\ voterLog' = [voterLog EXCEPT ![i] = -// voterLog[i] @@ (j :> m.mlog)] -// \/ /\ ~m.mvoteGranted -// /\ UNCHANGED <> -// /\ Discard(m) -// /\ UNCHANGED <> - + // TLA+ Spec + // HandleRequestVoteResponse(i, j, m) == + // \* This tallies votes even when the current state is not Candidate, but + // \* they won't be looked at, so it doesn't matter. + // /\ m.mterm = currentTerm[i] + // /\ votesResponded' = [votesResponded EXCEPT ![i] = + // votesResponded[i] \cup {j}] + // /\ \/ /\ m.mvoteGranted + // /\ votesGranted' = [votesGranted EXCEPT ![i] = + // votesGranted[i] \cup {j}] + // /\ voterLog' = [voterLog EXCEPT ![i] = + // voterLog[i] @@ (j :> m.mlog)] + // \/ /\ ~m.mvoteGranted + // /\ UNCHANGED <> + // /\ Discard(m) + // /\ UNCHANGED <> } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index e069091ad8..cba196db27 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,26 +1,19 @@ #include +#include "syncIO.h" #include "syncInt.h" int main() { - printf("test \n"); + tsAsyncLog = 0; + taosInitLog((char*)"syncTest.log", 100000, 10); - syncStartEnv(); + sDebug("sync test"); + syncStartEnv(); - char temp[100]; - snprintf(temp, 100, "./debug.log"); - taosInitLog(temp, 10000, 1); - tsAsyncLog = 0; + SSyncIO *syncIO = syncIOCreate(); + assert(syncIO != NULL); - for (int i = 0; i < 100; i++) { - sDebug("log:%d -------- \n", i); - } - - fflush(NULL); - //taosCloseLog(); - - while(1) { - sleep(3); - } - return 0; + while (1) { + sleep(3); + } + return 0; } - From c2a01bfd91e53b659c580bcca833dd10d4b580c2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Feb 2022 16:14:02 +0800 Subject: [PATCH 04/19] add sync io --- source/libs/sync/src/syncIO.c | 29 +++++++++++++++++++++++++---- source/libs/sync/test/syncTest.cpp | 21 +++++++++++++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index b20367cf56..023836f74f 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -14,7 +14,10 @@ */ #include "syncIO.h" +#include #include "syncOnMessage.h" +#include "tglobal.h" +#include "tutil.h" void *syncConsumer(void *param) { SSyncIO *io = param; @@ -32,7 +35,7 @@ void *syncConsumer(void *param) { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - sDebug("sync-io recv msg: %s", (char *)(pRpcMsg->pCont)); + sDebug("sync-io recv type:%d msg:%s", pRpcMsg->msgType, (char *)(pRpcMsg->pCont)); } taosResetQitems(qall); @@ -109,6 +112,8 @@ SSyncIO *syncIOCreate() { static int32_t syncIOStart(SSyncIO *io) { taosBlockSIGPIPE(); + tsRpcForceTcp = 1; + // cient rpc init { SRpcInit rpcInit; @@ -122,7 +127,7 @@ static int32_t syncIOStart(SSyncIO *io) { rpcInit.user = "sync-io"; rpcInit.secret = "sync-io"; rpcInit.ckey = "key"; - rpcInit.spi = 1; + rpcInit.spi = 0; rpcInit.connType = TAOS_CONN_CLIENT; io->clientRpc = rpcOpen(&rpcInit); @@ -155,7 +160,7 @@ static int32_t syncIOStart(SSyncIO *io) { // start consumer thread { - if (pthread_create(&io->tid, NULL, syncConsumer, NULL) != 0) { + if (pthread_create(&io->tid, NULL, syncConsumer, io) != 0) { sError("failed to create sync consumer thread since %s", strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -171,7 +176,23 @@ static int32_t syncIOStop(SSyncIO *io) { return 0; } -static int32_t syncIOPing(SSyncIO *io) { return 0; } +static int32_t syncIOPing(SSyncIO *io) { + SRpcMsg rpcMsg, rspMsg; + + rpcMsg.pCont = rpcMallocCont(10); + snprintf(rpcMsg.pCont, 10, "ping"); + rpcMsg.contLen = 10; + rpcMsg.handle = io; + rpcMsg.msgType = 1; + + SEpSet epSet; + epSet.inUse = 0; + addEpIntoEpSet(&epSet, "127.0.0.1", 38000); + + rpcSendRequest(io->clientRpc, &epSet, &rpcMsg, NULL); + + return 0; +} static int32_t syncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index cba196db27..955beea693 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -2,9 +2,19 @@ #include "syncIO.h" #include "syncInt.h" +void *pingFunc(void *param) { + SSyncIO *io = (SSyncIO *)param; + while (1) { + sDebug("io->ping"); + io->ping(io); + sleep(1); + } + return NULL; +} + int main() { tsAsyncLog = 0; - taosInitLog((char*)"syncTest.log", 100000, 10); + taosInitLog((char *)"syncTest.log", 100000, 10); sDebug("sync test"); syncStartEnv(); @@ -12,8 +22,15 @@ int main() { SSyncIO *syncIO = syncIOCreate(); assert(syncIO != NULL); + syncIO->start(syncIO); + + sleep(2); + + pthread_t tid; + pthread_create(&tid, NULL, pingFunc, syncIO); + while (1) { - sleep(3); + sleep(1); } return 0; } From 7a47508c0e804985e2470a01e588b071db87601b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Feb 2022 16:58:47 +0800 Subject: [PATCH 05/19] add sync io --- source/libs/sync/inc/syncIO.h | 7 ++++- source/libs/sync/src/syncIO.c | 49 +++++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index bf3b3d34c1..8775326bdd 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -37,6 +37,11 @@ typedef struct SSyncIO { pthread_t tid; int8_t isStart; + SEpSet epSet; + + void *syncTimer; +void *syncTimerManager; + int32_t (*start)(struct SSyncIO *ths); int32_t (*stop)(struct SSyncIO *ths); int32_t (*ping)(struct SSyncIO *ths); @@ -45,7 +50,7 @@ typedef struct SSyncIO { } SSyncIO; -SSyncIO * syncIOCreate(); +SSyncIO *syncIOCreate(); static int32_t syncIOStart(SSyncIO *io); static int32_t syncIOStop(SSyncIO *io); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 023836f74f..20c0f8038c 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -17,8 +17,30 @@ #include #include "syncOnMessage.h" #include "tglobal.h" +#include "ttimer.h" #include "tutil.h" +static void syncTick(void *param, void *tmrId) { + SSyncIO *io = (SSyncIO *)param; + sDebug("syncTick ... "); + + SRpcMsg rpcMsg; + rpcMsg.pCont = rpcMallocCont(10); + snprintf(rpcMsg.pCont, 10, "TICK"); + rpcMsg.contLen = 10; + rpcMsg.handle = io; + rpcMsg.msgType = 2; + + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); + + taosWriteQitem(io->pMsgQ, pTemp); + + io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); +} + void *syncConsumer(void *param) { SSyncIO *io = param; @@ -58,6 +80,7 @@ void *syncConsumer(void *param) { } taosFreeQall(qall); + return NULL; } static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { @@ -68,17 +91,8 @@ static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encryp } static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - /* -// SInfo *pInfo = (SInfo *)pMsg->ahandle; -sDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - pMsg->code); - -if (pEpSet) pInfo->epSet = *pEpSet; - -rpcFreeCont(pMsg->pCont); -// tsem_post(&pInfo->rspSem); -tsem_post(&pInfo->rspSem); -*/ + sDebug("processResponse ... "); + rpcFreeCont(pMsg->pCont); } static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -158,6 +172,9 @@ static int32_t syncIOStart(SSyncIO *io) { } } + io->epSet.inUse = 0; + addEpIntoEpSet(&io->epSet, "127.0.0.1", 38000); + // start consumer thread { if (pthread_create(&io->tid, NULL, syncConsumer, io) != 0) { @@ -167,6 +184,10 @@ static int32_t syncIOStart(SSyncIO *io) { } } + // start tmr thread + io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); + io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); + return 0; } @@ -185,11 +206,7 @@ static int32_t syncIOPing(SSyncIO *io) { rpcMsg.handle = io; rpcMsg.msgType = 1; - SEpSet epSet; - epSet.inUse = 0; - addEpIntoEpSet(&epSet, "127.0.0.1", 38000); - - rpcSendRequest(io->clientRpc, &epSet, &rpcMsg, NULL); + rpcSendRequest(io->clientRpc, &io->epSet, &rpcMsg, NULL); return 0; } From 6cf9bb57f5128479d73b817df8dfb84054bf5b2b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Feb 2022 17:17:02 +0800 Subject: [PATCH 06/19] add timer --- source/libs/sync/src/syncIO.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 20c0f8038c..8dadd1e492 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -28,7 +28,7 @@ static void syncTick(void *param, void *tmrId) { rpcMsg.pCont = rpcMallocCont(10); snprintf(rpcMsg.pCont, 10, "TICK"); rpcMsg.contLen = 10; - rpcMsg.handle = io; + rpcMsg.handle = NULL; rpcMsg.msgType = 2; SRpcMsg *pTemp; @@ -65,15 +65,15 @@ void *syncConsumer(void *param) { taosGetQitem(qall, (void **)&pRpcMsg); rpcFreeCont(pRpcMsg->pCont); - /* - int msgSize = 128; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - rpcMsg.handle = pRpcMsg->handle; - rpcMsg.code = 0; - rpcSendResponse(&rpcMsg); - */ + if (pRpcMsg->handle != NULL) { + int msgSize = 128; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + } taosFreeQitem(pRpcMsg); } From 700287508aff45bdccc04040a10bbd106b75b7d9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Feb 2022 20:14:37 +0800 Subject: [PATCH 07/19] add sync store --- source/libs/sync/inc/syncRaftStore.h | 3 +++ source/libs/sync/src/syncIO.c | 2 +- source/libs/sync/src/syncRaftStore.c | 33 ++++++++++++++++++++++++++++ source/libs/sync/test/syncTest.cpp | 5 +++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 4cb852f34a..2953066731 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -27,6 +27,9 @@ extern "C" { #include "syncRaft.h" #include "taosdef.h" +void testJson(); +void testJson2(); + int32_t currentTerm(SyncTerm *pCurrentTerm); int32_t persistCurrentTerm(SyncTerm currentTerm); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 8dadd1e492..b657597e9d 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -203,7 +203,7 @@ static int32_t syncIOPing(SSyncIO *io) { rpcMsg.pCont = rpcMallocCont(10); snprintf(rpcMsg.pCont, 10, "ping"); rpcMsg.contLen = 10; - rpcMsg.handle = io; + rpcMsg.handle = NULL; rpcMsg.msgType = 1; rpcSendRequest(io->clientRpc, &io->epSet, &rpcMsg, NULL); diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index d45e53132c..3c624c5ff7 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -14,8 +14,41 @@ */ #include "syncRaftStore.h" +#include "cJSON.h" #include "sync.h" +char *serialized; + +void testJson() { + FileFd raftStoreFd = taosOpenFileReadWrite("raft.store"); + + uint64_t currentTerm = 100; + uint64_t voteFor = 200; + + cJSON *pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "current_term", currentTerm); + cJSON_AddNumberToObject(pRoot, "vote_for", voteFor); + + serialized = cJSON_Print(pRoot); + int len = strlen(serialized); + printf("serialized: %s \n", serialized); + + taosWriteFile(raftStoreFd, serialized, len); + taosCloseFile(raftStoreFd); +} + +void testJson2() { + cJSON *pRoot = cJSON_Parse(serialized); + + cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); + uint64_t currentTerm = pCurrentTerm->valueint; + + cJSON *pVoteFor = cJSON_GetObjectItem(pRoot, "vote_for"); + uint64_t voteFor = pVoteFor->valueint; + + printf("read json: currentTerm:%lu, voteFor:%lu \n", currentTerm, voteFor); +} + int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; } int32_t persistCurrentTerm(SyncTerm currentTerm) { return 0; } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 955beea693..b5c24f896f 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,6 +1,7 @@ #include #include "syncIO.h" #include "syncInt.h" +#include "syncRaftStore.h" void *pingFunc(void *param) { SSyncIO *io = (SSyncIO *)param; @@ -13,6 +14,10 @@ void *pingFunc(void *param) { } int main() { + + testJson(); + testJson2(); + tsAsyncLog = 0; taosInitLog((char *)"syncTest.log", 100000, 10); From 8b71c4f5af6909f1df857a38002ff1aad4956c76 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Feb 2022 10:42:24 +0800 Subject: [PATCH 08/19] add sync store test --- source/libs/sync/src/syncRaftStore.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 3c624c5ff7..741c68ee44 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -17,10 +17,10 @@ #include "cJSON.h" #include "sync.h" -char *serialized; +FileFd raftStoreFd; void testJson() { - FileFd raftStoreFd = taosOpenFileReadWrite("raft.store"); + raftStoreFd = taosOpenFileReadWrite("raft.store"); uint64_t currentTerm = 100; uint64_t voteFor = 200; @@ -29,16 +29,22 @@ void testJson() { cJSON_AddNumberToObject(pRoot, "current_term", currentTerm); cJSON_AddNumberToObject(pRoot, "vote_for", voteFor); - serialized = cJSON_Print(pRoot); - int len = strlen(serialized); + char *serialized = cJSON_Print(pRoot); + int len = strlen(serialized); printf("serialized: %s \n", serialized); taosWriteFile(raftStoreFd, serialized, len); - taosCloseFile(raftStoreFd); } void testJson2() { - cJSON *pRoot = cJSON_Parse(serialized); + taosLSeekFile(raftStoreFd, 0, SEEK_SET); + + char buf[128]; + memset(buf, 0, sizeof(buf)); + taosReadFile(raftStoreFd, buf, sizeof(buf)); + printf("read file: %s \n", buf); + + cJSON *pRoot = cJSON_Parse(buf); cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); uint64_t currentTerm = pCurrentTerm->valueint; @@ -47,6 +53,8 @@ void testJson2() { uint64_t voteFor = pVoteFor->valueint; printf("read json: currentTerm:%lu, voteFor:%lu \n", currentTerm, voteFor); + + taosCloseFile(raftStoreFd); } int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; } From c67a14ad8ff0b2ba0ccaa9052b7157ba4f6e3eaf Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Feb 2022 20:22:55 +0800 Subject: [PATCH 09/19] add raft store --- source/libs/sync/inc/syncIO.h | 4 +- source/libs/sync/inc/syncRaft.h | 2 +- source/libs/sync/inc/syncRaftStore.h | 30 +++++-- source/libs/sync/src/syncIO.c | 2 +- source/libs/sync/src/syncRaftStore.c | 128 ++++++++++++++++++++------- source/libs/sync/test/syncTest.cpp | 18 +++- 6 files changed, 140 insertions(+), 44 deletions(-) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 8775326bdd..4bfeadb15a 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -37,10 +37,10 @@ typedef struct SSyncIO { pthread_t tid; int8_t isStart; - SEpSet epSet; + SEpSet epSet; void *syncTimer; -void *syncTimerManager; + void *syncTimerManager; int32_t (*start)(struct SSyncIO *ths); int32_t (*stop)(struct SSyncIO *ths); diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h index 0c7e573572..f11ef50b06 100644 --- a/source/libs/sync/inc/syncRaft.h +++ b/source/libs/sync/inc/syncRaft.h @@ -28,7 +28,7 @@ extern "C" { #include "taosdef.h" typedef struct SRaftId { - SyncNodeId nodeId; + SyncNodeId addr; SyncGroupId vgId; } SRaftId; diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 2953066731..bdaeb81aee 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -23,20 +23,36 @@ extern "C" { #include #include #include -#include "sync.h" +#include "cJSON.h" +#include "syncInt.h" #include "syncRaft.h" #include "taosdef.h" -void testJson(); -void testJson2(); +#define RAFT_STORE_BLOCK_SIZE 512 +#define RAFT_STORE_PATH_LEN 128 -int32_t currentTerm(SyncTerm *pCurrentTerm); +typedef struct SRaftStore { + SyncTerm currentTerm; + SRaftId voteFor; + FileFd fd; + char path[RAFT_STORE_PATH_LEN]; +} SRaftStore; -int32_t persistCurrentTerm(SyncTerm currentTerm); +SRaftStore *raftStoreOpen(const char *path); -int32_t voteFor(SRaftId *pRaftId); +static int32_t raftStoreInit(SRaftStore *pRaftStore); -int32_t persistVoteFor(SRaftId *pRaftId); +int32_t raftStoreClose(SRaftStore *pRaftStore); + +int32_t raftStorePersist(SRaftStore *pRaftStore); + +static bool raftStoreFileExist(char *path); + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); + +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); + +void raftStorePrint(SRaftStore *pRaftStore); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index b657597e9d..736dd6632c 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -45,7 +45,7 @@ void *syncConsumer(void *param) { SSyncIO *io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; int type; qall = taosAllocateQall(); diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 741c68ee44..7d9c812f8a 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -17,50 +17,116 @@ #include "cJSON.h" #include "sync.h" -FileFd raftStoreFd; +SRaftStore *raftStoreOpen(const char *path) { + int32_t ret; -void testJson() { - raftStoreFd = taosOpenFileReadWrite("raft.store"); + SRaftStore *pRaftStore = malloc(sizeof(SRaftStore)); + if (pRaftStore == NULL) { + sError("raftStoreOpen malloc error"); + return NULL; + } + memset(pRaftStore, 0, sizeof(*pRaftStore)); + snprintf(pRaftStore->path, sizeof(pRaftStore->path), "%s", path); - uint64_t currentTerm = 100; - uint64_t voteFor = 200; + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + memset(storeBuf, 0, sizeof(storeBuf)); + if (!raftStoreFileExist(pRaftStore->path)) { + ret = raftStoreInit(pRaftStore); + assert(ret == 0); + } + + pRaftStore->fd = taosOpenFileReadWrite(pRaftStore->path); + if (pRaftStore->fd < 0) { + return NULL; + } + + int len = taosReadFile(pRaftStore->fd, storeBuf, sizeof(storeBuf)); + assert(len == RAFT_STORE_BLOCK_SIZE); + + ret = raftStoreDeserialize(pRaftStore, storeBuf, len); + assert(ret == 0); + + return pRaftStore; +} + +static int32_t raftStoreInit(SRaftStore *pRaftStore) { + pRaftStore->fd = taosOpenFileCreateWrite(pRaftStore->path); + if (pRaftStore->fd < 0) { + return -1; + } + + pRaftStore->currentTerm = 0; + pRaftStore->voteFor.addr = 0; + pRaftStore->voteFor.vgId = 0; + + int32_t ret = raftStorePersist(pRaftStore); + assert(ret == 0); + + taosCloseFile(pRaftStore->fd); + return 0; +} + +int32_t raftStoreClose(SRaftStore *pRaftStore) { + taosCloseFile(pRaftStore->fd); + free(pRaftStore); + return 0; +} + +int32_t raftStorePersist(SRaftStore *pRaftStore) { + int32_t ret; + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + + ret = raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); + assert(ret == 0); + + taosLSeekFile(pRaftStore->fd, 0, SEEK_SET); + + ret = taosWriteFile(pRaftStore->fd, storeBuf, sizeof(storeBuf)); + assert(ret == RAFT_STORE_BLOCK_SIZE); + + fsync(pRaftStore->fd); + return 0; +} + +static bool raftStoreFileExist(char *path) { return taosStatFile(path, NULL, NULL) >= 0; } + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { cJSON *pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "current_term", currentTerm); - cJSON_AddNumberToObject(pRoot, "vote_for", voteFor); + cJSON_AddNumberToObject(pRoot, "current_term", pRaftStore->currentTerm); + cJSON_AddNumberToObject(pRoot, "vote_for_addr", pRaftStore->voteFor.addr); + cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); char *serialized = cJSON_Print(pRoot); - int len = strlen(serialized); - printf("serialized: %s \n", serialized); + int len2 = strlen(serialized); + assert(len2 < len); + memset(buf, 0, len); + snprintf(buf, len, "%s", serialized); + free(serialized); - taosWriteFile(raftStoreFd, serialized, len); + cJSON_Delete(pRoot); + return 0; } -void testJson2() { - taosLSeekFile(raftStoreFd, 0, SEEK_SET); - - char buf[128]; - memset(buf, 0, sizeof(buf)); - taosReadFile(raftStoreFd, buf, sizeof(buf)); - printf("read file: %s \n", buf); - +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { + assert(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); cJSON *pRoot = cJSON_Parse(buf); - cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); - uint64_t currentTerm = pCurrentTerm->valueint; + cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); + pRaftStore->currentTerm = pCurrentTerm->valueint; - cJSON *pVoteFor = cJSON_GetObjectItem(pRoot, "vote_for"); - uint64_t voteFor = pVoteFor->valueint; + cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); + pRaftStore->voteFor.addr = pVoteForAddr->valueint; - printf("read json: currentTerm:%lu, voteFor:%lu \n", currentTerm, voteFor); + cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); + pRaftStore->voteFor.vgId = pVoteForVgid->valueint; - taosCloseFile(raftStoreFd); + cJSON_Delete(pRoot); + return 0; } -int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; } - -int32_t persistCurrentTerm(SyncTerm currentTerm) { return 0; } - -int32_t voteFor(SRaftId *pRaftId) { return 0; } - -int32_t persistVoteFor(SRaftId *pRaftId) { return 0; } \ No newline at end of file +void raftStorePrint(SRaftStore *pRaftStore) { + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); + printf("%s\n", storeBuf); +} diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index b5c24f896f..6930faa5b9 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -14,9 +14,23 @@ void *pingFunc(void *param) { } int main() { + tsAsyncLog = 0; + taosInitLog((char *)"syncTest.log", 100000, 10); + + SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); + assert(pRaftStore != NULL); + + raftStorePrint(pRaftStore); + + pRaftStore->currentTerm = 100; + pRaftStore->voteFor.addr = 200; + pRaftStore->voteFor.vgId = 300; + + raftStorePrint(pRaftStore); + + raftStorePersist(pRaftStore); + - testJson(); - testJson2(); tsAsyncLog = 0; taosInitLog((char *)"syncTest.log", 100000, 10); From 51adf7e25edef904748ae307da9cac64ae6558f5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Feb 2022 21:03:25 +0800 Subject: [PATCH 10/19] add raft store --- include/libs/sync/sync.h | 2 -- source/libs/sync/inc/syncEnv.h | 14 ++++++++++++++ source/libs/sync/src/syncEnv.c | 24 ++++++++++++++++++------ source/libs/sync/src/syncMain.c | 6 +++++- source/libs/sync/test/syncTest.cpp | 4 +--- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 3d875a4be8..0ec741ec3e 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -153,8 +153,6 @@ int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); ESyncState syncGetMyRole(int64_t rid); void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); -int32_t syncStartEnv(); - extern int32_t sDebugFlag; #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index a6ab03e1ee..f1c4327b69 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -23,9 +23,23 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "taosdef.h" #include "trpc.h" +typedef struct SSyncEnv { + void *pTimer; + void *pTimerManager; +} SSyncEnv; + +int32_t syncEnvStart(); + +int32_t syncEnvStop(); + +static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv); + +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index a11ed86dc0..b314e89b44 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -14,13 +14,25 @@ */ #include "syncEnv.h" +#include #include "sync.h" #include "syncInt.h" -int32_t syncStartEnv() { - sInfo("log: syncStartEnv \n"); +SSyncEnv *gSyncEnv = NULL; - if (rpcInit() != 0) { - return -1; - } -} \ No newline at end of file +int32_t syncEnvStart() { + int32_t ret; + gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + assert(gSyncEnv != NULL); + ret = doSyncEnvStart(gSyncEnv); + return ret; +} + +int32_t syncEnvStop() { + int32_t ret = doSyncEnvStop(gSyncEnv); + return ret; +} + +static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { return 0; } + +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fbb969eb1c..9b5a7a0a38 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -15,9 +15,13 @@ #include #include "sync.h" +#include "syncEnv.h" #include "syncInt.h" -int32_t syncInit() { return 0; } +int32_t syncInit() { + int32_t ret = syncEnvStart(); + return ret; +} void syncCleanUp() {} diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 6930faa5b9..f247be2711 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -30,13 +30,11 @@ int main() { raftStorePersist(pRaftStore); - - tsAsyncLog = 0; taosInitLog((char *)"syncTest.log", 100000, 10); sDebug("sync test"); - syncStartEnv(); + SSyncIO *syncIO = syncIOCreate(); assert(syncIO != NULL); From 113618bea394cd8d82c4fa844c507ccd03423898 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 25 Feb 2022 14:38:12 +0800 Subject: [PATCH 11/19] add sync test --- source/libs/sync/inc/syncInt.h | 9 +++-- source/libs/sync/src/syncMain.c | 7 +++- source/libs/sync/test/CMakeLists.txt | 21 +++++++++++ source/libs/sync/test/syncEnvTest.cpp | 52 +++++++++++++++++++++++++++ source/libs/sync/test/syncTest.cpp | 16 ++++++--- 5 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 source/libs/sync/test/syncEnvTest.cpp diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index df8412ab23..e03835fc54 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -29,9 +29,6 @@ extern "C" { extern int32_t sDebugFlag; -#define sLog(...) \ - { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); } - #define sFatal(...) \ { \ if (sDebugFlag & DEBUG_FATAL) { \ @@ -79,6 +76,12 @@ typedef struct SSyncNode { int64_t rid; } SSyncNode; +SSyncNode* syncNodeStart(const SSyncInfo* pSyncInfo); +void syncNodeStop(SSyncNode* pSyncNode); + +// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); +int32_t syncNodeForwardToPeer(SSyncNode* pSyncNode, const SSyncBuffer* pBuf, bool isWeak); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9b5a7a0a38..4606496141 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -35,4 +35,9 @@ int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { r ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} \ No newline at end of file +void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} + +SSyncNode* syncNodeStart(const SSyncInfo* pSyncInfo) { return NULL; } +void syncNodeStop(SSyncNode* pSyncNode) {} + +int32_t syncNodeForwardToPeer(SSyncNode* pSyncNode, const SSyncBuffer* pBuf, bool isWeak) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index f24a1a9a46..17405989f5 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -1,18 +1,39 @@ add_executable(syncTest "") +add_executable(syncEnvTest "") + + target_sources(syncTest PRIVATE "syncTest.cpp" ) +target_sources(syncEnvTest + PRIVATE + "syncEnvTest.cpp" +) + + target_include_directories(syncTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncEnvTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + target_link_libraries(syncTest sync gtest_main ) +target_link_libraries(syncEnvTest + sync + gtest_main +) + + enable_testing() add_test( NAME sync_test diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp new file mode 100644 index 0000000000..5b8ccabdcc --- /dev/null +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -0,0 +1,52 @@ +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void *pingFunc(void *param) { + SSyncIO *io = (SSyncIO *)param; + while (1) { + sDebug("io->ping"); + io->ping(io); + sleep(1); + } + return NULL; +} + +int main() { + tsAsyncLog = 0; + taosInitLog((char *)"syncTest.log", 100000, 10); + + SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); + assert(pRaftStore != NULL); + + raftStorePrint(pRaftStore); + + pRaftStore->currentTerm = 100; + pRaftStore->voteFor.addr = 200; + pRaftStore->voteFor.vgId = 300; + + raftStorePrint(pRaftStore); + + raftStorePersist(pRaftStore); + + tsAsyncLog = 0; + taosInitLog((char *)"syncTest.log", 100000, 10); + + sDebug("sync test"); + + SSyncIO *syncIO = syncIOCreate(); + assert(syncIO != NULL); + + syncIO->start(syncIO); + + sleep(2); + + pthread_t tid; + pthread_create(&tid, NULL, pingFunc, syncIO); + + while (1) { + sleep(1); + } + return 0; +} diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index f247be2711..8e85278960 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -14,8 +14,18 @@ void *pingFunc(void *param) { } int main() { - tsAsyncLog = 0; taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + sTrace("sync log test: trace"); + sDebug("sync log test: debug"); + sInfo("sync log test: info"); + sWarn("sync log test: warn"); + sError("sync log test: error"); + sFatal("sync log test: fatal"); + + SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); assert(pRaftStore != NULL); @@ -30,12 +40,8 @@ int main() { raftStorePersist(pRaftStore); - tsAsyncLog = 0; - taosInitLog((char *)"syncTest.log", 100000, 10); - sDebug("sync test"); - SSyncIO *syncIO = syncIOCreate(); assert(syncIO != NULL); From 296d9abe2a3a2517f840221d6fa0fc5179e8459c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 25 Feb 2022 15:34:24 +0800 Subject: [PATCH 12/19] add sync code --- include/libs/sync/sync.h | 4 -- source/libs/sync/inc/syncAppendEntries.h | 1 + source/libs/sync/inc/syncAppendEntriesReply.h | 1 + source/libs/sync/inc/syncElection.h | 1 + source/libs/sync/inc/syncIO.h | 13 +++-- source/libs/sync/inc/syncMessage.h | 2 +- source/libs/sync/inc/syncRaft.h | 7 ++- source/libs/sync/inc/syncRaftEntry.h | 2 +- source/libs/sync/inc/syncRaftLog.h | 2 +- source/libs/sync/inc/syncReplication.h | 1 + source/libs/sync/inc/syncRequestVote.h | 1 + source/libs/sync/inc/syncRequestVoteReply.h | 1 + source/libs/sync/inc/syncSnapshot.h | 2 +- source/libs/sync/inc/syncTimeout.h | 1 + source/libs/sync/inc/syncVoteMgr.h | 34 ++++++++++++ source/libs/sync/src/syncAppendEntries.c | 1 - source/libs/sync/src/syncAppendEntriesReply.c | 1 - source/libs/sync/src/syncElection.c | 2 +- source/libs/sync/src/syncEnv.c | 2 - source/libs/sync/src/syncIO.c | 24 +++++---- source/libs/sync/src/syncMain.c | 1 - source/libs/sync/src/syncMessage.c | 1 - source/libs/sync/src/syncOnMessage.c | 2 +- source/libs/sync/src/syncRaft.c | 1 - source/libs/sync/src/syncRaftEntry.c | 2 +- source/libs/sync/src/syncRaftLog.c | 1 - source/libs/sync/src/syncRaftStore.c | 1 - source/libs/sync/src/syncReplication.c | 2 +- source/libs/sync/src/syncRequestVote.c | 1 - source/libs/sync/src/syncRequestVoteReply.c | 1 - source/libs/sync/src/syncSnapshot.c | 1 - source/libs/sync/src/syncTimeout.c | 1 - source/libs/sync/src/syncVoteMgr.c | 16 ++++++ source/libs/sync/test/syncEnvTest.cpp | 54 ++++++++----------- source/libs/sync/test/syncTest.cpp | 2 - 35 files changed, 114 insertions(+), 76 deletions(-) create mode 100644 source/libs/sync/inc/syncVoteMgr.h create mode 100644 source/libs/sync/src/syncVoteMgr.c diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 0ec741ec3e..f5881837c6 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -131,10 +131,6 @@ typedef struct SStateMgr { typedef struct { SyncGroupId vgId; SSyncCfg syncCfg; - SSyncLogStore logStore; - SStateMgr stateManager; - SSyncFSM syncFsm; - } SSyncInfo; struct SSyncNode; diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index 9ca0de19c5..b7c1c051cc 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 8b5cbf1da5..22f8eb464f 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 34dfdb3d09..0c86e9a6d3 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -24,6 +24,7 @@ extern "C" { #include #include #include "taosdef.h" +#include "syncInt.h" #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 4bfeadb15a..54a3d2b8c1 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -50,13 +50,16 @@ typedef struct SSyncIO { } SSyncIO; +int32_t syncIOStart(); +int32_t syncIOStop(); + SSyncIO *syncIOCreate(); -static int32_t syncIOStart(SSyncIO *io); -static int32_t syncIOStop(SSyncIO *io); -static int32_t syncIOPing(SSyncIO *io); -static int32_t syncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t syncIODestroy(SSyncIO *io); +static int32_t doSyncIOStart(SSyncIO *io); +static int32_t doSyncIOStop(SSyncIO *io); +static int32_t doSyncIOPing(SSyncIO *io); +static int32_t doSyncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t doSyncIODestroy(SSyncIO *io); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index f410c8cf6e..41a19eb49a 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "syncRaftEntry.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h index f11ef50b06..3964b95f28 100644 --- a/source/libs/sync/inc/syncRaft.h +++ b/source/libs/sync/inc/syncRaft.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "syncMessage.h" #include "taosdef.h" @@ -34,6 +34,11 @@ typedef struct SRaftId { typedef struct SRaft { SRaftId id; + + SSyncLogStore *logStore; + SStateMgr *stateManager; + SSyncFSM *syncFsm; + } SRaft; int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 65f77f3759..516bef4d48 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "taosdef.h" typedef struct SSyncRaftEntry { diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index 8c4b5116ea..ee971062cf 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "taosdef.h" int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 40c5ff790b..bfe071853c 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -24,6 +24,7 @@ extern "C" { #include #include #include "taosdef.h" +#include "syncInt.h" #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index 3ff96bbe8f..c2eca55151 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index 033ac89bc2..38068dd0e2 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 3b6121578a..89fcb230fb 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index 8159d2566c..2f5517069e 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -26,6 +26,7 @@ extern "C" { #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" +#include "syncInt.h" void onTimeout(SRaft *pRaft, void *pMsg); diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h new file mode 100644 index 0000000000..93729a859c --- /dev/null +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_VOTG_MGR_H +#define _TD_LIBS_SYNC_VOTG_MGR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "taosdef.h" +#include "syncInt.h" + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_VOTG_MGR_H*/ diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 65654564ab..2b9c59ec92 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -14,7 +14,6 @@ */ #include "syncAppendEntries.h" -#include "sync.h" void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 20235ef720..05734237b9 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -14,7 +14,6 @@ */ #include "syncAppendEntriesReply.h" -#include "sync.h" void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 738fc4c5e1..329105e2a1 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncElection.h" diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index b314e89b44..e71cf55cb1 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -15,8 +15,6 @@ #include "syncEnv.h" #include -#include "sync.h" -#include "syncInt.h" SSyncEnv *gSyncEnv = NULL; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 736dd6632c..0e32d9ac50 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -20,6 +20,10 @@ #include "ttimer.h" #include "tutil.h" +int32_t syncIOStart() { return 0; } + +int32_t syncIOStop() { return 0; } + static void syncTick(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; sDebug("syncTick ... "); @@ -114,16 +118,16 @@ SSyncIO *syncIOCreate() { io->pQset = taosOpenQset(); taosAddIntoQset(io->pQset, io->pMsgQ, NULL); - io->start = syncIOStart; - io->stop = syncIOStop; - io->ping = syncIOPing; - io->onMessage = syncIOOnMessage; - io->destroy = syncIODestroy; + io->start = doSyncIOStart; + io->stop = doSyncIOStop; + io->ping = doSyncIOPing; + io->onMessage = doSyncIOOnMessage; + io->destroy = doSyncIODestroy; return io; } -static int32_t syncIOStart(SSyncIO *io) { +static int32_t doSyncIOStart(SSyncIO *io) { taosBlockSIGPIPE(); tsRpcForceTcp = 1; @@ -191,13 +195,13 @@ static int32_t syncIOStart(SSyncIO *io) { return 0; } -static int32_t syncIOStop(SSyncIO *io) { +static int32_t doSyncIOStop(SSyncIO *io) { atomic_store_8(&io->isStart, 0); pthread_join(io->tid, NULL); return 0; } -static int32_t syncIOPing(SSyncIO *io) { +static int32_t doSyncIOPing(SSyncIO *io) { SRpcMsg rpcMsg, rspMsg; rpcMsg.pCont = rpcMallocCont(10); @@ -211,9 +215,9 @@ static int32_t syncIOPing(SSyncIO *io) { return 0; } -static int32_t syncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } +static int32_t doSyncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } -static int32_t syncIODestroy(SSyncIO *io) { +static int32_t doSyncIODestroy(SSyncIO *io) { int8_t start = atomic_load_8(&io->isStart); assert(start == 0); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4606496141..0e7d83d39a 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -14,7 +14,6 @@ */ #include -#include "sync.h" #include "syncEnv.h" #include "syncInt.h" diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index dcfc940f76..8937303725 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -14,7 +14,6 @@ */ #include "syncMessage.h" -#include "sync.h" #include "syncRaft.h" void onMessage(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncOnMessage.c b/source/libs/sync/src/syncOnMessage.c index 738fc4c5e1..19a97ee156 100644 --- a/source/libs/sync/src/syncOnMessage.c +++ b/source/libs/sync/src/syncOnMessage.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncOnMessage.h" diff --git a/source/libs/sync/src/syncRaft.c b/source/libs/sync/src/syncRaft.c index 85c2c6fe27..f0a29917e0 100644 --- a/source/libs/sync/src/syncRaft.c +++ b/source/libs/sync/src/syncRaft.c @@ -14,7 +14,6 @@ */ #include "syncRaft.h" -#include "sync.h" int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 738fc4c5e1..e525d3c7c2 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncRaftEntry.h" diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 4a5fc201b0..37bb3ce48c 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -14,7 +14,6 @@ */ #include "syncRaftLog.h" -#include "sync.h" int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 7d9c812f8a..4391f5d25c 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -15,7 +15,6 @@ #include "syncRaftStore.h" #include "cJSON.h" -#include "sync.h" SRaftStore *raftStoreOpen(const char *path) { int32_t ret; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 738fc4c5e1..4cea7c150e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncReplication.h" diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 88056c95ff..7aee47b8e4 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -14,7 +14,6 @@ */ #include "syncRequestVote.h" -#include "sync.h" void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 4ca1b1343f..a9c88a7975 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -14,7 +14,6 @@ */ #include "syncRequestVoteReply.h" -#include "sync.h" void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 8a27f097d1..da194780ff 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,7 +14,6 @@ */ #include "syncSnapshot.h" -#include "sync.h" #include "syncRaft.h" int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 206dd70046..e27df55d07 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -14,6 +14,5 @@ */ #include "syncTimeout.h" -#include "sync.h" void onTimeout(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c new file mode 100644 index 0000000000..02cf4ac033 --- /dev/null +++ b/source/libs/sync/src/syncVoteMgr.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncVoteMgr.h" diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index 5b8ccabdcc..c682284b25 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -1,52 +1,40 @@ +#include "syncEnv.h" #include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" -void *pingFunc(void *param) { - SSyncIO *io = (SSyncIO *)param; - while (1) { - sDebug("io->ping"); - io->ping(io); - sleep(1); - } - return NULL; +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); } int main() { + taosInitLog((char*)"syncEnvTest.log", 100000, 10); tsAsyncLog = 0; - taosInitLog((char *)"syncTest.log", 100000, 10); + sDebugFlag = 143 + 64; - SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); - assert(pRaftStore != NULL); + logTest(); - raftStorePrint(pRaftStore); + int32_t ret = syncEnvStart(); + assert(ret == 0); - pRaftStore->currentTerm = 100; - pRaftStore->voteFor.addr = 200; - pRaftStore->voteFor.vgId = 300; + ret = syncIOStart(); + assert(ret == 0); - raftStorePrint(pRaftStore); + SSyncInfo syncInfo; + syncInfo.vgId = 1; - raftStorePersist(pRaftStore); - - tsAsyncLog = 0; - taosInitLog((char *)"syncTest.log", 100000, 10); - - sDebug("sync test"); - - SSyncIO *syncIO = syncIOCreate(); - assert(syncIO != NULL); - - syncIO->start(syncIO); - - sleep(2); - - pthread_t tid; - pthread_create(&tid, NULL, pingFunc, syncIO); + SSyncNode* pSyncNode = syncNodeStart(&syncInfo); + assert(pSyncNode != NULL); while (1) { - sleep(1); + taosMsleep(1000); } + return 0; } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 8e85278960..12b0905fa0 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -25,8 +25,6 @@ int main() { sError("sync log test: error"); sFatal("sync log test: fatal"); - - SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); assert(pRaftStore != NULL); From b63995287d7a2848ce92aaf109ab90b1c5e60119 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 25 Feb 2022 17:55:20 +0800 Subject: [PATCH 13/19] add raft store --- include/libs/sync/sync.h | 20 ++++----- source/libs/sync/inc/syncElection.h | 2 +- source/libs/sync/inc/syncRaft.h | 6 +-- source/libs/sync/inc/syncReplication.h | 2 +- source/libs/sync/inc/syncTimeout.h | 2 +- source/libs/sync/inc/syncVoteMgr.h | 5 +-- source/libs/sync/test/CMakeLists.txt | 14 +++++++ source/libs/sync/test/syncEnvTest.cpp | 30 ++++++++++---- source/libs/sync/test/syncPingTest.cpp | 57 ++++++++++++++++++++++++++ 9 files changed, 112 insertions(+), 26 deletions(-) create mode 100644 source/libs/sync/test/syncPingTest.cpp diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index f5881837c6..03ec7c0eac 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -34,23 +34,23 @@ typedef enum { TAOS_SYNC_STATE_LEADER = 2, } ESyncState; -typedef struct { +typedef struct SSyncBuffer { void* data; size_t len; } SSyncBuffer; -typedef struct { - SyncNodeId nodeId; - uint16_t nodePort; // node sync Port - char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN +typedef struct SNodeInfo { + uint16_t nodePort; // node sync Port + char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN } SNodeInfo; -typedef struct { +typedef struct SSyncCfg { int32_t replicaNum; + int32_t myIndex; SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; } SSyncCfg; -typedef struct { +typedef struct SNodesRole { int32_t replicaNum; SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; ESyncState role[TSDB_MAX_REPLICA]; @@ -128,9 +128,9 @@ typedef struct SStateMgr { } SStateMgr; -typedef struct { - SyncGroupId vgId; - SSyncCfg syncCfg; +typedef struct SSyncInfo { + SyncGroupId vgId; + SSyncCfg syncCfg; } SSyncInfo; struct SSyncNode; diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 0c86e9a6d3..7e9e637854 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -23,8 +23,8 @@ extern "C" { #include #include #include -#include "taosdef.h" #include "syncInt.h" +#include "taosdef.h" #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h index 3964b95f28..5852c0ec30 100644 --- a/source/libs/sync/inc/syncRaft.h +++ b/source/libs/sync/inc/syncRaft.h @@ -35,9 +35,9 @@ typedef struct SRaftId { typedef struct SRaft { SRaftId id; - SSyncLogStore *logStore; - SStateMgr *stateManager; - SSyncFSM *syncFsm; + SSyncLogStore* logStore; + SStateMgr* stateManager; + SSyncFSM* syncFsm; } SRaft; diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index bfe071853c..7f97ae9e49 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -23,8 +23,8 @@ extern "C" { #include #include #include -#include "taosdef.h" #include "syncInt.h" +#include "taosdef.h" #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index 2f5517069e..d9d6a17939 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -23,10 +23,10 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" -#include "syncInt.h" void onTimeout(SRaft *pRaft, void *pMsg); diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index 93729a859c..cfcf58bee2 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -20,12 +20,11 @@ extern "C" { #endif +#include #include #include -#include -#include "taosdef.h" #include "syncInt.h" - +#include "taosdef.h" #ifdef __cplusplus } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 17405989f5..e655ac01be 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -1,5 +1,6 @@ add_executable(syncTest "") add_executable(syncEnvTest "") +add_executable(syncPingTest "") target_sources(syncTest @@ -10,6 +11,10 @@ target_sources(syncEnvTest PRIVATE "syncEnvTest.cpp" ) +target_sources(syncPingTest + PRIVATE + "syncPingTest.cpp" +) target_include_directories(syncTest @@ -22,6 +27,11 @@ target_include_directories(syncEnvTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncPingTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -32,6 +42,10 @@ target_link_libraries(syncEnvTest sync gtest_main ) +target_link_libraries(syncPingTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index c682284b25..31dad593e6 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -13,6 +13,26 @@ void logTest() { sFatal("--- sync log test: fatal"); } +void doSync() { + SSyncInfo syncInfo; + syncInfo.vgId = 1; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = 3; + + pCfg->nodeInfo[0].nodePort = 7010; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + pCfg->nodeInfo[1].nodePort = 7110; + taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + + pCfg->nodeInfo[2].nodePort = 7210; + taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + + SSyncNode* pSyncNode = syncNodeStart(&syncInfo); + assert(pSyncNode != NULL); +} + int main() { taosInitLog((char*)"syncEnvTest.log", 100000, 10); tsAsyncLog = 0; @@ -20,17 +40,13 @@ int main() { logTest(); - int32_t ret = syncEnvStart(); + int32_t ret = syncIOStart(); assert(ret == 0); - ret = syncIOStart(); + ret = syncEnvStart(); assert(ret == 0); - SSyncInfo syncInfo; - syncInfo.vgId = 1; - - SSyncNode* pSyncNode = syncNodeStart(&syncInfo); - assert(pSyncNode != NULL); + doSync(); while (1) { taosMsleep(1000); diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp new file mode 100644 index 0000000000..5ed72fd56b --- /dev/null +++ b/source/libs/sync/test/syncPingTest.cpp @@ -0,0 +1,57 @@ +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void doSync() { + SSyncInfo syncInfo; + syncInfo.vgId = 1; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = 0; + pCfg->replicaNum = 3; + + pCfg->nodeInfo[0].nodePort = 7010; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + pCfg->nodeInfo[1].nodePort = 7110; + taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + + pCfg->nodeInfo[2].nodePort = 7210; + taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + + SSyncNode* pSyncNode = syncNodeStart(&syncInfo); + assert(pSyncNode != NULL); +} + +int main() { + taosInitLog((char*)"syncPingTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret = syncIOStart(); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + doSync(); + + while (1) { + taosMsleep(1000); + } + + return 0; +} From e8780cebed850bddde87773aac033283293dfdf5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 26 Feb 2022 18:30:58 +0800 Subject: [PATCH 14/19] add sync code --- source/libs/sync/CMakeLists.txt | 6 +- source/libs/sync/inc/syncAppendEntries.h | 1 - source/libs/sync/inc/syncAppendEntriesReply.h | 1 - source/libs/sync/inc/syncElection.h | 1 - source/libs/sync/inc/syncEnv.h | 47 ---- source/libs/sync/inc/syncIO.h | 68 ----- source/libs/sync/inc/syncInt.h | 10 - source/libs/sync/inc/syncMessage.h | 20 +- source/libs/sync/inc/syncRaft.h | 45 +++- source/libs/sync/inc/syncRaftEntry.h | 8 +- source/libs/sync/inc/syncRaftLog.h | 2 +- source/libs/sync/inc/syncRaftStore.h | 29 +-- source/libs/sync/inc/syncReplication.h | 1 - source/libs/sync/inc/syncRequestVote.h | 1 - source/libs/sync/inc/syncRequestVoteReply.h | 1 - source/libs/sync/inc/syncSnapshot.h | 2 +- source/libs/sync/inc/syncTimeout.h | 1 - source/libs/sync/inc/syncVoteMgr.h | 33 --- source/libs/sync/src/syncAppendEntries.c | 1 + source/libs/sync/src/syncAppendEntriesReply.c | 1 + source/libs/sync/src/syncElection.c | 2 +- source/libs/sync/src/syncEnv.c | 36 --- source/libs/sync/src/syncIO.c | 245 ------------------ source/libs/sync/src/syncMain.c | 14 +- source/libs/sync/src/syncMessage.c | 1 + source/libs/sync/src/syncOnMessage.c | 2 +- source/libs/sync/src/syncRaft.c | 41 +++ source/libs/sync/src/syncRaftEntry.c | 2 +- source/libs/sync/src/syncRaftLog.c | 1 + source/libs/sync/src/syncRaftStore.c | 116 +-------- source/libs/sync/src/syncReplication.c | 2 +- source/libs/sync/src/syncRequestVote.c | 1 + source/libs/sync/src/syncRequestVoteReply.c | 1 + source/libs/sync/src/syncSnapshot.c | 1 + source/libs/sync/src/syncTimeout.c | 1 + source/libs/sync/src/syncVoteMgr.c | 16 -- source/libs/sync/test/CMakeLists.txt | 55 ---- source/libs/sync/test/syncEnvTest.cpp | 56 ---- source/libs/sync/test/syncPingTest.cpp | 57 ---- source/libs/sync/test/syncTest.cpp | 56 +--- 40 files changed, 127 insertions(+), 858 deletions(-) delete mode 100644 source/libs/sync/inc/syncEnv.h delete mode 100644 source/libs/sync/inc/syncIO.h delete mode 100644 source/libs/sync/inc/syncVoteMgr.h delete mode 100644 source/libs/sync/src/syncEnv.c delete mode 100644 source/libs/sync/src/syncIO.c delete mode 100644 source/libs/sync/src/syncVoteMgr.c delete mode 100644 source/libs/sync/test/CMakeLists.txt delete mode 100644 source/libs/sync/test/syncEnvTest.cpp delete mode 100644 source/libs/sync/test/syncPingTest.cpp diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index cb38d7e363..37ee5194c8 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -13,8 +13,4 @@ target_include_directories( sync PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) +) \ No newline at end of file diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index b7c1c051cc..9ca0de19c5 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -23,7 +23,6 @@ extern "C" { #include #include #include -#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 22f8eb464f..8b5cbf1da5 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -23,7 +23,6 @@ extern "C" { #include #include #include -#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 7e9e637854..34dfdb3d09 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -23,7 +23,6 @@ extern "C" { #include #include #include -#include "syncInt.h" #include "taosdef.h" #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h deleted file mode 100644 index f1c4327b69..0000000000 --- a/source/libs/sync/inc/syncEnv.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_ENV_H -#define _TD_LIBS_SYNC_ENV_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include "syncInt.h" -#include "taosdef.h" -#include "trpc.h" - -typedef struct SSyncEnv { - void *pTimer; - void *pTimerManager; -} SSyncEnv; - -int32_t syncEnvStart(); - -int32_t syncEnvStop(); - -static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv); - -static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_LIBS_SYNC_ENV_H*/ diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h deleted file mode 100644 index 54a3d2b8c1..0000000000 --- a/source/libs/sync/inc/syncIO.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_IO_H -#define _TD_LIBS_IO_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include "os.h" -#include "syncInt.h" -#include "taosdef.h" -#include "tqueue.h" -#include "trpc.h" - -typedef struct SSyncIO { - void * serverRpc; - void * clientRpc; - STaosQueue *pMsgQ; - STaosQset * pQset; - pthread_t tid; - int8_t isStart; - - SEpSet epSet; - - void *syncTimer; - void *syncTimerManager; - - int32_t (*start)(struct SSyncIO *ths); - int32_t (*stop)(struct SSyncIO *ths); - int32_t (*ping)(struct SSyncIO *ths); - int32_t (*onMessage)(struct SSyncIO *ths, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); - int32_t (*destroy)(struct SSyncIO *ths); - -} SSyncIO; - -int32_t syncIOStart(); -int32_t syncIOStop(); - -SSyncIO *syncIOCreate(); - -static int32_t doSyncIOStart(SSyncIO *io); -static int32_t doSyncIOStop(SSyncIO *io); -static int32_t doSyncIOPing(SSyncIO *io); -static int32_t doSyncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t doSyncIODestroy(SSyncIO *io); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_LIBS_IO_H*/ diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index e03835fc54..551ce83122 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -23,11 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" #include "taosdef.h" -#include "tlog.h" - -extern int32_t sDebugFlag; #define sFatal(...) \ { \ @@ -76,12 +72,6 @@ typedef struct SSyncNode { int64_t rid; } SSyncNode; -SSyncNode* syncNodeStart(const SSyncInfo* pSyncInfo); -void syncNodeStop(SSyncNode* pSyncNode); - -// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); -int32_t syncNodeForwardToPeer(SSyncNode* pSyncNode, const SSyncBuffer* pBuf, bool isWeak); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 41a19eb49a..dc74526c73 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "syncInt.h" +#include "sync.h" #include "syncRaftEntry.h" #include "taosdef.h" @@ -41,26 +41,28 @@ typedef enum ESyncMessageType { typedef struct SyncPing { ESyncMessageType msgType; const SSyncBuffer *pData; -} SyncPing; +} SyncPing, RaftPing; + typedef struct SyncPingReply { ESyncMessageType msgType; const SSyncBuffer *pData; -} SyncPingReply; +} SyncPingReply, RaftPingReply; typedef struct SyncClientRequest { ESyncMessageType msgType; const SSyncBuffer *pData; int64_t seqNum; bool isWeak; -} SyncClientRequest; +} SyncClientRequest, RaftClientRequest; typedef struct SyncClientRequestReply { ESyncMessageType msgType; int32_t errCode; const SSyncBuffer *pErrMsg; const SSyncBuffer *pLeaderHint; -} SyncClientRequestReply; +} SyncClientRequestReply, RaftClientRequestReply; + typedef struct SyncRequestVote { ESyncMessageType msgType; @@ -69,7 +71,7 @@ typedef struct SyncRequestVote { SyncGroupId vgId; SyncIndex lastLogIndex; SyncTerm lastLogTerm; -} SyncRequestVote; +} SyncRequestVote, RaftRequestVote; typedef struct SyncRequestVoteReply { ESyncMessageType msgType; @@ -77,7 +79,7 @@ typedef struct SyncRequestVoteReply { SyncNodeId nodeId; SyncGroupId vgId; bool voteGranted; -} SyncRequestVoteReply; +} SyncRequestVoteReply, RaftRequestVoteReply; typedef struct SyncAppendEntries { ESyncMessageType msgType; @@ -88,7 +90,7 @@ typedef struct SyncAppendEntries { int32_t entryCount; SSyncRaftEntry * logEntries; SyncIndex commitIndex; -} SyncAppendEntries; +} SyncAppendEntries, RaftAppendEntries; typedef struct SyncAppendEntriesReply { ESyncMessageType msgType; @@ -96,7 +98,7 @@ typedef struct SyncAppendEntriesReply { SyncNodeId nodeId; bool success; SyncIndex matchIndex; -} SyncAppendEntriesReply; +} SyncAppendEntriesReply, RaftAppendEntriesReply; #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h index 5852c0ec30..6a3140d930 100644 --- a/source/libs/sync/inc/syncRaft.h +++ b/source/libs/sync/inc/syncRaft.h @@ -23,24 +23,59 @@ extern "C" { #include #include #include -#include "syncInt.h" +#include "sync.h" #include "syncMessage.h" #include "taosdef.h" typedef struct SRaftId { - SyncNodeId addr; + SyncNodeId nodeId; SyncGroupId vgId; } SRaftId; typedef struct SRaft { SRaftId id; + void* data; - SSyncLogStore* logStore; - SStateMgr* stateManager; - SSyncFSM* syncFsm; + int32_t (*FpPing)(struct SRaft* ths, const RaftPing* pMsg); + + int32_t (*FpOnPing)(struct SRaft* ths, RaftPing* pMsg); + + int32_t (*FpOnPingReply)(struct SRaft* ths, RaftPingReply* pMsg); + + int32_t (*FpRequestVote)(struct SRaft* ths, const RaftRequestVote* pMsg); + + int32_t (*FpOnRequestVote)(struct SRaft* ths, RaftRequestVote* pMsg); + + int32_t (*FpOnRequestVoteReply)(struct SRaft* ths, RaftRequestVoteReply* pMsg); + + int32_t (*FpAppendEntries)(struct SRaft* ths, const RaftAppendEntries* pMsg); + + int32_t (*FpOnAppendEntries)(struct SRaft* ths, RaftAppendEntries* pMsg); + + int32_t (*FpOnAppendEntriesReply)(struct SRaft* ths, RaftAppendEntriesReply* pMsg); } SRaft; +SRaft* raftCreate(SRaftId raftId, void* data); + +static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg); + +static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg); + +static int32_t onRaftPingReply(struct SRaft* ths, RaftPingReply* pMsg); + +static int32_t doRaftRequestVote(struct SRaft* ths, const RaftRequestVote* pMsg); + +static int32_t onRaftRequestVote(struct SRaft* ths, RaftRequestVote* pMsg); + +static int32_t onRaftRequestVoteReply(struct SRaft* ths, RaftRequestVoteReply* pMsg); + +static int32_t doRaftAppendEntries(struct SRaft* ths, const RaftAppendEntries* pMsg); + +static int32_t onRaftAppendEntries(struct SRaft* ths, RaftAppendEntries* pMsg); + +static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesReply* pMsg); + int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak); static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 516bef4d48..adc82f2c5d 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_LIBS_SYNC_RAFT_ENTRY_H -#define _TD_LIBS_SYNC_RAFT_ENTRY_H +#ifndef _TD_LIBS_TPL_H +#define _TD_LIBS_TPL_H #ifdef __cplusplus extern "C" { @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "syncInt.h" +#include "sync.h" #include "taosdef.h" typedef struct SSyncRaftEntry { @@ -37,4 +37,4 @@ typedef struct SSyncRaftEntry { } #endif -#endif /*_TD_LIBS_SYNC_RAFT_ENTRY_H*/ +#endif /*_TD_LIBS_TPL_H*/ diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index ee971062cf..8c4b5116ea 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "syncInt.h" +#include "sync.h" #include "taosdef.h" int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index bdaeb81aee..4cb852f34a 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -23,36 +23,17 @@ extern "C" { #include #include #include -#include "cJSON.h" -#include "syncInt.h" +#include "sync.h" #include "syncRaft.h" #include "taosdef.h" -#define RAFT_STORE_BLOCK_SIZE 512 -#define RAFT_STORE_PATH_LEN 128 +int32_t currentTerm(SyncTerm *pCurrentTerm); -typedef struct SRaftStore { - SyncTerm currentTerm; - SRaftId voteFor; - FileFd fd; - char path[RAFT_STORE_PATH_LEN]; -} SRaftStore; +int32_t persistCurrentTerm(SyncTerm currentTerm); -SRaftStore *raftStoreOpen(const char *path); +int32_t voteFor(SRaftId *pRaftId); -static int32_t raftStoreInit(SRaftStore *pRaftStore); - -int32_t raftStoreClose(SRaftStore *pRaftStore); - -int32_t raftStorePersist(SRaftStore *pRaftStore); - -static bool raftStoreFileExist(char *path); - -int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); - -int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); - -void raftStorePrint(SRaftStore *pRaftStore); +int32_t persistVoteFor(SRaftId *pRaftId); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 7f97ae9e49..40c5ff790b 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -23,7 +23,6 @@ extern "C" { #include #include #include -#include "syncInt.h" #include "taosdef.h" #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index c2eca55151..3ff96bbe8f 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -23,7 +23,6 @@ extern "C" { #include #include #include -#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index 38068dd0e2..033ac89bc2 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -23,7 +23,6 @@ extern "C" { #include #include #include -#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 89fcb230fb..3b6121578a 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "syncInt.h" +#include "sync.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index d9d6a17939..8159d2566c 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -23,7 +23,6 @@ extern "C" { #include #include #include -#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h deleted file mode 100644 index cfcf58bee2..0000000000 --- a/source/libs/sync/inc/syncVoteMgr.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_VOTG_MGR_H -#define _TD_LIBS_SYNC_VOTG_MGR_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include "syncInt.h" -#include "taosdef.h" - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_LIBS_SYNC_VOTG_MGR_H*/ diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 2b9c59ec92..65654564ab 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -14,6 +14,7 @@ */ #include "syncAppendEntries.h" +#include "sync.h" void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 05734237b9..20235ef720 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -14,6 +14,7 @@ */ #include "syncAppendEntriesReply.h" +#include "sync.h" void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 329105e2a1..738fc4c5e1 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "syncElection.h" +#include "sync.h" diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c deleted file mode 100644 index e71cf55cb1..0000000000 --- a/source/libs/sync/src/syncEnv.c +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncEnv.h" -#include - -SSyncEnv *gSyncEnv = NULL; - -int32_t syncEnvStart() { - int32_t ret; - gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); - assert(gSyncEnv != NULL); - ret = doSyncEnvStart(gSyncEnv); - return ret; -} - -int32_t syncEnvStop() { - int32_t ret = doSyncEnvStop(gSyncEnv); - return ret; -} - -static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { return 0; } - -static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { return 0; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c deleted file mode 100644 index 0e32d9ac50..0000000000 --- a/source/libs/sync/src/syncIO.c +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncIO.h" -#include -#include "syncOnMessage.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tutil.h" - -int32_t syncIOStart() { return 0; } - -int32_t syncIOStop() { return 0; } - -static void syncTick(void *param, void *tmrId) { - SSyncIO *io = (SSyncIO *)param; - sDebug("syncTick ... "); - - SRpcMsg rpcMsg; - rpcMsg.pCont = rpcMallocCont(10); - snprintf(rpcMsg.pCont, 10, "TICK"); - rpcMsg.contLen = 10; - rpcMsg.handle = NULL; - rpcMsg.msgType = 2; - - SRpcMsg *pTemp; - - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); - memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); - - taosWriteQitem(io->pMsgQ, pTemp); - - io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); -} - -void *syncConsumer(void *param) { - SSyncIO *io = param; - - STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; - int type; - - qall = taosAllocateQall(); - - while (1) { - int numOfMsgs = taosReadAllQitemsFromQset(io->pQset, qall, NULL, NULL); - sDebug("%d sync-io msgs are received", numOfMsgs); - if (numOfMsgs <= 0) break; - - for (int i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - sDebug("sync-io recv type:%d msg:%s", pRpcMsg->msgType, (char *)(pRpcMsg->pCont)); - } - - taosResetQitems(qall); - for (int i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - rpcFreeCont(pRpcMsg->pCont); - - if (pRpcMsg->handle != NULL) { - int msgSize = 128; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - rpcMsg.handle = pRpcMsg->handle; - rpcMsg.code = 0; - rpcSendResponse(&rpcMsg); - } - - taosFreeQitem(pRpcMsg); - } - } - - taosFreeQall(qall); - return NULL; -} - -static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { - // app shall retrieve the auth info based on meterID from DB or a data file - // demo code here only for simple demo - int ret = 0; - return ret; -} - -static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - sDebug("processResponse ... "); - rpcFreeCont(pMsg->pCont); -} - -static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - SSyncIO *io = pParent; - SRpcMsg *pTemp; - - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); - memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - - sDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); - taosWriteQitem(io->pMsgQ, pTemp); -} - -SSyncIO *syncIOCreate() { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); - memset(io, 0, sizeof(*io)); - - io->pMsgQ = taosOpenQueue(); - io->pQset = taosOpenQset(); - taosAddIntoQset(io->pQset, io->pMsgQ, NULL); - - io->start = doSyncIOStart; - io->stop = doSyncIOStop; - io->ping = doSyncIOPing; - io->onMessage = doSyncIOOnMessage; - io->destroy = doSyncIODestroy; - - return io; -} - -static int32_t doSyncIOStart(SSyncIO *io) { - taosBlockSIGPIPE(); - - tsRpcForceTcp = 1; - - // cient rpc init - { - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "SYNC-IO-CLIENT"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processResponse; - rpcInit.sessions = 100; - rpcInit.idleTime = 100; - rpcInit.user = "sync-io"; - rpcInit.secret = "sync-io"; - rpcInit.ckey = "key"; - rpcInit.spi = 0; - rpcInit.connType = TAOS_CONN_CLIENT; - - io->clientRpc = rpcOpen(&rpcInit); - if (io->clientRpc == NULL) { - sError("failed to initialize RPC"); - return -1; - } - } - - // server rpc init - { - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 38000; - rpcInit.label = "SYNC-IO-SERVER"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processRequestMsg; - rpcInit.sessions = 1000; - rpcInit.idleTime = 2 * 1500; - rpcInit.afp = retrieveAuthInfo; - rpcInit.parent = io; - rpcInit.connType = TAOS_CONN_SERVER; - - void *pRpc = rpcOpen(&rpcInit); - if (pRpc == NULL) { - sError("failed to start RPC server"); - return -1; - } - } - - io->epSet.inUse = 0; - addEpIntoEpSet(&io->epSet, "127.0.0.1", 38000); - - // start consumer thread - { - if (pthread_create(&io->tid, NULL, syncConsumer, io) != 0) { - sError("failed to create sync consumer thread since %s", strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } - - // start tmr thread - io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); - io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); - - return 0; -} - -static int32_t doSyncIOStop(SSyncIO *io) { - atomic_store_8(&io->isStart, 0); - pthread_join(io->tid, NULL); - return 0; -} - -static int32_t doSyncIOPing(SSyncIO *io) { - SRpcMsg rpcMsg, rspMsg; - - rpcMsg.pCont = rpcMallocCont(10); - snprintf(rpcMsg.pCont, 10, "ping"); - rpcMsg.contLen = 10; - rpcMsg.handle = NULL; - rpcMsg.msgType = 1; - - rpcSendRequest(io->clientRpc, &io->epSet, &rpcMsg, NULL); - - return 0; -} - -static int32_t doSyncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } - -static int32_t doSyncIODestroy(SSyncIO *io) { - int8_t start = atomic_load_8(&io->isStart); - assert(start == 0); - - if (io->serverRpc != NULL) { - free(io->serverRpc); - io->serverRpc = NULL; - } - - if (io->clientRpc != NULL) { - free(io->clientRpc); - io->clientRpc = NULL; - } - - if (io->pMsgQ != NULL) { - free(io->pMsgQ); - io->pMsgQ = NULL; - } - - if (io->pQset != NULL) { - free(io->pQset); - io->pQset = NULL; - } - - return 0; -} \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0e7d83d39a..fbb969eb1c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -14,13 +14,10 @@ */ #include -#include "syncEnv.h" +#include "sync.h" #include "syncInt.h" -int32_t syncInit() { - int32_t ret = syncEnvStart(); - return ret; -} +int32_t syncInit() { return 0; } void syncCleanUp() {} @@ -34,9 +31,4 @@ int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { r ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} - -SSyncNode* syncNodeStart(const SSyncInfo* pSyncInfo) { return NULL; } -void syncNodeStop(SSyncNode* pSyncNode) {} - -int32_t syncNodeForwardToPeer(SSyncNode* pSyncNode, const SSyncBuffer* pBuf, bool isWeak) { return 0; } \ No newline at end of file +void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 8937303725..dcfc940f76 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -14,6 +14,7 @@ */ #include "syncMessage.h" +#include "sync.h" #include "syncRaft.h" void onMessage(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncOnMessage.c b/source/libs/sync/src/syncOnMessage.c index 19a97ee156..738fc4c5e1 100644 --- a/source/libs/sync/src/syncOnMessage.c +++ b/source/libs/sync/src/syncOnMessage.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "syncOnMessage.h" +#include "sync.h" diff --git a/source/libs/sync/src/syncRaft.c b/source/libs/sync/src/syncRaft.c index f0a29917e0..1a01aefb0b 100644 --- a/source/libs/sync/src/syncRaft.c +++ b/source/libs/sync/src/syncRaft.c @@ -14,6 +14,47 @@ */ #include "syncRaft.h" +#include "sync.h" + +SRaft* raftCreate(SRaftId raftId, void* data) { + SRaft* pRaft = (SRaft*)malloc(sizeof(SRaft)); + assert(pRaft != NULL); + + pRaft->id = raftId; + pRaft->data = data; + + pRaft->FpPing = doRaftPing; + pRaft->FpOnPing = onRaftPing; + pRaft->FpOnPingReply = onRaftPingReply; + + pRaft->FpRequestVote = doRaftRequestVote; + pRaft->FpOnRequestVote = onRaftRequestVote; + pRaft->FpOnRequestVoteReply = onRaftRequestVoteReply; + + pRaft->FpAppendEntries = doRaftAppendEntries; + pRaft->FpOnAppendEntries = onRaftAppendEntries; + pRaft->FpOnAppendEntriesReply = onRaftAppendEntriesReply; + + return pRaft; +} + +static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg) { return 0; } + +static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg) { return 0; } + +static int32_t onRaftPingReply(struct SRaft* ths, RaftPingReply* pMsg) { return 0; } + +static int32_t doRaftRequestVote(struct SRaft* ths, const RaftRequestVote* pMsg) { return 0; } + +static int32_t onRaftRequestVote(struct SRaft* ths, RaftRequestVote* pMsg) { return 0; } + +static int32_t onRaftRequestVoteReply(struct SRaft* ths, RaftRequestVoteReply* pMsg) { return 0; } + +static int32_t doRaftAppendEntries(struct SRaft* ths, const RaftAppendEntries* pMsg) { return 0; } + +static int32_t onRaftAppendEntries(struct SRaft* ths, RaftAppendEntries* pMsg) { return 0; } + +static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesReply* pMsg) { return 0; } int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index e525d3c7c2..738fc4c5e1 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "syncRaftEntry.h" +#include "sync.h" diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 37bb3ce48c..4a5fc201b0 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -14,6 +14,7 @@ */ #include "syncRaftLog.h" +#include "sync.h" int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 4391f5d25c..d45e53132c 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -14,118 +14,12 @@ */ #include "syncRaftStore.h" -#include "cJSON.h" +#include "sync.h" -SRaftStore *raftStoreOpen(const char *path) { - int32_t ret; +int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; } - SRaftStore *pRaftStore = malloc(sizeof(SRaftStore)); - if (pRaftStore == NULL) { - sError("raftStoreOpen malloc error"); - return NULL; - } - memset(pRaftStore, 0, sizeof(*pRaftStore)); - snprintf(pRaftStore->path, sizeof(pRaftStore->path), "%s", path); +int32_t persistCurrentTerm(SyncTerm currentTerm) { return 0; } - char storeBuf[RAFT_STORE_BLOCK_SIZE]; - memset(storeBuf, 0, sizeof(storeBuf)); +int32_t voteFor(SRaftId *pRaftId) { return 0; } - if (!raftStoreFileExist(pRaftStore->path)) { - ret = raftStoreInit(pRaftStore); - assert(ret == 0); - } - - pRaftStore->fd = taosOpenFileReadWrite(pRaftStore->path); - if (pRaftStore->fd < 0) { - return NULL; - } - - int len = taosReadFile(pRaftStore->fd, storeBuf, sizeof(storeBuf)); - assert(len == RAFT_STORE_BLOCK_SIZE); - - ret = raftStoreDeserialize(pRaftStore, storeBuf, len); - assert(ret == 0); - - return pRaftStore; -} - -static int32_t raftStoreInit(SRaftStore *pRaftStore) { - pRaftStore->fd = taosOpenFileCreateWrite(pRaftStore->path); - if (pRaftStore->fd < 0) { - return -1; - } - - pRaftStore->currentTerm = 0; - pRaftStore->voteFor.addr = 0; - pRaftStore->voteFor.vgId = 0; - - int32_t ret = raftStorePersist(pRaftStore); - assert(ret == 0); - - taosCloseFile(pRaftStore->fd); - return 0; -} - -int32_t raftStoreClose(SRaftStore *pRaftStore) { - taosCloseFile(pRaftStore->fd); - free(pRaftStore); - return 0; -} - -int32_t raftStorePersist(SRaftStore *pRaftStore) { - int32_t ret; - char storeBuf[RAFT_STORE_BLOCK_SIZE]; - - ret = raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); - assert(ret == 0); - - taosLSeekFile(pRaftStore->fd, 0, SEEK_SET); - - ret = taosWriteFile(pRaftStore->fd, storeBuf, sizeof(storeBuf)); - assert(ret == RAFT_STORE_BLOCK_SIZE); - - fsync(pRaftStore->fd); - return 0; -} - -static bool raftStoreFileExist(char *path) { return taosStatFile(path, NULL, NULL) >= 0; } - -int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { - cJSON *pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "current_term", pRaftStore->currentTerm); - cJSON_AddNumberToObject(pRoot, "vote_for_addr", pRaftStore->voteFor.addr); - cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); - - char *serialized = cJSON_Print(pRoot); - int len2 = strlen(serialized); - assert(len2 < len); - memset(buf, 0, len); - snprintf(buf, len, "%s", serialized); - free(serialized); - - cJSON_Delete(pRoot); - return 0; -} - -int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { - assert(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); - cJSON *pRoot = cJSON_Parse(buf); - - cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); - pRaftStore->currentTerm = pCurrentTerm->valueint; - - cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); - pRaftStore->voteFor.addr = pVoteForAddr->valueint; - - cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); - pRaftStore->voteFor.vgId = pVoteForVgid->valueint; - - cJSON_Delete(pRoot); - return 0; -} - -void raftStorePrint(SRaftStore *pRaftStore) { - char storeBuf[RAFT_STORE_BLOCK_SIZE]; - raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); - printf("%s\n", storeBuf); -} +int32_t persistVoteFor(SRaftId *pRaftId) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 4cea7c150e..738fc4c5e1 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "syncReplication.h" +#include "sync.h" diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 7aee47b8e4..88056c95ff 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -14,6 +14,7 @@ */ #include "syncRequestVote.h" +#include "sync.h" void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index a9c88a7975..4ca1b1343f 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -14,6 +14,7 @@ */ #include "syncRequestVoteReply.h" +#include "sync.h" void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index da194780ff..8a27f097d1 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,6 +14,7 @@ */ #include "syncSnapshot.h" +#include "sync.h" #include "syncRaft.h" int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index e27df55d07..206dd70046 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -14,5 +14,6 @@ */ #include "syncTimeout.h" +#include "sync.h" void onTimeout(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c deleted file mode 100644 index 02cf4ac033..0000000000 --- a/source/libs/sync/src/syncVoteMgr.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncVoteMgr.h" diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt deleted file mode 100644 index e655ac01be..0000000000 --- a/source/libs/sync/test/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -add_executable(syncTest "") -add_executable(syncEnvTest "") -add_executable(syncPingTest "") - - -target_sources(syncTest - PRIVATE - "syncTest.cpp" -) -target_sources(syncEnvTest - PRIVATE - "syncEnvTest.cpp" -) -target_sources(syncPingTest - PRIVATE - "syncPingTest.cpp" -) - - -target_include_directories(syncTest - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncEnvTest - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncPingTest - PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) - - -target_link_libraries(syncTest - sync - gtest_main -) -target_link_libraries(syncEnvTest - sync - gtest_main -) -target_link_libraries(syncPingTest - sync - gtest_main -) - - -enable_testing() -add_test( - NAME sync_test - COMMAND syncTest -) diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp deleted file mode 100644 index 31dad593e6..0000000000 --- a/source/libs/sync/test/syncEnvTest.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "syncEnv.h" -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -void doSync() { - SSyncInfo syncInfo; - syncInfo.vgId = 1; - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = 3; - - pCfg->nodeInfo[0].nodePort = 7010; - taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - - pCfg->nodeInfo[1].nodePort = 7110; - taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodePort = 7210; - taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); - - SSyncNode* pSyncNode = syncNodeStart(&syncInfo); - assert(pSyncNode != NULL); -} - -int main() { - taosInitLog((char*)"syncEnvTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - logTest(); - - int32_t ret = syncIOStart(); - assert(ret == 0); - - ret = syncEnvStart(); - assert(ret == 0); - - doSync(); - - while (1) { - taosMsleep(1000); - } - - return 0; -} diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp deleted file mode 100644 index 5ed72fd56b..0000000000 --- a/source/libs/sync/test/syncPingTest.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -void doSync() { - SSyncInfo syncInfo; - syncInfo.vgId = 1; - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = 0; - pCfg->replicaNum = 3; - - pCfg->nodeInfo[0].nodePort = 7010; - taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - - pCfg->nodeInfo[1].nodePort = 7110; - taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodePort = 7210; - taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); - - SSyncNode* pSyncNode = syncNodeStart(&syncInfo); - assert(pSyncNode != NULL); -} - -int main() { - taosInitLog((char*)"syncPingTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - logTest(); - - int32_t ret = syncIOStart(); - assert(ret == 0); - - ret = syncEnvStart(); - assert(ret == 0); - - doSync(); - - while (1) { - taosMsleep(1000); - } - - return 0; -} diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 12b0905fa0..47566d537e 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,57 +1,7 @@ #include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" - -void *pingFunc(void *param) { - SSyncIO *io = (SSyncIO *)param; - while (1) { - sDebug("io->ping"); - io->ping(io); - sleep(1); - } - return NULL; -} int main() { - taosInitLog((char *)"syncTest.log", 100000, 10); - tsAsyncLog = 0; - sDebugFlag = 143 + 64; - - sTrace("sync log test: trace"); - sDebug("sync log test: debug"); - sInfo("sync log test: info"); - sWarn("sync log test: warn"); - sError("sync log test: error"); - sFatal("sync log test: fatal"); - - SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); - assert(pRaftStore != NULL); - - raftStorePrint(pRaftStore); - - pRaftStore->currentTerm = 100; - pRaftStore->voteFor.addr = 200; - pRaftStore->voteFor.vgId = 300; - - raftStorePrint(pRaftStore); - - raftStorePersist(pRaftStore); - - sDebug("sync test"); - - SSyncIO *syncIO = syncIOCreate(); - assert(syncIO != NULL); - - syncIO->start(syncIO); - - sleep(2); - - pthread_t tid; - pthread_create(&tid, NULL, pingFunc, syncIO); - - while (1) { - sleep(1); - } - return 0; + printf("test \n"); + return 0; } + From dbe28cd4ffdef3161c89ceb7ee684132d4925a60 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 27 Feb 2022 00:02:18 +0800 Subject: [PATCH 15/19] add sync code --- include/libs/sync/sync.h | 3 ++ source/libs/sync/inc/syncInt.h | 59 ++++++++++++++++++++--- source/libs/sync/inc/syncMessage.h | 2 - source/libs/sync/inc/syncRaft.h | 8 ++-- source/libs/sync/src/syncMain.c | 77 +++++++++++++++++++++++++++++- source/libs/sync/src/syncRaft.c | 9 +++- 6 files changed, 142 insertions(+), 16 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 03ec7c0eac..41e1491aec 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -131,6 +131,9 @@ typedef struct SStateMgr { typedef struct SSyncInfo { SyncGroupId vgId; SSyncCfg syncCfg; + char path[TSDB_FILENAME_LEN]; + SSyncFSM* pFsm; + } SSyncInfo; struct SSyncNode; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 551ce83122..5c23b585e0 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncMessage.h" #include "taosdef.h" #define sFatal(...) \ @@ -62,16 +63,60 @@ extern "C" { } \ } +struct SRaft; + typedef struct SSyncNode { - char path[TSDB_FILENAME_LEN]; - int8_t replica; - int8_t quorum; - int8_t selfIndex; - uint32_t vgId; - int32_t refCount; - int64_t rid; + int8_t replica; + int8_t quorum; + + SyncGroupId vgId; + SSyncCfg syncCfg; + char path[TSDB_FILENAME_LEN]; + + struct SRaft* pRaft; + + int32_t (*FpPing)(struct SSyncNode* ths, const SyncPing* pMsg); + + int32_t (*FpOnPing)(struct SSyncNode* ths, SyncPing* pMsg); + + int32_t (*FpOnPingReply)(struct SSyncNode* ths, SyncPingReply* pMsg); + + int32_t (*FpRequestVote)(struct SSyncNode* ths, const SyncRequestVote* pMsg); + + int32_t (*FpOnRequestVote)(struct SSyncNode* ths, SyncRequestVote* pMsg); + + int32_t (*FpOnRequestVoteReply)(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); + + int32_t (*FpAppendEntries)(struct SSyncNode* ths, const SyncAppendEntries* pMsg); + + int32_t (*FpOnAppendEntries)(struct SSyncNode* ths, SyncAppendEntries* pMsg); + + int32_t (*FpOnAppendEntriesReply)(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); + } SSyncNode; +SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); + +void syncNodeClose(SSyncNode* pSyncNode); + +static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg); + +static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg); + +static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg); + +static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg); + +static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg); + +static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); + +static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg); + +static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg); + +static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index dc74526c73..2ee5e0109c 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -43,7 +43,6 @@ typedef struct SyncPing { const SSyncBuffer *pData; } SyncPing, RaftPing; - typedef struct SyncPingReply { ESyncMessageType msgType; const SSyncBuffer *pData; @@ -63,7 +62,6 @@ typedef struct SyncClientRequestReply { const SSyncBuffer *pLeaderHint; } SyncClientRequestReply, RaftClientRequestReply; - typedef struct SyncRequestVote { ESyncMessageType msgType; SyncTerm currentTerm; diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h index 6a3140d930..4ea7849c6b 100644 --- a/source/libs/sync/inc/syncRaft.h +++ b/source/libs/sync/inc/syncRaft.h @@ -33,8 +33,8 @@ typedef struct SRaftId { } SRaftId; typedef struct SRaft { - SRaftId id; - void* data; + SRaftId id; + SSyncFSM* pFsm; int32_t (*FpPing)(struct SRaft* ths, const RaftPing* pMsg); @@ -56,7 +56,9 @@ typedef struct SRaft { } SRaft; -SRaft* raftCreate(SRaftId raftId, void* data); +SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm); + +void raftClose(SRaft* pRaft); static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fbb969eb1c..d34095728f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -16,12 +16,17 @@ #include #include "sync.h" #include "syncInt.h" +#include "syncRaft.h" int32_t syncInit() { return 0; } void syncCleanUp() {} -int64_t syncStart(const SSyncInfo* pSyncInfo) { return 0; } +int64_t syncStart(const SSyncInfo* pSyncInfo) { + SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); + assert(pSyncNode != NULL); + return 0; +} void syncStop(int64_t rid) {} @@ -31,4 +36,72 @@ int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { r ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} \ No newline at end of file +void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} + +SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { + SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); + assert(pSyncNode != NULL); + + pSyncNode->FpPing = doSyncNodePing; + pSyncNode->FpOnPing = onSyncNodePing; + pSyncNode->FpOnPingReply = onSyncNodePingReply; + pSyncNode->FpRequestVote = doSyncNodeRequestVote; + pSyncNode->FpOnRequestVote = onSyncNodeRequestVote; + pSyncNode->FpOnRequestVoteReply = onSyncNodeRequestVoteReply; + pSyncNode->FpAppendEntries = doSyncNodeAppendEntries; + pSyncNode->FpOnAppendEntries = onSyncNodeAppendEntries; + pSyncNode->FpOnAppendEntriesReply = onSyncNodeAppendEntriesReply; + + return pSyncNode; +} + +void syncNodeClose(SSyncNode* pSyncNode) { + assert(pSyncNode != NULL); + raftClose(pSyncNode->pRaft); + free(pSyncNode); +} + +static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg) { + int32_t ret = ths->pRaft->FpPing(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg) { + int32_t ret = ths->pRaft->FpOnPing(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg) { + int32_t ret = ths->pRaft->FpOnPingReply(ths->pRaft, pMsg); + return ret; +} + +static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg) { + int32_t ret = ths->pRaft->FpRequestVote(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = ths->pRaft->FpOnRequestVote(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = ths->pRaft->FpOnRequestVoteReply(ths->pRaft, pMsg); + return ret; +} + +static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg) { + int32_t ret = ths->pRaft->FpAppendEntries(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = ths->pRaft->FpOnAppendEntries(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = ths->pRaft->FpOnAppendEntriesReply(ths->pRaft, pMsg); + return ret; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaft.c b/source/libs/sync/src/syncRaft.c index 1a01aefb0b..9f139730d1 100644 --- a/source/libs/sync/src/syncRaft.c +++ b/source/libs/sync/src/syncRaft.c @@ -16,12 +16,12 @@ #include "syncRaft.h" #include "sync.h" -SRaft* raftCreate(SRaftId raftId, void* data) { +SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm) { SRaft* pRaft = (SRaft*)malloc(sizeof(SRaft)); assert(pRaft != NULL); pRaft->id = raftId; - pRaft->data = data; + pRaft->pFsm = pFsm; pRaft->FpPing = doRaftPing; pRaft->FpOnPing = onRaftPing; @@ -38,6 +38,11 @@ SRaft* raftCreate(SRaftId raftId, void* data) { return pRaft; } +void raftClose(SRaft* pRaft) { + assert(pRaft != NULL); + free(pRaft); +} + static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg) { return 0; } static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg) { return 0; } From b55cf2bc7ef58715eb471889e85e15c863e93b6c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 27 Feb 2022 02:24:50 +0800 Subject: [PATCH 16/19] add sync code --- source/libs/sync/CMakeLists.txt | 6 +- source/libs/sync/inc/syncAppendEntries.h | 1 + source/libs/sync/inc/syncAppendEntriesReply.h | 1 + source/libs/sync/inc/syncElection.h | 1 + source/libs/sync/inc/syncEnv.h | 47 ++++ source/libs/sync/inc/syncIO.h | 68 +++++ source/libs/sync/inc/syncInt.h | 28 +- source/libs/sync/inc/syncRaft.h | 2 +- source/libs/sync/inc/syncRaftEntry.h | 8 +- source/libs/sync/inc/syncRaftLog.h | 2 +- source/libs/sync/inc/syncRaftStore.h | 30 ++- source/libs/sync/inc/syncReplication.h | 1 + source/libs/sync/inc/syncRequestVote.h | 1 + source/libs/sync/inc/syncRequestVoteReply.h | 1 + source/libs/sync/inc/syncSnapshot.h | 2 +- source/libs/sync/inc/syncTimeout.h | 1 + source/libs/sync/inc/syncVoteMgr.h | 33 +++ source/libs/sync/src/syncAppendEntries.c | 1 - source/libs/sync/src/syncAppendEntriesReply.c | 1 - source/libs/sync/src/syncElection.c | 2 +- source/libs/sync/src/syncEnv.c | 36 +++ source/libs/sync/src/syncIO.c | 245 ++++++++++++++++++ source/libs/sync/src/syncMain.c | 2 + source/libs/sync/src/syncMessage.c | 1 - source/libs/sync/src/syncOnMessage.c | 2 +- source/libs/sync/src/syncRaftEntry.c | 2 +- source/libs/sync/src/syncRaftLog.c | 1 - source/libs/sync/src/syncRaftStore.c | 116 ++++++++- source/libs/sync/src/syncReplication.c | 2 +- source/libs/sync/src/syncRequestVote.c | 1 - source/libs/sync/src/syncRequestVoteReply.c | 1 - source/libs/sync/src/syncSnapshot.c | 1 - source/libs/sync/src/syncTimeout.c | 1 - source/libs/sync/src/syncVoteMgr.c | 16 ++ source/libs/sync/test/CMakeLists.txt | 55 ++++ source/libs/sync/test/syncEnvTest.cpp | 56 ++++ source/libs/sync/test/syncPingTest.cpp | 57 ++++ source/libs/sync/test/syncTest.cpp | 56 +++- 38 files changed, 853 insertions(+), 35 deletions(-) create mode 100644 source/libs/sync/inc/syncEnv.h create mode 100644 source/libs/sync/inc/syncIO.h create mode 100644 source/libs/sync/inc/syncVoteMgr.h create mode 100644 source/libs/sync/src/syncEnv.c create mode 100644 source/libs/sync/src/syncIO.c create mode 100644 source/libs/sync/src/syncVoteMgr.c create mode 100644 source/libs/sync/test/CMakeLists.txt create mode 100644 source/libs/sync/test/syncEnvTest.cpp create mode 100644 source/libs/sync/test/syncPingTest.cpp diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index 37ee5194c8..cb38d7e363 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -13,4 +13,8 @@ target_include_directories( sync PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) \ No newline at end of file +) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index 9ca0de19c5..b7c1c051cc 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 8b5cbf1da5..22f8eb464f 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 34dfdb3d09..7e9e637854 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "taosdef.h" #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h new file mode 100644 index 0000000000..f1c4327b69 --- /dev/null +++ b/source/libs/sync/inc/syncEnv.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_ENV_H +#define _TD_LIBS_SYNC_ENV_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncInt.h" +#include "taosdef.h" +#include "trpc.h" + +typedef struct SSyncEnv { + void *pTimer; + void *pTimerManager; +} SSyncEnv; + +int32_t syncEnvStart(); + +int32_t syncEnvStop(); + +static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv); + +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_ENV_H*/ diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h new file mode 100644 index 0000000000..54a3d2b8c1 --- /dev/null +++ b/source/libs/sync/inc/syncIO.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_IO_H +#define _TD_LIBS_IO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "os.h" +#include "syncInt.h" +#include "taosdef.h" +#include "tqueue.h" +#include "trpc.h" + +typedef struct SSyncIO { + void * serverRpc; + void * clientRpc; + STaosQueue *pMsgQ; + STaosQset * pQset; + pthread_t tid; + int8_t isStart; + + SEpSet epSet; + + void *syncTimer; + void *syncTimerManager; + + int32_t (*start)(struct SSyncIO *ths); + int32_t (*stop)(struct SSyncIO *ths); + int32_t (*ping)(struct SSyncIO *ths); + int32_t (*onMessage)(struct SSyncIO *ths, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); + int32_t (*destroy)(struct SSyncIO *ths); + +} SSyncIO; + +int32_t syncIOStart(); +int32_t syncIOStop(); + +SSyncIO *syncIOCreate(); + +static int32_t doSyncIOStart(SSyncIO *io); +static int32_t doSyncIOStop(SSyncIO *io); +static int32_t doSyncIOPing(SSyncIO *io); +static int32_t doSyncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t doSyncIODestroy(SSyncIO *io); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_IO_H*/ diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 5c23b585e0..9bd8606ee6 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -23,8 +23,11 @@ extern "C" { #include #include #include -#include "syncMessage.h" +#include "sync.h" #include "taosdef.h" +#include "tlog.h" + +extern int32_t sDebugFlag; #define sFatal(...) \ { \ @@ -64,6 +67,25 @@ extern "C" { } struct SRaft; +typedef struct SRaft SRaft; + +struct SyncPing; +typedef struct SyncPing SyncPing; + +struct SyncPingReply; +typedef struct SyncPingReply SyncPingReply; + +struct SyncRequestVote; +typedef struct SyncRequestVote SyncRequestVote; + +struct SyncRequestVoteReply; +typedef struct SyncRequestVoteReply SyncRequestVoteReply; + +struct SyncAppendEntries; +typedef struct SyncAppendEntries SyncAppendEntries; + +struct SyncAppendEntriesReply; +typedef struct SyncAppendEntriesReply SyncAppendEntriesReply; typedef struct SSyncNode { int8_t replica; @@ -73,7 +95,7 @@ typedef struct SSyncNode { SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - struct SRaft* pRaft; + SRaft* pRaft; int32_t (*FpPing)(struct SSyncNode* ths, const SyncPing* pMsg); @@ -117,6 +139,8 @@ static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); + + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h index 4ea7849c6b..a247a29fc4 100644 --- a/source/libs/sync/inc/syncRaft.h +++ b/source/libs/sync/inc/syncRaft.h @@ -28,7 +28,7 @@ extern "C" { #include "taosdef.h" typedef struct SRaftId { - SyncNodeId nodeId; + SyncNodeId addr; SyncGroupId vgId; } SRaftId; diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index adc82f2c5d..516bef4d48 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_LIBS_TPL_H -#define _TD_LIBS_TPL_H +#ifndef _TD_LIBS_SYNC_RAFT_ENTRY_H +#define _TD_LIBS_SYNC_RAFT_ENTRY_H #ifdef __cplusplus extern "C" { @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "taosdef.h" typedef struct SSyncRaftEntry { @@ -37,4 +37,4 @@ typedef struct SSyncRaftEntry { } #endif -#endif /*_TD_LIBS_TPL_H*/ +#endif /*_TD_LIBS_SYNC_RAFT_ENTRY_H*/ diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index 8c4b5116ea..ee971062cf 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "taosdef.h" int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 4cb852f34a..610f0c2487 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -23,17 +23,37 @@ extern "C" { #include #include #include -#include "sync.h" +#include "cJSON.h" +#include "syncInt.h" #include "syncRaft.h" #include "taosdef.h" -int32_t currentTerm(SyncTerm *pCurrentTerm); +#define RAFT_STORE_BLOCK_SIZE 512 +#define RAFT_STORE_PATH_LEN 128 -int32_t persistCurrentTerm(SyncTerm currentTerm); +typedef struct SRaftStore { + SyncTerm currentTerm; + SRaftId voteFor; + FileFd fd; + char path[RAFT_STORE_PATH_LEN]; +} SRaftStore; -int32_t voteFor(SRaftId *pRaftId); +SRaftStore *raftStoreOpen(const char *path); + +static int32_t raftStoreInit(SRaftStore *pRaftStore); + +int32_t raftStoreClose(SRaftStore *pRaftStore); + +int32_t raftStorePersist(SRaftStore *pRaftStore); + +static bool raftStoreFileExist(char *path); + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); + +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); + +void raftStorePrint(SRaftStore *pRaftStore); -int32_t persistVoteFor(SRaftId *pRaftId); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 40c5ff790b..7f97ae9e49 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "taosdef.h" #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index 3ff96bbe8f..c2eca55151 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index 033ac89bc2..38068dd0e2 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 3b6121578a..89fcb230fb 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index 8159d2566c..d9d6a17939 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h new file mode 100644 index 0000000000..cfcf58bee2 --- /dev/null +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_VOTG_MGR_H +#define _TD_LIBS_SYNC_VOTG_MGR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "syncInt.h" +#include "taosdef.h" + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_VOTG_MGR_H*/ diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 65654564ab..2b9c59ec92 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -14,7 +14,6 @@ */ #include "syncAppendEntries.h" -#include "sync.h" void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 20235ef720..05734237b9 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -14,7 +14,6 @@ */ #include "syncAppendEntriesReply.h" -#include "sync.h" void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 738fc4c5e1..329105e2a1 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncElection.h" diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c new file mode 100644 index 0000000000..e71cf55cb1 --- /dev/null +++ b/source/libs/sync/src/syncEnv.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncEnv.h" +#include + +SSyncEnv *gSyncEnv = NULL; + +int32_t syncEnvStart() { + int32_t ret; + gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + assert(gSyncEnv != NULL); + ret = doSyncEnvStart(gSyncEnv); + return ret; +} + +int32_t syncEnvStop() { + int32_t ret = doSyncEnvStop(gSyncEnv); + return ret; +} + +static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { return 0; } + +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { return 0; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c new file mode 100644 index 0000000000..0e32d9ac50 --- /dev/null +++ b/source/libs/sync/src/syncIO.c @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncIO.h" +#include +#include "syncOnMessage.h" +#include "tglobal.h" +#include "ttimer.h" +#include "tutil.h" + +int32_t syncIOStart() { return 0; } + +int32_t syncIOStop() { return 0; } + +static void syncTick(void *param, void *tmrId) { + SSyncIO *io = (SSyncIO *)param; + sDebug("syncTick ... "); + + SRpcMsg rpcMsg; + rpcMsg.pCont = rpcMallocCont(10); + snprintf(rpcMsg.pCont, 10, "TICK"); + rpcMsg.contLen = 10; + rpcMsg.handle = NULL; + rpcMsg.msgType = 2; + + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); + + taosWriteQitem(io->pMsgQ, pTemp); + + io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); +} + +void *syncConsumer(void *param) { + SSyncIO *io = param; + + STaosQall *qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + + qall = taosAllocateQall(); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(io->pQset, qall, NULL, NULL); + sDebug("%d sync-io msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + sDebug("sync-io recv type:%d msg:%s", pRpcMsg->msgType, (char *)(pRpcMsg->pCont)); + } + + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + rpcFreeCont(pRpcMsg->pCont); + + if (pRpcMsg->handle != NULL) { + int msgSize = 128; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + } + + taosFreeQitem(pRpcMsg); + } + } + + taosFreeQall(qall); + return NULL; +} + +static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int ret = 0; + return ret; +} + +static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + sDebug("processResponse ... "); + rpcFreeCont(pMsg->pCont); +} + +static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SSyncIO *io = pParent; + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + sDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(io->pMsgQ, pTemp); +} + +SSyncIO *syncIOCreate() { + SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + memset(io, 0, sizeof(*io)); + + io->pMsgQ = taosOpenQueue(); + io->pQset = taosOpenQset(); + taosAddIntoQset(io->pQset, io->pMsgQ, NULL); + + io->start = doSyncIOStart; + io->stop = doSyncIOStop; + io->ping = doSyncIOPing; + io->onMessage = doSyncIOOnMessage; + io->destroy = doSyncIODestroy; + + return io; +} + +static int32_t doSyncIOStart(SSyncIO *io) { + taosBlockSIGPIPE(); + + tsRpcForceTcp = 1; + + // cient rpc init + { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "SYNC-IO-CLIENT"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processResponse; + rpcInit.sessions = 100; + rpcInit.idleTime = 100; + rpcInit.user = "sync-io"; + rpcInit.secret = "sync-io"; + rpcInit.ckey = "key"; + rpcInit.spi = 0; + rpcInit.connType = TAOS_CONN_CLIENT; + + io->clientRpc = rpcOpen(&rpcInit); + if (io->clientRpc == NULL) { + sError("failed to initialize RPC"); + return -1; + } + } + + // server rpc init + { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 38000; + rpcInit.label = "SYNC-IO-SERVER"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processRequestMsg; + rpcInit.sessions = 1000; + rpcInit.idleTime = 2 * 1500; + rpcInit.afp = retrieveAuthInfo; + rpcInit.parent = io; + rpcInit.connType = TAOS_CONN_SERVER; + + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + sError("failed to start RPC server"); + return -1; + } + } + + io->epSet.inUse = 0; + addEpIntoEpSet(&io->epSet, "127.0.0.1", 38000); + + // start consumer thread + { + if (pthread_create(&io->tid, NULL, syncConsumer, io) != 0) { + sError("failed to create sync consumer thread since %s", strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } + + // start tmr thread + io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); + io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); + + return 0; +} + +static int32_t doSyncIOStop(SSyncIO *io) { + atomic_store_8(&io->isStart, 0); + pthread_join(io->tid, NULL); + return 0; +} + +static int32_t doSyncIOPing(SSyncIO *io) { + SRpcMsg rpcMsg, rspMsg; + + rpcMsg.pCont = rpcMallocCont(10); + snprintf(rpcMsg.pCont, 10, "ping"); + rpcMsg.contLen = 10; + rpcMsg.handle = NULL; + rpcMsg.msgType = 1; + + rpcSendRequest(io->clientRpc, &io->epSet, &rpcMsg, NULL); + + return 0; +} + +static int32_t doSyncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } + +static int32_t doSyncIODestroy(SSyncIO *io) { + int8_t start = atomic_load_8(&io->isStart); + assert(start == 0); + + if (io->serverRpc != NULL) { + free(io->serverRpc); + io->serverRpc = NULL; + } + + if (io->clientRpc != NULL) { + free(io->clientRpc); + io->clientRpc = NULL; + } + + if (io->pMsgQ != NULL) { + free(io->pMsgQ); + io->pMsgQ = NULL; + } + + if (io->pQset != NULL) { + free(io->pQset); + io->pQset = NULL; + } + + return 0; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d34095728f..1e13c6125e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -38,6 +38,7 @@ ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} + SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); @@ -55,6 +56,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { return pSyncNode; } + void syncNodeClose(SSyncNode* pSyncNode) { assert(pSyncNode != NULL); raftClose(pSyncNode->pRaft); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index dcfc940f76..8937303725 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -14,7 +14,6 @@ */ #include "syncMessage.h" -#include "sync.h" #include "syncRaft.h" void onMessage(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncOnMessage.c b/source/libs/sync/src/syncOnMessage.c index 738fc4c5e1..19a97ee156 100644 --- a/source/libs/sync/src/syncOnMessage.c +++ b/source/libs/sync/src/syncOnMessage.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncOnMessage.h" diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 738fc4c5e1..e525d3c7c2 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncRaftEntry.h" diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 4a5fc201b0..37bb3ce48c 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -14,7 +14,6 @@ */ #include "syncRaftLog.h" -#include "sync.h" int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index d45e53132c..4391f5d25c 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -14,12 +14,118 @@ */ #include "syncRaftStore.h" -#include "sync.h" +#include "cJSON.h" -int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; } +SRaftStore *raftStoreOpen(const char *path) { + int32_t ret; -int32_t persistCurrentTerm(SyncTerm currentTerm) { return 0; } + SRaftStore *pRaftStore = malloc(sizeof(SRaftStore)); + if (pRaftStore == NULL) { + sError("raftStoreOpen malloc error"); + return NULL; + } + memset(pRaftStore, 0, sizeof(*pRaftStore)); + snprintf(pRaftStore->path, sizeof(pRaftStore->path), "%s", path); -int32_t voteFor(SRaftId *pRaftId) { return 0; } + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + memset(storeBuf, 0, sizeof(storeBuf)); -int32_t persistVoteFor(SRaftId *pRaftId) { return 0; } \ No newline at end of file + if (!raftStoreFileExist(pRaftStore->path)) { + ret = raftStoreInit(pRaftStore); + assert(ret == 0); + } + + pRaftStore->fd = taosOpenFileReadWrite(pRaftStore->path); + if (pRaftStore->fd < 0) { + return NULL; + } + + int len = taosReadFile(pRaftStore->fd, storeBuf, sizeof(storeBuf)); + assert(len == RAFT_STORE_BLOCK_SIZE); + + ret = raftStoreDeserialize(pRaftStore, storeBuf, len); + assert(ret == 0); + + return pRaftStore; +} + +static int32_t raftStoreInit(SRaftStore *pRaftStore) { + pRaftStore->fd = taosOpenFileCreateWrite(pRaftStore->path); + if (pRaftStore->fd < 0) { + return -1; + } + + pRaftStore->currentTerm = 0; + pRaftStore->voteFor.addr = 0; + pRaftStore->voteFor.vgId = 0; + + int32_t ret = raftStorePersist(pRaftStore); + assert(ret == 0); + + taosCloseFile(pRaftStore->fd); + return 0; +} + +int32_t raftStoreClose(SRaftStore *pRaftStore) { + taosCloseFile(pRaftStore->fd); + free(pRaftStore); + return 0; +} + +int32_t raftStorePersist(SRaftStore *pRaftStore) { + int32_t ret; + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + + ret = raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); + assert(ret == 0); + + taosLSeekFile(pRaftStore->fd, 0, SEEK_SET); + + ret = taosWriteFile(pRaftStore->fd, storeBuf, sizeof(storeBuf)); + assert(ret == RAFT_STORE_BLOCK_SIZE); + + fsync(pRaftStore->fd); + return 0; +} + +static bool raftStoreFileExist(char *path) { return taosStatFile(path, NULL, NULL) >= 0; } + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { + cJSON *pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "current_term", pRaftStore->currentTerm); + cJSON_AddNumberToObject(pRoot, "vote_for_addr", pRaftStore->voteFor.addr); + cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); + + char *serialized = cJSON_Print(pRoot); + int len2 = strlen(serialized); + assert(len2 < len); + memset(buf, 0, len); + snprintf(buf, len, "%s", serialized); + free(serialized); + + cJSON_Delete(pRoot); + return 0; +} + +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { + assert(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); + cJSON *pRoot = cJSON_Parse(buf); + + cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); + pRaftStore->currentTerm = pCurrentTerm->valueint; + + cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); + pRaftStore->voteFor.addr = pVoteForAddr->valueint; + + cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); + pRaftStore->voteFor.vgId = pVoteForVgid->valueint; + + cJSON_Delete(pRoot); + return 0; +} + +void raftStorePrint(SRaftStore *pRaftStore) { + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); + printf("%s\n", storeBuf); +} diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 738fc4c5e1..4cea7c150e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncReplication.h" diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 88056c95ff..7aee47b8e4 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -14,7 +14,6 @@ */ #include "syncRequestVote.h" -#include "sync.h" void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 4ca1b1343f..a9c88a7975 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -14,7 +14,6 @@ */ #include "syncRequestVoteReply.h" -#include "sync.h" void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg) { // TLA+ Spec diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 8a27f097d1..da194780ff 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,7 +14,6 @@ */ #include "syncSnapshot.h" -#include "sync.h" #include "syncRaft.h" int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 206dd70046..e27df55d07 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -14,6 +14,5 @@ */ #include "syncTimeout.h" -#include "sync.h" void onTimeout(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c new file mode 100644 index 0000000000..02cf4ac033 --- /dev/null +++ b/source/libs/sync/src/syncVoteMgr.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncVoteMgr.h" diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt new file mode 100644 index 0000000000..e655ac01be --- /dev/null +++ b/source/libs/sync/test/CMakeLists.txt @@ -0,0 +1,55 @@ +add_executable(syncTest "") +add_executable(syncEnvTest "") +add_executable(syncPingTest "") + + +target_sources(syncTest + PRIVATE + "syncTest.cpp" +) +target_sources(syncEnvTest + PRIVATE + "syncEnvTest.cpp" +) +target_sources(syncPingTest + PRIVATE + "syncPingTest.cpp" +) + + +target_include_directories(syncTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncEnvTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncPingTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + + +target_link_libraries(syncTest + sync + gtest_main +) +target_link_libraries(syncEnvTest + sync + gtest_main +) +target_link_libraries(syncPingTest + sync + gtest_main +) + + +enable_testing() +add_test( + NAME sync_test + COMMAND syncTest +) diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp new file mode 100644 index 0000000000..14821ca520 --- /dev/null +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -0,0 +1,56 @@ +#include "syncEnv.h" +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void doSync() { + SSyncInfo syncInfo; + syncInfo.vgId = 1; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = 3; + + pCfg->nodeInfo[0].nodePort = 7010; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + pCfg->nodeInfo[1].nodePort = 7110; + taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + + pCfg->nodeInfo[2].nodePort = 7210; + taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); +} + +int main() { + taosInitLog((char*)"syncEnvTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret = syncIOStart(); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + doSync(); + + while (1) { + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp new file mode 100644 index 0000000000..436a25cb5e --- /dev/null +++ b/source/libs/sync/test/syncPingTest.cpp @@ -0,0 +1,57 @@ +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void doSync() { + SSyncInfo syncInfo; + syncInfo.vgId = 1; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = 0; + pCfg->replicaNum = 3; + + pCfg->nodeInfo[0].nodePort = 7010; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + pCfg->nodeInfo[1].nodePort = 7110; + taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + + pCfg->nodeInfo[2].nodePort = 7210; + taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); +} + +int main() { + taosInitLog((char*)"syncPingTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret = syncIOStart(); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + doSync(); + + while (1) { + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 47566d537e..12b0905fa0 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,7 +1,57 @@ #include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" -int main() { - printf("test \n"); - return 0; +void *pingFunc(void *param) { + SSyncIO *io = (SSyncIO *)param; + while (1) { + sDebug("io->ping"); + io->ping(io); + sleep(1); + } + return NULL; } +int main() { + taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + sTrace("sync log test: trace"); + sDebug("sync log test: debug"); + sInfo("sync log test: info"); + sWarn("sync log test: warn"); + sError("sync log test: error"); + sFatal("sync log test: fatal"); + + SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); + assert(pRaftStore != NULL); + + raftStorePrint(pRaftStore); + + pRaftStore->currentTerm = 100; + pRaftStore->voteFor.addr = 200; + pRaftStore->voteFor.vgId = 300; + + raftStorePrint(pRaftStore); + + raftStorePersist(pRaftStore); + + sDebug("sync test"); + + SSyncIO *syncIO = syncIOCreate(); + assert(syncIO != NULL); + + syncIO->start(syncIO); + + sleep(2); + + pthread_t tid; + pthread_create(&tid, NULL, pingFunc, syncIO); + + while (1) { + sleep(1); + } + return 0; +} From 4afcb4387c99c42550d7120fa6359e9d88e4580c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 27 Feb 2022 10:22:15 +0800 Subject: [PATCH 17/19] add sync io --- include/libs/sync/sync.h | 3 +++ source/libs/sync/inc/syncIO.h | 14 ++++++++++---- source/libs/sync/inc/syncInt.h | 4 ++-- source/libs/sync/inc/syncRaftStore.h | 1 - source/libs/sync/src/syncIO.c | 10 +++++++--- source/libs/sync/src/syncMain.c | 4 ++-- source/libs/sync/test/syncPingTest.cpp | 8 ++++++++ 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 41e1491aec..a619e66622 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -21,7 +21,9 @@ extern "C" { #endif #include +#include #include "taosdef.h" +#include "trpc.h" typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; @@ -133,6 +135,7 @@ typedef struct SSyncInfo { SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; SSyncFSM* pFsm; + int32_t (*FpSendMsg)(void* handle, const SEpSet* pEpSet, SRpcMsg* pMsg); } SSyncInfo; diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 54a3d2b8c1..4b788efd79 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -45,20 +45,26 @@ typedef struct SSyncIO { int32_t (*start)(struct SSyncIO *ths); int32_t (*stop)(struct SSyncIO *ths); int32_t (*ping)(struct SSyncIO *ths); - int32_t (*onMessage)(struct SSyncIO *ths, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); + + int32_t (*onMsg)(struct SSyncIO *ths, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); int32_t (*destroy)(struct SSyncIO *ths); + void *pSyncNode; + int32_t (*FpOnPing)(struct SSyncNode *ths, SyncPing *pMsg); + } SSyncIO; -int32_t syncIOStart(); -int32_t syncIOStop(); +extern SSyncIO *gSyncIO; +int32_t syncIOStart(); +int32_t syncIOStop(); +int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg); SSyncIO *syncIOCreate(); static int32_t doSyncIOStart(SSyncIO *io); static int32_t doSyncIOStop(SSyncIO *io); static int32_t doSyncIOPing(SSyncIO *io); -static int32_t doSyncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t doSyncIODestroy(SSyncIO *io); #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 9bd8606ee6..ad8484662a 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -115,6 +115,8 @@ typedef struct SSyncNode { int32_t (*FpOnAppendEntriesReply)(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); + int32_t (*FpSendMsg)(void* handle, const SEpSet* pEpSet, SRpcMsg* pMsg); + } SSyncNode; SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); @@ -139,8 +141,6 @@ static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); - - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 610f0c2487..bdaeb81aee 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -54,7 +54,6 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); void raftStorePrint(SRaftStore *pRaftStore); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 0e32d9ac50..bb20d11e37 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -20,6 +20,10 @@ #include "ttimer.h" #include "tutil.h" +SSyncIO *gSyncIO = NULL; + +int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg) { return 0; } + int32_t syncIOStart() { return 0; } int32_t syncIOStop() { return 0; } @@ -121,7 +125,7 @@ SSyncIO *syncIOCreate() { io->start = doSyncIOStart; io->stop = doSyncIOStop; io->ping = doSyncIOPing; - io->onMessage = doSyncIOOnMessage; + io->onMsg = doSyncIOOnMsg; io->destroy = doSyncIODestroy; return io; @@ -215,7 +219,7 @@ static int32_t doSyncIOPing(SSyncIO *io) { return 0; } -static int32_t doSyncIOOnMessage(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } +static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } static int32_t doSyncIODestroy(SSyncIO *io) { int8_t start = atomic_load_8(&io->isStart); @@ -242,4 +246,4 @@ static int32_t doSyncIODestroy(SSyncIO *io) { } return 0; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 1e13c6125e..bd2952505e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -38,11 +38,12 @@ ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} - SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); + pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; + pSyncNode->FpPing = doSyncNodePing; pSyncNode->FpOnPing = onSyncNodePing; pSyncNode->FpOnPingReply = onSyncNodePingReply; @@ -56,7 +57,6 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { return pSyncNode; } - void syncNodeClose(SSyncNode* pSyncNode) { assert(pSyncNode != NULL); raftClose(pSyncNode->pRaft); diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 436a25cb5e..b69b102b54 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -14,8 +14,13 @@ void logTest() { } void doSync() { + SSyncFSM* pFsm; + SSyncInfo syncInfo; syncInfo.vgId = 1; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = 0; @@ -32,6 +37,9 @@ void doSync() { SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); + + gSyncIO->FpOnPing = pSyncNode->FpOnPing; + gSyncIO->pSyncNode = pSyncNode; } int main() { From c12c0df80e79805ba105759d66913098df942854 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 27 Feb 2022 14:17:21 +0800 Subject: [PATCH 18/19] comment log store --- source/libs/sync/src/syncRaftStore.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 4391f5d25c..964cc78490 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -16,6 +16,28 @@ #include "syncRaftStore.h" #include "cJSON.h" +// to complie success: FileIO interface is modified + +SRaftStore *raftStoreOpen(const char *path) { return NULL;} + +static int32_t raftStoreInit(SRaftStore *pRaftStore) { return 0;} + +int32_t raftStoreClose(SRaftStore *pRaftStore) { return 0;} + +int32_t raftStorePersist(SRaftStore *pRaftStore) { return 0;} + +static bool raftStoreFileExist(char *path) { return 0;} + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0;} + +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0;} + +void raftStorePrint(SRaftStore *pRaftStore) {} + + + +#if 0 + SRaftStore *raftStoreOpen(const char *path) { int32_t ret; @@ -129,3 +151,5 @@ void raftStorePrint(SRaftStore *pRaftStore) { raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); printf("%s\n", storeBuf); } + +#endif From b6803ae9cc109d9d0f99b137383b627fd7bae7dd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 27 Feb 2022 14:27:06 +0800 Subject: [PATCH 19/19] comment log code --- source/libs/sync/inc/syncRaftStore.h | 2 +- source/libs/sync/test/syncEnvTest.cpp | 2 +- source/libs/sync/test/syncPingTest.cpp | 2 +- source/libs/sync/test/syncTest.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index bdaeb81aee..0fdbd7a150 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -34,7 +34,7 @@ extern "C" { typedef struct SRaftStore { SyncTerm currentTerm; SRaftId voteFor; - FileFd fd; + //FileFd fd; char path[RAFT_STORE_PATH_LEN]; } SRaftStore; diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index 14821ca520..1d050e7094 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -34,7 +34,7 @@ void doSync() { } int main() { - taosInitLog((char*)"syncEnvTest.log", 100000, 10); + //taosInitLog((char*)"syncEnvTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index b69b102b54..e62d051946 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -43,7 +43,7 @@ void doSync() { } int main() { - taosInitLog((char*)"syncPingTest.log", 100000, 10); + //taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 12b0905fa0..035ed54629 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -14,7 +14,7 @@ void *pingFunc(void *param) { } int main() { - taosInitLog((char *)"syncTest.log", 100000, 10); + //taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64;