diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 81ddc68d5d..dba80f5397 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -195,6 +195,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL) TD_NEW_MSG_SEG(TDMT_QND_MSG) @@ -234,6 +235,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_SYNC_COMMON_RESPONSE, "sync-common-response", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_APPLY_MSG, "sync-apply-msg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_CONFIG_CHANGE, "sync-config-change", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_SNAPSHOT_SEND, "sync-snapshot-send", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_SNAPSHOT_RSP, "sync-snapshot-rsp", NULL, NULL) #if defined(TD_MSG_NUMBER_) TDMT_MAX diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6b5eb3b491..31d13d6cf4 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -308,9 +308,11 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc if (pTask->sinkType == TASK_SINK__TABLE) { ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); + taosFreeQitem(pBlock); } else if (pTask->sinkType == TASK_SINK__SMA) { ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); + taosFreeQitem(pBlock); } else { ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); taosWriteQitem(pTask->outputQueue->queue, pBlock); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a587ad6ef2..10ece0b219 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -88,11 +88,16 @@ typedef struct SReConfigCbMeta { } SReConfigCbMeta; typedef struct SSnapshot { - void *data; + void* data; SyncIndex lastApplyIndex; SyncTerm lastApplyTerm; + SyncIndex lastConfigIndex; } SSnapshot; +typedef struct SSnapshotMeta { + SyncIndex lastConfigIndex; +} SSnapshotMeta; + typedef struct SSyncFSM { void* data; @@ -141,10 +146,28 @@ typedef struct SSyncLogStore { // return commit index of log SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); + // refactor, log[0 .. n] ==> log[m .. n] + int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); + int32_t (*syncLogResetBeginIndex)(struct SSyncLogStore* pLogStore); + SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore); + SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); + bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); + int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore); + bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index); + + SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore); + SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore); + SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore); + + int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); + int32_t (*syncLogGetEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); + int32_t (*syncLogTruncate)(struct SSyncLogStore* pLogStore, SyncIndex fromIndex); + } SSyncLogStore; typedef struct SSyncInfo { bool isStandBy; + bool snapshotEnable; SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; @@ -172,6 +195,8 @@ bool syncEnvIsStart(); const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); +int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta); + // to be moved to static void syncStartNormal(int64_t rid); void syncStartStandBy(int64_t rid); diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index bd396edf55..bb50fc141c 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -301,6 +301,7 @@ typedef struct SyncAppendEntries { SyncIndex prevLogIndex; SyncTerm prevLogTerm; SyncIndex commitIndex; + SyncTerm privateTerm; uint32_t dataLen; char data[]; } SyncAppendEntries; @@ -332,6 +333,7 @@ typedef struct SyncAppendEntriesReply { SRaftId destId; // private data SyncTerm term; + SyncTerm privateTerm; bool success; SyncIndex matchIndex; } SyncAppendEntriesReply; @@ -385,6 +387,75 @@ void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg); void syncApplyMsgLog(const SyncApplyMsg* pMsg); void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg); +// --------------------------------------------- +typedef struct SyncSnapshotSend { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + SyncTerm term; + SyncIndex lastIndex; // lastIndex of snapshot + SyncTerm lastTerm; // lastTerm of snapshot + SyncTerm privateTerm; + int32_t seq; + uint32_t dataLen; + char data[]; +} SyncSnapshotSend; + +SyncSnapshotSend* syncSnapshotSendBuild(uint32_t dataLen, int32_t vgId); +void syncSnapshotSendDestroy(SyncSnapshotSend* pMsg); +void syncSnapshotSendSerialize(const SyncSnapshotSend* pMsg, char* buf, uint32_t bufLen); +void syncSnapshotSendDeserialize(const char* buf, uint32_t len, SyncSnapshotSend* pMsg); +char* syncSnapshotSendSerialize2(const SyncSnapshotSend* pMsg, uint32_t* len); +SyncSnapshotSend* syncSnapshotSendDeserialize2(const char* buf, uint32_t len); +void syncSnapshotSend2RpcMsg(const SyncSnapshotSend* pMsg, SRpcMsg* pRpcMsg); +void syncSnapshotSendFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotSend* pMsg); +SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg); +char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg); + +// for debug ---------------------- +void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg); +void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg); +void syncSnapshotSendLog(const SyncSnapshotSend* pMsg); +void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg); + +// --------------------------------------------- +typedef struct SyncSnapshotRsp { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + SyncTerm term; + SyncIndex lastIndex; + SyncTerm lastTerm; + SyncTerm privateTerm; + int32_t ack; + int32_t code; +} SyncSnapshotRsp; + +SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId); +void syncSnapshotRspDestroy(SyncSnapshotRsp* pMsg); +void syncSnapshotRspSerialize(const SyncSnapshotRsp* pMsg, char* buf, uint32_t bufLen); +void syncSnapshotRspDeserialize(const char* buf, uint32_t len, SyncSnapshotRsp* pMsg); +char* syncSnapshotRspSerialize2(const SyncSnapshotRsp* pMsg, uint32_t* len); +SyncSnapshotRsp* syncSnapshotRspDeserialize2(const char* buf, uint32_t len); +void syncSnapshotRsp2RpcMsg(const SyncSnapshotRsp* pMsg, SRpcMsg* pRpcMsg); +void syncSnapshotRspFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotRsp* pMsg); +SyncSnapshotRsp* syncSnapshotRspFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg); +char* syncSnapshotRsp2Str(const SyncSnapshotRsp* pMsg); + +// for debug ---------------------- +void syncSnapshotRspPrint(const SyncSnapshotRsp* pMsg); +void syncSnapshotRspPrint2(char* s, const SyncSnapshotRsp* pMsg); +void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg); +void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg); + // on message ---------------------- int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); @@ -395,6 +466,29 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg); +int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); +int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg); +int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); + +int32_t syncNodeOnSnapshotSendCb(SSyncNode* ths, SyncSnapshotSend* pMsg); +int32_t syncNodeOnSnapshotRspCb(SSyncNode* ths, SyncSnapshotRsp* pMsg); + +// ----------------------------------------- +typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); +typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); +typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg); +typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg); +typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg); +typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); +typedef int32_t (*FpOnAppendEntriesReplyCb)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +typedef int32_t (*FpOnTimeoutCb)(SSyncNode* pSyncNode, SyncTimeout* pMsg); +typedef int32_t (*FpOnSnapshotSendCb)(SSyncNode* ths, SyncSnapshotSend* pMsg); +typedef int32_t (*FpOnSnapshotRspCb)(SSyncNode* ths, SyncSnapshotRsp* pMsg); + +// option ---------------------------------- +bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); + // --------------------------------------------- #ifdef __cplusplus diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 95af8ac306..c7d1ccd3de 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -141,6 +141,8 @@ typedef struct SWal { // ctl int64_t refId; TdThreadMutex mutex; + // ref + SHashObj *pRefHash; // ref -> SWalRef // path char path[WAL_PATH_LEN]; // reusable write head @@ -184,7 +186,7 @@ int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely int32_t walBeginSnapshot(SWal *, int64_t ver); int32_t walEndSnapshot(SWal *); -void walRestoreFromSnapshot(SWal *, int64_t ver); +int32_t walRestoreFromSnapshot(SWal *, int64_t ver); // int32_t walDataCorrupted(SWal*); // read @@ -199,6 +201,16 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead); int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead); int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead); +typedef struct { + int64_t refId; + int64_t ver; +} SWalRef; + +SWalRef *walOpenRef(SWal *); +void walCloseRef(SWalRef *); +int32_t walRefVer(SWalRef *, int64_t ver); +int32_t walUnrefVer(SWal *); + // deprecated #if 0 int32_t walRead(SWal *, SWalHead **, int64_t ver); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5d6425264d..aa679b9414 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1779,6 +1779,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } // assign data + // TODO ret = taosMemoryCalloc(1, cap + 46); ret = POINTER_SHIFT(ret, 46); ret->header.vgId = vgId; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 6c2783cb5c..264dc74e36 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -215,8 +215,12 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; @@ -226,9 +230,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 42fa7b718e..53943b61b0 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -71,7 +71,7 @@ static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { } static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); + dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 7cd4ddd713..1540f10ba4 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -360,6 +360,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 6661347e42..4869a19856 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -19,6 +19,7 @@ #include "mndDef.h" #include "sdb.h" +#include "sync.h" #include "syncTools.h" #include "tcache.h" #include "tdatablock.h" @@ -75,7 +76,6 @@ typedef struct { } STelemMgmt; typedef struct { - SWal *pWal; sem_t syncSem; int64_t sync; bool standby; @@ -108,6 +108,7 @@ typedef struct SMnode { SQHandle *pQuery; SHashObj *infosMeta; SHashObj *perfsMeta; + SWal *pWal; SShowMgmt showMgmt; SProfileMgmt profileMgmt; STelemMgmt telemMgmt; diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 89f93d30b5..c50279889e 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -41,6 +41,7 @@ int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pV int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo); int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray); int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId); +int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray); void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby); void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 61d6e7be6c..e6c93a9bfd 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -636,57 +636,6 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { - if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) { - return -1; - } - } else { - SVgObj newVgroup = {0}; - memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); - mndTransSetSerial(pTrans); - - if (newVgroup.replica < pDb->cfg.replications) { - mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId, - pVgroup->vnodeGid[0].dnodeId); - - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - } else { - mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); - - SVnodeGid del1 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - - SVnodeGid del2 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - } - - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { - sdbFreeRaw(pVgRaw); - return -1; - } - sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); - } - - return 0; -} - static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 7be4939337..3fab870277 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -566,9 +566,11 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM pRaw = NULL; if (pMObj != NULL) { + mDebug("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id); if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER; } if (numOfVnodes > 0) { + mDebug("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id); if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER; } if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 69300c0889..813e4c30b5 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -139,10 +139,40 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) { return 0; } +static int32_t mndInitWal(SMnode *pMnode) { + char path[PATH_MAX + 20] = {0}; + snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); + SWalCfg cfg = { + .vgId = 1, + .fsyncPeriod = 0, + .rollPeriod = -1, + .segSize = -1, + .retentionPeriod = -1, + .retentionSize = -1, + .level = TAOS_WAL_FSYNC, + }; + + pMnode->pWal = walOpen(path, &cfg); + if (pMnode->pWal == NULL) { + mError("failed to open wal since %s", terrstr()); + return -1; + } + + return 0; +} + +static void mndCloseWal(SMnode *pMnode) { + if (pMnode->pWal != NULL) { + walClose(pMnode->pWal); + pMnode->pWal = NULL; + } +} + static int32_t mndInitSdb(SMnode *pMnode) { SSdbOpt opt = {0}; opt.path = pMnode->path; opt.pMnode = pMnode; + opt.pWal = pMnode->pWal; pMnode->pSdb = sdbInit(&opt); if (pMnode->pSdb == NULL) { @@ -156,7 +186,6 @@ static int32_t mndOpenSdb(SMnode *pMnode) { if (!pMnode->deploy) { return sdbReadFile(pMnode->pSdb); } else { - // return sdbDeploy(pMnode->pSdb);; return 0; } } @@ -182,6 +211,7 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle } static int32_t mndInitSteps(SMnode *pMnode) { + if (mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; @@ -201,7 +231,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-stb", mndInitSma, mndCleanupSma) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1; if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; @@ -376,41 +406,93 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncRpcMsgLog2(logBuf, pMsg); taosMemoryFree(syncNodeStr); - if (pMsg->msgType == TDMT_SYNC_TIMEOUT) { - SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); - code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); - syncTimeoutDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_SYNC_PING) { - SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); - code = syncNodeOnPingCb(pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) { - SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); - code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); - syncPingReplyDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); - syncClientRequestDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { - SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); - code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); - code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { - SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); - code = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); - code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); + // ToDo: ugly! use function pointer + if (syncNodeSnapshotEnable(pSyncNode)) { + if (pMsg->msgType == TDMT_SYNC_TIMEOUT) { + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); + code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_PING) { + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); + code = syncNodeOnPingCb(pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) { + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); + code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); + code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); + + } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteSnapshotCb(pSyncNode, pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteReplySnapshotCb(pSyncNode, pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesSnapshotCb(pSyncNode, pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesReplySnapshotCb(pSyncNode, pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + + } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotSendCb(pSyncNode, pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotRspCb(pSyncNode, pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + + } else { + mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + } + } else { - mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + if (pMsg->msgType == TDMT_SYNC_TIMEOUT) { + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); + code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_PING) { + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); + code = syncNodeOnPingCb(pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) { + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); + code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); + code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + } else { + mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + } } mndReleaseSyncRef(pMnode); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index ce025d5547..a0daa72d9a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -17,15 +17,27 @@ #include "mndSync.h" #include "mndTrans.h" -int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { +static int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { SMsgHead *pHead = pMsg->pCont; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); - return tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); + int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; } -int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return tmsgSendReq(pEpSet, pMsg); } +static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { + int32_t code = tmsgSendReq(pEpSet, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; +} void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SMnode *pMnode = pFsm->data; @@ -34,7 +46,7 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); pMgmt->errCode = cbMeta.code; - mTrace("trans:%d, is proposed, savedTransId:%d code:0x%x, ver:%" PRId64 " term:%" PRId64 " role:%s raw:%p", transId, + mDebug("trans:%d, is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64 " role:%s raw:%p", transId, pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term, syncStr(cbMeta.state), pRaw); if (pMgmt->errCode == 0) { @@ -50,6 +62,10 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM tsem_post(&pMgmt->syncSem); } else { if (cbMeta.index - sdbGetApplyIndex(pMnode->pSdb) > 100) { + SSnapshotMeta sMeta = {0}; + if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); + } sdbWriteFile(pMnode->pSdb); } } @@ -57,13 +73,20 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM int32_t mndSyncGetSnapshot(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { SMnode *pMnode = pFsm->data; - pSnapshot->lastApplyIndex = sdbGetApplyIndex(pMnode->pSdb); - pSnapshot->lastApplyTerm = sdbGetApplyTerm(pMnode->pSdb); + pSnapshot->lastApplyIndex = sdbGetCommitIndex(pMnode->pSdb); + pSnapshot->lastApplyTerm = sdbGetCommitTerm(pMnode->pSdb); + pSnapshot->lastConfigIndex = sdbGetCurConfig(pMnode->pSdb); return 0; } void mndRestoreFinish(struct SSyncFSM *pFsm) { SMnode *pMnode = pFsm->data; + + SSnapshotMeta sMeta = {0}; + if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); + } + if (!pMnode->deploy) { mInfo("mnode sync restore finished, and will handle outstanding transactions"); mndTransPullup(pMnode); @@ -78,8 +101,8 @@ void mndReConfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) SSyncMgmt *pMgmt = &pMnode->syncMgmt; pMgmt->errCode = cbMeta.code; - mInfo("trans:-1, sync reconfig is proposed, savedTransId:%d code:0x%x, curTerm:%" PRId64 " term:%" PRId64, - pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term); + mInfo("trans:-1, sync reconfig is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64, pMgmt->transId, + cbMeta.code, cbMeta.index, cbMeta.term); if (pMgmt->transId == -1) { if (pMgmt->errCode != 0) { @@ -144,29 +167,12 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { int32_t mndInitSync(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); - SWalCfg cfg = { - .vgId = 1, - .fsyncPeriod = 0, - .rollPeriod = -1, - .segSize = -1, - .retentionPeriod = -1, - .retentionSize = -1, - .level = TAOS_WAL_FSYNC, - }; - - pMgmt->pWal = walOpen(path, &cfg); - if (pMgmt->pWal == NULL) { - mError("failed to open wal since %s", terrstr()); - return -1; - } - SSyncInfo syncInfo = {.vgId = 1, .FpSendMsg = mndSyncSendMsg, .FpEqMsg = mndSyncEqMsg}; snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP); - syncInfo.pWal = pMgmt->pWal; + syncInfo.pWal = pMnode->pWal; syncInfo.pFsm = mndSyncMakeFsm(pMnode); syncInfo.isStandBy = pMgmt->standby; + syncInfo.snapshotEnable = true; SSyncCfg *pCfg = &syncInfo.syncCfg; pCfg->replicaNum = pMnode->replica; @@ -196,10 +202,6 @@ void mndCleanupSync(SMnode *pMnode) { mDebug("mnode sync is stopped, id:%" PRId64, pMgmt->sync); tsem_destroy(&pMgmt->syncSem); - if (pMgmt->pWal != NULL) { - walClose(pMgmt->pWal); - } - memset(pMgmt, 0, sizeof(SSyncMgmt)); } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c2097c069b..1e98a3bbf9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -922,7 +922,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio char detail[1024] = {0}; int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(pAction->msgType), pAction->epSet.numOfEps, pAction->epSet.inUse); - for (int32_t i = 0; i < pTrans->lastErrorEpset.numOfEps; ++i) { + for (int32_t i = 0; i < pAction->epSet.numOfEps; ++i) { len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, pAction->epSet.eps[i].fqdn, pAction->epSet.eps[i].port); } @@ -1085,6 +1085,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } if (code == 0) { + if (!pMnode->deploy && !mndIsMaster(pMnode)) break; + pTrans->code = 0; pTrans->redoActionPos++; mDebug("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage), @@ -1386,6 +1388,10 @@ void mndTransPullup(SMnode *pMnode) { mndReleaseTrans(pMnode, pTrans); } + SSnapshotMeta sMeta = {0}; + if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); + } sdbWriteFile(pMnode->pSdb); taosArrayDestroy(pArray); } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 5244bc657b..76e65ddd92 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -55,6 +55,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIRM_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_HASHRANGE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); @@ -1166,7 +1167,155 @@ _OVER: return code; } -static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { return 0; } +int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { + if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) { + return -1; + } + } else { + SVgObj newVgroup = {0}; + memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); + mndTransSetSerial(pTrans); + + if (newVgroup.replica < pDb->cfg.replications) { + mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId, + pVgroup->vnodeGid[0].dnodeId); + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + } else if (newVgroup.replica > pDb->cfg.replications) { + mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); + + SVnodeGid del1 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + + SVnodeGid del2 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + } else { + } + + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + } + + return 0; +} + +static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + return 0; +} + +static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { + int32_t code = -1; + SSdbRaw *pRaw = NULL; + STrans *pTrans = NULL; + SArray *pArray = mndBuildDnodesArray(pMnode, 0); + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); + mDebug("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId); + + SVgObj newVg1 = {0}; + memcpy(&newVg1, pVgroup, sizeof(SVgObj)); + mInfo("vgId:%d, vgroup info before split, replica:%d hashBegin:%u hashEnd:%u", newVg1.vgId, newVg1.replica, + newVg1.hashBegin, newVg1.hashEnd); + for (int32_t i = 0; i < newVg1.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId); + } + + if (newVg1.replica == 1) { + if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1], true) != 0) goto _OVER; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_REPLICA) != 0) goto _OVER; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; + } else if (newVg1.replica == 3) { + SVnodeGid del1 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_REPLICA) != 0) goto _OVER; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; + } else { + goto _OVER; + } + + SVgObj newVg2 = {0}; + memcpy(&newVg1, &newVg2, sizeof(SVgObj)); + newVg1.replica = 1; + newVg1.hashEnd = (newVg1.hashBegin + newVg1.hashEnd) / 2; + memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid)); + + newVg2.replica = 1; + newVg2.hashBegin = newVg1.hashEnd + 1; + memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid)); + memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid)); + + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg2, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER; + + // adjust vgroup + if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg1, pArray) != 0) goto _OVER; + if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg2, pArray) != 0) goto _OVER; + +_OVER: + mndTransDrop(pTrans); + sdbFreeRaw(pRaw); + return code; +} + +static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + int32_t vgId = 2; + SUserObj *pUser = NULL; + SVgObj *pVgroup = NULL; + SDbObj *pDb = NULL; + + mDebug("vgId:%d, start to split", vgId); + + pVgroup = mndAcquireVgroup(pMnode, vgId); + if (pVgroup == NULL) goto _OVER; + + pDb = mndAcquireDb(pMnode, pVgroup->dbName); + if (pDb == NULL) goto _OVER; + + pUser = mndAcquireUser(pMnode, pReq->conn.user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto _OVER; + } + + if (mndCheckNodeAuth(pUser) != 0) { + goto _OVER; + } + + code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup); + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + +_OVER: + mndReleaseUser(pMnode, pUser); + mndReleaseVgroup(pMnode, pVgroup); + mndReleaseDb(pMnode, pDb); + return code; +} static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pSrc, SDnodeObj *pDst) { diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 9444543804..8536c451b7 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -22,6 +22,7 @@ #include "tlockfree.h" #include "tlog.h" #include "tmsg.h" +#include "wal.h" #ifdef __cplusplus extern "C" { @@ -165,12 +166,14 @@ typedef struct SSdbRow { typedef struct SSdb { SMnode *pMnode; + SWal *pWal; char *currDir; char *tmpDir; int64_t lastCommitVer; int64_t lastCommitTerm; int64_t curVer; int64_t curTerm; + int64_t curConfig; int64_t tableVer[SDB_MAX]; int64_t maxId[SDB_MAX]; EKeyType keyTypes[SDB_MAX]; @@ -205,6 +208,7 @@ typedef struct { typedef struct SSdbOpt { const char *path; SMnode *pMnode; + SWal *pWal; } SSdbOpt; /** @@ -358,9 +362,13 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type); * @return int32_t The current index of sdb */ void sdbSetApplyIndex(SSdb *pSdb, int64_t index); -int64_t sdbGetApplyIndex(SSdb *pSdb); void sdbSetApplyTerm(SSdb *pSdb, int64_t term); +void sdbSetCurConfig(SSdb *pSdb, int64_t config); +int64_t sdbGetApplyIndex(SSdb *pSdb); int64_t sdbGetApplyTerm(SSdb *pSdb); +int64_t sdbGetCommitIndex(SSdb *pSdb); +int64_t sdbGetCommitTerm(SSdb *pSdb); +int64_t sdbGetCurConfig(SSdb *pSdb); SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); void sdbFreeRaw(SSdbRaw *pRaw); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 0526ea5c2d..c11aaaaa8a 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -52,10 +52,12 @@ SSdb *sdbInit(SSdbOpt *pOption) { pSdb->keyTypes[i] = SDB_KEY_INT32; } + pSdb->pWal = pOption->pWal; pSdb->curVer = -1; pSdb->curTerm = -1; pSdb->lastCommitVer = -1; pSdb->lastCommitTerm = -1; + pSdb->curConfig = -1; pSdb->pMnode = pOption->pMnode; taosThreadMutexInit(&pSdb->filelock, NULL); mDebug("sdb init successfully"); @@ -159,8 +161,16 @@ static int32_t sdbCreateDir(SSdb *pSdb) { void sdbSetApplyIndex(SSdb *pSdb, int64_t index) { pSdb->curVer = index; } -int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; } - void sdbSetApplyTerm(SSdb *pSdb, int64_t term) { pSdb->curTerm = term; } +void sdbSetCurConfig(SSdb *pSdb, int64_t config) { pSdb->curConfig = config; } + +int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; } + int64_t sdbGetApplyTerm(SSdb *pSdb) { return pSdb->curTerm; } + +int64_t sdbGetCommitIndex(SSdb *pSdb) { return pSdb->lastCommitVer; } + +int64_t sdbGetCommitTerm(SSdb *pSdb) { return pSdb->lastCommitTerm; } + +int64_t sdbGetCurConfig(SSdb *pSdb) { return pSdb->curConfig; } \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 83135491a9..f98ecf5343 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -110,6 +110,16 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { return -1; } + ret = taosReadFile(pFile, &pSdb->curConfig, sizeof(int64_t)); + if (ret < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (ret != sizeof(int64_t)) { + terrno = TSDB_CODE_FILE_CORRUPTED; + return -1; + } + for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = 0; ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); @@ -173,6 +183,11 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { return -1; } + if (taosWriteFile(pFile, &pSdb->curConfig, sizeof(int64_t)) != sizeof(int64_t)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = 0; if (i < SDB_MAX) { @@ -288,8 +303,8 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { pSdb->lastCommitVer = pSdb->curVer; pSdb->lastCommitTerm = pSdb->curTerm; memcpy(pSdb->tableVer, tableVer, sizeof(tableVer)); - mDebug("read sdb file:%s successfully, ver:%" PRId64 " term:%" PRId64, file, pSdb->lastCommitVer, - pSdb->lastCommitTerm); + mDebug("read sdb file:%s successfully, index:%" PRId64 " term:%" PRId64 " config:%" PRId64, file, pSdb->lastCommitVer, + pSdb->lastCommitTerm, pSdb->curConfig); _OVER: taosCloseFile(&pFile); @@ -426,12 +441,23 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } int32_t sdbWriteFile(SSdb *pSdb) { + int32_t code = 0; if (pSdb->curVer == pSdb->lastCommitVer) { return 0; } taosThreadMutexLock(&pSdb->filelock); - int32_t code = sdbWriteFileImp(pSdb); + if (pSdb->pWal != NULL) { + code = walBeginSnapshot(pSdb->pWal, pSdb->curVer); + } + if (code == 0) { + code = sdbWriteFileImp(pSdb); + } + if (code == 0) { + if (pSdb->pWal != NULL) { + code = walEndSnapshot(pSdb->pWal); + } + } if (code != 0) { mError("failed to write sdb file since %s", terrstr()); } @@ -496,6 +522,9 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) { snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); taosThreadMutexLock(&pSdb->filelock); + int64_t commitIndex = pSdb->lastCommitVer; + int64_t commitTerm = pSdb->lastCommitTerm; + int64_t curConfig = pSdb->curConfig; if (taosCopyFile(datafile, pIter->name) < 0) { taosThreadMutexUnlock(&pSdb->filelock); terrno = TAOS_SYSTEM_ERROR(errno); @@ -514,7 +543,8 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) { } *ppIter = pIter; - mInfo("sdbiter:%p, is created to read snapshot, file:%s", pIter, pIter->name); + mInfo("sdbiter:%p, is created to read snapshot, index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", pIter, + commitIndex, commitTerm, curConfig, pIter->name); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 55ab5f5729..eb489a6d81 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -24,6 +24,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRpcMsg *pRsp); static int32_t vnodeProcessExpWndsClrReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -164,6 +165,9 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_ALTER_CONFIRM: vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); break; + case TDMT_VND_ALTER_HASHRANGE: + vnodeProcessAlterHasnRangeReq(pVnode, version, pReq, len, pRsp); + break; case TDMT_VND_ALTER_CONFIG: break; default: @@ -933,4 +937,14 @@ _err: vError("vgId:%d, success to process expWnds clear for tsma %" PRIi64 " version %" PRIi64 " since %s", TD_VID(pVnode), req.indexUid, req.version, terrstr()); return -1; +} + +static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode)); + + // todo + // 1. stop work + // 2. adjust hash range / compact / remove wals / rename vgroups + // 3. reload sync + return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 203eecd8ab..816c0cfac9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -252,6 +252,8 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncInfo syncInfo = { + .isStandBy = false, + .snapshotEnable = false, .vgId = pVnode->config.vgId, .isStandBy = pVnode->config.standby, .syncCfg = pVnode->config.syncCfg, diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 6c95eb987b..8b5885d58b 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -62,25 +62,25 @@ typedef struct CacheTerm { } CacheTerm; // -IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type); +IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type); -void indexCacheForceToMerge(void* cache); -void indexCacheDestroy(void* cache); -void indexCacheBroadcast(void* cache); -void indexCacheWait(void* cache); +void idxCacheForceToMerge(void* cache); +void idxCacheDestroy(void* cache); +void idxCacheBroadcast(void* cache); +void idxCacheWait(void* cache); -Iterate* indexCacheIteratorCreate(IndexCache* cache); +Iterate* idxCacheIteratorCreate(IndexCache* cache); void idxCacheIteratorDestroy(Iterate* iiter); -int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid); +int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid); // int indexCacheGet(void *cache, uint64_t *rst); -int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s); +int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s); -void indexCacheRef(IndexCache* cache); -void indexCacheUnRef(IndexCache* cache); +void idxCacheRef(IndexCache* cache); +void idxCacheUnRef(IndexCache* cache); -void indexCacheDebug(IndexCache* cache); +void idxCacheDebug(IndexCache* cache); void idxCacheDestroyImm(IndexCache* cache); #ifdef __cplusplus diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index bccba98116..09fd2f3555 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -34,11 +34,11 @@ typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); -__compar_fn_t indexGetCompar(int8_t type); +__compar_fn_t idxGetCompar(int8_t type); TExeCond tCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b, int8_t dType); TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b); -_cache_range_compare indexGetCompare(RangeType ty); +_cache_range_compare idxGetCompare(RangeType ty); int32_t idxConvertData(void* src, int8_t type, void** dst); int32_t idxConvertDataToStr(void* src, int8_t type, void** dst); diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 47f7260d3a..906cbb6a20 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -133,24 +133,24 @@ typedef struct TFileCacheKey { } ICacheKey; int idxFlushCacheToTFile(SIndex* sIdx, void*, bool quit); -int64_t indexAddRef(void* p); -int32_t indexRemoveRef(int64_t ref); -void indexAcquireRef(int64_t ref); -void indexReleaseRef(int64_t ref); +int64_t idxAddRef(void* p); +int32_t idxRemoveRef(int64_t ref); +void idxAcquireRef(int64_t ref); +void idxReleaseRef(int64_t ref); -int32_t indexSerialCacheKey(ICacheKey* key, char* buf); +int32_t idxSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); -#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) +#define IDX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) -#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) +#define IDX_TYPE_GET_TYPE(ty) (ty & 0x0F) -#define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ - do { \ - uint8_t oldTy = ty; \ - ty = (ty >> 4) | exTy; \ - ty = (ty << 4) | oldTy; \ +#define IDX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ + do { \ + uint8_t oldTy = ty; \ + ty = (ty >> 4) | exTy; \ + ty = (ty << 4) | oldTy; \ } while (0) #ifdef __cplusplus diff --git a/source/libs/index/inc/indexTfile.h b/source/libs/index/inc/indexTfile.h index ca55aa93da..6cfea5bc0b 100644 --- a/source/libs/index/inc/indexTfile.h +++ b/source/libs/index/inc/indexTfile.h @@ -117,10 +117,10 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order); int tfileWriterFinish(TFileWriter* tw); // -IndexTFile* indexTFileCreate(const char* path); -void indexTFileDestroy(IndexTFile* tfile); -int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); -int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* tr); +IndexTFile* idxTFileCreate(const char* path); +void idxTFileDestroy(IndexTFile* tfile); +int idxTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); +int idxTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* tr); Iterate* tfileIteratorCreate(TFileReader* reader); void tfileIteratorDestroy(Iterate* iterator); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9b8bee5623..04d7e04b30 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -90,7 +90,7 @@ static void idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateV // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); -static void indexPost(void* idx) { +static void idxPost(void* idx) { SIndex* pIdx = idx; tsem_post(&pIdx->sem); } @@ -106,8 +106,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { return -1; } - // sIdx->cache = (void*)indexCacheCreate(sIdx); - sIdx->tindex = indexTFileCreate(path); + // sIdx->cache = (void*)idxCacheCreate(sIdx); + sIdx->tindex = idxTFileCreate(path); if (sIdx->tindex == NULL) { goto END; } @@ -118,8 +118,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadMutexInit(&sIdx->mtx, NULL); tsem_init(&sIdx->sem, 0, 0); - sIdx->refId = indexAddRef(sIdx); - indexAcquireRef(sIdx->refId); + sIdx->refId = idxAddRef(sIdx); + idxAcquireRef(sIdx->refId); *index = sIdx; return 0; @@ -136,7 +136,7 @@ void indexDestroy(void* handle) { SIndex* sIdx = handle; taosThreadMutexDestroy(&sIdx->mtx); tsem_destroy(&sIdx->sem); - indexTFileDestroy(sIdx->tindex); + idxTFileDestroy(sIdx->tindex); taosMemoryFree(sIdx->path); taosMemoryFree(sIdx); return; @@ -147,33 +147,33 @@ void indexClose(SIndex* sIdx) { void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - indexCacheForceToMerge((void*)(*pCache)); + idxCacheForceToMerge((void*)(*pCache)); indexInfo("%s wait to merge", (*pCache)->colName); indexWait((void*)(sIdx)); indexInfo("%s finish to wait", (*pCache)->colName); iter = taosHashIterate(sIdx->colObj, iter); - indexCacheUnRef(*pCache); + idxCacheUnRef(*pCache); } taosHashCleanup(sIdx->colObj); sIdx->colObj = NULL; } - indexReleaseRef(sIdx->refId); - indexRemoveRef(sIdx->refId); + idxReleaseRef(sIdx->refId); + idxRemoveRef(sIdx->refId); } -int64_t indexAddRef(void* p) { +int64_t idxAddRef(void* p) { // impl return taosAddRef(indexRefMgt, p); } -int32_t indexRemoveRef(int64_t ref) { +int32_t idxRemoveRef(int64_t ref) { // impl later return taosRemoveRef(indexRefMgt, ref); } -void indexAcquireRef(int64_t ref) { +void idxAcquireRef(int64_t ref) { // impl taosAcquireRef(indexRefMgt, ref); } -void indexReleaseRef(int64_t ref) { +void idxReleaseRef(int64_t ref) { // impl taosReleaseRef(indexRefMgt, ref); } @@ -186,11 +186,11 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { char buf[128] = {0}; ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; - int32_t sz = indexSerialCacheKey(&key, buf); + int32_t sz = idxSerialCacheKey(&key, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { - IndexCache* pCache = indexCacheCreate(index, p->suid, p->colName, p->colType); + IndexCache* pCache = idxCacheCreate(index, p->suid, p->colName, p->colType); taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } @@ -201,12 +201,12 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { char buf[128] = {0}; ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; - int32_t sz = indexSerialCacheKey(&key, buf); + int32_t sz = idxSerialCacheKey(&key, buf); indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); - int ret = indexCachePut(*cache, p, uid); + int ret = idxCachePut(*cache, p, uid); if (ret != 0) { return ret; } @@ -289,7 +289,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy tm->nColName = nColName; char* buf = NULL; - int32_t len = idxConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf); + int32_t len = idxConvertDataToStr((void*)colVal, IDX_TYPE_GET_TYPE(colType), (void**)&buf); assert(len != -1); tm->colVal = buf; @@ -331,7 +331,7 @@ static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) ICacheKey key = { .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); - int32_t sz = indexSerialCacheKey(&key, buf); + int32_t sz = idxSerialCacheKey(&key, buf); taosThreadMutexLock(&sIdx->mtx); IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); @@ -345,14 +345,14 @@ static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) int64_t st = taosGetTimestampUs(); SIdxTRslt* tr = idxTRsltCreate(); - if (0 == indexCacheSearch(cache, query, tr, &s)) { + if (0 == idxCacheSearch(cache, query, tr, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by", term->colName); // coloum already drop by other oper, no need to query tindex return 0; } else { st = taosGetTimestampUs(); - if (0 != indexTFileSearch(sIdx->tindex, query, tr)) { + if (0 != idxTFileSearch(sIdx->tindex, query, tr)) { indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); goto END; } @@ -465,23 +465,23 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { IndexCache* pCache = (IndexCache*)cache; - while (quit && atomic_load_32(&pCache->merging) == 1) { - } + while (quit && atomic_load_32(&pCache->merging) == 1) + ; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); if (pReader == NULL) { indexWarn("empty tfile reader found"); } // handle flush - Iterate* cacheIter = indexCacheIteratorCreate(pCache); + Iterate* cacheIter = idxCacheIteratorCreate(pCache); if (cacheIter == NULL) { indexError("%p immtable is empty, ignore merge opera", pCache); idxCacheDestroyImm(pCache); tfileReaderUnRef(pReader); atomic_store_32(&pCache->merging, 0); if (quit) { - indexPost(sIdx); + idxPost(sIdx); } - indexReleaseRef(sIdx->refId); + idxReleaseRef(sIdx->refId); return 0; } @@ -532,7 +532,7 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { tfileIteratorDestroy(tfileIter); tfileReaderUnRef(pReader); - indexCacheUnRef(pCache); + idxCacheUnRef(pCache); int64_t cost = taosGetTimestampUs() - st; if (ret != 0) { @@ -542,9 +542,9 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { } atomic_store_32(&pCache->merging, 0); if (quit) { - indexPost(sIdx); + idxPost(sIdx); } - indexReleaseRef(sIdx->refId); + idxReleaseRef(sIdx->refId); return ret; } @@ -561,7 +561,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { value->colVal = NULL; } -static int64_t indexGetAvaialbleVer(SIndex* sIdx, IndexCache* cache) { +static int64_t idxGetAvailableVer(SIndex* sIdx, IndexCache* cache) { ICacheKey key = {.suid = cache->suid, .colName = cache->colName, .nColName = strlen(cache->colName)}; int64_t ver = CACHE_VERSION(cache); @@ -579,7 +579,7 @@ static int64_t indexGetAvaialbleVer(SIndex* sIdx, IndexCache* cache) { return ver; } static int idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { - int64_t version = indexGetAvaialbleVer(sIdx, cache); + int64_t version = idxGetAvailableVer(sIdx, cache); indexInfo("file name version: %" PRId64 "", version); uint8_t colType = cache->type; @@ -620,8 +620,8 @@ END: return -1; } -int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); +int32_t idxSerialCacheKey(ICacheKey* key, char* buf) { + bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); char* p = buf; char tbuf[65] = {0}; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 20cd9c8b4c..040e8ed830 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -68,7 +68,7 @@ static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTRsl cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON, cacheSearchRange_JSON}}; -static void doMergeWork(SSchedMsg* msg); +static void idxDoMergeWork(SSchedMsg* msg); static bool idxCacheIteratorNext(Iterate* itera); static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { @@ -127,7 +127,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* MemTable* mem = cache; IndexCache* pCache = mem->pCache; - _cache_range_compare cmpFn = indexGetCompare(type); + _cache_range_compare cmpFn = idxGetCompare(type); CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); pCt->colVal = term->colVal; @@ -187,7 +187,7 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr pCt->version = atomic_load_64(&pCache->version); char* exBuf = NULL; - if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { exBuf = idxPackJsonData(term); pCt->colVal = exBuf; } @@ -257,7 +257,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR if (cache == NULL) { return 0; } - _cache_range_compare cmpFn = indexGetCompare(type); + _cache_range_compare cmpFn = idxGetCompare(type); MemTable* mem = cache; IndexCache* pCache = mem->pCache; @@ -266,7 +266,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR pCt->colVal = term->colVal; pCt->version = atomic_load_64(&pCache->version); - int8_t dType = INDEX_TYPE_GET_TYPE(term->colType); + int8_t dType = IDX_TYPE_GET_TYPE(term->colType); int skip = 0; char* exBuf = NULL; if (type == CONTAINS) { @@ -331,9 +331,9 @@ static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTRslt* tr, ST // impl later return 0; } -static IterateValue* indexCacheIteratorGetValue(Iterate* iter); +static IterateValue* idxCacheIteratorGetValue(Iterate* iter); -IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { +IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { IndexCache* cache = taosMemoryCalloc(1, sizeof(IndexCache)); if (cache == NULL) { indexError("failed to create index cache"); @@ -342,7 +342,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in cache->mem = idxInternalCacheCreate(type); cache->mem->pCache = cache; - cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); + cache->colName = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; cache->index = idx; cache->version = 0; @@ -352,13 +352,13 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in taosThreadMutexInit(&cache->mtx, NULL); taosThreadCondInit(&cache->finished, NULL); - indexCacheRef(cache); + idxCacheRef(cache); if (idx != NULL) { - indexAcquireRef(idx->refId); + idxAcquireRef(idx->refId); } return cache; } -void indexCacheDebug(IndexCache* cache) { +void idxCacheDebug(IndexCache* cache) { MemTable* tbl = NULL; taosThreadMutexLock(&cache->mtx); @@ -405,7 +405,7 @@ void indexCacheDebug(IndexCache* cache) { } } -void indexCacheDestroySkiplist(SSkipList* slt) { +void idxCacheDestroySkiplist(SSkipList* slt) { SSkipListIterator* iter = tSkipListCreateIter(slt); while (iter != NULL && tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); @@ -418,11 +418,11 @@ void indexCacheDestroySkiplist(SSkipList* slt) { tSkipListDestroyIter(iter); tSkipListDestroy(slt); } -void indexCacheBroadcast(void* cache) { +void idxCacheBroadcast(void* cache) { IndexCache* pCache = cache; taosThreadCondBroadcast(&pCache->finished); } -void indexCacheWait(void* cache) { +void idxCacheWait(void* cache) { IndexCache* pCache = cache; taosThreadCondWait(&pCache->finished, &pCache->mtx); } @@ -435,14 +435,14 @@ void idxCacheDestroyImm(IndexCache* cache) { tbl = cache->imm; cache->imm = NULL; // or throw int bg thread - indexCacheBroadcast(cache); + idxCacheBroadcast(cache); taosThreadMutexUnlock(&cache->mtx); idxMemUnRef(tbl); idxMemUnRef(tbl); } -void indexCacheDestroy(void* cache) { +void idxCacheDestroy(void* cache) { IndexCache* pCache = cache; if (pCache == NULL) { return; @@ -455,12 +455,12 @@ void indexCacheDestroy(void* cache) { taosThreadMutexDestroy(&pCache->mtx); taosThreadCondDestroy(&pCache->finished); if (pCache->index != NULL) { - indexReleaseRef(((SIndex*)pCache->index)->refId); + idxReleaseRef(((SIndex*)pCache->index)->refId); } taosMemoryFree(pCache); } -Iterate* indexCacheIteratorCreate(IndexCache* cache) { +Iterate* idxCacheIteratorCreate(IndexCache* cache) { if (cache->imm == NULL) { return NULL; } @@ -477,7 +477,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { iiter->val.colVal = NULL; iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL; iiter->next = idxCacheIteratorNext; - iiter->getValue = indexCacheIteratorGetValue; + iiter->getValue = idxCacheIteratorGetValue; taosThreadMutexUnlock(&cache->mtx); @@ -492,30 +492,30 @@ void idxCacheIteratorDestroy(Iterate* iter) { taosMemoryFree(iter); } -int indexCacheSchedToMerge(IndexCache* pCache, bool notify) { +int idxCacheSchedToMerge(IndexCache* pCache, bool notify) { SSchedMsg schedMsg = {0}; - schedMsg.fp = doMergeWork; + schedMsg.fp = idxDoMergeWork; schedMsg.ahandle = pCache; if (notify) { schedMsg.thandle = taosMemoryMalloc(1); } schedMsg.msg = NULL; - indexAcquireRef(pCache->index->refId); + idxAcquireRef(pCache->index->refId); taosScheduleTask(indexQhandle, &schedMsg); return 0; } -static void indexCacheMakeRoomForWrite(IndexCache* cache) { +static void idxCacheMakeRoomForWrite(IndexCache* cache) { while (true) { if (cache->occupiedMem * MEM_ESTIMATE_RADIO < MEM_THRESHOLD) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - indexCacheWait(cache); + idxCacheWait(cache); } else { bool quit = cache->occupiedMem >= MEM_SIGNAL_QUIT ? true : false; - indexCacheRef(cache); + idxCacheRef(cache); cache->imm = cache->mem; cache->mem = idxInternalCacheCreate(cache->type); cache->mem->pCache = cache; @@ -525,18 +525,18 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } // sched to merge // unref cache in bgwork - indexCacheSchedToMerge(cache, quit); + idxCacheSchedToMerge(cache, quit); } } } -int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { +int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; } - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); + bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); IndexCache* pCache = cache; - indexCacheRef(pCache); + idxCacheRef(pCache); // encode data CacheTerm* ct = taosMemoryCalloc(1, sizeof(CacheTerm)); if (cache == NULL) { @@ -559,7 +559,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { taosThreadMutexLock(&pCache->mtx); pCache->occupiedMem += estimate; - indexCacheMakeRoomForWrite(pCache); + idxCacheMakeRoomForWrite(pCache); MemTable* tbl = pCache->mem; idxMemRef(tbl); tSkipListPut(tbl->mem, (char*)ct); @@ -567,29 +567,29 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { taosThreadMutexUnlock(&pCache->mtx); - indexCacheUnRef(pCache); + idxCacheUnRef(pCache); return 0; // encode end } -void indexCacheForceToMerge(void* cache) { +void idxCacheForceToMerge(void* cache) { IndexCache* pCache = cache; - indexCacheRef(pCache); + idxCacheRef(pCache); taosThreadMutexLock(&pCache->mtx); indexInfo("%p is forced to merge into tfile", pCache); pCache->occupiedMem += MEM_SIGNAL_QUIT; - indexCacheMakeRoomForWrite(pCache); + idxCacheMakeRoomForWrite(pCache); taosThreadMutexUnlock(&pCache->mtx); - indexCacheUnRef(pCache); + idxCacheUnRef(pCache); return; } -int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { +int idxCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { IndexCache* pCache = cache; return 0; } -static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s) { +static int32_t idxQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s) { if (mem == NULL) { return 0; } @@ -597,13 +597,13 @@ static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* t SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { return cacheSearch[1][qtype](mem, term, tr, s); } else { return cacheSearch[0][qtype](mem, term, tr, s); } } -int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) { +int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) { int64_t st = taosGetTimestampUs(); if (cache == NULL) { return 0; @@ -618,10 +618,10 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe idxMemRef(imm); taosThreadMutexUnlock(&pCache->mtx); - int ret = (mem && mem->mem) ? indexQueryMem(mem, query, result, s) : 0; + int ret = (mem && mem->mem) ? idxQueryMem(mem, query, result, s) : 0; if (ret == 0 && *s != kTypeDeletion) { // continue search in imm - ret = (imm && imm->mem) ? indexQueryMem(imm, query, result, s) : 0; + ret = (imm && imm->mem) ? idxQueryMem(imm, query, result, s) : 0; } idxMemUnRef(mem); @@ -631,20 +631,20 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe return ret; } -void indexCacheRef(IndexCache* cache) { +void idxCacheRef(IndexCache* cache) { if (cache == NULL) { return; } int ref = T_REF_INC(cache); UNUSED(ref); } -void indexCacheUnRef(IndexCache* cache) { +void idxCacheUnRef(IndexCache* cache) { if (cache == NULL) { return; } int ref = T_REF_DEC(cache); if (ref == 0) { - indexCacheDestroy(cache); + idxCacheDestroy(cache); } } @@ -662,7 +662,7 @@ void idxMemUnRef(MemTable* tbl) { int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; - indexCacheDestroySkiplist(slt); + idxCacheDestroySkiplist(slt); taosMemoryFree(tbl); } } @@ -693,15 +693,15 @@ static int32_t idxCacheTermCompare(const void* l, const void* r) { return cmp; } -static int indexFindCh(char* a, char c) { +static int idxFindCh(char* a, char c) { char* p = a; while (*p != 0 && *p++ != c) { } return p - a; } static int idxCacheJsonTermCompareImpl(char* a, char* b) { - // int alen = indexFindCh(a, '&'); - // int blen = indexFindCh(b, '&'); + // int alen = idxFindCh(a, '&'); + // int blen = idxFindCh(b, '&'); // int cmp = strncmp(a, b, MIN(alen, blen)); // if (cmp == 0) { @@ -730,9 +730,9 @@ static int32_t idxCacheJsonTermCompare(const void* l, const void* r) { return cmp; } static MemTable* idxInternalCacheCreate(int8_t type) { - int ttype = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY; + int ttype = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY; int32_t (*cmpFn)(const void* l, const void* r) = - INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare; + IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare; MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); idxMemRef(tbl); @@ -742,7 +742,7 @@ static MemTable* idxInternalCacheCreate(int8_t type) { return tbl; } -static void doMergeWork(SSchedMsg* msg) { +static void idxDoMergeWork(SSchedMsg* msg) { IndexCache* pCache = msg->ahandle; SIndex* sidx = (SIndex*)pCache->index; @@ -771,7 +771,7 @@ static bool idxCacheIteratorNext(Iterate* itera) { return next; } -static IterateValue* indexCacheIteratorGetValue(Iterate* iter) { +static IterateValue* idxCacheIteratorGetValue(Iterate* iter) { // opt later return &iter->val; } diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index be9243b8dd..99b49f97bd 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -75,35 +75,35 @@ char* idxInt2str(int64_t val, char* dst, int radix) { ; return dst - 1; } -__compar_fn_t indexGetCompar(int8_t type) { +__compar_fn_t idxGetCompar(int8_t type) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { return (__compar_fn_t)strcmp; } return getComparFunc(type, 0); } static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_THAN, a, b, type); } static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_EQUAL, a, b, type); } static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_THAN, a, b, type); } static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } static TExeCond tCompareContains(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } static TExeCond tCompareEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { @@ -205,14 +205,14 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = { tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual, tCompareContains, tCompareEqual}; -_cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; } +_cache_range_compare idxGetCompare(RangeType ty) { return rangeCompare[ty]; } char* idxPackJsonData(SIndexTerm* itm) { /* * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ - uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); @@ -240,7 +240,7 @@ char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ - uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); @@ -267,7 +267,7 @@ char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) { * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ - uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index e4af4a7a3f..bd78ec574a 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -318,7 +318,7 @@ int sifLessThan(void *a, void *b, int16_t dtype) { } int sifEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); - //__compar_fn_t func = indexGetCompar(dtype); + //__compar_fn_t func = idxGetCompar(dtype); return (int)tDoCompare(func, QUERY_TERM, a, b); } static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { diff --git a/source/libs/index/src/indexJson.c b/source/libs/index/src/indexJson.c index 88b3d907bb..8ce625dfb9 100644 --- a/source/libs/index/src/indexJson.c +++ b/source/libs/index/src/indexJson.c @@ -30,7 +30,7 @@ int indexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { } else { p->colType = TSDB_DATA_TYPE_DOUBLE; } - INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + IDX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); } // handle put return indexPut(index, terms, uid); @@ -48,7 +48,7 @@ int indexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *res } else { p->colType = TSDB_DATA_TYPE_DOUBLE; } - INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + IDX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); } // handle search return indexSearch(index, tq, result); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index b64db1dde4..d632540ee1 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -118,7 +118,7 @@ TFileCache* tfileCacheCreate(const char* path) { ICacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = (int32_t)strlen(header->colName)}; char buf[128] = {0}; - int32_t sz = indexSerialCacheKey(&key, buf); + int32_t sz = idxSerialCacheKey(&key, buf); assert(sz < sizeof(buf)); taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); @@ -149,7 +149,7 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { char buf[128] = {0}; - int32_t sz = indexSerialCacheKey(key, buf); + int32_t sz = idxSerialCacheKey(key, buf); assert(sz < sizeof(buf)); TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); if (reader == NULL || *reader == NULL) { @@ -161,7 +161,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { } void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { char buf[128] = {0}; - int32_t sz = indexSerialCacheKey(key, buf); + int32_t sz = idxSerialCacheKey(key, buf); // remove last version index reader TFileReader** p = taosHashGet(tcache->tableCache, buf, sz); if (p != NULL && *p != NULL) { @@ -281,7 +281,7 @@ static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { return 0; } static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); int ret = 0; char* p = tem->colVal; @@ -305,7 +305,7 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, int ret = 0; char* p = tem->colVal; int skip = 0; - _cache_range_compare cmpFn = indexGetCompare(type); + _cache_range_compare cmpFn = idxGetCompare(type); SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); @@ -431,7 +431,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt p = idxPackJsonDataPrefix(tem, &skip); } - _cache_range_compare cmpFn = indexGetCompare(ctype); + _cache_range_compare cmpFn = idxGetCompare(ctype); SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); @@ -457,7 +457,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt } else if (0 != strncmp(ch, p, skip)) { continue; } - cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); + cond = cmpFn(ch + skip, tem->colVal, IDX_TYPE_GET_TYPE(tem->colType)); } if (MATCH == cond) { tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); @@ -476,7 +476,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; int ret = 0; - if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { ret = tfSearch[1][qtype](reader, term, tr); } else { ret = tfSearch[0][qtype](reader, term, tr); @@ -536,7 +536,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { __compar_fn_t fn; int8_t colType = tw->header.colType; - colType = INDEX_TYPE_GET_TYPE(colType); + colType = IDX_TYPE_GET_TYPE(colType); if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { fn = tfileStrCompare; } else { @@ -620,7 +620,7 @@ void tfileWriterDestroy(TFileWriter* tw) { taosMemoryFree(tw); } -IndexTFile* indexTFileCreate(const char* path) { +IndexTFile* idxTFileCreate(const char* path) { TFileCache* cache = tfileCacheCreate(path); if (cache == NULL) { return NULL; @@ -635,7 +635,7 @@ IndexTFile* indexTFileCreate(const char* path) { tfile->cache = cache; return tfile; } -void indexTFileDestroy(IndexTFile* tfile) { +void idxTFileDestroy(IndexTFile* tfile) { if (tfile == NULL) { return; } @@ -644,7 +644,7 @@ void indexTFileDestroy(IndexTFile* tfile) { taosMemoryFree(tfile); } -int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) { +int idxTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) { int ret = -1; if (tfile == NULL) { return ret; @@ -667,7 +667,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) { return tfileReaderSearch(reader, query, result); } -int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { +int idxTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = // term->nColName, .version = 1}; @@ -845,7 +845,7 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) { TFileHeader* header = &write->header; uint8_t colType = header->colType; - colType = INDEX_TYPE_GET_TYPE(colType); + colType = IDX_TYPE_GET_TYPE(colType); FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal)); if (fstBuilderInsert(write->fb, key, tval->offset)) { fstSliceDestroy(&key); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 90dea3a377..e18297cd25 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -521,10 +521,10 @@ class CacheObj { public: CacheObj() { // TODO - cache = indexCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY); + cache = idxCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY); } int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { - int ret = indexCachePut(cache, term, uid); + int ret = idxCachePut(cache, term, uid); if (ret != 0) { // std::cout << "failed to put into cache: " << ret << std::endl; @@ -533,12 +533,12 @@ class CacheObj { } void Debug() { // - indexCacheDebug(cache); + idxCacheDebug(cache); } int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) { SIdxTRslt* tr = idxTRsltCreate(); - int ret = indexCacheSearch(cache, query, tr, s); + int ret = idxCacheSearch(cache, query, tr, s); idxTRsltMergeTo(tr, result); idxTRsltDestroy(tr); @@ -549,7 +549,7 @@ class CacheObj { } ~CacheObj() { // TODO - indexCacheDestroy(cache); + idxCacheDestroy(cache); } private: diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index 5999ef8300..98df22d51b 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -93,6 +93,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); +int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index c0c1f76707..e509a50dc4 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -41,6 +41,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 85a82dcfb7..128dbf4050 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -39,6 +39,8 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode); +int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode); + int32_t syncNodeElect(SSyncNode* pSyncNode); int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg); diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index b69c087b5f..88d3065375 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -57,6 +57,9 @@ typedef struct SSyncIO { int32_t (*FpOnSyncAppendEntriesReply)(SSyncNode *pSyncNode, SyncAppendEntriesReply *pMsg); int32_t (*FpOnSyncTimeout)(SSyncNode *pSyncNode, SyncTimeout *pMsg); + int32_t (*FpOnSyncSnapshotSend)(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg); + int32_t (*FpOnSyncSnapshotRsp)(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg); + int8_t isStart; } SSyncIO; diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 0a6e2428fe..1f60a9d57e 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -30,6 +30,7 @@ extern "C" { typedef struct SSyncIndexMgr { SRaftId (*replicas)[TSDB_MAX_REPLICA]; SyncIndex index[TSDB_MAX_REPLICA]; + SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function int32_t replicaNum; SSyncNode *pSyncNode; } SSyncIndexMgr; @@ -43,6 +44,9 @@ SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); +// void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term); +// SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); + // for debug ------------------- void syncIndexMgrPrint(SSyncIndexMgr *pObj); void syncIndexMgrPrint2(char *s, SSyncIndexMgr *pObj); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 4100aa0216..10218f69e6 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -58,6 +58,8 @@ typedef struct SSyncRespMgr SSyncRespMgr; typedef struct SSyncSnapshotSender SSyncSnapshotSender; typedef struct SSyncSnapshotReceiver SSyncSnapshotReceiver; +extern bool gRaftDetailLog; + typedef struct SSyncNode { // init by SSyncInfo SyncGroupId vgId; @@ -137,24 +139,27 @@ typedef struct SSyncNode { uint64_t heartbeatTimerCounter; // callback - int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg); - int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg); - int32_t (*FpOnClientRequest)(SSyncNode* ths, SyncClientRequest* pMsg); - int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg); - int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg); - int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg); - int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); - int32_t (*FpOnTimeout)(SSyncNode* pSyncNode, SyncTimeout* pMsg); + FpOnPingCb FpOnPing; + FpOnPingReplyCb FpOnPingReply; + FpOnClientRequestCb FpOnClientRequest; + FpOnTimeoutCb FpOnTimeout; + FpOnRequestVoteCb FpOnRequestVote; + FpOnRequestVoteReplyCb FpOnRequestVoteReply; + FpOnAppendEntriesCb FpOnAppendEntries; + FpOnAppendEntriesReplyCb FpOnAppendEntriesReply; + FpOnSnapshotSendCb FpOnSnapshotSend; + FpOnSnapshotRspCb FpOnSnapshotRsp; // tools SSyncRespMgr* pSyncRespMgr; // restore state - // sem_t restoreSem; - bool restoreFinish; - SSnapshot* pSnapshot; - SSyncSnapshotSender* pSender; - SSyncSnapshotReceiver* pReceiver; + bool restoreFinish; + // SSnapshot* pSnapshot; + SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; + SSyncSnapshotReceiver* pNewNodeReceiver; + + SSnapshotMeta sMeta; } SSyncNode; @@ -164,6 +169,9 @@ void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); +// option +bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); + // ping -------------- int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg); int32_t syncNodePingSelf(SSyncNode* pSyncNode); @@ -205,6 +213,25 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode); void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForSelf(SSyncNode* pSyncNode); +// snapshot -------------- +bool syncNodeHasSnapshot(SSyncNode* pSyncNode); +bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index); + +SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode); +SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode); +int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm); + +SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode); + +SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index); +SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index); +int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); + +int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag); + +bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId); +SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 1061e8bdc4..86c5fab87c 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -34,6 +34,7 @@ typedef struct SRaftCfg { TdFilePtr pFile; char path[TSDB_FILENAME_LEN * 2]; int8_t isStandBy; + int8_t snapshotEnable; } SRaftCfg; SRaftCfg *raftCfgOpen(const char *path); @@ -50,7 +51,12 @@ char * raftCfg2Str(SRaftCfg *pRaftCfg); int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg); int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg); -int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path); +typedef struct SRaftCfgMeta { + int8_t isStandBy; + int8_t snapshotEnable; +} SRaftCfgMeta; + +int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path); // for debug ---------------------- void syncCfgPrint(SSyncCfg *pCfg); diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index df5cd3f36c..aec1f77b42 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -30,6 +30,7 @@ extern "C" { typedef struct SSyncLogStoreData { SSyncNode* pSyncNode; SWal* pWal; + SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 } SSyncLogStoreData; SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); @@ -39,14 +40,7 @@ char* logStore2Str(SSyncLogStore* pLogStore); cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); char* logStoreSimple2Str(SSyncLogStore* pLogStore); -// SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); -// SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore); -// SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore); -// SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index); -// int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); -// int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex); -// int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index); -// SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); +SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore); // for debug void logStorePrint(SSyncLogStore* pLogStore); diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index e0cbcf0744..9f03ac3e55 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -49,8 +49,8 @@ void raftStoreClearVote(SRaftStore *pRaftStore); void raftStoreNextTerm(SRaftStore *pRaftStore); void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson); -cJSON * raftStore2Json(SRaftStore *pRaftStore); -char * raftStore2Str(SRaftStore *pRaftStore); +cJSON *raftStore2Json(SRaftStore *pRaftStore); +char *raftStore2Str(SRaftStore *pRaftStore); // for debug ------------------- void raftStorePrint(SRaftStore *pObj); diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 6fe18dae38..4b1f5b4638 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -52,6 +52,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode); +int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode); int32_t syncNodeReplicate(SSyncNode* pSyncNode); int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg); diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index fd4ccd5371..3fe8dc0237 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -50,6 +50,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg); +int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index bcaf71a541..ac47a8d026 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -45,6 +45,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); +int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 43d1c0c0c3..b16e47b51e 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -25,40 +25,64 @@ extern "C" { #include #include "cJSON.h" #include "syncInt.h" +#include "syncMessage.h" #include "taosdef.h" +#define SYNC_SNAPSHOT_SEQ_INVALID -1 +#define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -2 +#define SYNC_SNAPSHOT_SEQ_BEGIN 0 +#define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF + +#define SYNC_SNAPSHOT_RETRY_MS 5000 + typedef struct SSyncSnapshotSender { - int32_t sending; - int32_t received; - bool finish; - void * pCurrentBlock; + bool start; + int32_t seq; + int32_t ack; + void *pReader; + void *pCurrentBlock; int32_t blockLen; + SSnapshot snapshot; int64_t sendingMS; - SSnapshot *pSnapshot; SSyncNode *pSyncNode; + int32_t replicaIndex; + SyncTerm term; + SyncTerm privateTerm; + bool finish; } SSyncSnapshotSender; -SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode); +SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotSenderDestroy(SSyncSnapshotSender *pSender); +bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); +void snapshotSenderStart(SSyncSnapshotSender *pSender); +void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); +int32_t snapshotReSend(SSyncSnapshotSender *pSender); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { - bool start; - int32_t received; - int32_t progressIndex; - void * pCurrentBlock; - int32_t len; - SSnapshot *pSnapshot; + bool start; + + int32_t ack; + void *pWriter; + SyncTerm term; + SyncTerm privateTerm; + SSyncNode *pSyncNode; + int32_t replicaIndex; } SSyncSnapshotReceiver; -SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode); +SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); -int32_t snapshotReceive(SSyncSnapshotReceiver *pReceiver); -cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm); +bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); + +int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); +int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 1b08d3f7a1..7ecff7ae97 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -61,6 +61,7 @@ bool syncUtilIsData(tmsg_t msgType); bool syncUtilUserPreCommit(tmsg_t msgType); bool syncUtilUserCommit(tmsg_t msgType); bool syncUtilUserRollback(tmsg_t msgType); +void syncUtilJson2Line(char* jsonStr); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 89dcd8a476..3c558b60c8 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -18,8 +18,10 @@ #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncSnapshot.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "wal.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == @@ -335,8 +337,12 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { cbMeta.currentTerm = ths->pRaftStore->currentTerm; cbMeta.flag = 0x11; + SSnapshot snapshot; + ASSERT(ths->pFsm->FpGetSnapshot != NULL); + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + bool needExecute = true; - if (ths->pSnapshot != NULL && cbMeta.index <= ths->pSnapshot->lastApplyIndex) { + if (cbMeta.index <= snapshot.lastApplyIndex) { needExecute = false; } @@ -427,3 +433,332 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } + +static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t code; + + SyncIndex delBegin = pMsg->prevLogIndex + 1; + SyncIndex delEnd = ths->pLogStore->syncLogLastIndex(ths->pLogStore); + + // invert roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry; + code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, index, &pRollBackEntry); + ASSERT(code == 0); + ASSERT(pRollBackEntry != NULL); + + if (syncUtilUserRollback(pRollBackEntry->msgType)) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + + SFsmCbMeta cbMeta; + cbMeta.index = pRollBackEntry->index; + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); + } + + syncEntryDestory(pRollBackEntry); + } + } + + // delete confict entries + code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); + ASSERT(code == 0); + sInfo("sync event log truncate, from %ld to %ld", delBegin, delEnd); + logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); + + return code; +} + +static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 2; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + return 0; +} + +// really pre log match +// prevLogIndex == -1 +static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { + if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { + if (gRaftDetailLog) { + sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex); + } + return true; + } + + SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); + if (pMsg->prevLogIndex > myLastIndex) { + if (gRaftDetailLog) { + sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex, + myLastIndex); + } + return false; + } + + SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); + if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { + if (gRaftDetailLog) { + sTrace( + "syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " + "myPreLogTerm:%lu", + pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); + } + return true; + } + + if (gRaftDetailLog) { + sTrace( + "syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " + "myPreLogTerm:%lu", + pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); + } + + return false; +} + +int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = 0; + int32_t code = 0; + + // print log + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, vgId:%d, term:%lu", ths->vgId, + ths->pRaftStore->currentTerm); + syncAppendEntriesLog2(logBuf, pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + sInfo("recv SyncAppendEntries maybe replica already dropped"); + return ret; + } + + // maybe update term + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + + // reset elect timer + if (pMsg->term == ths->pRaftStore->currentTerm) { + ths->leaderCache = pMsg->srcId; + syncNodeResetElectTimer(ths); + } + ASSERT(pMsg->dataLen >= 0); + + // candidate to follower + // + // operation: + // to follower + do { + bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE; + if (condition) { + sTrace("recv SyncAppendEntries, candidate to follower"); + + syncNodeBecomeFollower(ths); + // do not reply? + return ret; + } + } while (0); + + // fake match + // + // condition1: + // I have snapshot, no log, preIndex > myLastIndex + // + // condition2: + // I have snapshot, have log, log <= snapshot, preIndex > myLastIndex + // + // condition3: + // I have snapshot, preIndex < snapshot.lastApplyIndex + // + // condition4: + // I have snapshot, preIndex == snapshot.lastApplyIndex, no data + // + // operation: + // match snapshot.lastApplyIndex - 1; + // no operation on log + do { + SyncIndex myLastIndex = syncNodeGetLastIndex(ths); + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + + bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && + syncNodeHasSnapshot(ths); + bool condition1 = + condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); + bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) && + (pMsg->prevLogIndex > myLastIndex); + bool condition3 = condition0 && (pMsg->prevLogIndex < snapshot.lastApplyIndex); + bool condition4 = condition0 && (pMsg->prevLogIndex == snapshot.lastApplyIndex) && (pMsg->dataLen == 0); + bool condition = condition1 || condition2 || condition3 || condition4; + + if (condition) { + sTrace( + "recv SyncAppendEntries, fake match, myLastIndex:%ld, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, " + "condition1:%d, condition2:%d, condition3:%d, condition4:%d", + myLastIndex, ths->pLogStore->syncLogBeginIndex(ths->pLogStore), + ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3, condition4); + + // prepare response msg + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; + pReply->success = true; + pReply->matchIndex = snapshot.lastApplyIndex; + + // send response + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + } while (0); + + // calculate logOK here, before will coredump, due to fake match + bool logOK = syncNodeOnAppendEntriesLogOK(ths, pMsg); + + // not match + // + // condition1: + // term < myTerm + // + // condition2: + // !logOK + // + // operation: + // not match + // no operation on log + do { + bool condition1 = pMsg->term < ths->pRaftStore->currentTerm; + bool condition2 = + (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK; + bool condition = condition1 || condition2; + + if (condition) { + sTrace( + "recv SyncAppendEntries, not match, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, condition1:%d, " + "condition2:%d, logOK:%d", + ths->pLogStore->syncLogBeginIndex(ths->pLogStore), ths->pLogStore->syncLogEndIndex(ths->pLogStore), + condition1, condition2, logOK); + + // prepare response msg + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + // send response + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + } while (0); + + // really match + // + // condition: + // logOK + // + // operation: + // match + // make log same + do { + bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && logOK; + if (condition) { + // has extra entries (> preIndex) in local log + SyncIndex myLastIndex = syncNodeGetLastIndex(ths); + bool hasExtraEntries = myLastIndex > pMsg->prevLogIndex; + + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; + + sTrace("recv SyncAppendEntries, match, myLastIndex:%ld, hasExtraEntries:%d, hasAppendEntries:%d", myLastIndex, + hasExtraEntries, hasAppendEntries); + + if (hasExtraEntries) { + // make log same, rollback deleted entries + code = syncNodeMakeLogSame(ths, pMsg); + ASSERT(code == 0); + } + + if (hasAppendEntries) { + // append entry + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + ASSERT(pAppendEntry != NULL); + + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); + ASSERT(code == 0); + + // pre commit + code = syncNodePreCommit(ths, pAppendEntry); + ASSERT(code == 0); + + syncEntryDestory(pAppendEntry); + } + + // prepare response msg + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; + pReply->success = true; + pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + 1 : pMsg->prevLogIndex; + + // send response + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + // maybe update commit index, leader notice me + if (pMsg->commitIndex > ths->commitIndex) { + // has commit entry in local + if (pMsg->commitIndex <= ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { + SyncIndex beginIndex = ths->commitIndex + 1; + SyncIndex endIndex = pMsg->commitIndex; + + // update commit index + ths->commitIndex = pMsg->commitIndex; + + // call back Wal + code = ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + ASSERT(code == 0); + + code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); + ASSERT(code == 0); + } + } + return ret; + } + } while (0); + + return ret; +} diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 4e6d870e19..5a543e1605 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -17,8 +17,10 @@ #include "syncCommit.h" #include "syncIndexMgr.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncSnapshot.h" #include "syncUtil.h" #include "syncVoteMgr.h" @@ -94,3 +96,116 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } + +int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = 0; + + // print log + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, vgId:%d, term:%lu", ths->vgId, + ths->pRaftStore->currentTerm); + syncAppendEntriesReplyLog2(logBuf, pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + sInfo("recv SyncAppendEntriesReply, maybe replica already dropped"); + return ret; + } + + // drop stale response + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("recv SyncAppendEntriesReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + return ret; + } + + syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex); + syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex); + { + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", + snapshot.lastApplyIndex, snapshot.lastApplyTerm); + } + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, error term, receive_term:%lu current_term:%lu", + pMsg->term, ths->pRaftStore->currentTerm); + syncNodeLog2(logBuf, ths); + sError("%s", logBuf); + return ret; + } + + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + + if (pMsg->success) { + // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); + sTrace("update next match, index:%ld, success:%d", pMsg->matchIndex + 1, pMsg->success); + + // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + + // maybe commit + if (ths->state == TAOS_SYNC_STATE_LEADER) { + syncMaybeAdvanceCommitIndex(ths); + } + + } else { + SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + sTrace("update next not match, begin, index:%ld, success:%d", nextIndex, pMsg->success); + + // notice! int64, uint64 + if (nextIndex > SYNC_INDEX_BEGIN) { + --nextIndex; + + // get sender + SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); + ASSERT(pSender != NULL); + bool hasSnapshot = syncNodeHasSnapshot(ths); + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + + // start sending snapshot first time + // start here, stop by receiver + if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) && + pMsg->privateTerm < pSender->privateTerm) { + snapshotSenderStart(pSender); + + char* s = snapshotSender2Str(pSender); + sInfo("sync event snapshot send start sender first time, sender:%s", s); + taosMemoryFree(s); + } + + SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1; + + // update nextIndex to sentryIndex + if (nextIndex <= sentryIndex) { + nextIndex = sentryIndex; + } + + } else { + nextIndex = SYNC_INDEX_BEGIN; + } + + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + sTrace("update next not match, end, index:%ld, success:%d", nextIndex, pMsg->success); + } + + syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); + syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); + { + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + sTrace("recv SyncAppendEntriesReply, after snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", + snapshot.lastApplyIndex, snapshot.lastApplyTerm); + } + + return ret; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index c6376495a4..c092b31adf 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -94,109 +94,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // execute fsm if (pSyncNode->pFsm != NULL) { - for (SyncIndex i = beginIndex; i <= endIndex; ++i) { - if (i != SYNC_INDEX_INVALID) { - SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, i); - assert(pEntry != NULL); - - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pEntry, &rpcMsg); - - if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { - SFsmCbMeta cbMeta; - cbMeta.index = pEntry->index; - cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = 0; - cbMeta.state = pSyncNode->state; - cbMeta.seqNum = pEntry->seqNum; - cbMeta.term = pEntry->term; - cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm; - cbMeta.flag = 0x1; - - bool needExecute = true; - if (pSyncNode->pSnapshot != NULL && cbMeta.index <= pSyncNode->pSnapshot->lastApplyIndex) { - needExecute = false; - } - - if (needExecute) { - pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta); - } - } - - // config change - if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE) { - SSyncCfg oldSyncCfg = pSyncNode->pRaftCfg->cfg; - - SSyncCfg newSyncCfg; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); - ASSERT(ret == 0); - - // update new config myIndex - bool hit = false; - for (int i = 0; i < newSyncCfg.replicaNum; ++i) { - if (strcmp(pSyncNode->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && - pSyncNode->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { - newSyncCfg.myIndex = i; - hit = true; - break; - } - } - - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - ASSERT(hit == true); - } - - bool isDrop; - syncNodeUpdateConfig(pSyncNode, &newSyncCfg, &isDrop); - - // change isStandBy to normal - if (!isDrop) { - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(pSyncNode); - } else { - syncNodeBecomeFollower(pSyncNode); - } - } - - char* sOld = syncCfg2Str(&oldSyncCfg); - char* sNew = syncCfg2Str(&newSyncCfg); - sInfo("==config change== 0x1 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); - taosMemoryFree(sOld); - taosMemoryFree(sNew); - - if (pSyncNode->pFsm->FpReConfigCb != NULL) { - SReConfigCbMeta cbMeta = {0}; - cbMeta.code = 0; - cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm; - cbMeta.index = pEntry->index; - cbMeta.term = pEntry->term; - cbMeta.oldCfg = oldSyncCfg; - cbMeta.flag = 0x1; - cbMeta.isDrop = isDrop; - pSyncNode->pFsm->FpReConfigCb(pSyncNode->pFsm, newSyncCfg, cbMeta); - } - } - - // restore finish - if (pEntry->index == pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore)) { - if (pSyncNode->restoreFinish == false) { - if (pSyncNode->pFsm->FpRestoreFinishCb != NULL) { - pSyncNode->pFsm->FpRestoreFinishCb(pSyncNode->pFsm); - } - pSyncNode->restoreFinish = true; - sInfo("==syncMaybeAdvanceCommitIndex== restoreFinish set true %p vgId:%d", pSyncNode, pSyncNode->vgId); - - /* - tsem_post(&pSyncNode->restoreSem); - sInfo("==syncMaybeAdvanceCommitIndex== RestoreFinish tsem_post %p", pSyncNode); - */ - } - } - - rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pEntry); - } - } + int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); + ASSERT(code == 0); } } } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 5101344b84..fdebbe3990 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -15,6 +15,7 @@ #include "syncElection.h" #include "syncMessage.h" +#include "syncRaftCfg.h" #include "syncRaftStore.h" #include "syncVoteMgr.h" @@ -49,6 +50,26 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { return ret; } +int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode) { + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SyncRequestVote* pMsg = syncRequestVoteBuild(pSyncNode->vgId); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = pSyncNode->peersId[i]; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + + ret = syncNodeGetLastIndexTerm(pSyncNode, &(pMsg->lastLogIndex), &(pMsg->lastLogTerm)); + ASSERT(ret == 0); + + ret = syncNodeRequestVote(pSyncNode, &pSyncNode->peersId[i], pMsg); + ASSERT(ret == 0); + syncRequestVoteDestroy(pMsg); + } + return ret; +} + int32_t syncNodeElect(SSyncNode* pSyncNode) { int32_t ret = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { @@ -71,7 +92,12 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { return ret; } - ret = syncNodeRequestVotePeers(pSyncNode); + if (pSyncNode->pRaftCfg->snapshotEnable) { + ret = syncNodeRequestVotePeersSnapshot(pSyncNode); + } else { + ret = syncNodeRequestVotePeers(pSyncNode); + } + assert(ret == 0); syncNodeResetElectTimer(pSyncNode); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index aa8484de99..0b5a9685c0 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -75,7 +75,8 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { syncUtilMsgNtoH(pMsg->pCont); char logBuf[256] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port); + snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d msgType:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port, + pMsg->msgType); syncRpcMsgLog2(logBuf, pMsg); syncUtilMsgHtoN(pMsg->pCont); @@ -89,8 +90,10 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t syncIOEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { int32_t ret = 0; - char logBuf[128] = {0}; - syncRpcMsgLog2((char *)"==syncIOEqMsg==", pMsg); + + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncIOEqMsg== msgType:%d", pMsg->msgType); + syncRpcMsgLog2(logBuf, pMsg); SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); @@ -253,7 +256,9 @@ static void *syncIOConsumerFunc(void *param) { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncIOConsumMsg== msgType:%d", pRpcMsg->msgType); + syncRpcMsgLog2(logBuf, pRpcMsg); // use switch case instead of if else if (pRpcMsg->msgType == TDMT_SYNC_PING) { @@ -319,6 +324,23 @@ static void *syncIOConsumerFunc(void *param) { io->FpOnSyncTimeout(io->pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); } + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { + if (io->FpOnSyncSnapshotSend != NULL) { + SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + io->FpOnSyncSnapshotSend(io->pSyncNode, pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + } + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { + if (io->FpOnSyncSnapshotRsp != NULL) { + SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + io->FpOnSyncSnapshotRsp(io->pSyncNode, pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + } + } else { sTrace("unknown msgType:%d, no operator", pRpcMsg->msgType); } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 4d556d21dd..ecc1c8f1e2 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -46,6 +46,7 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) { memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index)); + memset(pSyncIndexMgr->privateTerm, 0, sizeof(pSyncIndexMgr->privateTerm)); /* for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { pSyncIndexMgr->index[i] = 0; @@ -62,7 +63,7 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, } // maybe config change - // assert(0); + assert(0); } SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { @@ -86,14 +87,27 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); } - int respondNum = 0; - int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - arr[i] = pSyncIndexMgr->index[i]; + + { + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + arr[i] = pSyncIndexMgr->index[i]; + } + cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "index", pIndex); } - cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "index", pIndex); + + { + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + arr[i] = pSyncIndexMgr->privateTerm[i]; + } + cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "privateTerm", pIndex); + } + snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); } @@ -105,7 +119,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -132,7 +146,31 @@ void syncIndexMgrLog(SSyncIndexMgr *pObj) { } void syncIndexMgrLog2(char *s, SSyncIndexMgr *pObj) { - char *serialized = syncIndexMgr2Str(pObj); - sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char *serialized = syncIndexMgr2Str(pObj); + sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + (pSyncIndexMgr->privateTerm)[i] = term; + return; + } + } + + // maybe config change + assert(0); +} + +SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + SyncTerm term = (pSyncIndexMgr->privateTerm)[i]; + return term; + } + } + assert(0); } \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 795d3e3c27..9516df64da 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -29,11 +29,14 @@ #include "syncRequestVote.h" #include "syncRequestVoteReply.h" #include "syncRespMgr.h" +#include "syncSnapshot.h" #include "syncTimeout.h" #include "syncUtil.h" #include "syncVoteMgr.h" #include "tref.h" +bool gRaftDetailLog = false; + static int32_t tsNodeRefId = -1; // ------ local funciton --------- @@ -213,6 +216,18 @@ bool syncIsRestoreFinish(int64_t rid) { return b; } +int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return -1; + } + assert(rid == pSyncNode->rid); + *sMeta = pSyncNode->sMeta; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return 0; +} + const char* syncGetMyRoleStr(int64_t rid) { const char* s = syncUtilState2String(syncGetMyRole(rid)); return s; @@ -411,8 +426,11 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path); if (!taosCheckExistFile(pSyncNode->configPath)) { - // create raft config file - ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), pSyncInfo->isStandBy, pSyncNode->configPath); + // create a new raft config file + SRaftCfgMeta meta; + meta.isStandBy = pSyncInfo->isStandBy; + meta.snapshotEnable = pSyncInfo->snapshotEnable; + ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath); assert(ret == 0); } else { @@ -552,35 +570,64 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { pSyncNode->FpOnPing = syncNodeOnPingCb; pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb; pSyncNode->FpOnClientRequest = syncNodeOnClientRequestCb; - pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; - pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; - pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; - pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; pSyncNode->FpOnTimeout = syncNodeOnTimeoutCb; + pSyncNode->FpOnSnapshotSend = syncNodeOnSnapshotSendCb; + pSyncNode->FpOnSnapshotRsp = syncNodeOnSnapshotRspCb; + + if (pSyncNode->pRaftCfg->snapshotEnable) { + sInfo("sync node use snapshot"); + pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteSnapshotCb; + pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplySnapshotCb; + pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesSnapshotCb; + pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplySnapshotCb; + + } else { + sInfo("sync node do not use snapshot"); + pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; + pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; + pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; + pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; + } + // tools pSyncNode->pSyncRespMgr = syncRespMgrCreate(NULL, 0); assert(pSyncNode->pSyncRespMgr != NULL); // restore state pSyncNode->restoreFinish = false; - pSyncNode->pSnapshot = NULL; - if (pSyncNode->pFsm->FpGetSnapshot != NULL) { - pSyncNode->pSnapshot = taosMemoryMalloc(sizeof(SSnapshot)); - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, pSyncNode->pSnapshot); - } + + // pSyncNode->pSnapshot = NULL; + // if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + // pSyncNode->pSnapshot = taosMemoryMalloc(sizeof(SSnapshot)); + // pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, pSyncNode->pSnapshot); + // } // tsem_init(&(pSyncNode->restoreSem), 0, 0); + // snapshot senders + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i); + // ASSERT(pSender != NULL); + (pSyncNode->senders)[i] = pSender; + } + + // snapshot receivers + pSyncNode->pNewNodeReceiver = snapshotReceiverCreate(pSyncNode, 100); + // start in syncNodeStart // start raft // syncNodeBecomeFollower(pSyncNode); + // snapshot meta + pSyncNode->sMeta.lastConfigIndex = -1; + return pSyncNode; } void syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { + raftStoreNextTerm(pSyncNode->pRaftStore); syncNodeBecomeLeader(pSyncNode); syncNodeLog2("==state change become leader immediately==", pSyncNode); @@ -662,9 +709,23 @@ void syncNodeClose(SSyncNode* pSyncNode) { taosMemoryFree(pSyncNode->pFsm); } + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + if ((pSyncNode->senders)[i] != NULL) { + snapshotSenderDestroy((pSyncNode->senders)[i]); + (pSyncNode->senders)[i] = NULL; + } + } + + if (pSyncNode->pNewNodeReceiver != NULL) { + snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); + pSyncNode->pNewNodeReceiver = NULL; + } + + /* if (pSyncNode->pSnapshot != NULL) { taosMemoryFree(pSyncNode->pSnapshot); } + */ // tsem_destroy(&pSyncNode->restoreSem); @@ -672,6 +733,9 @@ void syncNodeClose(SSyncNode* pSyncNode) { // taosMemoryFree(pSyncNode); } +// option +bool syncNodeSnapshotEnable(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotEnable; } + // ping -------------- int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { syncPingLog2((char*)"==syncNodePing==", pMsg); @@ -762,7 +826,13 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { int32_t ret = 0; - int32_t electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); + int32_t electMS; + + if (pSyncNode->pRaftCfg->isStandBy) { + electMS = TIMER_MAX_MS; + } else { + electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); + } ret = syncNodeRestartElectTimer(pSyncNode, electMS); return ret; } @@ -788,6 +858,13 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); if (pSyncNode->FpSendMsg != NULL) { + if (gRaftDetailLog) { + char* JsonStr = syncRpcMsg2Str(pMsg); + syncUtilJson2Line(JsonStr); + sTrace("sync send msg, vgId:%d, type:%d, msg:%s", pSyncNode->vgId, pMsg->msgType, JsonStr); + taosMemoryFree(JsonStr); + } + // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -952,6 +1029,20 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); + + // restoreFinish + cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish); + + // snapshot senders + cJSON* pSenders = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "senders", pSenders); + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i])); + } + + // snapshot receivers + cJSON* pReceivers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); } cJSON* pJson = cJSON_CreateObject(); @@ -973,10 +1064,10 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { "syncNode2SimpleStr vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " "electTimerLogicClock:%lu, " "electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "electTimerMS:%d, replicaNum:%d", pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftCfg->isStandBy, pSyncNode->electTimerLogicClock, - pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, pSyncNode->replicaNum); return s; } @@ -1013,6 +1104,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, bool* isDro voteGrantedUpdate(pSyncNode->pVotesGranted, pSyncNode); votesRespondUpdate(pSyncNode->pVotesRespond, pSyncNode); + pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); + // isDrop *isDrop = true; bool IamInOld, IamInNew; @@ -1103,7 +1196,15 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode) { for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { // maybe overwrite myself, no harm // just do it! - pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + + // pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + + // maybe wal is deleted + SyncIndex lastIndex; + SyncTerm lastTerm; + int32_t code = syncNodeGetLastIndexTerm(pSyncNode, &lastIndex, &lastTerm); + ASSERT(code == 0); + pSyncNode->pNextIndex->index[i] = lastIndex + 1; } for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { @@ -1112,6 +1213,17 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode) { pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; } + // update sender private term + SSyncSnapshotSender* pMySender = syncNodeGetSnapshotSender(pSyncNode, &(pSyncNode->myRaftId)); + if (pMySender != NULL) { + for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { + if ((pSyncNode->senders)[i]->privateTerm > pMySender->privateTerm) { + pMySender->privateTerm = (pSyncNode->senders)[i]->privateTerm; + } + } + (pMySender->privateTerm) += 100; + } + // stop elect timer syncNodeStopElectTimer(pSyncNode); @@ -1186,6 +1298,153 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { syncRequestVoteReplyDestroy(pMsg); } +// snapshot -------------- +bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { + bool ret = false; + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN) { + ret = true; + } + } + return ret; +} + +bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { + ASSERT(syncNodeHasSnapshot(pSyncNode)); + ASSERT(pSyncNode->pFsm->FpGetSnapshot != NULL); + ASSERT(index >= SYNC_INDEX_BEGIN); + + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + bool b = (index <= snapshot.lastApplyIndex); + return b; +} + +SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + + SyncIndex lastIndex = logLastIndex > snapshot.lastApplyIndex ? logLastIndex : snapshot.lastApplyIndex; + return lastIndex; +} + +SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) { + SyncTerm lastTerm = 0; + if (syncNodeHasSnapshot(pSyncNode)) { + // has snapshot + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + if (logLastIndex > snapshot.lastApplyIndex) { + lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore); + } else { + lastTerm = snapshot.lastApplyTerm; + } + + } else { + // no snapshot + lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore); + } + + return lastTerm; +} + +// get last index and term along with snapshot +int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm) { + *pLastIndex = syncNodeGetLastIndex(pSyncNode); + *pLastTerm = syncNodeGetLastTerm(pSyncNode); + return 0; +} + +SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) { + SyncIndex syncStartIndex = syncNodeGetLastIndex(pSyncNode) + 1; + return syncStartIndex; +} + +SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { + ASSERT(index >= SYNC_INDEX_BEGIN); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + ASSERT(index <= syncStartIndex); + + SyncIndex preIndex = index - 1; + return preIndex; +} + +SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { + ASSERT(index >= SYNC_INDEX_BEGIN); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + ASSERT(index <= syncStartIndex); + + if (index == SYNC_INDEX_BEGIN) { + return 0; + } + + SyncTerm preTerm = 0; + if (syncNodeHasSnapshot(pSyncNode)) { + // has snapshot + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + + if (index > snapshot.lastApplyIndex + 1) { + // should be log preTerm + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); + ASSERT(code == 0); + ASSERT(pPreEntry != NULL); + + preTerm = pPreEntry->term; + taosMemoryFree(pPreEntry); + + } else if (index == snapshot.lastApplyIndex + 1) { + preTerm = snapshot.lastApplyTerm; + + } else { + // maybe snapshot change + sError("sync get pre term, bad scene. index:%ld", index); + logStoreLog2("sync get pre term, bad scene", pSyncNode->pLogStore); + + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); + ASSERT(code == 0); + ASSERT(pPreEntry != NULL); + + preTerm = pPreEntry->term; + taosMemoryFree(pPreEntry); + } + + } else { + // no snapshot + ASSERT(index > SYNC_INDEX_BEGIN); + + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); + ASSERT(code == 0); + ASSERT(pPreEntry != NULL); + + preTerm = pPreEntry->term; + taosMemoryFree(pPreEntry); + } + + return preTerm; +} + +// get pre index and term of "index" +int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { + *pPreIndex = syncNodeGetPreIndex(pSyncNode, index); + *pPreTerm = syncNodeGetPreTerm(pSyncNode, index); + return 0; +} + // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); @@ -1327,7 +1586,8 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { assert(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { - ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + // ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); syncNodeReplicate(ths); } @@ -1383,13 +1643,14 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { int32_t ret = 0; syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); - SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = ths->pRaftStore->currentTerm; SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index); assert(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { - ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + // ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); // start replicate right now! syncNodeReplicate(ths); @@ -1459,3 +1720,137 @@ const char* syncStr(ESyncState state) { return "error"; } } + +int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { + int32_t code = 0; + ESyncState state = flag; + sInfo("sync event commit from index:%" PRId64 " to index:%" PRId64 ", %s", beginIndex, endIndex, + syncUtilState2String(state)); + + // maybe execute by leader, skip snapshot + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (ths->pFsm->FpGetSnapshot != NULL) { + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + } + if (beginIndex <= snapshot.lastApplyIndex) { + beginIndex = snapshot.lastApplyIndex + 1; + } + + // execute fsm + if (ths->pFsm != NULL) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + if (i != SYNC_INDEX_INVALID) { + SSyncRaftEntry* pEntry; + code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); + ASSERT(code == 0); + ASSERT(pEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.term = pEntry->term; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.flag = flag; + + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + + // config change + if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE) { + SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; + + SSyncCfg newSyncCfg; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + ASSERT(ret == 0); + + // update new config myIndex + bool hit = false; + for (int i = 0; i < newSyncCfg.replicaNum; ++i) { + if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && + ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { + newSyncCfg.myIndex = i; + hit = true; + break; + } + } + + SReConfigCbMeta cbMeta = {0}; + bool isDrop; + + // I am in newConfig + if (hit) { + syncNodeUpdateConfig(ths, &newSyncCfg, &isDrop); + + // change isStandBy to normal + if (!isDrop) { + if (ths->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(ths); + } else { + syncNodeBecomeFollower(ths); + } + } + + char* sOld = syncCfg2Str(&oldSyncCfg); + char* sNew = syncCfg2Str(&newSyncCfg); + sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); + taosMemoryFree(sOld); + taosMemoryFree(sNew); + } + + // always call FpReConfigCb + if (ths->pFsm->FpReConfigCb != NULL) { + cbMeta.code = 0; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.index = pEntry->index; + cbMeta.term = pEntry->term; + cbMeta.oldCfg = oldSyncCfg; + cbMeta.flag = 0x11; + cbMeta.isDrop = isDrop; + ths->pFsm->FpReConfigCb(ths->pFsm, newSyncCfg, cbMeta); + } + } + + // restore finish + if (pEntry->index == ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { + if (ths->restoreFinish == false) { + if (ths->pFsm->FpRestoreFinishCb != NULL) { + ths->pFsm->FpRestoreFinishCb(ths->pFsm); + } + ths->restoreFinish = true; + sInfo("restore finish %p vgId:%d", ths, ths->vgId); + } + } + + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pEntry); + } + } + } + return 0; +} + +bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { + for (int i = 0; i < ths->replicaNum; ++i) { + if (syncUtilSameId(&((ths->replicasId)[i]), pRaftId)) { + return true; + } + } + return false; +} + +SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) { + SSyncSnapshotSender* pSender = NULL; + for (int i = 0; i < ths->replicaNum; ++i) { + if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { + pSender = (ths->senders)[i]; + } + } + return pSender; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 6871e6b3ed..af04a0f649 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -65,6 +65,16 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { pRoot = syncAppendEntriesReply2Json(pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotSend2Json(pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotRsp2Json(pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { pRoot = cJSON_CreateObject(); char* s; @@ -135,9 +145,11 @@ void syncRpcMsgLog(SRpcMsg* pMsg) { } void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncRpcMsg2Str(pMsg); + sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncTimeout---- @@ -264,9 +276,11 @@ void syncTimeoutLog(const SyncTimeout* pMsg) { } void syncTimeoutLog2(char* s, const SyncTimeout* pMsg) { - char* serialized = syncTimeout2Str(pMsg); - sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncTimeout2Str(pMsg); + sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncPing---- @@ -524,9 +538,11 @@ void syncPingLog(const SyncPing* pMsg) { } void syncPingLog2(char* s, const SyncPing* pMsg) { - char* serialized = syncPing2Str(pMsg); - sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncPing2Str(pMsg); + sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncPingReply---- @@ -784,9 +800,11 @@ void syncPingReplyLog(const SyncPingReply* pMsg) { } void syncPingReplyLog2(char* s, const SyncPingReply* pMsg) { - char* serialized = syncPingReply2Str(pMsg); - sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncPingReply2Str(pMsg); + sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncClientRequest---- @@ -925,9 +943,11 @@ void syncClientRequestLog(const SyncClientRequest* pMsg) { } void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncClientRequest2Str(pMsg); + sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncRequestVote---- @@ -1074,9 +1094,11 @@ void syncRequestVoteLog(const SyncRequestVote* pMsg) { } void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) { - char* serialized = syncRequestVote2Str(pMsg); - sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncRequestVote2Str(pMsg); + sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncRequestVoteReply---- @@ -1220,9 +1242,11 @@ void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg) { } void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) { - char* serialized = syncRequestVoteReply2Str(pMsg); - sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncRequestVoteReply2Str(pMsg); + sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncAppendEntries---- @@ -1333,6 +1357,9 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->prevLogIndex); cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); @@ -1386,9 +1413,11 @@ void syncAppendEntriesLog(const SyncAppendEntries* pMsg) { } void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { - char* serialized = syncAppendEntries2Str(pMsg); - sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncAppendEntries2Str(pMsg); + sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncAppendEntriesReply---- @@ -1494,6 +1523,9 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "success", pMsg->success); @@ -1535,9 +1567,11 @@ void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg) { } void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { - char* serialized = syncAppendEntriesReply2Str(pMsg); - sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncAppendEntriesReply2Str(pMsg); + sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncApplyMsg---- @@ -1686,7 +1720,339 @@ void syncApplyMsgLog(const SyncApplyMsg* pMsg) { } void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) { - char* serialized = syncApplyMsg2Str(pMsg); - sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + if (gRaftDetailLog) { + char* serialized = syncApplyMsg2Str(pMsg); + sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// --------------------------------------------- +SyncSnapshotSend* syncSnapshotSendBuild(uint32_t dataLen, int32_t vgId) { + uint32_t bytes = sizeof(SyncSnapshotSend) + dataLen; + SyncSnapshotSend* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_SNAPSHOT_SEND; + pMsg->dataLen = dataLen; + return pMsg; +} + +void syncSnapshotSendDestroy(SyncSnapshotSend* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncSnapshotSendSerialize(const SyncSnapshotSend* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncSnapshotSendDeserialize(const char* buf, uint32_t len, SyncSnapshotSend* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); + assert(pMsg->bytes == sizeof(SyncSnapshotSend) + pMsg->dataLen); +} + +char* syncSnapshotSendSerialize2(const SyncSnapshotSend* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + assert(buf != NULL); + syncSnapshotSendSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncSnapshotSend* syncSnapshotSendDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncSnapshotSend* pMsg = taosMemoryMalloc(bytes); + assert(pMsg != NULL); + syncSnapshotSendDeserialize(buf, len, pMsg); + assert(len == pMsg->bytes); + return pMsg; +} + +void syncSnapshotSend2RpcMsg(const SyncSnapshotSend* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncSnapshotSendSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncSnapshotSendFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotSend* pMsg) { + syncSnapshotSendDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncSnapshotSend* pMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + assert(pMsg != NULL); + return pMsg; +} + +cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); + cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm); + cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot); + return pJson; +} + +char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { + cJSON* pJson = syncSnapshotSend2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); taosMemoryFree(serialized); } + +void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// --------------------------------------------- +SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncSnapshotRsp); + SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_SNAPSHOT_RSP; + return pMsg; +} + +void syncSnapshotRspDestroy(SyncSnapshotRsp* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncSnapshotRspSerialize(const SyncSnapshotRsp* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncSnapshotRspDeserialize(const char* buf, uint32_t len, SyncSnapshotRsp* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +char* syncSnapshotRspSerialize2(const SyncSnapshotRsp* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + assert(buf != NULL); + syncSnapshotRspSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncSnapshotRsp* syncSnapshotRspDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); + assert(pMsg != NULL); + syncSnapshotRspDeserialize(buf, len, pMsg); + assert(len == pMsg->bytes); + return pMsg; +} + +void syncSnapshotRsp2RpcMsg(const SyncSnapshotRsp* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncSnapshotRspSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncSnapshotRspFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotRsp* pMsg) { + syncSnapshotRspDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncSnapshotRsp* syncSnapshotRspFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncSnapshotRsp* pMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + assert(pMsg != NULL); + return pMsg; +} + +cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); + cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm); + cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack); + cJSON_AddNumberToObject(pRoot, "code", pMsg->code); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncSnapshotRsp", pRoot); + return pJson; +} + +char* syncSnapshotRsp2Str(const SyncSnapshotRsp* pMsg) { + cJSON* pJson = syncSnapshotRsp2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncSnapshotRspPrint(const SyncSnapshotRsp* pMsg) { + char* serialized = syncSnapshotRsp2Str(pMsg); + printf("syncSnapshotRspPrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotRspPrint2(char* s, const SyncSnapshotRsp* pMsg) { + char* serialized = syncSnapshotRsp2Str(pMsg); + printf("syncSnapshotRspPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg) { + char* serialized = syncSnapshotRsp2Str(pMsg); + sTrace("syncSnapshotRspLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncSnapshotRsp2Str(pMsg); + sTrace("syncSnapshotRspLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 3e1931e2c3..95eec5d98f 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -148,6 +148,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { cJSON *pRoot = cJSON_CreateObject(); cJSON_AddItemToObject(pRoot, "SSyncCfg", syncCfg2Json(&(pRaftCfg->cfg))); cJSON_AddNumberToObject(pRoot, "isStandBy", pRaftCfg->isStandBy); + cJSON_AddNumberToObject(pRoot, "snapshotEnable", pRaftCfg->snapshotEnable); cJSON *pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "RaftCfg", pRoot); @@ -161,7 +162,7 @@ char *raftCfg2Str(SRaftCfg *pRaftCfg) { return serialized; } -int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path) { +int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { assert(pCfg != NULL); TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE); @@ -169,7 +170,8 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path) { SRaftCfg raftCfg; raftCfg.cfg = *pCfg; - raftCfg.isStandBy = isStandBy; + raftCfg.isStandBy = meta.isStandBy; + raftCfg.snapshotEnable = meta.snapshotEnable; char *s = raftCfg2Str(&raftCfg); char buf[CONFIG_FILE_LEN] = {0}; @@ -194,6 +196,9 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { cJSON *pJsonIsStandBy = cJSON_GetObjectItem(pJson, "isStandBy"); pRaftCfg->isStandBy = cJSON_GetNumberValue(pJsonIsStandBy); + cJSON *pJsonSnapshotEnable = cJSON_GetObjectItem(pJson, "snapshotEnable"); + pRaftCfg->snapshotEnable = cJSON_GetNumberValue(pJsonSnapshotEnable); + cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index b353ed85db..49509ae979 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -16,6 +16,23 @@ #include "syncRaftLog.h" #include "wal.h" +// refactor, log[0 .. n] ==> log[m .. n] +static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); +static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); +static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); +static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); +static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); +static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); +static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index); +static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); +static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); +static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); +static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); +static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex); + +static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry); + +//------------------------------- static SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); static SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore); static SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore); @@ -25,6 +42,202 @@ static int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex from static int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index); static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); +// refactor, log[0 .. n] ==> log[m .. n] +static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) { + sTrace("raftLogSetBeginIndex beginIndex:%ld", beginIndex); + + // if beginIndex == 0, donot need call this funciton + ASSERT(beginIndex > 0); + + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + pData->beginIndex = beginIndex; + walRestoreFromSnapshot(pWal, beginIndex - 1); + return 0; +} + +int32_t raftLogResetBeginIndex(struct SSyncLogStore* pLogStore) { return 0; } + +static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + return pData->beginIndex; +} + +static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } + +static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + SyncIndex endIndex = raftLogEndIndex(pLogStore); + return (endIndex < beginIndex); +} + +static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + SyncIndex endIndex = raftLogEndIndex(pLogStore); + int32_t count = endIndex - beginIndex + 1; + return count > 0 ? count : 0; +} + +static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index) { + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + SyncIndex endIndex = raftLogEndIndex(pLogStore); + if (index >= beginIndex && index <= endIndex) { + return true; + } else { + return false; + } +} + +static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { + SyncIndex lastIndex; + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + SyncIndex lastVer = walGetLastVer(pWal); + SyncIndex firstVer = walGetFirstVer(pWal); + + if (lastVer < firstVer) { + // no record + lastIndex = -1; + + } else { + if (firstVer >= 0) { + lastIndex = lastVer; + } else if (firstVer == -1) { + lastIndex = -1; + } else { + ASSERT(0); + } + } + + return lastIndex; +} + +static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + SyncIndex lastVer = walGetLastVer(pWal); + return lastVer + 1; +} + +static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { + SyncTerm lastTerm = 0; + if (raftLogEntryCount(pLogStore) == 0) { + lastTerm = 0; + } else { + SSyncRaftEntry* pLastEntry; + int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry); + ASSERT(code == 0); + if (pLastEntry != NULL) { + lastTerm = pLastEntry->term; + taosMemoryFree(pLastEntry); + } + } + return lastTerm; +} + +static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + + SyncIndex writeIndex = raftLogWriteIndex(pLogStore); + ASSERT(pEntry->index == writeIndex); + + int code = 0; + SSyncLogMeta syncMeta; + syncMeta.isWeek = pEntry->isWeak; + syncMeta.seqNum = pEntry->seqNum; + syncMeta.term = pEntry->term; + code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("raftLogAppendEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); + ASSERT(0); + } + + walFsync(pWal, true); + + sTrace("sync event write index:%" PRId64, pEntry->index); + + return code; +} + +static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + int32_t code; + + *ppEntry = NULL; + if (raftLogInRange(pLogStore, index)) { + SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + ASSERT(pWalHandle != NULL); + + code = walReadWithHandle(pWalHandle, index); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("raftLogGetEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); + ASSERT(0); + walCloseReadHandle(pWalHandle); + return code; + } + + *ppEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); + ASSERT(*ppEntry != NULL); + (*ppEntry)->msgType = TDMT_SYNC_CLIENT_REQUEST; + (*ppEntry)->originalRpcType = pWalHandle->pHead->head.msgType; + (*ppEntry)->seqNum = pWalHandle->pHead->head.syncMeta.seqNum; + (*ppEntry)->isWeak = pWalHandle->pHead->head.syncMeta.isWeek; + (*ppEntry)->term = pWalHandle->pHead->head.syncMeta.term; + (*ppEntry)->index = index; + ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen); + memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); + + // need to hold, do not new every time!! + walCloseReadHandle(pWalHandle); + + } else { + // index not in range + code = 0; + } + + return code; +} + +static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + int32_t code = walRollback(pWal, fromIndex); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("raftLogTruncate error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); + ASSERT(0); + } + return code; +} + +static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { + *ppLastEntry = NULL; + if (raftLogEntryCount(pLogStore) == 0) { + return 0; + } + SyncIndex lastIndex = raftLogLastIndex(pLogStore); + int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); + return code; +} + +//------------------------------- SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore)); assert(pLogStore != NULL); @@ -36,6 +249,16 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pData->pSyncNode = pSyncNode; pData->pWal = pSyncNode->pWal; + SyncIndex firstVer = walGetFirstVer(pData->pWal); + SyncIndex lastVer = walGetLastVer(pData->pWal); + if (firstVer >= 0) { + pData->beginIndex = firstVer; + } else if (firstVer == -1) { + pData->beginIndex = lastVer + 1; + } else { + ASSERT(0); + } + pLogStore->appendEntry = logStoreAppendEntry; pLogStore->getEntry = logStoreGetEntry; pLogStore->truncate = logStoreTruncate; @@ -43,6 +266,20 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->getLastTerm = logStoreLastTerm; pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex; + + pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex; + pLogStore->syncLogBeginIndex = raftLogBeginIndex; + pLogStore->syncLogEndIndex = raftLogEndIndex; + pLogStore->syncLogIsEmpty = raftLogIsEmpty; + pLogStore->syncLogEntryCount = raftLogEntryCount; + pLogStore->syncLogInRange = raftLogInRange; + pLogStore->syncLogLastIndex = raftLogLastIndex; + pLogStore->syncLogLastTerm = raftLogLastTerm; + pLogStore->syncLogAppendEntry = raftLogAppendEntry; + pLogStore->syncLogGetEntry = raftLogGetEntry; + pLogStore->syncLogTruncate = raftLogTruncate; + pLogStore->syncLogWriteIndex = raftLogWriteIndex; + return pLogStore; } @@ -53,6 +290,7 @@ void logStoreDestory(SSyncLogStore* pLogStore) { } } +//------------------------------- int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -78,6 +316,8 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { // assert(code == 0); walFsync(pWal, true); + + sTrace("sync event old write wal: %ld", pEntry->index); return code; } @@ -136,7 +376,7 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { linuxErrMsg); ASSERT(0); } - return 0; // to avoid compiler error + return 0; } SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore) { @@ -169,7 +409,7 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } - return 0; // to avoid compiler error + return 0; } SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore) { @@ -199,15 +439,32 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); + + snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); cJSON* pEntries = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "pEntries", pEntries); - SyncIndex lastIndex = logStoreLastIndex(pLogStore); - for (SyncIndex i = 0; i <= lastIndex; ++i) { + + for (SyncIndex i = pData->beginIndex; i <= endIndex; ++i) { SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); syncEntryDestory(pEntry); @@ -236,9 +493,26 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); + + snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); } @@ -254,6 +528,12 @@ char* logStoreSimple2Str(SSyncLogStore* pLogStore) { return serialized; } +SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + return walGetFirstVer(pWal); +} + // for debug ----------------- void logStorePrint(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); @@ -303,7 +583,9 @@ void logStoreSimpleLog(SSyncLogStore* pLogStore) { } void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index d17e64d936..ff39b0b13d 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -16,9 +16,11 @@ #include "syncReplication.h" #include "syncIndexMgr.h" #include "syncMessage.h" +#include "syncRaftCfg.h" #include "syncRaftEntry.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncSnapshot.h" #include "syncUtil.h" // TLA+ Spec @@ -59,6 +61,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { // set prevLogIndex SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + SyncIndex preLogIndex = nextIndex - 1; // set preLogTerm @@ -113,9 +116,87 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { return ret; } +int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + + syncIndexMgrLog2("begin append entries peers pNextIndex:", pSyncNode->pNextIndex); + syncIndexMgrLog2("begin append entries peers pMatchIndex:", pSyncNode->pMatchIndex); + logStoreSimpleLog2("begin append entries peers LogStore:", pSyncNode->pLogStore); + { + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + sTrace("begin append entries peers, snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", + snapshot.lastApplyIndex, snapshot.lastApplyTerm); + } + + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId* pDestId = &(pSyncNode->peersId[i]); + + // next index + SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + + // pre index, pre term + SyncIndex preLogIndex = syncNodeGetPreIndex(pSyncNode, nextIndex); + SyncTerm preLogTerm = syncNodeGetPreTerm(pSyncNode, nextIndex); + + // batch optimized + // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); + + // prepare entry + SyncAppendEntries* pMsg = NULL; + + SSyncRaftEntry* pEntry; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, nextIndex, &pEntry); + ASSERT(code == 0); + + if (pEntry != NULL) { + pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); + ASSERT(pMsg != NULL); + + // add pEntry into msg + uint32_t len; + char* serialized = syncEntrySerialize(pEntry, &len); + assert(len == pEntry->bytes); + memcpy(pMsg->data, serialized, len); + + taosMemoryFree(serialized); + syncEntryDestory(pEntry); + + } else { + // no entry in log + pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); + ASSERT(pMsg != NULL); + } + + // prepare msg + ASSERT(pMsg != NULL); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = *pDestId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->prevLogIndex = preLogIndex; + pMsg->prevLogTerm = preLogTerm; + pMsg->commitIndex = pSyncNode->commitIndex; + pMsg->privateTerm = 0; + // pMsg->privateTerm = syncIndexMgrGetTerm(pSyncNode->pNextIndex, pDestId); + + // send msg + syncNodeAppendEntries(pSyncNode, pDestId, pMsg); + syncAppendEntriesDestroy(pMsg); + } + + return ret; +} + int32_t syncNodeReplicate(SSyncNode* pSyncNode) { // start replicate - int32_t ret = syncNodeAppendEntriesPeers(pSyncNode); + int32_t ret = 0; + + if (pSyncNode->pRaftCfg->snapshotEnable) { + ret = syncNodeAppendEntriesPeersSnapshot(pSyncNode); + } else { + ret = syncNodeAppendEntriesPeers(pSyncNode); + } return ret; } diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 2656771292..9ed7f00982 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -15,6 +15,7 @@ #include "syncRequestVote.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" @@ -62,6 +63,9 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { // maybe has already voted for pMsg->srcId // vote again, no harm raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + + // forbid elect for this round + syncNodeResetElectTimer(ths); } SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); @@ -77,3 +81,64 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return ret; } + +static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) { + SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); + SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); + + if (pMsg->lastLogTerm > myLastTerm) { + return true; + } + if (pMsg->lastLogTerm == myLastTerm && pMsg->lastLogIndex >= myLastIndex) { + return true; + } + + return false; +} + +int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = 0; + + // print log + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVote, currentTerm:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteLog2(logBuf, pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + sInfo("recv SyncRequestVote maybe replica already dropped"); + return ret; + } + + // maybe update term + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + + bool logOK = syncNodeOnRequestVoteLogOK(ths, pMsg); + bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && + ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + if (grant) { + // maybe has already voted for pMsg->srcId + // vote again, no harm + raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + + // forbid elect for this round + syncNodeResetElectTimer(ths); + } + + // send msg + SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->voteGranted = grant; + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncRequestVoteReplyDestroy(pReply); + + return ret; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 75236aee2b..5d041cefcd 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -15,6 +15,7 @@ #include "syncRequestVoteReply.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" @@ -92,3 +93,68 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } + +int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = 0; + + // print log + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteReplyLog2(logBuf, pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + sInfo("recv SyncRequestVoteReply, maybe replica already dropped"); + return ret; + } + + // drop stale response + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + return ret; + } + + // assert(!(pMsg->term > ths->pRaftStore->currentTerm)); + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, error term, receive_term:%lu current_term:%lu", + pMsg->term, ths->pRaftStore->currentTerm); + syncNodePrint2(logBuf, ths); + sError("%s", logBuf); + return ret; + } + + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + + // This tallies votes even when the current state is not Candidate, + // but they won't be looked at, so it doesn't matter. + if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { + votesRespondAdd(ths->pVotesRespond, pMsg); + if (pMsg->voteGranted) { + // add vote + voteGrantedVote(ths->pVotesGranted, pMsg); + + // maybe to leader + if (voteGrantedMajority(ths->pVotesGranted)) { + if (!ths->pVotesGranted->toLeader) { + syncNodeCandidate2Leader(ths); + + // prevent to leader again! + ths->pVotesGranted->toLeader = true; + } + } + } else { + ; + // do nothing + // UNCHANGED <> + } + } + + return ret; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index ccb0e6071b..a68312d07f 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,23 +14,598 @@ */ #include "syncSnapshot.h" +#include "syncIndexMgr.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" -SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode) { return NULL; } +static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm); -void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {} +//---------------------------------- +SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { + bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && + (pSyncNode->pFsm->FpSnapshotDoRead != NULL); -int32_t snapshotSend(SSyncSnapshotSender *pSender) { return 0; } + SSyncSnapshotSender *pSender = NULL; + if (condition) { + pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); + ASSERT(pSender != NULL); + memset(pSender, 0, sizeof(*pSender)); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { return NULL; } + pSender->start = false; + pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; + pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + pSender->pReader = NULL; + pSender->pCurrentBlock = NULL; + pSender->blockLen = 0; + pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; + pSender->pSyncNode = pSyncNode; + pSender->replicaIndex = replicaIndex; + pSender->term = pSyncNode->pRaftStore->currentTerm; + pSender->privateTerm = taosGetTimestampMs() + 100; + pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); + pSender->finish = false; + } else { + sError("snapshotSenderCreate cannot create sender"); + } + return pSender; +} -char *snapshotSender2Str(SSyncSnapshotSender *pSender) { return NULL; } +void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { + if (pSender != NULL) { + if (pSender->pCurrentBlock != NULL) { + taosMemoryFree(pSender->pCurrentBlock); + } + taosMemoryFree(pSender); + } +} -SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode) { return NULL; } +bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } -void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {} +// begin send snapshot (current term, seq begin) +void snapshotSenderStart(SSyncSnapshotSender *pSender) { + ASSERT(!snapshotSenderIsStart(pSender)); -int32_t snapshotReceive(SSyncSnapshotReceiver *pReceiver) { return 0; } + pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; + pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { return NULL; } + // open snapshot reader + ASSERT(pSender->pReader == NULL); + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStartRead(pSender->pSyncNode->pFsm, &(pSender->pReader)); + ASSERT(ret == 0); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { return NULL; } + if (pSender->pCurrentBlock != NULL) { + taosMemoryFree(pSender->pCurrentBlock); + } + + pSender->blockLen = 0; + + // get current snapshot info + pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); + + pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; + pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + ++(pSender->privateTerm); + pSender->finish = false; + pSender->start = true; + + // build begin msg + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->seq = pSender->seq; // SYNC_SNAPSHOT_SEQ_BEGIN + pMsg->privateTerm = pSender->privateTerm; + + // send msg + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); + sTrace("sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", host, + port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, msgStr); + taosMemoryFree(msgStr); + + syncSnapshotSendDestroy(pMsg); +} + +#if 0 +// when entry in snapshot, start sender +void snapshotSenderStart(SSyncSnapshotSender *pSender) { + if (!(pSender->start)) { + // start + snapshotSenderDoStart(pSender); + pSender->start = true; + } else { + // already start + ASSERT(pSender->pSyncNode->pRaftStore->currentTerm >= pSender->term); + + // if current term is higher, need start again + if (pSender->pSyncNode->pRaftStore->currentTerm > pSender->term) { + // force peer rollback + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->seq = SYNC_SNAPSHOT_SEQ_FORCE_CLOSE; + + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("snapshot send force close seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); + taosMemoryFree(msgStr); + + syncSnapshotSendDestroy(pMsg); + + // close reader + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); + ASSERT(ret == 0); + pSender->pReader = NULL; + + // start again + snapshotSenderDoStart(pSender); + pSender->start = true; + } else { + // current term, do nothing + ASSERT(pSender->pSyncNode->pRaftStore->currentTerm == pSender->term); + } + } + + char *s = snapshotSender2Str(pSender); + sInfo("snapshotSenderStart %s", s); + taosMemoryFree(s); +} +#endif + +void snapshotSenderStop(SSyncSnapshotSender *pSender) { + if (pSender->pReader != NULL) { + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); + ASSERT(ret == 0); + pSender->pReader = NULL; + } + + if (pSender->pCurrentBlock != NULL) { + taosMemoryFree(pSender->pCurrentBlock); + pSender->pCurrentBlock = NULL; + pSender->blockLen = 0; + } + + pSender->start = false; + + char *s = snapshotSender2Str(pSender); + sInfo("snapshotSenderStop %s", s); + taosMemoryFree(s); +} + +// when sender receiver ack, call this function to send msg from seq +// seq = ack + 1, already updated +int32_t snapshotSend(SSyncSnapshotSender *pSender) { + // free memory last time (seq - 1) + if (pSender->pCurrentBlock != NULL) { + taosMemoryFree(pSender->pCurrentBlock); + pSender->pCurrentBlock = NULL; + pSender->blockLen = 0; + } + + // read data + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, + &(pSender->pCurrentBlock), &(pSender->blockLen)); + ASSERT(ret == 0); + if (pSender->blockLen > 0) { + // has read data + } else { + // read finish + pSender->seq = SYNC_SNAPSHOT_SEQ_END; + } + + // build msg + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->seq = pSender->seq; + pMsg->privateTerm = pSender->privateTerm; + memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + + // send msg + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); + if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { + sTrace("sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", + host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, + msgStr); + } else { + sTrace("sync event snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", + host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, + msgStr); + } + taosMemoryFree(msgStr); + + syncSnapshotSendDestroy(pMsg); + return 0; +} + +// send snapshot data from cache +int32_t snapshotReSend(SSyncSnapshotSender *pSender) { + if (pSender->pCurrentBlock != NULL) { + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->seq = pSender->seq; + memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); + sTrace("sync event snapshot send to %s:%d resend seq:%d ack:%d send msg:%s", host, port, pSender->seq, pSender->ack, + msgStr); + taosMemoryFree(msgStr); + + syncSnapshotSendDestroy(pMsg); + } + return 0; +} + +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pSender != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pSender->start); + cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); + cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); + cJSON_AddStringToObject(pRoot, "pReader", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); + cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); + + if (pSender->pCurrentBlock != NULL) { + char *s; + s = syncUtilprintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); + taosMemoryFree(s); + s = syncUtilprintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); + taosMemoryFree(s); + } + + cJSON *pSnapshot = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); + cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); + + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS); + cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); + return pJson; +} + +char *snapshotSender2Str(SSyncSnapshotSender *pSender) { + cJSON *pJson = snapshotSender2Json(pSender); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// ------------------------------------- +SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { + bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && + (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); + + SSyncSnapshotReceiver *pReceiver = NULL; + if (condition) { + pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); + ASSERT(pReceiver != NULL); + memset(pReceiver, 0, sizeof(*pReceiver)); + + pReceiver->start = false; + pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; + pReceiver->pWriter = NULL; + pReceiver->pSyncNode = pSyncNode; + pReceiver->replicaIndex = replicaIndex; + pReceiver->term = pSyncNode->pRaftStore->currentTerm; + pReceiver->privateTerm = 0; + + } else { + sInfo("snapshotReceiverCreate cannot create receiver"); + } + + return pReceiver; +} + +void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { + if (pReceiver != NULL) { + taosMemoryFree(pReceiver); + } +} + +bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } + +// begin receive snapshot msg (current term, seq begin) +static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm) { + pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; + pReceiver->privateTerm = privateTerm; + pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; + + ASSERT(pReceiver->pWriter == NULL); + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); + ASSERT(ret == 0); +} + +// if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver +// if already start, force close, start again +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm) { + if (!snapshotReceiverIsStart(pReceiver)) { + // start + snapshotReceiverDoStart(pReceiver, privateTerm); + pReceiver->start = true; + + } else { + // already start + + // force close, abandon incomplete data + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; + + // start again + snapshotReceiverDoStart(pReceiver, privateTerm); + pReceiver->start = true; + + ASSERT(0); + } + + char *s = snapshotReceiver2Str(pReceiver); + sInfo("snapshotReceiverStart %s", s); + taosMemoryFree(s); +} + +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { + if (pReceiver->pWriter != NULL) { + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; + } + + pReceiver->start = false; + + if (apply) { + ++(pReceiver->privateTerm); + } + + char *s = snapshotReceiver2Str(pReceiver); + sInfo("snapshotReceiverStop %s", s); + taosMemoryFree(s); +} + +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pReceiver != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); + cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); + cJSON_AddStringToObject(pRoot, "pWriter", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "replicaIndex", pReceiver->replicaIndex); + snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); + return pJson; +} + +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { + cJSON *pJson = snapshotReceiver2Json(pReceiver); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// receiver do something +int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { + // get receiver + SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; + bool needRsp = false; + int32_t writeCode = 0; + + // state, term, seq/ack + if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { + if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { + if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { + // begin + snapshotReceiverStart(pReceiver, pMsg->privateTerm); + pReceiver->ack = pMsg->seq; + needRsp = true; + + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sTrace("sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, port, + pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + taosMemoryFree(msgStr); + + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { + // end, finish FSM + writeCode = pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + ASSERT(writeCode == 0); + + pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); + + pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); + char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + char host[128]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sInfo( + "sync event snapshot recv from %s:%d finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, " + "snapshot.lastApplyTerm:%lu, raft log:%s", + host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); + taosMemoryFree(logSimpleStr); + + pReceiver->pWriter = NULL; + snapshotReceiverStop(pReceiver, true); + pReceiver->ack = pMsg->seq; + needRsp = true; + + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, port, + pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + taosMemoryFree(msgStr); + + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { + pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); + snapshotReceiverStop(pReceiver, false); + needRsp = false; + + char host[128]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, + port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + + taosMemoryFree(msgStr); + + } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { + // transfering + if (pMsg->seq == pReceiver->ack + 1) { + writeCode = + pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + ASSERT(writeCode == 0); + pReceiver->ack = pMsg->seq; + } + needRsp = true; + + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sTrace("sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, + port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + taosMemoryFree(msgStr); + + } else { + ASSERT(0); + } + + if (needRsp) { + SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); + pRspMsg->srcId = pSyncNode->myRaftId; + pRspMsg->destId = pMsg->srcId; + pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->lastIndex = pMsg->lastIndex; + pRspMsg->lastTerm = pMsg->lastTerm; + pRspMsg->ack = pReceiver->ack; + pRspMsg->code = writeCode; + pRspMsg->privateTerm = pReceiver->privateTerm; + + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); + syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); + + syncSnapshotRspDestroy(pRspMsg); + } + } + } else { + syncNodeLog2("syncNodeOnSnapshotSendCb not follower", pSyncNode); + } + + return 0; +} + +// sender receives ack, set seq = ack + 1, send msg from seq +// if ack == SYNC_SNAPSHOT_SEQ_END, stop sender +int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { + // get sender + SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId)); + ASSERT(pSender != NULL); + + // state, term, seq/ack + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { + // receiver ack is finish, close sender + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { + pSender->finish = true; + snapshotSenderStop(pSender); + return 0; + } + + // send next msg + if (pMsg->ack == pSender->seq) { + // update sender ack + pSender->ack = pMsg->ack; + (pSender->seq)++; + snapshotSend(pSender); + + } else if (pMsg->ack == pSender->seq - 1) { + snapshotReSend(pSender); + + } else { + ASSERT(0); + } + } + } else { + syncNodeLog2("syncNodeOnSnapshotRspCb not leader", pSyncNode); + } + + return 0; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 48567b75c2..f6ff521e01 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -240,4 +240,26 @@ bool syncUtilUserRollback(tmsg_t msgType) { return true; } return false; +} + +void syncUtilJson2Line(char* jsonStr) { + int p, q, len; + p = 0; + q = 1; + len = strlen(jsonStr); + while (1) { + if (jsonStr[q] == '\0') { + jsonStr[p + 1] = '\0'; + break; + } + + if (jsonStr[q] == '\n' || jsonStr[q] == ' ' || jsonStr[q] == '\t') { + q++; + continue; + } else { + jsonStr[p + 1] = jsonStr[q]; + p++; + q++; + } + } } \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index cfbdf0e961..c68c6349fb 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -38,6 +38,15 @@ add_executable(syncRespMgrTest "") add_executable(syncSnapshotTest "") add_executable(syncApplyMsgTest "") add_executable(syncConfigChangeTest "") +add_executable(syncConfigChangeSnapshotTest "") +add_executable(syncSnapshotSendTest "") +add_executable(syncSnapshotRspTest "") +add_executable(syncSnapshotSenderTest "") +add_executable(syncSnapshotReceiverTest "") +add_executable(syncTestTool "") +add_executable(syncRaftLogTest "") +add_executable(syncRaftLogTest2 "") +add_executable(syncRaftLogTest3 "") target_sources(syncTest @@ -200,6 +209,42 @@ target_sources(syncConfigChangeTest PRIVATE "syncConfigChangeTest.cpp" ) +target_sources(syncConfigChangeSnapshotTest + PRIVATE + "syncConfigChangeSnapshotTest.cpp" +) +target_sources(syncSnapshotSendTest + PRIVATE + "syncSnapshotSendTest.cpp" +) +target_sources(syncSnapshotRspTest + PRIVATE + "syncSnapshotRspTest.cpp" +) +target_sources(syncSnapshotSenderTest + PRIVATE + "syncSnapshotSenderTest.cpp" +) +target_sources(syncSnapshotReceiverTest + PRIVATE + "syncSnapshotReceiverTest.cpp" +) +target_sources(syncTestTool + PRIVATE + "syncTestTool.cpp" +) +target_sources(syncRaftLogTest + PRIVATE + "syncRaftLogTest.cpp" +) +target_sources(syncRaftLogTest2 + PRIVATE + "syncRaftLogTest2.cpp" +) +target_sources(syncRaftLogTest3 + PRIVATE + "syncRaftLogTest3.cpp" +) target_include_directories(syncTest @@ -402,6 +447,51 @@ target_include_directories(syncConfigChangeTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncConfigChangeSnapshotTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncSnapshotSendTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncSnapshotRspTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncSnapshotSenderTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncSnapshotReceiverTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncTestTool + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncRaftLogTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncRaftLogTest2 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncRaftLogTest3 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -564,6 +654,42 @@ target_link_libraries(syncConfigChangeTest sync gtest_main ) +target_link_libraries(syncConfigChangeSnapshotTest + sync + gtest_main +) +target_link_libraries(syncSnapshotSendTest + sync + gtest_main +) +target_link_libraries(syncSnapshotRspTest + sync + gtest_main +) +target_link_libraries(syncSnapshotSenderTest + sync + gtest_main +) +target_link_libraries(syncSnapshotReceiverTest + sync + gtest_main +) +target_link_libraries(syncTestTool + sync + gtest_main +) +target_link_libraries(syncRaftLogTest + sync + gtest_main +) +target_link_libraries(syncRaftLogTest2 + sync + gtest_main +) +target_link_libraries(syncRaftLogTest3 + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index a90259bc3a..d41e99a3cd 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -22,6 +22,8 @@ SyncAppendEntriesReply *createMsg() { pMsg->destId.vgId = 100; pMsg->success = true; pMsg->matchIndex = 77; + pMsg->term = 33; + pMsg->privateTerm = 44; return pMsg; } diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index bb9f306a1c..98b392274e 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -23,6 +23,7 @@ SyncAppendEntries *createMsg() { pMsg->prevLogIndex = 11; pMsg->prevLogTerm = 22; pMsg->commitIndex = 33; + pMsg->privateTerm = 44; strcpy(pMsg->data, "hello world"); return pMsg; } diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp new file mode 100644 index 0000000000..781c168da9 --- /dev/null +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -0,0 +1,366 @@ +#include +#include +#include "os.h" +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t gPorts[] = {7010, 7110, 7210, 7310, 7410}; +const char* gDir = "./syncReplicateTest"; +int32_t gVgId = 1234; +SyncIndex gSnapshotLastApplyIndex; +SyncIndex gSnapshotLastApplyTerm; + +void init() { + int code = walInit(); + assert(code == 0); + + code = syncInit(); + assert(code == 0); + + sprintf(tsTempDir, "%s", "."); +} + +void cleanup() { walCleanUp(); } + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + SyncIndex beginIndex = SYNC_INDEX_INVALID; + if (pFsm->FpGetSnapshot != NULL) { + SSnapshot snapshot; + pFsm->FpGetSnapshot(pFsm, &snapshot); + beginIndex = snapshot.lastApplyIndex; + } + + if (cbMeta.index > beginIndex) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); + } else { + sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + } +} + +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; + return 0; +} + +int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) { + *ppReader = (void*)0xABCD; + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartRead== pFsm:%p, *ppReader:%p", pFsm, *ppReader); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopRead== pFsm:%p, pReader:%p", pFsm, pReader); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { + static int readIter = 0; + + if (readIter == 5) { + *len = 0; + *ppBuf = NULL; + } else if (readIter < 5) { + *len = 20; + *ppBuf = taosMemoryMalloc(*len); + snprintf((char*)*ppBuf, *len, "data iter:%d", readIter); + } + + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==SnapshotDoRead== pFsm:%p, pReader:%p, *len:%d, *ppBuf:%s, readIter:%d", pFsm, pReader, *len, + (char*)(*ppBuf), readIter); + sTrace("%s", logBuf); + + readIter++; + return 0; +} + +int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { + *ppWriter = (void*)0xCDEF; + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartWrite== pFsm:%p, *ppWriter:%p", pFsm, *ppWriter); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d", pFsm, pWriter, + isApply); + sTrace("%s", logBuf); + + if (isApply) { + gSnapshotLastApplyIndex = 10; + gSnapshotLastApplyTerm = 1; + } + + return 0; +} + +int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotDoWrite== pFsm:%p, pWriter:%p, len:%d pBuf:%s", pFsm, + pWriter, len, (char*)pBuf); + sTrace("%s", logBuf); + return 0; +} + +void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb=="); } + +void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { + sTrace("==callback== ==ReConfigCb== flag:0x%lX, isDrop:%d, index:%ld, code:%d, currentTerm:%lu, term:%lu", + cbMeta.flag, cbMeta.isDrop, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term); +} + +SSyncFSM* createFsm() { + SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + memset(pFsm, 0, sizeof(*pFsm)); + + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; + + pFsm->FpGetSnapshot = GetSnapshotCb; + pFsm->FpRestoreFinishCb = RestoreFinishCb; + pFsm->FpSnapshotStartRead = SnapshotStartRead; + pFsm->FpSnapshotStopRead = SnapshotStopRead; + pFsm->FpSnapshotDoRead = SnapshotDoRead; + pFsm->FpSnapshotStartWrite = SnapshotStartWrite; + pFsm->FpSnapshotStopWrite = SnapshotStopWrite; + pFsm->FpSnapshotDoWrite = SnapshotDoWrite; + + pFsm->FpReConfigCb = ReConfigCb; + + return pFsm; +} + +SWal* createWal(char* path, int32_t vgId) { + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal* pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + return pWal; +} + +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) { + SSyncInfo syncInfo; + syncInfo.vgId = vgId; + syncInfo.msgcb = &gSyncIO->msgcb; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = createFsm(); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); + syncInfo.pWal = pWal; + syncInfo.isStandBy = isStandBy; + syncInfo.snapshotEnable = true; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + + if (isStandBy) { + pCfg->myIndex = 0; + pCfg->replicaNum = 1; + pCfg->nodeInfo[0].nodePort = gPorts[myIndex]; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + } else { + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } + } + + int64_t rid = syncOpen(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + + gSyncIO->FpOnSyncSnapshotSend = pSyncNode->FpOnSnapshotSend; + gSyncIO->FpOnSyncSnapshotRsp = pSyncNode->FpOnSnapshotRsp; + + gSyncIO->pSyncNode = pSyncNode; + syncNodeRelease(pSyncNode); + + return rid; +} + +void configChange(int64_t rid, int32_t replicaNum, int32_t myIndex) { + SSyncCfg syncCfg; + + syncCfg.myIndex = myIndex; + syncCfg.replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + syncCfg.nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn); + } + + syncReconfig(rid, &syncCfg); +} + +void usage(char* exe) { + printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange lastApplyTerm \n", exe); +} + +SRpcMsg* createRpcMsg(int i, int count, int myIndex) { + SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 256; + pMsg->pCont = rpcMallocCont(pMsg->contLen); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + return pMsg; +} + +int main(int argc, char** argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_INFO; + if (argc != 8) { + usage(argv[0]); + exit(-1); + } + + int32_t replicaNum = atoi(argv[1]); + int32_t myIndex = atoi(argv[2]); + int32_t lastApplyIndex = atoi(argv[3]); + int32_t writeRecordNum = atoi(argv[4]); + bool isStandBy = atoi(argv[5]); + bool isConfigChange = atoi(argv[6]); + int32_t lastApplyTerm = atoi(argv[7]); + + sTrace( + "args: replicaNum:%d, myIndex:%d, lastApplyIndex:%d, writeRecordNum:%d, isStandBy:%d, isConfigChange:%d, " + "lastApplyTerm:%d", + replicaNum, myIndex, lastApplyIndex, writeRecordNum, isStandBy, isConfigChange, lastApplyTerm); + + gSnapshotLastApplyIndex = lastApplyIndex; + gSnapshotLastApplyTerm = lastApplyTerm; + + if (!isStandBy) { + assert(replicaNum >= 1 && replicaNum <= 5); + assert(myIndex >= 0 && myIndex < replicaNum); + assert(lastApplyIndex >= -1); + assert(writeRecordNum >= 0); + } + + init(); + int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); + assert(ret == 0); + + char walPath[128]; + snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); + SWal* pWal = createWal(walPath, gVgId); + + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy); + assert(rid > 0); + + syncStart(rid); + + /* + if (isStandBy) { + syncStartStandBy(rid); + } else { + syncStart(rid); + } + */ + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + if (isConfigChange) { + configChange(rid, 2, myIndex); + } + + //--------------------------- + int32_t alreadySend = 0; + while (1) { + char* s = syncNode2SimpleStr(pSyncNode); + + if (alreadySend < writeRecordNum) { + SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); + int32_t ret = syncPropose(rid, pRpcMsg, false); + if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + sTrace("%s value%d write not leader", s, alreadySend); + } else { + assert(ret == 0); + sTrace("%s value%d write ok", s, alreadySend); + } + alreadySend++; + + rpcFreeCont(pRpcMsg->pCont); + taosMemoryFree(pRpcMsg); + } else { + sTrace("%s", s); + } + + taosMsleep(1000); + taosMemoryFree(s); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + syncStop(rid); + walClose(pWal); + syncIOStop(); + cleanup(); + return 0; +} diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index 1ab3ce203a..c9d9ca48aa 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -93,7 +93,6 @@ SSyncFSM* createFsm() { pFsm->FpGetSnapshot = GetSnapshotCb; pFsm->FpRestoreFinishCb = RestoreFinishCb; - pFsm->FpReConfigCb = ReConfigCb; return pFsm; diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 7fcce2bc4f..0ad69f0f51 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -22,55 +22,23 @@ int32_t replicaNum = 3; int32_t myIndex = 0; SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; SSyncNode* pSyncNode; SSyncNode* syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.msgcb = &gSyncIO->msgcb; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; - + pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->replicaNum = replicaNum; for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + pSyncNode->replicasId[i].addr = syncUtilAddr2U64("127.0.0.1", ports[i]); + pSyncNode->replicasId[i].vgId = 1234; + + ids[i].addr = pSyncNode->replicasId[i].addr; + ids[i].vgId = pSyncNode->replicasId[i].vgId; } - pSyncNode = syncNodeOpen(&syncInfo); - assert(pSyncNode != NULL); - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->pSyncNode = pSyncNode; - return pSyncNode; } -SSyncNode* syncInitTest() { return syncNodeInit(); } - -void initRaftId(SSyncNode* pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char* s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; @@ -80,58 +48,52 @@ int main(int argc, char** argv) { myIndex = atoi(argv[1]); } - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - ret = syncEnvStart(); - assert(ret == 0); - - SSyncNode* pSyncNode = syncInitTest(); + SSyncNode* pSyncNode = syncNodeInit(); assert(pSyncNode != NULL); - char* serialized = syncNode2Str(pSyncNode); - printf("%s\n", serialized); - taosMemoryFree(serialized); - - initRaftId(pSyncNode); - + printf("---------------------------------------\n"); SSyncIndexMgr* pSyncIndexMgr = syncIndexMgrCreate(pSyncNode); assert(pSyncIndexMgr != NULL); - - printf("---------------------------------------\n"); { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); taosMemoryFree(serialized); } + printf("---------------------------------------\n"); + printf("---------------------------------------\n"); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300); - - printf("---------------------------------------\n"); + // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[0], 700); + // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[1], 800); + // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[2], 900); { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); taosMemoryFree(serialized); } + printf("---------------------------------------\n"); printf("---------------------------------------\n"); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]); - printf("index %d : %lu \n", i, idx); + // SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); + // printf("%d: index:%ld term:%lu \n", i, idx, term); } - - syncIndexMgrClear(pSyncIndexMgr); printf("---------------------------------------\n"); + + printf("---------------------------------------\n"); + syncIndexMgrClear(pSyncIndexMgr); { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); taosMemoryFree(serialized); } + printf("---------------------------------------\n"); syncIndexMgrDestroy(pSyncIndexMgr); return 0; diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index f5b24db651..564cbdb69a 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -71,7 +71,10 @@ void test3() { if (taosCheckExistFile(s)) { printf("%s file: %s already exist! \n", (char*)__FUNCTION__, s); } else { - raftCfgCreateFile(pCfg, 7, s); + SRaftCfgMeta meta; + meta.isStandBy = 7; + meta.snapshotEnable = 9; + raftCfgCreateFile(pCfg, meta, s); printf("%s create json file: %s \n", (char*)__FUNCTION__, s); } @@ -94,6 +97,7 @@ void test5() { pCfg->cfg.myIndex = taosGetTimestampSec(); pCfg->isStandBy += 2; + pCfg->snapshotEnable += 3; raftCfgPersist(pCfg); printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex); diff --git a/source/libs/sync/test/syncRaftLogTest.cpp b/source/libs/sync/test/syncRaftLogTest.cpp new file mode 100644 index 0000000000..7903e86749 --- /dev/null +++ b/source/libs/sync/test/syncRaftLogTest.cpp @@ -0,0 +1,172 @@ +#include "syncRaftLog.h" +//#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +const char *gWalPath = "./syncLogStoreTest_wal"; + +void init() { walInit(); } + +void test1() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void test2() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + for (int i = 0; i < 5; ++i) { + int code = walWrite(pWal, i, 100, "aa", 3); + if (code != 0) { + printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i); + assert(0); + } + } + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void test3() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + walRestoreFromSnapshot(pWal, 5); + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void test4() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + walRestoreFromSnapshot(pWal, 5); + + for (int i = 6; i < 10; ++i) { + int code = walWrite(pWal, i, 100, "aa", 3); + if (code != 0) { + printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i); + assert(0); + } + } + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void test5() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + walRestoreFromSnapshot(pWal, 5); + walRestoreFromSnapshot(pWal, 7); + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void cleanup() { walCleanUp(); } + +int main(int argc, char **argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + init(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + cleanup(); + return 0; +} diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp new file mode 100644 index 0000000000..64e1da51a1 --- /dev/null +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -0,0 +1,437 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SSyncNode* pSyncNode; +SWal* pWal; +SSyncLogStore* pLogStore; +const char* pWalPath = "./syncLogStoreTest_wal"; + +SyncIndex gSnapshotLastApplyIndex; +SyncIndex gSnapshotLastApplyTerm; + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; + return 0; +} + +bool gAssert = true; + +void init() { + walInit(); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pWalPath, &walCfg); + assert(pWal != NULL); + + pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->pWal = pWal; + + pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + pSyncNode->pFsm->FpGetSnapshot = GetSnapshotCb; +} + +void cleanup() { + walClose(pWal); + walCleanUp(); + taosMemoryFree(pSyncNode); +} + +void test1() { + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest1 ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + + logStoreDestory(pLogStore); + cleanup(); + + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest1 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test2() { + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pLogStore->syncLogSetBeginIndex(pLogStore, 5); + logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + + logStoreDestory(pLogStore); + cleanup(); + + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest2 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test3() { + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest3 ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + + for (int i = 0; i <= 4; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test3 after appendEntry", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == 4); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == 4); + assert(pLogStore->syncLogLastTerm(pLogStore) == 104); + } + + logStoreDestory(pLogStore); + cleanup(); + + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest3 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == 4); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == 4); + assert(pLogStore->syncLogLastTerm(pLogStore) == 104); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test4() { + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); + pLogStore->syncLogSetBeginIndex(pLogStore, 5); + + for (int i = 5; i <= 9; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test4 after appendEntry", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 9); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == 9); + assert(pLogStore->syncLogLastTerm(pLogStore) == 109); + } + + logStoreDestory(pLogStore); + cleanup(); + + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest4 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 9); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == 9); + assert(pLogStore->syncLogLastTerm(pLogStore) == 109); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test5() { + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); + pLogStore->syncLogSetBeginIndex(pLogStore, 5); + + for (int i = 5; i <= 9; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test5 after appendEntry", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 9); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == 9); + assert(pLogStore->syncLogLastTerm(pLogStore) == 109); + } + + pLogStore->syncLogTruncate(pLogStore, 7); + logStoreLog2((char*)"after truncate 7", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 6); + assert(pLogStore->syncLogEntryCount(pLogStore) == 2); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 7); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == 6); + assert(pLogStore->syncLogLastTerm(pLogStore) == 106); + } + + logStoreDestory(pLogStore); + cleanup(); + + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest5 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 6); + assert(pLogStore->syncLogEntryCount(pLogStore) == 2); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 7); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == 6); + assert(pLogStore->syncLogLastTerm(pLogStore) == 106); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test6() { + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore); + pLogStore->syncLogSetBeginIndex(pLogStore, 5); + + for (int i = 5; i <= 9; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test6 after appendEntry", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 9); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == 9); + assert(pLogStore->syncLogLastTerm(pLogStore) == 109); + } + + pLogStore->syncLogTruncate(pLogStore, 5); + logStoreLog2((char*)"after truncate 5", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + + logStoreDestory(pLogStore); + cleanup(); + + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +int main(int argc, char** argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + + if (argc == 2) { + gAssert = atoi(argv[1]); + } + sTrace("gAssert : %d", gAssert); + + test1(); + test2(); + test3(); + test4(); + test5(); + test6(); + + return 0; +} diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp new file mode 100644 index 0000000000..b47f8c96c5 --- /dev/null +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -0,0 +1,388 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +bool gAssert = true; + +SSyncNode* pSyncNode; +SWal* pWal; +SSyncLogStore* pLogStore; +const char* pWalPath = "./syncLogStoreTest_wal"; + +SyncIndex gSnapshotLastApplyIndex; +SyncIndex gSnapshotLastApplyTerm; + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; + return 0; +} + +void init() { + walInit(); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pWalPath, &walCfg); + assert(pWal != NULL); + + pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->pWal = pWal; + + pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + pSyncNode->pFsm->FpGetSnapshot = GetSnapshotCb; +} + +void cleanup() { + walClose(pWal); + walCleanUp(); + taosMemoryFree(pSyncNode); +} + +void test1() { + // no snapshot + // no log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest1 ----- ", pLogStore); + + gSnapshotLastApplyIndex = -1; + gSnapshotLastApplyTerm = 0; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + SyncIndex testIndex = 0; + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, testIndex); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, testIndex); + + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + + sTrace("test1"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); + sTrace("%ld's preIndex: %ld", testIndex, preIndex); + sTrace("%ld's preTerm: %lu", testIndex, preTerm); + + if (gAssert) { + assert(lastIndex == -1); + assert(lastTerm == 0); + assert(syncStartIndex == 0); + assert(preIndex == -1); + assert(preTerm == 0); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test2() { + // no snapshot + // whole log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); + + for (int i = 0; i <= 10; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test2 after appendEntry", pLogStore); + + gSnapshotLastApplyIndex = -1; + gSnapshotLastApplyTerm = 0; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + + sTrace("test2"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); + + if (gAssert) { + assert(lastIndex == 10); + assert(lastTerm == 110); + assert(syncStartIndex == 11); + } + + for (SyncIndex i = 11; i >= 0; --i) { + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); + + sTrace("%ld's preIndex: %ld", i, preIndex); + sTrace("%ld's preTerm: %lu", i, preTerm); + + if (gAssert) { + SyncIndex preIndexArr[12] = {-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + SyncTerm preTermArr[12] = {0, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110}; + + assert(preIndex == preIndexArr[i]); + assert(preTerm == preTermArr[i]); + } + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test3() { + // has snapshot + // no log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest3 ----- ", pLogStore); + + gSnapshotLastApplyIndex = 5; + gSnapshotLastApplyTerm = 100; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, 6); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, 6); + + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + + sTrace("test3"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); + sTrace("%d's preIndex: %ld", 6, preIndex); + sTrace("%d's preTerm: %lu", 6, preTerm); + + if (gAssert) { + assert(lastIndex == 5); + assert(lastTerm == 100); + assert(syncStartIndex == 6); + assert(preIndex == 5); + assert(preTerm == 100); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test4() { + // has snapshot + // whole log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); + + for (int i = 0; i <= 10; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test4 after appendEntry", pLogStore); + + gSnapshotLastApplyIndex = 5; + gSnapshotLastApplyTerm = 100; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + + sTrace("test4"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); + + if (gAssert) { + assert(lastIndex == 10); + assert(lastTerm == 110); + assert(syncStartIndex == 11); + } + + for (SyncIndex i = 11; i >= 6; --i) { + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); + + sTrace("%ld's preIndex: %ld", i, preIndex); + sTrace("%ld's preTerm: %lu", i, preTerm); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test5() { + // has snapshot + // partial log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); + + pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); + for (int i = 6; i <= 10; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test5 after appendEntry", pLogStore); + + gSnapshotLastApplyIndex = 5; + gSnapshotLastApplyTerm = 100; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + + sTrace("test5"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); + + for (SyncIndex i = 11; i >= 6; --i) { + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); + + sTrace("%ld's preIndex: %ld", i, preIndex); + sTrace("%ld's preTerm: %lu", i, preTerm); + + if (gAssert) { + SyncIndex preIndexArr[12] = {9999, 9999, 9999, 9999, 9999, 9999, 5, 6, 7, 8, 9, 10}; + SyncTerm preTermArr[12] = {9999, 9999, 9999, 9999, 9999, 9999, 100, 106, 107, 108, 109, 110}; + + assert(preIndex == preIndexArr[i]); + assert(preTerm == preTermArr[i]); + } + } + + logStoreDestory(pLogStore); + cleanup(); +} + +int main(int argc, char** argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + + if (argc == 2) { + gAssert = atoi(argv[1]); + } + sTrace("gAssert : %d", gAssert); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp new file mode 100644 index 0000000000..69670f09a6 --- /dev/null +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -0,0 +1,63 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftStore.h" +#include "syncSnapshot.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} + +void RestoreFinishCb(struct SSyncFSM* pFsm) {} +void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {} + +int32_t GetSnapshot(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { return 0; } + +int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) { return 0; } +int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { return 0; } +int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { return 0; } + +int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { return 0; } +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { return 0; } +int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { return 0; } + +SSyncSnapshotReceiver* createReceiver() { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); + pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); + pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); + pSyncNode->pFsm->FpSnapshotStartWrite = SnapshotStartWrite; + pSyncNode->pFsm->FpSnapshotStopWrite = SnapshotStopWrite; + pSyncNode->pFsm->FpSnapshotDoWrite = SnapshotDoWrite; + + SSyncSnapshotReceiver* pReceiver = snapshotReceiverCreate(pSyncNode, 2); + pReceiver->start = true; + pReceiver->ack = 20; + pReceiver->pWriter = (void*)0x11; + pReceiver->term = 66; + pReceiver->privateTerm = 99; + + return pReceiver; +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + SSyncSnapshotReceiver* pReceiver = createReceiver(); + sTrace("%s", snapshotReceiver2Str(pReceiver)); + + return 0; +} diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp new file mode 100644 index 0000000000..f689d47aaf --- /dev/null +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -0,0 +1,101 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncSnapshotRsp *createMsg() { + SyncSnapshotRsp *pMsg = syncSnapshotRspBuild(1000); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 11; + pMsg->privateTerm = 99; + pMsg->lastIndex = 22; + pMsg->lastTerm = 33; + pMsg->ack = 44; + pMsg->code = 55; + return pMsg; +} + +void test1() { + SyncSnapshotRsp *pMsg = createMsg(); + syncSnapshotRspLog2((char *)"test1:", pMsg); + syncSnapshotRspDestroy(pMsg); +} + +void test2() { + SyncSnapshotRsp *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncSnapshotRspSerialize(pMsg, serialized, len); + SyncSnapshotRsp *pMsg2 = syncSnapshotRspBuild(1000); + syncSnapshotRspDeserialize(serialized, len, pMsg2); + syncSnapshotRspLog2((char *)"test2: syncSnapshotRspSerialize -> syncSnapshotRspDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncSnapshotRspDestroy(pMsg); + syncSnapshotRspDestroy(pMsg2); +} + +void test3() { + SyncSnapshotRsp *pMsg = createMsg(); + uint32_t len; + char * serialized = syncSnapshotRspSerialize2(pMsg, &len); + SyncSnapshotRsp *pMsg2 = syncSnapshotRspDeserialize2(serialized, len); + syncSnapshotRspLog2((char *)"test3: syncSnapshotRspSerialize2 -> syncSnapshotRspDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncSnapshotRspDestroy(pMsg); + syncSnapshotRspDestroy(pMsg2); +} + +void test4() { + SyncSnapshotRsp *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pMsg, &rpcMsg); + SyncSnapshotRsp *pMsg2 = (SyncSnapshotRsp *)taosMemoryMalloc(rpcMsg.contLen); + syncSnapshotRspFromRpcMsg(&rpcMsg, pMsg2); + syncSnapshotRspLog2((char *)"test4: syncSnapshotRsp2RpcMsg -> syncSnapshotRspFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncSnapshotRspDestroy(pMsg); + syncSnapshotRspDestroy(pMsg2); +} + +void test5() { + SyncSnapshotRsp *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pMsg, &rpcMsg); + SyncSnapshotRsp *pMsg2 = syncSnapshotRspFromRpcMsg2(&rpcMsg); + syncSnapshotRspLog2((char *)"test5: syncSnapshotRsp2RpcMsg -> syncSnapshotRspFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncSnapshotRspDestroy(pMsg); + syncSnapshotRspDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp new file mode 100644 index 0000000000..01d3264693 --- /dev/null +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -0,0 +1,101 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncSnapshotSend *createMsg() { + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(20, 1000); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 11; + pMsg->privateTerm = 99; + pMsg->lastIndex = 22; + pMsg->lastTerm = 33; + pMsg->seq = 44; + strcpy(pMsg->data, "hello world"); + return pMsg; +} + +void test1() { + SyncSnapshotSend *pMsg = createMsg(); + syncSnapshotSendLog2((char *)"test1:", pMsg); + syncSnapshotSendDestroy(pMsg); +} + +void test2() { + SyncSnapshotSend *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncSnapshotSendSerialize(pMsg, serialized, len); + SyncSnapshotSend *pMsg2 = syncSnapshotSendBuild(pMsg->dataLen, 1000); + syncSnapshotSendDeserialize(serialized, len, pMsg2); + syncSnapshotSendLog2((char *)"test2: syncSnapshotSendSerialize -> syncSnapshotSendDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncSnapshotSendDestroy(pMsg); + syncSnapshotSendDestroy(pMsg2); +} + +void test3() { + SyncSnapshotSend *pMsg = createMsg(); + uint32_t len; + char * serialized = syncSnapshotSendSerialize2(pMsg, &len); + SyncSnapshotSend *pMsg2 = syncSnapshotSendDeserialize2(serialized, len); + syncSnapshotSendLog2((char *)"test3: syncSnapshotSendSerialize2 -> syncSnapshotSendDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncSnapshotSendDestroy(pMsg); + syncSnapshotSendDestroy(pMsg2); +} + +void test4() { + SyncSnapshotSend *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + SyncSnapshotSend *pMsg2 = (SyncSnapshotSend *)taosMemoryMalloc(rpcMsg.contLen); + syncSnapshotSendFromRpcMsg(&rpcMsg, pMsg2); + syncSnapshotSendLog2((char *)"test4: syncSnapshotSend2RpcMsg -> syncSnapshotSendFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncSnapshotSendDestroy(pMsg); + syncSnapshotSendDestroy(pMsg2); +} + +void test5() { + SyncSnapshotSend *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + SyncSnapshotSend *pMsg2 = syncSnapshotSendFromRpcMsg2(&rpcMsg); + syncSnapshotSendLog2((char *)"test5: syncSnapshotSend2RpcMsg -> syncSnapshotSendFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncSnapshotSendDestroy(pMsg); + syncSnapshotSendDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp new file mode 100644 index 0000000000..404ba2acae --- /dev/null +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -0,0 +1,72 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftStore.h" +#include "syncSnapshot.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} + +void RestoreFinishCb(struct SSyncFSM* pFsm) {} +void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {} + +int32_t GetSnapshot(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { return 0; } + +int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) { return 0; } +int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { return 0; } +int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { return 0; } + +int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { return 0; } +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { return 0; } +int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { return 0; } + +SSyncSnapshotSender* createSender() { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); + pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); + pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); + pSyncNode->pFsm->FpSnapshotStartRead = SnapshotStartRead; + pSyncNode->pFsm->FpSnapshotStopRead = SnapshotStopRead; + pSyncNode->pFsm->FpSnapshotDoRead = SnapshotDoRead; + pSyncNode->pFsm->FpGetSnapshot = GetSnapshot; + + SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, 2); + pSender->start = true; + pSender->seq = 10; + pSender->ack = 20; + pSender->pReader = (void*)0x11; + pSender->blockLen = 20; + pSender->pCurrentBlock = taosMemoryMalloc(pSender->blockLen); + snprintf((char*)(pSender->pCurrentBlock), pSender->blockLen, "%s", "hello"); + + pSender->snapshot.lastApplyIndex = 99; + pSender->snapshot.lastApplyTerm = 88; + pSender->sendingMS = 77; + pSender->term = 66; + pSender->privateTerm = 99; + + return pSender; +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + SSyncSnapshotSender* pSender = createSender(); + sTrace("%s", snapshotSender2Str(pSender)); + + return 0; +} diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index ffe8b81571..97de81572a 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -50,14 +50,16 @@ void test4() { } int main(int argc, char** argv) { - // taosInitLog("tmp/syncTest.log", 100); + taosInitLog("/tmp/syncTest.log", 100); tsAsyncLog = 0; + sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; test1(); test2(); test3(); test4(); + /* if (argc == 2) { bool bTaosDirExist = taosDirExist(argv[1]); printf("%s bTaosDirExist:%d \n", argv[1], bTaosDirExist); @@ -65,7 +67,8 @@ int main(int argc, char** argv) { bool bTaosCheckExistFile = taosCheckExistFile(argv[1]); printf("%s bTaosCheckExistFile:%d \n", argv[1], bTaosCheckExistFile); } + */ - // taosCloseLog(); + taosCloseLog(); return 0; } diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp new file mode 100644 index 0000000000..782baf3c97 --- /dev/null +++ b/source/libs/sync/test/syncTestTool.cpp @@ -0,0 +1,399 @@ +#include +#include +#include "os.h" +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftCfg.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t gPorts[] = {7000, 7001, 7002, 7003, 7004}; +const char* gDir = "./syncTestTool"; +int32_t gVgId = 1234; +SyncIndex gSnapshotLastApplyIndex; +SyncIndex gSnapshotLastApplyTerm; +int gIterTimes = 0; + +SyncIndex gFinishLastApplyIndex; +SyncIndex gFinishLastApplyTerm; + +void init() { + int code = walInit(); + assert(code == 0); + + code = syncInit(); + assert(code == 0); +} + +void cleanup() { walCleanUp(); } + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " + "currentTerm:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " + "currentTerm:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " + "currentTerm:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; + return 0; +} + +int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) { + *ppReader = (void*)0xABCD; + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartRead== pFsm:%p, *ppReader:%p", pFsm, *ppReader); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopRead== pFsm:%p, pReader:%p", pFsm, pReader); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { + static int readIter = 0; + + if (readIter == gIterTimes) { + *len = 0; + *ppBuf = NULL; + } else if (readIter < gIterTimes) { + *len = 20; + *ppBuf = taosMemoryMalloc(*len); + snprintf((char*)*ppBuf, *len, "data iter:%d", readIter); + } + + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==SnapshotDoRead== pFsm:%p, pReader:%p, *len:%d, *ppBuf:[%s], readIter:%d", pFsm, pReader, + *len, (char*)(*ppBuf), readIter); + sTrace("%s", logBuf); + + readIter++; + return 0; +} + +int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { + *ppWriter = (void*)0xCDEF; + char logBuf[256] = {0}; + + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartWrite== pFsm:%p, *ppWriter:%p", pFsm, *ppWriter); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { + if (isApply) { + gSnapshotLastApplyIndex = gFinishLastApplyIndex; + gSnapshotLastApplyTerm = gFinishLastApplyTerm; + } + + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d, gSnapshotLastApplyIndex:%ld, " + "gSnapshotLastApplyTerm:%ld", + pFsm, pWriter, isApply, gSnapshotLastApplyIndex, gSnapshotLastApplyTerm); + sTrace("%s", logBuf); + + return 0; +} + +int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotDoWrite== pFsm:%p, pWriter:%p, len:%d pBuf:[%s]", pFsm, + pWriter, len, (char*)pBuf); + sTrace("%s", logBuf); + return 0; +} + +void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb== pFsm:%p", pFsm); } + +void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { + char* s = syncCfg2Str(&newCfg); + sTrace("==callback== ==ReConfigCb== flag:0x%lX, isDrop:%d, index:%ld, code:%d, currentTerm:%lu, term:%lu, newCfg:%s", + cbMeta.flag, cbMeta.isDrop, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term, s); + taosMemoryFree(s); +} + +SSyncFSM* createFsm() { + SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + memset(pFsm, 0, sizeof(*pFsm)); + + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; + + pFsm->FpReConfigCb = ReConfigCb; + pFsm->FpGetSnapshot = GetSnapshotCb; + pFsm->FpRestoreFinishCb = RestoreFinishCb; + + pFsm->FpSnapshotStartRead = SnapshotStartRead; + pFsm->FpSnapshotStopRead = SnapshotStopRead; + pFsm->FpSnapshotDoRead = SnapshotDoRead; + pFsm->FpSnapshotStartWrite = SnapshotStartWrite; + pFsm->FpSnapshotStopWrite = SnapshotStopWrite; + pFsm->FpSnapshotDoWrite = SnapshotDoWrite; + + return pFsm; +} + +SWal* createWal(char* path, int32_t vgId) { + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal* pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + return pWal; +} + +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy, + bool enableSnapshot) { + SSyncInfo syncInfo; + syncInfo.vgId = vgId; + syncInfo.msgcb = &gSyncIO->msgcb; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = createFsm(); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); + syncInfo.pWal = pWal; + syncInfo.isStandBy = isStandBy; + syncInfo.snapshotEnable = enableSnapshot; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + +#if 0 + { + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } + } +#endif + + if (isStandBy) { + pCfg->myIndex = 0; + pCfg->replicaNum = 1; + pCfg->nodeInfo[0].nodePort = gPorts[myIndex]; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + } else { + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } + } + + + int64_t rid = syncOpen(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + + gSyncIO->FpOnSyncSnapshotSend = pSyncNode->FpOnSnapshotSend; + gSyncIO->FpOnSyncSnapshotRsp = pSyncNode->FpOnSnapshotRsp; + + gSyncIO->pSyncNode = pSyncNode; + syncNodeRelease(pSyncNode); + + return rid; +} + +void configChange(int64_t rid, int32_t newReplicaNum, int32_t myIndex) { + SSyncCfg syncCfg; + + syncCfg.myIndex = myIndex; + syncCfg.replicaNum = newReplicaNum; + + for (int i = 0; i < newReplicaNum; ++i) { + syncCfg.nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn); + } + + syncReconfig(rid, &syncCfg); +} + +void usage(char* exe) { + printf( + "usage: %s replicaNum(1-5) myIndex(0-..) enableSnapshot(0/1) lastApplyIndex(>=-1) lastApplyTerm(>=0) " + "writeRecordNum(>=0) " + "isStandBy(0/1) isConfigChange(0-5) iterTimes(>=0) finishLastApplyIndex(>=-1) finishLastApplyTerm(>=0) \n", + exe); +} + +SRpcMsg* createRpcMsg(int i, int count, int myIndex) { + SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 256; + pMsg->pCont = rpcMallocCont(pMsg->contLen); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + return pMsg; +} + +int main(int argc, char** argv) { + sprintf(tsTempDir, "%s", "."); + tsAsyncLog = 0; + sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; + + if (argc != 12) { + usage(argv[0]); + exit(-1); + } + + int32_t replicaNum = atoi(argv[1]); + int32_t myIndex = atoi(argv[2]); + bool enableSnapshot = atoi(argv[3]); + int32_t lastApplyIndex = atoi(argv[4]); + int32_t lastApplyTerm = atoi(argv[5]); + int32_t writeRecordNum = atoi(argv[6]); + bool isStandBy = atoi(argv[7]); + int32_t isConfigChange = atoi(argv[8]); + int32_t iterTimes = atoi(argv[9]); + int32_t finishLastApplyIndex = atoi(argv[10]); + int32_t finishLastApplyTerm = atoi(argv[11]); + + sTrace( + "args: replicaNum:%d, myIndex:%d, enableSnapshot:%d, lastApplyIndex:%d, lastApplyTerm:%d, writeRecordNum:%d, " + "isStandBy:%d, isConfigChange:%d, iterTimes:%d, finishLastApplyIndex:%d, finishLastApplyTerm:%d", + replicaNum, myIndex, enableSnapshot, lastApplyIndex, lastApplyTerm, writeRecordNum, isStandBy, isConfigChange, + iterTimes, finishLastApplyIndex, finishLastApplyTerm); + + // check parameter + assert(replicaNum >= 1 && replicaNum <= 5); + // assert(myIndex >= 0 && myIndex < replicaNum); + assert(lastApplyIndex >= -1); + assert(lastApplyTerm >= 0); + assert(writeRecordNum >= 0); + assert(isConfigChange >= 0 && isConfigChange <= 5); + assert(iterTimes >= 0); + assert(finishLastApplyIndex >= -1); + assert(finishLastApplyTerm >= 0); + + char logFile[256]; + snprintf(logFile, sizeof(logFile), "/tmp/%s-replicaNum%d-myIndex%d.log", gDir, replicaNum, myIndex); + taosInitLog(logFile, 100); + sTrace("logFile : %s", logFile); + + gSnapshotLastApplyIndex = lastApplyIndex; + gSnapshotLastApplyTerm = lastApplyTerm; + gIterTimes = iterTimes; + + gFinishLastApplyIndex = finishLastApplyIndex; + gFinishLastApplyTerm = finishLastApplyTerm; + + init(); + int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); + assert(ret == 0); + + char walPath[128]; + snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); + SWal* pWal = createWal(walPath, gVgId); + + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy, enableSnapshot); + assert(rid > 0); + syncStart(rid); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + if (isConfigChange > 0) { + configChange(rid, isConfigChange, myIndex); + } + + //--------------------------- + int32_t alreadySend = 0; + while (1) { + char* simpleStr = syncNode2SimpleStr(pSyncNode); + + if (alreadySend < writeRecordNum) { + SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); + int32_t ret = syncPropose(rid, pRpcMsg, false); + if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + sTrace("%s value%d write not leader", simpleStr, alreadySend); + } else { + assert(ret == 0); + sTrace("%s value%d write ok", simpleStr, alreadySend); + } + alreadySend++; + + rpcFreeCont(pRpcMsg->pCont); + taosMemoryFree(pRpcMsg); + } else { + sTrace("%s", simpleStr); + } + + taosMsleep(1000); + taosMemoryFree(simpleStr); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + syncStop(rid); + walClose(pWal); + syncIOStop(); + cleanup(); + taosCloseLog(); + return 0; +} diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index 30f25bd1d8..e60fabe38b 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -78,6 +78,26 @@ void test5() { syncTimeoutDestroy(pMsg2); } +void test6() { + SyncTimeout *pMsg = createMsg(); + char * jsonStr = syncTimeout2Str(pMsg); + sTrace("jsonStr: %s", jsonStr); + + syncUtilJson2Line(jsonStr); + sTrace("jsonStr: %s", jsonStr); + + char str[10]; + snprintf(str, sizeof(str), "%s", "{}"); + sTrace("str: %s", str); + syncUtilJson2Line(str); + sTrace("str: %s", str); + + snprintf(str, sizeof(str), "%s", ""); + sTrace("str: %s", str); + syncUtilJson2Line(str); + sTrace("str: %s", str); +} + int main() { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; @@ -88,6 +108,7 @@ int main() { test3(); test4(); test5(); + test6(); return 0; } diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 84fe2814ff..7ca105ff2b 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -132,6 +132,7 @@ static inline void walResetVer(SWalVer* pVer) { int walLoadMeta(SWal* pWal); int walSaveMeta(SWal* pWal); +int walRemoveMeta(SWal* pWal); int walRollFileInfo(SWal* pWal); int walCheckAndRepairMeta(SWal* pWal); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 9aa848a7bb..8e9cb3a84b 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -419,3 +419,12 @@ int walLoadMeta(SWal* pWal) { taosMemoryFree(buf); return code; } + +int walRemoveMeta(SWal* pWal) { + int metaVer = walFindCurMetaVer(pWal); + if (metaVer == -1) return 0; + char fnameStr[WAL_FILE_LEN]; + walBuildMetaName(pWal, metaVer, fnameStr); + taosRemoveFile(fnameStr); + return 0; +} diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 71cd6de73f..9505b02806 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -75,7 +75,7 @@ void walCleanUp() { } SWal *walOpen(const char *path, SWalCfg *pCfg) { - SWal *pWal = taosMemoryMalloc(sizeof(SWal)); + SWal *pWal = taosMemoryCalloc(1, sizeof(SWal)); if (pWal == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -92,6 +92,13 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } + // init ref + pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (pWal->pRefHash == NULL) { + taosMemoryFree(pWal); + return NULL; + } + // open meta walResetVer(&pWal->vers); pWal->pWriteLogTFile = NULL; @@ -100,6 +107,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); + taosHashCleanup(pWal->pRefHash); taosMemoryFree(pWal); return NULL; } @@ -115,12 +123,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); + taosHashCleanup(pWal->pRefHash); taosMemoryFree(pWal); return NULL; } pWal->refId = taosAddRef(tsWal.refSetId, pWal); if (pWal->refId < 0) { + taosHashCleanup(pWal->pRefHash); taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); taosMemoryFree(pWal); @@ -130,6 +140,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { + taosHashCleanup(pWal->pRefHash); taosRemoveRef(tsWal.refSetId, pWal->refId); taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); @@ -175,6 +186,7 @@ void walClose(SWal *pWal) { walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; + taosHashCleanup(pWal->pRefHash); taosThreadMutexUnlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 793ab8b2fb..d30e0b6844 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -18,12 +18,47 @@ #include "tchecksum.h" #include "walInt.h" -void walRestoreFromSnapshot(SWal *pWal, int64_t ver) { - /*pWal->vers.firstVer = -1;*/ +int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { + taosThreadMutexLock(&pWal->mutex); + + void *pIter = NULL; + while (1) { + taosHashIterate(pWal->pRefHash, pIter); + if (pIter == NULL) break; + SWalRef *pRef = (SWalRef *)pIter; + if (pRef->ver != -1) { + taosHashCancelIterate(pWal->pRefHash, pIter); + return -1; + } + } + + taosCloseFile(&pWal->pWriteLogTFile); + taosCloseFile(&pWal->pWriteIdxTFile); + + if (pWal->vers.firstVer != -1) { + int32_t fileSetSize = taosArrayGetSize(pWal->fileInfoSet); + for (int32_t i = 0; i < fileSetSize; i++) { + SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + char fnameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); + taosRemoveFile(fnameStr); + } + } + walRemoveMeta(pWal); + + pWal->writeCur = -1; + pWal->totSize = 0; + pWal->lastRollSeq = -1; + + taosArrayClear(pWal->fileInfoSet); + pWal->vers.firstVer = -1; pWal->vers.lastVer = ver; pWal->vers.commitVer = ver - 1; pWal->vers.snapshotVer = ver - 1; pWal->vers.verInSnapshotting = -1; + + taosThreadMutexUnlock(&pWal->mutex); + return 0; } int32_t walCommit(SWal *pWal, int64_t ver) { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 28bc98a972..5a8cf562a0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -22,6 +22,7 @@ # ---- dnode ./test.sh -f tsim/dnode/create_dnode.sim +./test.sh -f tsim/dnode/drop_dnode_mnode.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim @@ -56,7 +57,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim -./test.sh -f tsim/mnode/basic2.sim +#./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim diff --git a/tests/script/tsim/dnode/drop_dnode_mnode.sim b/tests/script/tsim/dnode/drop_dnode_mnode.sim new file mode 100644 index 0000000000..e0a85b9803 --- /dev/null +++ b/tests/script/tsim/dnode/drop_dnode_mnode.sim @@ -0,0 +1,52 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +sql create dnode $hostname port 7300 +sql drop dnode 3 +sql_error drop dnode 1 + +print =============== step2: create mnode +sql create mnode on dnode 2 + +print =============== step3: drop dnode 3 +sql drop dnode 2 +sql show dnodes; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +return +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index bc70cd7a85..dec036faaf 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -39,8 +39,11 @@ endi print =============== step2: create mnode 2 sql create mnode on dnode 2 sql create mnode on dnode 3 +return +system sh/exec.sh -n dnode1 -s stop -x SIGKILL sql_error create mnode on dnode 4 + $x = 0 step2: $x = $x + 1 @@ -147,4 +150,4 @@ endi system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop system sh/exec.sh -n dnode3 -s stop -system sh/exec.sh -n dnode4 -s stop \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop